From 05b79fd2e0e519f1fa5d67ed70a574b80736cb06 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Mon, 6 May 2013 16:47:37 -0400 Subject: [PATCH] split rt provider into two --- .../HttpClientManager/HttpClientManager.cs | 31 ++- .../HttpClientManager/HttpResponseInfo.cs | 6 + .../MediaBrowser.Controller.csproj | 1 + .../Movies/RottenTomatoesMovieProvider.cs | 95 +------ .../RottenTomatoesMovieReviewsProvider.cs | 236 ++++++++++++++++++ Nuget/MediaBrowser.Common.Internal.nuspec | 6 +- Nuget/MediaBrowser.Common.nuspec | 2 +- Nuget/MediaBrowser.Server.Core.nuspec | 4 +- 8 files changed, 285 insertions(+), 96 deletions(-) create mode 100644 MediaBrowser.Controller/Providers/Movies/RottenTomatoesMovieReviewsProvider.cs diff --git a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs index ca19e88442..23433ac75b 100644 --- a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs +++ b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs @@ -134,11 +134,15 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager if (cachedInfo != null) { - var isCacheValid = (!cachedInfo.MustRevalidate && !string.IsNullOrEmpty(cachedInfo.Etag)) - || (cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > DateTime.UtcNow); + var now = DateTime.UtcNow; + + var isCacheValid = (!cachedInfo.MustRevalidate && !string.IsNullOrEmpty(cachedInfo.Etag) && (now - cachedInfo.RequestDate).TotalDays < 14) + || (cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > now); if (isCacheValid) { + _logger.Debug("Cache is still valid for {0}", options.Url); + try { return GetCachedResponse(cachedReponsePath); @@ -180,26 +184,38 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager var response = await GetHttpClient(GetHostFromUrl(options.Url)).SendAsync(message, HttpCompletionOption.ResponseHeadersRead, options.CancellationToken).ConfigureAwait(false); - EnsureSuccessStatusCode(response); - - options.CancellationToken.ThrowIfCancellationRequested(); - if (options.EnableResponseCache) { + if (response.StatusCode != HttpStatusCode.NotModified) + { + EnsureSuccessStatusCode(response); + } + + options.CancellationToken.ThrowIfCancellationRequested(); + cachedInfo = UpdateInfoCache(cachedInfo, options.Url, cachedInfoPath, response); if (response.StatusCode == HttpStatusCode.NotModified) { + _logger.Debug("Server indicates not modified for {0}. Returning cached result.", options.Url); + return GetCachedResponse(cachedReponsePath); } - if (!string.IsNullOrEmpty(cachedInfo.Etag) || cachedInfo.LastModified.HasValue || (cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > DateTime.UtcNow)) + if (!string.IsNullOrEmpty(cachedInfo.Etag) || cachedInfo.LastModified.HasValue || + (cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > DateTime.UtcNow)) { await UpdateResponseCache(response, cachedReponsePath).ConfigureAwait(false); return GetCachedResponse(cachedReponsePath); } } + else + { + EnsureSuccessStatusCode(response); + + options.CancellationToken.ThrowIfCancellationRequested(); + } return await response.Content.ReadAsStreamAsync().ConfigureAwait(false); } @@ -284,6 +300,7 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager } cachedInfo.Url = url; + cachedInfo.RequestDate = DateTime.UtcNow; var etag = response.Headers.ETag; if (etag != null) diff --git a/MediaBrowser.Common.Implementations/HttpClientManager/HttpResponseInfo.cs b/MediaBrowser.Common.Implementations/HttpClientManager/HttpResponseInfo.cs index 240e99d796..4a4612ffb5 100644 --- a/MediaBrowser.Common.Implementations/HttpClientManager/HttpResponseInfo.cs +++ b/MediaBrowser.Common.Implementations/HttpClientManager/HttpResponseInfo.cs @@ -36,5 +36,11 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager /// /// true if [must revalidate]; otherwise, false. public bool MustRevalidate { get; set; } + + /// + /// Gets or sets the request date. + /// + /// The request date. + public DateTime RequestDate { get; set; } } } diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index afed9fa0b7..cb7f635adf 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -117,6 +117,7 @@ + diff --git a/MediaBrowser.Controller/Providers/Movies/RottenTomatoesMovieProvider.cs b/MediaBrowser.Controller/Providers/Movies/RottenTomatoesMovieProvider.cs index d306cc0a62..acf8c8da6a 100644 --- a/MediaBrowser.Controller/Providers/Movies/RottenTomatoesMovieProvider.cs +++ b/MediaBrowser.Controller/Providers/Movies/RottenTomatoesMovieProvider.cs @@ -23,18 +23,17 @@ namespace MediaBrowser.Controller.Providers.Movies /// /// The API key /// - private const string ApiKey = "x9wjnvv39ntjmt9zs95nm7bg"; + internal const string ApiKey = "x9wjnvv39ntjmt9zs95nm7bg"; - private const string BasicUrl = @"http://api.rottentomatoes.com/api/public/v1.0/"; - private const string Movie = @"movies/{1}.json?apikey={0}"; + internal const string BasicUrl = @"http://api.rottentomatoes.com/api/public/v1.0/"; private const string MovieImdb = @"movie_alias.json?id={1}&type=imdb&apikey={0}"; - private const string MovieSearch = @"movies.json?q={1}&apikey={0}&page_limit=20&page={2}"; - private const string MoviesReviews = @"movies/{1}/reviews.json?review_type=top_critic&page_limit=10&page=1&country=us&apikey={0}"; + + internal static RottenTomatoesMovieProvider Current { get; private set; } /// /// The _rotten tomatoes resource pool /// - private readonly SemaphoreSlim _rottenTomatoesResourcePool = new SemaphoreSlim(1, 1); + internal readonly SemaphoreSlim RottenTomatoesResourcePool = new SemaphoreSlim(1, 1); /// /// Gets the json serializer. @@ -60,6 +59,7 @@ namespace MediaBrowser.Controller.Providers.Movies { JsonSerializer = jsonSerializer; HttpClient = httpClient; + Current = this; } /// @@ -136,7 +136,7 @@ namespace MediaBrowser.Controller.Providers.Movies get { // Run after moviedb and xml providers - return MetadataProviderPriority.Last; + return MetadataProviderPriority.Third; } } @@ -183,60 +183,23 @@ namespace MediaBrowser.Controller.Providers.Movies return true; } - RTMovieSearchResult hit = null; - // Have IMDB Id using (var stream = await HttpClient.Get(new HttpRequestOptions { Url = GetMovieImdbUrl(imdbId), - ResourcePool = _rottenTomatoesResourcePool, + ResourcePool = RottenTomatoesResourcePool, CancellationToken = cancellationToken, EnableResponseCache = true }).ConfigureAwait(false)) { - var result = JsonSerializer.DeserializeFromStream(stream); + var hit = JsonSerializer.DeserializeFromStream(stream); - if (!string.IsNullOrEmpty(result.id)) + if (!string.IsNullOrEmpty(hit.id)) { // Got a result - hit = result; - } - } - - // If we found any results, that's great! - if (hit != null) - { - item.CriticRatingSummary = hit.critics_consensus; - item.CriticRating = float.Parse(hit.ratings.critics_score); - - using (var stream = await HttpClient.Get(new HttpRequestOptions - { - Url = GetMovieReviewsUrl(hit.id), - ResourcePool = _rottenTomatoesResourcePool, - CancellationToken = cancellationToken, - EnableResponseCache = true - - }).ConfigureAwait(false)) - { - - var result = JsonSerializer.DeserializeFromStream(stream); - - item.CriticReviews = result.reviews.Select(rtReview => new ItemReview - { - ReviewerName = rtReview.critic, - Publisher = rtReview.publication, - Date = DateTime.Parse(rtReview.date).ToUniversalTime(), - Caption = rtReview.quote, - Url = rtReview.links.review, - Likes = string.Equals(rtReview.freshness, "fresh", StringComparison.OrdinalIgnoreCase) - - }).ToList(); - - if (data == null) - { - data = new BaseProviderInfo(); - } + item.CriticRatingSummary = hit.critics_consensus; + item.CriticRating = float.Parse(hit.ratings.critics_score); data.Data = GetComparisonData(hit.alternate_ids.imdb); @@ -244,13 +207,6 @@ namespace MediaBrowser.Controller.Providers.Movies item.SetProviderId(MetadataProviders.RottenTomatoes, hit.id); } } - else - { - // Nothing found on RT - Logger.Info("Nothing found on RottenTomatoes for Movie \"{0}\"", item.Name); - - // TODO: When alternative names are implemented search for those instead - } data.Data = GetComparisonData(imdbId); data.LastRefreshStatus = ProviderRefreshStatus.Success; @@ -267,11 +223,6 @@ namespace MediaBrowser.Controller.Providers.Movies return BasicUrl + string.Format(MovieImdb, ApiKey, imdbId.TrimStart('t')); } - private string GetMovieReviewsUrl(string rtId) - { - return BasicUrl + string.Format(MoviesReviews, ApiKey, rtId); - } - // Data contract classes for use with the Rotten Tomatoes API protected class RTSearchResults @@ -312,27 +263,5 @@ namespace MediaBrowser.Controller.Providers.Movies { public string imdb { get; set; } } - - protected class RTReviewList - { - public int total { get; set; } - public List reviews { get; set; } - } - - protected class RTReview - { - public string critic { get; set; } - public string date { get; set; } - public string freshness { get; set; } - public string publication { get; set; } - public string quote { get; set; } - public RTReviewLink links { get; set; } - public string original_score { get; set; } - } - - protected class RTReviewLink - { - public string review { get; set; } - } } } \ No newline at end of file diff --git a/MediaBrowser.Controller/Providers/Movies/RottenTomatoesMovieReviewsProvider.cs b/MediaBrowser.Controller/Providers/Movies/RottenTomatoesMovieReviewsProvider.cs new file mode 100644 index 0000000000..3e2a5b558c --- /dev/null +++ b/MediaBrowser.Controller/Providers/Movies/RottenTomatoesMovieReviewsProvider.cs @@ -0,0 +1,236 @@ +using MediaBrowser.Common.Extensions; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.Movies; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Serialization; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Controller.Providers.Movies +{ + /// + /// Class RottenTomatoesMovieProvider + /// + public class RottenTomatoesMovieReviewsProvider : BaseMetadataProvider + { + // http://developer.rottentomatoes.com/iodocs + + private const string MoviesReviews = @"movies/{1}/reviews.json?review_type=top_critic&page_limit=10&page=1&country=us&apikey={0}"; + + /// + /// Gets the json serializer. + /// + /// The json serializer. + protected IJsonSerializer JsonSerializer { get; private set; } + + /// + /// Gets the HTTP client. + /// + /// The HTTP client. + protected IHttpClient HttpClient { get; private set; } + + /// + /// Initializes a new instance of the class. + /// + /// The log manager. + /// The configuration manager. + /// The json serializer. + /// The HTTP client. + public RottenTomatoesMovieReviewsProvider(ILogManager logManager, IServerConfigurationManager configurationManager, IJsonSerializer jsonSerializer, IHttpClient httpClient) + : base(logManager, configurationManager) + { + JsonSerializer = jsonSerializer; + HttpClient = httpClient; + } + + /// + /// Gets the provider version. + /// + /// The provider version. + protected override string ProviderVersion + { + get + { + return "5"; + } + } + + /// + /// Gets a value indicating whether [requires internet]. + /// + /// true if [requires internet]; otherwise, false. + public override bool RequiresInternet + { + get + { + return true; + } + } + + /// + /// Gets a value indicating whether [refresh on version change]. + /// + /// true if [refresh on version change]; otherwise, false. + protected override bool RefreshOnVersionChange + { + get + { + return true; + } + } + + /// + /// Supports the specified item. + /// + /// The item. + /// true if XXXX, false otherwise + public override bool Supports(BaseItem item) + { + return false; + var trailer = item as Trailer; + + if (trailer != null) + { + return !trailer.IsLocalTrailer; + } + + // Don't support local trailers + return item is Movie; + } + + /// + /// Gets the comparison data. + /// + /// The imdb id. + /// Guid. + private Guid GetComparisonData(string imdbId) + { + return string.IsNullOrEmpty(imdbId) ? Guid.Empty : imdbId.GetMD5(); + } + + /// + /// Gets the priority. + /// + /// The priority. + public override MetadataProviderPriority Priority + { + get + { + // Run after moviedb and xml providers + return MetadataProviderPriority.Last; + } + } + + /// + /// Needses the refresh internal. + /// + /// The item. + /// The provider info. + /// true if XXXX, false otherwise + protected override bool NeedsRefreshInternal(BaseItem item, BaseProviderInfo providerInfo) + { + // Refresh if rt id has changed + if (providerInfo.Data != GetComparisonData(item.GetProviderId(MetadataProviders.RottenTomatoes))) + { + return true; + } + + return base.NeedsRefreshInternal(item, providerInfo); + } + + /// + /// Fetches metadata and returns true or false indicating if any work that requires persistence was done + /// + /// The item. + /// if set to true [force]. + /// The cancellation token. + /// Task{System.Boolean}. + public override async Task FetchAsync(BaseItem item, bool force, CancellationToken cancellationToken) + { + BaseProviderInfo data; + + if (!item.ProviderData.TryGetValue(Id, out data)) + { + data = new BaseProviderInfo(); + item.ProviderData[Id] = data; + } + + var rottenTomatoesId = item.GetProviderId(MetadataProviders.RottenTomatoes); + + + if (string.IsNullOrEmpty(rottenTomatoesId)) + { + data.Data = GetComparisonData(rottenTomatoesId); + data.LastRefreshStatus = ProviderRefreshStatus.Success; + return true; + } + + using (var stream = await HttpClient.Get(new HttpRequestOptions + { + Url = GetMovieReviewsUrl(rottenTomatoesId), + ResourcePool = RottenTomatoesMovieProvider.Current.RottenTomatoesResourcePool, + CancellationToken = cancellationToken, + EnableResponseCache = true + + }).ConfigureAwait(false)) + { + + var result = JsonSerializer.DeserializeFromStream(stream); + + item.CriticReviews = result.reviews.Select(rtReview => new ItemReview + { + ReviewerName = rtReview.critic, + Publisher = rtReview.publication, + Date = DateTime.Parse(rtReview.date).ToUniversalTime(), + Caption = rtReview.quote, + Url = rtReview.links.review, + Likes = string.Equals(rtReview.freshness, "fresh", StringComparison.OrdinalIgnoreCase) + + }).ToList(); + } + + data.Data = GetComparisonData(rottenTomatoesId); + data.LastRefreshStatus = ProviderRefreshStatus.Success; + SetLastRefreshed(item, DateTime.UtcNow); + + return true; + } + + // Utility functions to get the URL of the API calls + + private string GetMovieReviewsUrl(string rtId) + { + return RottenTomatoesMovieProvider.BasicUrl + string.Format(MoviesReviews, RottenTomatoesMovieProvider.ApiKey, rtId); + } + + // Data contract classes for use with the Rotten Tomatoes API + + protected class RTReviewList + { + public int total { get; set; } + public List reviews { get; set; } + } + + protected class RTReview + { + public string critic { get; set; } + public string date { get; set; } + public string freshness { get; set; } + public string publication { get; set; } + public string quote { get; set; } + public RTReviewLink links { get; set; } + public string original_score { get; set; } + } + + protected class RTReviewLink + { + public string review { get; set; } + } + } +} \ No newline at end of file diff --git a/Nuget/MediaBrowser.Common.Internal.nuspec b/Nuget/MediaBrowser.Common.Internal.nuspec index b15974fe6f..b1688a11dc 100644 --- a/Nuget/MediaBrowser.Common.Internal.nuspec +++ b/Nuget/MediaBrowser.Common.Internal.nuspec @@ -2,7 +2,7 @@ MediaBrowser.Common.Internal - 3.0.91 + 3.0.92 MediaBrowser.Common.Internal Luke ebr,Luke,scottisafool @@ -12,9 +12,9 @@ Contains common components shared by Media Browser Theatre and Media Browser Server. Not intended for plugin developer consumption. Copyright © Media Browser 2013 - + - + diff --git a/Nuget/MediaBrowser.Common.nuspec b/Nuget/MediaBrowser.Common.nuspec index 97210cd627..acfb08736f 100644 --- a/Nuget/MediaBrowser.Common.nuspec +++ b/Nuget/MediaBrowser.Common.nuspec @@ -2,7 +2,7 @@ MediaBrowser.Common - 3.0.91 + 3.0.92 MediaBrowser.Common Media Browser Team ebr,Luke,scottisafool diff --git a/Nuget/MediaBrowser.Server.Core.nuspec b/Nuget/MediaBrowser.Server.Core.nuspec index 0bd9c18cf5..11c79207a9 100644 --- a/Nuget/MediaBrowser.Server.Core.nuspec +++ b/Nuget/MediaBrowser.Server.Core.nuspec @@ -2,7 +2,7 @@ MediaBrowser.Server.Core - 3.0.91 + 3.0.92 Media Browser.Server.Core Media Browser Team ebr,Luke,scottisafool @@ -12,7 +12,7 @@ Contains core components required to build plugins for Media Browser Server. Copyright © Media Browser 2013 - +