using MediaBrowser.Common.IO; using MediaBrowser.Common.Kernel; using MediaBrowser.Model.Net; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Cache; using System.Net.Http; using System.Text; using System.Threading; using System.Threading.Tasks; namespace MediaBrowser.Common.Net { /// /// Class HttpManager /// public class HttpManager : BaseManager { /// /// Initializes a new instance of the class. /// /// The kernel. public HttpManager(IKernel kernel) : base(kernel) { } /// /// Holds a dictionary of http clients by host. Use GetHttpClient(host) to retrieve or create a client for web requests. /// DON'T dispose it after use. /// /// The HTTP clients. private readonly ConcurrentDictionary _httpClients = new ConcurrentDictionary(); /// /// Gets /// /// The host. /// HttpClient. /// host private HttpClient GetHttpClient(string host) { if (string.IsNullOrEmpty(host)) { throw new ArgumentNullException("host"); } HttpClient client; if (!_httpClients.TryGetValue(host, out client)) { var handler = new WebRequestHandler { AutomaticDecompression = DecompressionMethods.Deflate, CachePolicy = new RequestCachePolicy(RequestCacheLevel.Revalidate) }; client = new HttpClient(handler); client.DefaultRequestHeaders.Add("Accept", "application/json,image/*"); client.Timeout = TimeSpan.FromSeconds(15); _httpClients.TryAdd(host, client); } return client; } /// /// Performs a GET request and returns the resulting stream /// /// The URL. /// The resource pool. /// The cancellation token. /// Task{Stream}. /// public async Task Get(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken) { ValidateParams(url, resourcePool, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); await resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); Logger.Info("HttpManager.Get url: {0}", url); try { cancellationToken.ThrowIfCancellationRequested(); var msg = await GetHttpClient(GetHostFromUrl(url)).GetAsync(url, cancellationToken).ConfigureAwait(false); EnsureSuccessStatusCode(msg); return await msg.Content.ReadAsStreamAsync().ConfigureAwait(false); } catch (OperationCanceledException ex) { throw GetCancellationException(url, cancellationToken, ex); } catch (HttpRequestException ex) { Logger.ErrorException("Error getting response from " + url, ex); throw new HttpException(ex.Message, ex); } finally { resourcePool.Release(); } } /// /// Performs a POST request /// /// The URL. /// Params to add to the POST data. /// The resource pool. /// The cancellation token. /// stream on success, null on failure /// postData /// public async Task Post(string url, Dictionary postData, SemaphoreSlim resourcePool, CancellationToken cancellationToken) { ValidateParams(url, resourcePool, cancellationToken); if (postData == null) { throw new ArgumentNullException("postData"); } cancellationToken.ThrowIfCancellationRequested(); var strings = postData.Keys.Select(key => string.Format("{0}={1}", key, postData[key])); var postContent = string.Join("&", strings.ToArray()); var content = new StringContent(postContent, Encoding.UTF8, "application/x-www-form-urlencoded"); await resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); Logger.Info("HttpManager.Post url: {0}", url); try { cancellationToken.ThrowIfCancellationRequested(); var msg = await GetHttpClient(GetHostFromUrl(url)).PostAsync(url, content, cancellationToken).ConfigureAwait(false); EnsureSuccessStatusCode(msg); return await msg.Content.ReadAsStreamAsync().ConfigureAwait(false); } catch (OperationCanceledException ex) { throw GetCancellationException(url, cancellationToken, ex); } catch (HttpRequestException ex) { Logger.ErrorException("Error getting response from " + url, ex); throw new HttpException(ex.Message, ex); } finally { resourcePool.Release(); } } /// /// Downloads the contents of a given url into a temporary location /// /// The URL. /// The resource pool. /// The cancellation token. /// The progress. /// The user agent. /// Task{System.String}. /// progress /// public async Task FetchToTempFile(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken, IProgress progress, string userAgent = null) { ValidateParams(url, resourcePool, cancellationToken); if (progress == null) { throw new ArgumentNullException("progress"); } cancellationToken.ThrowIfCancellationRequested(); var tempFile = Path.Combine(Kernel.ApplicationPaths.TempDirectory, Guid.NewGuid() + ".tmp"); var message = new HttpRequestMessage(HttpMethod.Get, url); if (!string.IsNullOrEmpty(userAgent)) { message.Headers.Add("User-Agent", userAgent); } await resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); Logger.Info("HttpManager.FetchToTempFile url: {0}, temp file: {1}", url, tempFile); try { cancellationToken.ThrowIfCancellationRequested(); using (var response = await GetHttpClient(GetHostFromUrl(url)).SendAsync(message, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false)) { EnsureSuccessStatusCode(response); cancellationToken.ThrowIfCancellationRequested(); IEnumerable lengthValues; if (!response.Headers.TryGetValues("content-length", out lengthValues) && !response.Content.Headers.TryGetValues("content-length", out lengthValues)) { // We're not able to track progress using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) { using (var fs = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) { await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); } } } else { var length = long.Parse(string.Join(string.Empty, lengthValues.ToArray())); using (var stream = ProgressStream.CreateReadProgressStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false), progress.Report, length)) { using (var fs = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) { await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); } } } progress.Report(100); cancellationToken.ThrowIfCancellationRequested(); } return tempFile; } catch (OperationCanceledException ex) { // Cleanup if (File.Exists(tempFile)) { File.Delete(tempFile); } throw GetCancellationException(url, cancellationToken, ex); } catch (HttpRequestException ex) { Logger.ErrorException("Error getting response from " + url, ex); // Cleanup if (File.Exists(tempFile)) { File.Delete(tempFile); } throw new HttpException(ex.Message, ex); } catch (Exception ex) { Logger.ErrorException("Error getting response from " + url, ex); // Cleanup if (File.Exists(tempFile)) { File.Delete(tempFile); } throw; } finally { resourcePool.Release(); } } /// /// Downloads the contents of a given url into a MemoryStream /// /// The URL. /// The resource pool. /// The cancellation token. /// Task{MemoryStream}. /// public async Task FetchToMemoryStream(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken) { ValidateParams(url, resourcePool, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); var message = new HttpRequestMessage(HttpMethod.Get, url); await resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); var ms = new MemoryStream(); Logger.Info("HttpManager.FetchToMemoryStream url: {0}", url); try { cancellationToken.ThrowIfCancellationRequested(); using (var response = await GetHttpClient(GetHostFromUrl(url)).SendAsync(message, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false)) { EnsureSuccessStatusCode(response); cancellationToken.ThrowIfCancellationRequested(); using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) { await stream.CopyToAsync(ms, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); } cancellationToken.ThrowIfCancellationRequested(); } ms.Position = 0; return ms; } catch (OperationCanceledException ex) { ms.Dispose(); throw GetCancellationException(url, cancellationToken, ex); } catch (HttpRequestException ex) { Logger.ErrorException("Error getting response from " + url, ex); ms.Dispose(); throw new HttpException(ex.Message, ex); } catch (Exception ex) { Logger.ErrorException("Error getting response from " + url, ex); ms.Dispose(); throw; } finally { resourcePool.Release(); } } /// /// Validates the params. /// /// The URL. /// The resource pool. /// The cancellation token. /// url private void ValidateParams(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(url)) { throw new ArgumentNullException("url"); } if (resourcePool == null) { throw new ArgumentNullException("resourcePool"); } if (cancellationToken == null) { throw new ArgumentNullException("cancellationToken"); } } /// /// Gets the host from URL. /// /// The URL. /// System.String. private string GetHostFromUrl(string url) { var start = url.IndexOf("://", StringComparison.OrdinalIgnoreCase) + 3; var len = url.IndexOf('/', start) - start; return url.Substring(start, len); } /// /// Releases unmanaged and - optionally - managed resources. /// /// true to release both managed and unmanaged resources; false to release only unmanaged resources. protected override void Dispose(bool dispose) { if (dispose) { foreach (var client in _httpClients.Values.ToList()) { client.Dispose(); } _httpClients.Clear(); } base.Dispose(dispose); } /// /// Throws the cancellation exception. /// /// The URL. /// The cancellation token. /// The exception. /// Exception. private Exception GetCancellationException(string url, CancellationToken cancellationToken, OperationCanceledException exception) { // If the HttpClient's timeout is reached, it will cancel the Task internally if (!cancellationToken.IsCancellationRequested) { var msg = string.Format("Connection to {0} timed out", url); Logger.Error(msg); // Throw an HttpException so that the caller doesn't think it was cancelled by user code return new HttpException(msg, exception) { IsTimedOut = true }; } return exception; } /// /// Ensures the success status code. /// /// The response. /// private void EnsureSuccessStatusCode(HttpResponseMessage response) { if (!response.IsSuccessStatusCode) { throw new HttpException(response.ReasonPhrase) { StatusCode = response.StatusCode }; } } } }