From d405a400aaa5f9676cc2ce9159b562f94233dcd5 Mon Sep 17 00:00:00 2001 From: Bond_009 Date: Fri, 14 Jun 2019 16:32:37 +0200 Subject: [PATCH] Fixes issues with HttpClientManager --- Emby.Dlna/PlayTo/SsdpHttpClient.cs | 22 +-- .../ApplicationHost.cs | 13 +- .../HttpClientManager/HttpClientManager.cs | 143 +++++++++--------- .../LiveTv/EmbyTV/DirectRecorder.cs | 2 +- .../LiveTv/Listings/SchedulesDirect.cs | 20 +-- .../LiveTv/Listings/XmlTvListingsProvider.cs | 4 +- .../LiveTv/TunerHosts/SharedHttpStream.cs | 2 +- Jellyfin.Server/Program.cs | 12 ++ MediaBrowser.Common/Net/HttpRequestOptions.cs | 29 ++-- MediaBrowser.Common/Net/HttpResponseInfo.cs | 12 +- 10 files changed, 119 insertions(+), 140 deletions(-) diff --git a/Emby.Dlna/PlayTo/SsdpHttpClient.cs b/Emby.Dlna/PlayTo/SsdpHttpClient.cs index 1ad99fac5b..780b0a8897 100644 --- a/Emby.Dlna/PlayTo/SsdpHttpClient.cs +++ b/Emby.Dlna/PlayTo/SsdpHttpClient.cs @@ -34,16 +34,13 @@ namespace Emby.Dlna.PlayTo { var cancellationToken = CancellationToken.None; - using (var response = await PostSoapDataAsync(NormalizeServiceUrl(baseUrl, service.ControlUrl), "\"" + service.ServiceType + "#" + command + "\"", postData, header, logRequest, cancellationToken) + var url = NormalizeServiceUrl(baseUrl, service.ControlUrl); + using (var response = await PostSoapDataAsync(url, '\"' + service.ServiceType + '#' + command + '\"', postData, header, logRequest, cancellationToken) .ConfigureAwait(false)) + using (var stream = response.Content) + using (var reader = new StreamReader(stream, Encoding.UTF8)) { - using (var stream = response.Content) - { - using (var reader = new StreamReader(stream, Encoding.UTF8)) - { - return XDocument.Parse(reader.ReadToEnd(), LoadOptions.PreserveWhitespace); - } - } + return XDocument.Parse(reader.ReadToEnd(), LoadOptions.PreserveWhitespace); } } @@ -121,15 +118,18 @@ namespace Emby.Dlna.PlayTo } } - private Task PostSoapDataAsync(string url, + private Task PostSoapDataAsync( + string url, string soapAction, string postData, string header, bool logRequest, CancellationToken cancellationToken) { - if (!soapAction.StartsWith("\"")) - soapAction = "\"" + soapAction + "\""; + if (soapAction[0] != '\"') + { + soapAction = '\"' + soapAction + '\"'; + } var options = new HttpRequestOptions { diff --git a/Emby.Server.Implementations/ApplicationHost.cs b/Emby.Server.Implementations/ApplicationHost.cs index 02d661d15a..c64922a931 100644 --- a/Emby.Server.Implementations/ApplicationHost.cs +++ b/Emby.Server.Implementations/ApplicationHost.cs @@ -430,7 +430,7 @@ namespace Emby.Server.Implementations /// Gets the current application user agent /// /// The application user agent. - public string ApplicationUserAgent => Name.Replace(' ','-') + "/" + ApplicationVersion; + public string ApplicationUserAgent => Name.Replace(' ','-') + '/' + ApplicationVersion; /// /// Gets the email address for use within a comment section of a user agent field. @@ -690,11 +690,6 @@ namespace Emby.Server.Implementations await HttpServer.RequestHandler(req, request.GetDisplayUrl(), request.Host.ToString(), localPath, CancellationToken.None).ConfigureAwait(false); } - protected virtual IHttpClient CreateHttpClient() - { - return new HttpClientManager.HttpClientManager(ApplicationPaths, LoggerFactory, FileSystemManager, () => ApplicationUserAgent); - } - public static IStreamHelper StreamHelper { get; set; } /// @@ -720,7 +715,11 @@ namespace Emby.Server.Implementations serviceCollection.AddSingleton(FileSystemManager); serviceCollection.AddSingleton(); - HttpClient = CreateHttpClient(); + HttpClient = new HttpClientManager.HttpClientManager( + ApplicationPaths, + LoggerFactory.CreateLogger(), + FileSystemManager, + () => ApplicationUserAgent); serviceCollection.AddSingleton(HttpClient); serviceCollection.AddSingleton(NetworkManager); diff --git a/Emby.Server.Implementations/HttpClientManager/HttpClientManager.cs b/Emby.Server.Implementations/HttpClientManager/HttpClientManager.cs index b82d55d0e6..987657bcb1 100644 --- a/Emby.Server.Implementations/HttpClientManager/HttpClientManager.cs +++ b/Emby.Server.Implementations/HttpClientManager/HttpClientManager.cs @@ -23,11 +23,6 @@ namespace Emby.Server.Implementations.HttpClientManager /// public class HttpClientManager : IHttpClient { - /// - /// When one request to a host times out, we'll ban all other requests for this period of time, to prevent scans from stalling - /// - private const int TimeoutSeconds = 30; - /// /// The _logger /// @@ -46,7 +41,7 @@ namespace Emby.Server.Implementations.HttpClientManager /// public HttpClientManager( IApplicationPaths appPaths, - ILoggerFactory loggerFactory, + ILogger logger, IFileSystem fileSystem, Func defaultUserAgentFn) { @@ -55,18 +50,15 @@ namespace Emby.Server.Implementations.HttpClientManager throw new ArgumentNullException(nameof(appPaths)); } - if (loggerFactory == null) + if (logger == null) { - throw new ArgumentNullException(nameof(loggerFactory)); + throw new ArgumentNullException(nameof(logger)); } - _logger = loggerFactory.CreateLogger(nameof(HttpClientManager)); + _logger = logger; _fileSystem = fileSystem; _appPaths = appPaths; _defaultUserAgentFn = defaultUserAgentFn; - - // http://stackoverflow.com/questions/566437/http-post-returns-the-error-417-expectation-failed-c - ServicePointManager.Expect100Continue = false; } /// @@ -83,13 +75,12 @@ namespace Emby.Server.Implementations.HttpClientManager /// if set to true [enable HTTP compression]. /// HttpClient. /// host - private HttpClient GetHttpClient(string url, bool enableHttpCompression) + private HttpClient GetHttpClient(string url) { - var key = GetHostFromUrl(url) + enableHttpCompression; + var key = GetHostFromUrl(url); if (!_httpClients.TryGetValue(key, out var client)) { - client = new HttpClient() { BaseAddress = new Uri(url) @@ -109,24 +100,27 @@ namespace Emby.Server.Implementations.HttpClientManager if (!string.IsNullOrWhiteSpace(userInfo)) { _logger.LogWarning("Found userInfo in url: {0} ... url: {1}", userInfo, url); - url = url.Replace(userInfo + "@", string.Empty); + url = url.Replace(userInfo + '@', string.Empty); } var request = new HttpRequestMessage(method, url); AddRequestHeaders(request, options); - if (options.EnableHttpCompression) + switch (options.DecompressionMethod) { - if (options.DecompressionMethod.HasValue - && options.DecompressionMethod.Value == CompressionMethod.Gzip) - { + case CompressionMethod.Deflate | CompressionMethod.Gzip: request.Headers.Add(HeaderNames.AcceptEncoding, new[] { "gzip", "deflate" }); - } - else - { + break; + case CompressionMethod.Deflate: request.Headers.Add(HeaderNames.AcceptEncoding, "deflate"); - } + break; + case CompressionMethod.Gzip: + request.Headers.Add(HeaderNames.AcceptEncoding, "gzip"); + break; + case 0: + default: + break; } if (options.EnableKeepAlive) @@ -134,20 +128,8 @@ namespace Emby.Server.Implementations.HttpClientManager request.Headers.Add(HeaderNames.Connection, "Keep-Alive"); } - if (!string.IsNullOrEmpty(options.Host)) - { - request.Headers.Add(HeaderNames.Host, options.Host); - } - - if (!string.IsNullOrEmpty(options.Referer)) - { - request.Headers.Add(HeaderNames.Referer, options.Referer); - } - //request.Headers.Add(HeaderNames.CacheControl, "no-cache"); - //request.Headers.Add(HeaderNames., options.TimeoutMs; - /* if (!string.IsNullOrWhiteSpace(userInfo)) { @@ -188,9 +170,7 @@ namespace Emby.Server.Implementations.HttpClientManager /// The options. /// Task{HttpResponseInfo}. public Task GetResponse(HttpRequestOptions options) - { - return SendAsync(options, HttpMethod.Get); - } + => SendAsync(options, HttpMethod.Get); /// /// Performs a GET request and returns the resulting stream @@ -324,18 +304,29 @@ namespace Emby.Server.Implementations.HttpClientManager options.CancellationToken.ThrowIfCancellationRequested(); - var client = GetHttpClient(options.Url, options.EnableHttpCompression); + var client = GetHttpClient(options.Url); var httpWebRequest = GetRequestMessage(options, httpMethod); - if (options.RequestContentBytes != null || - !string.IsNullOrEmpty(options.RequestContent) || - httpMethod == HttpMethod.Post) + if (options.RequestContentBytes != null + || !string.IsNullOrEmpty(options.RequestContent) + || httpMethod == HttpMethod.Post) { try { - httpWebRequest.Content = new StringContent(Encoding.UTF8.GetString(options.RequestContentBytes) ?? options.RequestContent ?? string.Empty); - + if (options.RequestContentBytes != null) + { + httpWebRequest.Content = new ByteArrayContent(options.RequestContentBytes); + } + else if (options.RequestContent != null) + { + httpWebRequest.Content = new StringContent(options.RequestContent); + } + else + { + httpWebRequest.Content = new ByteArrayContent(Array.Empty()); + } + /* var contentType = options.RequestContentType ?? "application/x-www-form-urlencoded"; if (options.AppendCharsetToMimeType) @@ -343,8 +334,11 @@ namespace Emby.Server.Implementations.HttpClientManager contentType = contentType.TrimEnd(';') + "; charset=\"utf-8\""; } - httpWebRequest.Headers.Add(HeaderNames.ContentType, contentType); - await client.SendAsync(httpWebRequest).ConfigureAwait(false); + httpWebRequest.Headers.Add(HeaderNames.ContentType, contentType);*/ + using (var response = await client.SendAsync(httpWebRequest).ConfigureAwait(false)) + { + return await HandleResponseAsync(response, options).ConfigureAwait(false); + } } catch (Exception ex) { @@ -374,18 +368,7 @@ namespace Emby.Server.Implementations.HttpClientManager using (var response = await client.SendAsync(httpWebRequest).ConfigureAwait(false)) { - await EnsureSuccessStatusCode(response, options).ConfigureAwait(false); - - options.CancellationToken.ThrowIfCancellationRequested(); - - using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) - { - var memoryStream = new MemoryStream(); - await stream.CopyToAsync(memoryStream).ConfigureAwait(false); - memoryStream.Position = 0; - - return GetResponseInfo(response, memoryStream, memoryStream.Length, null); - } + return await HandleResponseAsync(response, options).ConfigureAwait(false); } } catch (OperationCanceledException ex) @@ -394,9 +377,25 @@ namespace Emby.Server.Implementations.HttpClientManager } } - private HttpResponseInfo GetResponseInfo(HttpResponseMessage httpResponse, Stream content, long? contentLength, IDisposable disposable) + private async Task HandleResponseAsync(HttpResponseMessage response, HttpRequestOptions options) { - var responseInfo = new HttpResponseInfo(disposable) + await EnsureSuccessStatusCode(response, options).ConfigureAwait(false); + + options.CancellationToken.ThrowIfCancellationRequested(); + + using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + { + var memoryStream = new MemoryStream(); + await stream.CopyToAsync(memoryStream, 81920, options.CancellationToken).ConfigureAwait(false); + memoryStream.Position = 0; + + return GetResponseInfo(response, memoryStream, memoryStream.Length); + } + } + + private HttpResponseInfo GetResponseInfo(HttpResponseMessage httpResponse, Stream content, long? contentLength) + { + var responseInfo = new HttpResponseInfo() { Content = content, StatusCode = httpResponse.StatusCode, @@ -433,16 +432,14 @@ namespace Emby.Server.Implementations.HttpClientManager private static void SetHeaders(HttpContentHeaders headers, HttpResponseInfo responseInfo) { - foreach (var key in headers) + foreach (var header in headers) { - responseInfo.Headers[key.Key] = string.Join(", ", key.Value); + responseInfo.Headers[header.Key] = string.Join(", ", header.Value); } } public Task Post(HttpRequestOptions options) - { - return SendAsync(options, HttpMethod.Post); - } + => SendAsync(options, HttpMethod.Post); /// /// Downloads the contents of a given url into a temporary location @@ -451,10 +448,8 @@ namespace Emby.Server.Implementations.HttpClientManager /// Task{System.String}. public async Task GetTempFile(HttpRequestOptions options) { - using (var response = await GetTempFileResponse(options).ConfigureAwait(false)) - { - return response.TempFilePath; - } + var response = await GetTempFileResponse(options).ConfigureAwait(false); + return response.TempFilePath; } public async Task GetTempFileResponse(HttpRequestOptions options) @@ -481,13 +476,13 @@ namespace Emby.Server.Implementations.HttpClientManager _logger.LogDebug("HttpClientManager.GetTempFileResponse url: {0}", options.Url); } - var client = GetHttpClient(options.Url, options.EnableHttpCompression); + var client = GetHttpClient(options.Url); try { options.CancellationToken.ThrowIfCancellationRequested(); - using (var response = (await client.SendAsync(httpWebRequest).ConfigureAwait(false))) + using (var response = (await client.SendAsync(httpWebRequest, options.CancellationToken).ConfigureAwait(false))) { await EnsureSuccessStatusCode(response, options).ConfigureAwait(false); @@ -530,7 +525,7 @@ namespace Emby.Server.Implementations.HttpClientManager { if (options.LogErrors) { - _logger.LogError(webException, "Error {status} getting response from {url}", webException.Status, options.Url); + _logger.LogError(webException, "Error {Status} getting response from {Url}", webException.Status, options.Url); } var exception = new HttpException(webException.Message, webException); @@ -565,7 +560,7 @@ namespace Emby.Server.Implementations.HttpClientManager if (options.LogErrors) { - _logger.LogError(ex, "Error getting response from {url}", options.Url); + _logger.LogError(ex, "Error getting response from {Url}", options.Url); } return ex; diff --git a/Emby.Server.Implementations/LiveTv/EmbyTV/DirectRecorder.cs b/Emby.Server.Implementations/LiveTv/EmbyTV/DirectRecorder.cs index dd636e6cdf..8dee7046e7 100644 --- a/Emby.Server.Implementations/LiveTv/EmbyTV/DirectRecorder.cs +++ b/Emby.Server.Implementations/LiveTv/EmbyTV/DirectRecorder.cs @@ -71,7 +71,7 @@ namespace Emby.Server.Implementations.LiveTv.EmbyTV UserAgent = "Emby/3.0", // Shouldn't matter but may cause issues - EnableHttpCompression = false + DecompressionMethod = CompressionMethod.None }; using (var response = await _httpClient.SendAsync(httpRequestOptions, "GET").ConfigureAwait(false)) diff --git a/Emby.Server.Implementations/LiveTv/Listings/SchedulesDirect.cs b/Emby.Server.Implementations/LiveTv/Listings/SchedulesDirect.cs index f3f7477180..f5dffc22af 100644 --- a/Emby.Server.Implementations/LiveTv/Listings/SchedulesDirect.cs +++ b/Emby.Server.Implementations/LiveTv/Listings/SchedulesDirect.cs @@ -627,15 +627,7 @@ namespace Emby.Server.Implementations.LiveTv.Listings ListingsProviderInfo providerInfo) { // Schedules direct requires that the client support compression and will return a 400 response without it - options.EnableHttpCompression = true; - - // On windows 7 under .net core, this header is not getting added -#if NETSTANDARD2_0 - if (Environment.OSVersion.Platform == PlatformID.Win32NT) - { - options.RequestHeaders[HeaderNames.AcceptEncoding] = "deflate"; - } -#endif + options.DecompressionMethod = CompressionMethod.Deflate; try { @@ -665,15 +657,7 @@ namespace Emby.Server.Implementations.LiveTv.Listings ListingsProviderInfo providerInfo) { // Schedules direct requires that the client support compression and will return a 400 response without it - options.EnableHttpCompression = true; - - // On windows 7 under .net core, this header is not getting added -#if NETSTANDARD2_0 - if (Environment.OSVersion.Platform == PlatformID.Win32NT) - { - options.RequestHeaders[HeaderNames.AcceptEncoding] = "deflate"; - } -#endif + options.DecompressionMethod = CompressionMethod.Deflate; try { diff --git a/Emby.Server.Implementations/LiveTv/Listings/XmlTvListingsProvider.cs b/Emby.Server.Implementations/LiveTv/Listings/XmlTvListingsProvider.cs index 69b10e6daa..d39c917835 100644 --- a/Emby.Server.Implementations/LiveTv/Listings/XmlTvListingsProvider.cs +++ b/Emby.Server.Implementations/LiveTv/Listings/XmlTvListingsProvider.cs @@ -73,11 +73,9 @@ namespace Emby.Server.Implementations.LiveTv.Listings CancellationToken = cancellationToken, Url = path, Progress = new SimpleProgress(), - DecompressionMethod = CompressionMethod.Gzip, - // It's going to come back gzipped regardless of this value // So we need to make sure the decompression method is set to gzip - EnableHttpCompression = true, + DecompressionMethod = CompressionMethod.Gzip, UserAgent = "Emby/3.0" diff --git a/Emby.Server.Implementations/LiveTv/TunerHosts/SharedHttpStream.cs b/Emby.Server.Implementations/LiveTv/TunerHosts/SharedHttpStream.cs index e8b34da0ca..60c914898c 100644 --- a/Emby.Server.Implementations/LiveTv/TunerHosts/SharedHttpStream.cs +++ b/Emby.Server.Implementations/LiveTv/TunerHosts/SharedHttpStream.cs @@ -47,7 +47,7 @@ namespace Emby.Server.Implementations.LiveTv.TunerHosts CancellationToken = CancellationToken.None, BufferContent = false, - EnableHttpCompression = false, + DecompressionMethod = CompressionMethod.None, }; foreach (var header in mediaSource.RequiredHttpHeaders) diff --git a/Jellyfin.Server/Program.cs b/Jellyfin.Server/Program.cs index d4b10c8c84..95ce08f269 100644 --- a/Jellyfin.Server/Program.cs +++ b/Jellyfin.Server/Program.cs @@ -118,8 +118,20 @@ namespace Jellyfin.Server SQLitePCL.Batteries_V2.Init(); + // Increase the max http request limit + // The default connection limit is 10 for ASP.NET hosted applications and 2 for all others. + ServicePointManager.DefaultConnectionLimit = Math.Max(96, ServicePointManager.DefaultConnectionLimit); + + // Disable the "Expect: 100-Continue" header by default + // http://stackoverflow.com/questions/566437/http-post-returns-the-error-417-expectation-failed-c + ServicePointManager.Expect100Continue = false; + +// CA5359: Do Not Disable Certificate Validation +#pragma warning disable CA5359 + // Allow all https requests ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(delegate { return true; }); +#pragma warning restore CA5359 var fileSystem = new ManagedFileSystem(_loggerFactory, appPaths); diff --git a/MediaBrowser.Common/Net/HttpRequestOptions.cs b/MediaBrowser.Common/Net/HttpRequestOptions.cs index 38e0ff0f55..432e389d3b 100644 --- a/MediaBrowser.Common/Net/HttpRequestOptions.cs +++ b/MediaBrowser.Common/Net/HttpRequestOptions.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading; using Microsoft.Net.Http.Headers; @@ -17,7 +16,7 @@ namespace MediaBrowser.Common.Net /// The URL. public string Url { get; set; } - public CompressionMethod? DecompressionMethod { get; set; } + public CompressionMethod DecompressionMethod { get; set; } /// /// Gets or sets the accept header. @@ -49,13 +48,21 @@ namespace MediaBrowser.Common.Net /// Gets or sets the referrer. /// /// The referrer. - public string Referer { get; set; } + public string Referer + { + get => GetHeaderValue(HeaderNames.Referer); + set => RequestHeaders[HeaderNames.Referer] = value; + } /// /// Gets or sets the host. /// /// The host. - public string Host { get; set; } + public string Host + { + get => GetHeaderValue(HeaderNames.Host); + set => RequestHeaders[HeaderNames.Host] = value; + } /// /// Gets or sets the progress. @@ -63,12 +70,6 @@ namespace MediaBrowser.Common.Net /// The progress. public IProgress Progress { get; set; } - /// - /// Gets or sets a value indicating whether [enable HTTP compression]. - /// - /// true if [enable HTTP compression]; otherwise, false. - public bool EnableHttpCompression { get; set; } - public Dictionary RequestHeaders { get; private set; } public string RequestContentType { get; set; } @@ -104,13 +105,12 @@ namespace MediaBrowser.Common.Net /// public HttpRequestOptions() { - EnableHttpCompression = true; - RequestHeaders = new Dictionary(StringComparer.OrdinalIgnoreCase); LogRequest = true; LogErrors = true; CacheMode = CacheMode.None; + DecompressionMethod = CompressionMethod.Deflate; } } @@ -122,7 +122,8 @@ namespace MediaBrowser.Common.Net public enum CompressionMethod { - Deflate, - Gzip + None = 0b00000001, + Deflate = 0b00000010, + Gzip = 0b00000100 } } diff --git a/MediaBrowser.Common/Net/HttpResponseInfo.cs b/MediaBrowser.Common/Net/HttpResponseInfo.cs index 1866741676..aa496adaca 100644 --- a/MediaBrowser.Common/Net/HttpResponseInfo.cs +++ b/MediaBrowser.Common/Net/HttpResponseInfo.cs @@ -52,13 +52,6 @@ namespace MediaBrowser.Common.Net /// The headers. public Dictionary Headers { get; set; } - private readonly IDisposable _disposable; - - public HttpResponseInfo(IDisposable disposable) - { - _disposable = disposable; - Headers = new Dictionary(StringComparer.OrdinalIgnoreCase); - } public HttpResponseInfo() { Headers = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -66,10 +59,7 @@ namespace MediaBrowser.Common.Net public void Dispose() { - if (_disposable != null) - { - _disposable.Dispose(); - } + // Only IDisposable for backwards compatibility } } }