From a1dde468877cbd9f27775eb932bc9c8d1321324d Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sun, 14 Jul 2019 19:20:06 -0500 Subject: [PATCH] simplify validator a bit --- ...UrlValidator.cs => IEntityUrlValidator.cs} | 2 +- .../Features/Urls/ValidateUrls.cs | 8 +- .../Infrastructure/Web/EntityUrlValidator.cs | 72 ++++++++++++ .../Web/ServiceCollectionExtensions.cs | 2 +- .../Infrastructure/Web/UrlValidator.cs | 104 ------------------ 5 files changed, 78 insertions(+), 110 deletions(-) rename src/FilterLists.Agent/Core/Urls/{IUrlValidator.cs => IEntityUrlValidator.cs} (83%) create mode 100644 src/FilterLists.Agent/Infrastructure/Web/EntityUrlValidator.cs delete mode 100644 src/FilterLists.Agent/Infrastructure/Web/UrlValidator.cs diff --git a/src/FilterLists.Agent/Core/Urls/IUrlValidator.cs b/src/FilterLists.Agent/Core/Urls/IEntityUrlValidator.cs similarity index 83% rename from src/FilterLists.Agent/Core/Urls/IUrlValidator.cs rename to src/FilterLists.Agent/Core/Urls/IEntityUrlValidator.cs index 849ef3851..04cf340a6 100644 --- a/src/FilterLists.Agent/Core/Urls/IUrlValidator.cs +++ b/src/FilterLists.Agent/Core/Urls/IEntityUrlValidator.cs @@ -3,7 +3,7 @@ namespace FilterLists.Agent.Core.Urls { - public interface IUrlValidator + public interface IEntityUrlValidator { Task ValidateAsync(EntityUrl entityUrl, CancellationToken cancellationToken); } diff --git a/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs b/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs index 4203d4927..2d50755c1 100644 --- a/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs +++ b/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs @@ -23,11 +23,11 @@ public Command(IEnumerable entityUrls) public class Handler : IRequestHandler> { private const int MaxDegreeOfParallelism = 5; - private readonly IUrlValidator _urlValidator; + private readonly IEntityUrlValidator _entityUrlValidator; - public Handler(IUrlValidator urlValidator) + public Handler(IEntityUrlValidator entityUrlValidator) { - _urlValidator = urlValidator; + _entityUrlValidator = entityUrlValidator; } public async Task> Handle(Command request, CancellationToken cancellationToken) @@ -46,7 +46,7 @@ public async Task> Handle(Command request, CancellationTo private TransformBlock BuildValidator(CancellationToken cancellationToken) { return new TransformBlock( - e => _urlValidator.ValidateAsync(e, cancellationToken), + e => _entityUrlValidator.ValidateAsync(e, cancellationToken), new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism} ); } diff --git a/src/FilterLists.Agent/Infrastructure/Web/EntityUrlValidator.cs b/src/FilterLists.Agent/Infrastructure/Web/EntityUrlValidator.cs new file mode 100644 index 000000000..ec3738a69 --- /dev/null +++ b/src/FilterLists.Agent/Infrastructure/Web/EntityUrlValidator.cs @@ -0,0 +1,72 @@ +using System; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using FilterLists.Agent.Core.Urls; + +namespace FilterLists.Agent.Infrastructure.Web +{ + public class EntityUrlValidator : IEntityUrlValidator + { + private readonly HttpClient _httpClient; + + public EntityUrlValidator(HttpClient httpClient) + { + httpClient.Timeout = TimeSpan.FromSeconds(90); + var header = new ProductHeaderValue("FilterLists.Agent"); + var userAgent = new ProductInfoHeaderValue(header); + httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent); + _httpClient = httpClient; + } + + public async Task ValidateAsync(EntityUrl entityUrl, CancellationToken cancellationToken) + { + var url = entityUrl.Url; + if (!Uri.IsWellFormedUriString(url.OriginalString, UriKind.Absolute)) + { + entityUrl.SetBroken(); + return entityUrl; + } + + if (url.Scheme == Uri.UriSchemeHttp) + if (await IsHttpsSupported(url, cancellationToken)) + entityUrl.SetSupportsHttps(); + try + { + using var response = await GetAsync(url, cancellationToken); + if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400) + entityUrl.SetRedirectsTo(response.Headers.Location); + else if (!response.IsSuccessStatusCode) + entityUrl.SetBroken(); + } + catch (Exception ex) when (ex is HttpRequestException || + ex is TaskCanceledException) + { + entityUrl.SetBroken(); + } + + return entityUrl; + } + + private async Task IsHttpsSupported(Uri url, CancellationToken cancellationToken) + { + var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps, Port = -1}.Uri; + try + { + using var response = await GetAsync(httpsUrl, cancellationToken); + return response.IsSuccessStatusCode; + } + catch (Exception ex) when (ex is HttpRequestException || + ex is TaskCanceledException) + { + return false; + } + } + + private async Task GetAsync(Uri url, CancellationToken cancellationToken) + { + return await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/Infrastructure/Web/ServiceCollectionExtensions.cs b/src/FilterLists.Agent/Infrastructure/Web/ServiceCollectionExtensions.cs index 860b826b0..67f592917 100644 --- a/src/FilterLists.Agent/Infrastructure/Web/ServiceCollectionExtensions.cs +++ b/src/FilterLists.Agent/Infrastructure/Web/ServiceCollectionExtensions.cs @@ -11,7 +11,7 @@ public static void AddWebServices(this IServiceCollection services) { services.AddHttpClient() .AddPolicyHandlerFromRegistry("waitAndRetryTooManyRequests"); - services.AddHttpClient().ConfigureHttpMessageHandlerBuilder(b => + services.AddHttpClient().ConfigureHttpMessageHandlerBuilder(b => { b.PrimaryHandler = new HttpClientHandler {AllowAutoRedirect = false}; b.Build(); diff --git a/src/FilterLists.Agent/Infrastructure/Web/UrlValidator.cs b/src/FilterLists.Agent/Infrastructure/Web/UrlValidator.cs deleted file mode 100644 index a9c9d5674..000000000 --- a/src/FilterLists.Agent/Infrastructure/Web/UrlValidator.cs +++ /dev/null @@ -1,104 +0,0 @@ -using System; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Threading; -using System.Threading.Tasks; -using FilterLists.Agent.Core.Urls; -using Microsoft.Extensions.Logging; - -namespace FilterLists.Agent.Infrastructure.Web -{ - public class UrlValidator : IUrlValidator - { - private readonly HttpClient _httpClient; - private readonly ILogger _logger; - - public UrlValidator(HttpClient httpClient, ILogger logger) - { - httpClient.Timeout = TimeSpan.FromSeconds(90); - var header = new ProductHeaderValue("FilterLists.Agent"); - var userAgent = new ProductInfoHeaderValue(header); - httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent); - _httpClient = httpClient; - - _logger = logger; - } - - public async Task ValidateAsync(EntityUrl entityUrl, CancellationToken cancellationToken) - { - var url = entityUrl.Url; - if (!Uri.IsWellFormedUriString(url.OriginalString, UriKind.Absolute)) - { - entityUrl.SetBroken(); - _logger.LogError( - $"Url validation for ({url.OriginalString}) failed because it appears to be improperly formatted."); - return entityUrl; - } - - try - { - using var response = - await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); - if (url.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(url, cancellationToken)) - entityUrl.SetSupportsHttps(); - if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400) - { - entityUrl.SetRedirectsTo(response.Headers.Location); - } - else if (!response.IsSuccessStatusCode) - { - entityUrl.SetBroken(); - _logger.LogError( - $"Url validation for ({url.AbsoluteUri}) failed with status code: {response.StatusCode}."); - } - } - catch (HttpRequestException ex) - { - entityUrl.SetBroken(); - _logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex); - } - catch (TaskCanceledException ex) - { - entityUrl.SetBroken(); - _logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex); - } - catch (InvalidOperationException ex) - { - entityUrl.SetBroken(); - _logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex); - } - - return entityUrl; - } - - private async Task IsHttpsSupported(Uri url, CancellationToken cancellationToken) - { - var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps, Port = -1}.Uri; - try - { - using var response = await _httpClient.GetAsync(httpsUrl, HttpCompletionOption.ResponseHeadersRead, - cancellationToken); - if (response.IsSuccessStatusCode) - return true; - _logger.LogInformation( - $"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}."); - return response.IsSuccessStatusCode; - } - catch (HttpRequestException ex) - { - _logger.LogInformation($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); - return false; - } - catch (TaskCanceledException ex) - { - _logger.LogInformation($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); - return false; - } - catch (InvalidOperationException ex) - { - _logger.LogInformation($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); - return false; - } - } - } -} \ No newline at end of file