From deaf5e550f6dc9893c3386f9edfe3ad43046864e Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sun, 7 Jul 2019 20:56:43 -0500 Subject: [PATCH] mv url validation logic to UrlService --- .../Core/Interfaces/Services/IUrlService.cs | 7 +- .../Features/Urls/ValidateUrls.cs | 91 +---------------- .../Infrastructure/Services/UrlService.cs | 98 ++++++++++++++++++- 3 files changed, 102 insertions(+), 94 deletions(-) diff --git a/src/FilterLists.Agent/Core/Interfaces/Services/IUrlService.cs b/src/FilterLists.Agent/Core/Interfaces/Services/IUrlService.cs index 95bac666e..5886ad84b 100644 --- a/src/FilterLists.Agent/Core/Interfaces/Services/IUrlService.cs +++ b/src/FilterLists.Agent/Core/Interfaces/Services/IUrlService.cs @@ -1,9 +1,12 @@ -using System.Net.Http; +using System; +using System.Threading; +using System.Threading.Tasks; +using FilterLists.Agent.Features.Urls.Models.ValidationResults; namespace FilterLists.Agent.Core.Interfaces.Services { public interface IUrlService { - HttpClient HttpClient { get; } + Task ValidateAsync(Uri u, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs b/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs index a25cf036c..386eb74f2 100644 --- a/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs +++ b/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net.Http; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; @@ -9,7 +8,6 @@ using FilterLists.Agent.Extensions; using FilterLists.Agent.Features.Urls.Models.ValidationResults; using MediatR; -using Microsoft.Extensions.Logging; namespace FilterLists.Agent.Features.Urls { @@ -28,13 +26,11 @@ public Command(IEnumerable urls) public class Handler : IRequestHandler> { private const int MaxDegreeOfParallelism = 5; - private readonly HttpClient _httpClient; - private readonly ILogger _logger; + private readonly IUrlService _urlService; - public Handler(IUrlService urlService, ILogger logger) + public Handler(IUrlService urlService) { - _httpClient = urlService.HttpClient; - _logger = logger; + _urlService = urlService; } public async Task> Handle(Command request, CancellationToken cancellationToken) @@ -59,89 +55,10 @@ public async Task> Handle(Command request, Cancellatio private TransformBlock BuildValidator(CancellationToken cancellationToken) { return new TransformBlock( - async u => - { - var result = new UrlValidationResult(u); - if (!u.IsValidUrl()) - { - result.SetBroken(); - _logger.LogError($"{u.OriginalString}) is not a valid URL."); - return result; - } - - try - { - var response = await _httpClient.GetAsync(u, HttpCompletionOption.ResponseHeadersRead, - cancellationToken); - if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken)) - result.SetSupportsHttps(); - if (response.IsSuccessStatusCode) - return result; - if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400) - { - result.SetRedirectsTo(response.Headers.Location); - } - else - { - result.SetBroken(); - _logger.LogError( - $"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}."); - } - - return result; - } - catch (HttpRequestException ex) - { - result.SetBroken(); - _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); - return result; - } - catch (TaskCanceledException ex) - { - result.SetBroken(); - _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); - return result; - } - catch (InvalidOperationException ex) - { - result.SetBroken(); - _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); - return result; - } - }, + async u => await _urlService.ValidateAsync(u, cancellationToken), new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism} ); } - - private async Task IsHttpsSupported(Uri url, CancellationToken cancellationToken) - { - var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps}.Uri; - try - { - var response = await _httpClient.GetAsync(httpsUrl, HttpCompletionOption.ResponseHeadersRead, - cancellationToken); - if (response.IsSuccessStatusCode) - return true; - _logger.LogError( - $"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}."); - return response.IsSuccessStatusCode; - } - catch (HttpRequestException ex) - { - _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); - return false; - } - catch (TaskCanceledException ex) - { - _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); - return false; - } - catch (InvalidOperationException ex) - { - _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); - return false; - } - } } } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Infrastructure/Services/UrlService.cs b/src/FilterLists.Agent/Infrastructure/Services/UrlService.cs index 4704c705e..448d1f035 100644 --- a/src/FilterLists.Agent/Infrastructure/Services/UrlService.cs +++ b/src/FilterLists.Agent/Infrastructure/Services/UrlService.cs @@ -1,25 +1,113 @@ using System; using System.Net.Http; using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; using FilterLists.Agent.Core.Interfaces.Services; +using FilterLists.Agent.Extensions; +using FilterLists.Agent.Features.Urls.Models.ValidationResults; using JetBrains.Annotations; +using Microsoft.Extensions.Logging; namespace FilterLists.Agent.Infrastructure.Services { [UsedImplicitly] public class UrlService : IUrlService { - public UrlService(HttpClient client) + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + + public UrlService(HttpClient httpClient, ILogger logger) { - client.Timeout = TimeSpan.FromSeconds(90); + httpClient.Timeout = TimeSpan.FromSeconds(90); var header = new ProductHeaderValue("FilterLists.Agent"); var userAgent = new ProductInfoHeaderValue(header); - client.DefaultRequestHeaders.UserAgent.Add(userAgent); + httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent); - HttpClient = client; + _httpClient = httpClient; + _logger = logger; } - public HttpClient HttpClient { get; } + public async Task ValidateAsync(Uri u, CancellationToken cancellationToken) + { + var result = new UrlValidationResult(u); + if (!u.IsValidUrl()) + { + result.SetBroken(); + _logger.LogError($"{u.OriginalString}) is not a valid URL."); + return result; + } + + try + { + var response = + await _httpClient.GetAsync(u, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken)) + result.SetSupportsHttps(); + if (response.IsSuccessStatusCode) + return result; + if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400) + { + result.SetRedirectsTo(response.Headers.Location); + } + else + { + result.SetBroken(); + _logger.LogError( + $"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}."); + } + + return result; + } + catch (HttpRequestException ex) + { + result.SetBroken(); + _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); + return result; + } + catch (TaskCanceledException ex) + { + result.SetBroken(); + _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); + return result; + } + catch (InvalidOperationException ex) + { + result.SetBroken(); + _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); + return result; + } + } + + private async Task IsHttpsSupported(Uri url, CancellationToken cancellationToken) + { + var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps}.Uri; + try + { + var response = await _httpClient.GetAsync(httpsUrl, HttpCompletionOption.ResponseHeadersRead, + cancellationToken); + if (response.IsSuccessStatusCode) + return true; + _logger.LogError( + $"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}."); + return response.IsSuccessStatusCode; + } + catch (HttpRequestException ex) + { + _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); + return false; + } + catch (TaskCanceledException ex) + { + _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); + return false; + } + catch (InvalidOperationException ex) + { + _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); + return false; + } + } } } \ No newline at end of file