diff --git a/src/FilterLists.Agent/Core/Interfaces/Repositories/IUrlRepository.cs b/src/FilterLists.Agent/Core/Interfaces/Repositories/IUrlRepository.cs index f9bdf66fa..be20d8c4d 100644 --- a/src/FilterLists.Agent/Core/Interfaces/Repositories/IUrlRepository.cs +++ b/src/FilterLists.Agent/Core/Interfaces/Repositories/IUrlRepository.cs @@ -1,11 +1,15 @@ using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; +using FilterLists.Agent.Features.Urls.Models.ValidationResults; namespace FilterLists.Agent.Core.Interfaces.Repositories { public interface IUrlRepository { Task> GetAllAsync(); + + Task ValidateAsync(Uri u, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Core/Interfaces/Services/IUrlService.cs b/src/FilterLists.Agent/Core/Interfaces/Services/IUrlService.cs deleted file mode 100644 index 5886ad84b..000000000 --- a/src/FilterLists.Agent/Core/Interfaces/Services/IUrlService.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using FilterLists.Agent.Features.Urls.Models.ValidationResults; - -namespace FilterLists.Agent.Core.Interfaces.Services -{ - public interface IUrlService - { - Task ValidateAsync(Uri u, CancellationToken cancellationToken); - } -} \ No newline at end of file diff --git a/src/FilterLists.Agent/Extensions/ServiceCollectionExtensions.cs b/src/FilterLists.Agent/Extensions/ServiceCollectionExtensions.cs index f77923a94..a8556cc83 100644 --- a/src/FilterLists.Agent/Extensions/ServiceCollectionExtensions.cs +++ b/src/FilterLists.Agent/Extensions/ServiceCollectionExtensions.cs @@ -4,10 +4,8 @@ using CommandLine; using FilterLists.Agent.AppSettings; using FilterLists.Agent.Core.Interfaces.Repositories; -using FilterLists.Agent.Core.Interfaces.Services; using FilterLists.Agent.Infrastructure.Clients; using FilterLists.Agent.Infrastructure.Repositories; -using FilterLists.Agent.Infrastructure.Services; using LibGit2Sharp; using MediatR; using Microsoft.Extensions.Configuration; @@ -32,11 +30,10 @@ public static void RegisterAgentServices(this IServiceCollection services) services.AddMediatR(AppDomain.CurrentDomain.GetAssemblies()); services.AddSingleton(); services.AddGitHubClient(); - services.AddUrlService(); + services.AddUrlRepository(); services.AddListRepository(); services.AddArchiveRepository(); services.AddTransient(); - services.AddTransient(); services.AddTransient(); } @@ -86,9 +83,9 @@ private static void AddGitHubClient(this IServiceCollection services) }); } - private static void AddUrlService(this IServiceCollection services) + private static void AddUrlRepository(this IServiceCollection services) { - services.AddHttpClient() + services.AddHttpClient() .ConfigureHttpMessageHandlerBuilder(b => { b.PrimaryHandler = new HttpClientHandler {AllowAutoRedirect = false}; diff --git a/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs b/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs index 386eb74f2..4473b2640 100644 --- a/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs +++ b/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; -using FilterLists.Agent.Core.Interfaces.Services; +using FilterLists.Agent.Core.Interfaces.Repositories; using FilterLists.Agent.Extensions; using FilterLists.Agent.Features.Urls.Models.ValidationResults; using MediatR; @@ -26,11 +26,11 @@ public Command(IEnumerable urls) public class Handler : IRequestHandler> { private const int MaxDegreeOfParallelism = 5; - private readonly IUrlService _urlService; + private readonly IUrlRepository _repo; - public Handler(IUrlService urlService) + public Handler(IUrlRepository urlRepository) { - _urlService = urlService; + _repo = urlRepository; } public async Task> Handle(Command request, CancellationToken cancellationToken) @@ -55,7 +55,7 @@ public async Task> Handle(Command request, Cancellatio private TransformBlock BuildValidator(CancellationToken cancellationToken) { return new TransformBlock( - async u => await _urlService.ValidateAsync(u, cancellationToken), + async u => await _repo.ValidateAsync(u, cancellationToken), new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism} ); } diff --git a/src/FilterLists.Agent/Infrastructure/Repositories/UrlRepository.cs b/src/FilterLists.Agent/Infrastructure/Repositories/UrlRepository.cs index 241528460..44d886f25 100644 --- a/src/FilterLists.Agent/Infrastructure/Repositories/UrlRepository.cs +++ b/src/FilterLists.Agent/Infrastructure/Repositories/UrlRepository.cs @@ -2,15 +2,23 @@ using System.Collections.Generic; using System.ComponentModel; using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; using System.Threading.Tasks; using FilterLists.Agent.Core.Interfaces.Repositories; +using FilterLists.Agent.Extensions; using FilterLists.Agent.Features.Urls.Models.DataFileUrls; +using FilterLists.Agent.Features.Urls.Models.ValidationResults; using FilterLists.Agent.Infrastructure.Clients; +using JetBrains.Annotations; using Microsoft.Extensions.Localization; +using Microsoft.Extensions.Logging; using RestSharp; namespace FilterLists.Agent.Infrastructure.Repositories { + [UsedImplicitly] public class UrlRepository : IUrlRepository { private static readonly Dictionary EntityUrlsEndpoints = new Dictionary @@ -23,12 +31,23 @@ public class UrlRepository : IUrlRepository }; private readonly IFilterListsApiClient _apiClient; + private readonly HttpClient _httpClient; private readonly IStringLocalizer _localizer; + private readonly ILogger _logger; - public UrlRepository(IFilterListsApiClient apiClient, IStringLocalizer stringLocalizer) + public UrlRepository(IFilterListsApiClient apiClient, HttpClient httpClient, + IStringLocalizer stringLocalizer, ILogger logger) { _apiClient = apiClient; + + httpClient.Timeout = TimeSpan.FromSeconds(90); + var header = new ProductHeaderValue("FilterLists.Agent"); + var userAgent = new ProductInfoHeaderValue(header); + httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent); + _httpClient = httpClient; + _localizer = stringLocalizer; + _logger = logger; } public async Task> GetAllAsync() @@ -40,5 +59,87 @@ public async Task> GetAllAsync() return response.SelectMany(r => r.GetType().GetProperties().Where(p => p.GetValue(r) != null).Select(p => (Uri)p.GetValue(r))); } + + + public async Task ValidateAsync(Uri u, CancellationToken cancellationToken) + { + var result = new UrlValidationResult(u); + if (!u.IsValidUrl()) + { + result.SetBroken(); + _logger.LogError($"{u.OriginalString}) is not a valid URL."); + return result; + } + + try + { + var response = + await _httpClient.GetAsync(u, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken)) + result.SetSupportsHttps(); + if (response.IsSuccessStatusCode) + return result; + if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400) + { + result.SetRedirectsTo(response.Headers.Location); + } + else + { + result.SetBroken(); + _logger.LogError( + $"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}."); + } + + return result; + } + catch (HttpRequestException ex) + { + result.SetBroken(); + _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); + return result; + } + catch (TaskCanceledException ex) + { + result.SetBroken(); + _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); + return result; + } + catch (InvalidOperationException ex) + { + result.SetBroken(); + _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); + return result; + } + } + + private async Task IsHttpsSupported(Uri url, CancellationToken cancellationToken) + { + var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps}.Uri; + try + { + var response = await _httpClient.GetAsync(httpsUrl, HttpCompletionOption.ResponseHeadersRead, + cancellationToken); + if (response.IsSuccessStatusCode) + return true; + _logger.LogError( + $"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}."); + return response.IsSuccessStatusCode; + } + catch (HttpRequestException ex) + { + _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); + return false; + } + catch (TaskCanceledException ex) + { + _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); + return false; + } + catch (InvalidOperationException ex) + { + _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); + return false; + } + } } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Infrastructure/Services/UrlService.cs b/src/FilterLists.Agent/Infrastructure/Services/UrlService.cs deleted file mode 100644 index 448d1f035..000000000 --- a/src/FilterLists.Agent/Infrastructure/Services/UrlService.cs +++ /dev/null @@ -1,113 +0,0 @@ -using System; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Threading; -using System.Threading.Tasks; -using FilterLists.Agent.Core.Interfaces.Services; -using FilterLists.Agent.Extensions; -using FilterLists.Agent.Features.Urls.Models.ValidationResults; -using JetBrains.Annotations; -using Microsoft.Extensions.Logging; - -namespace FilterLists.Agent.Infrastructure.Services -{ - [UsedImplicitly] - public class UrlService : IUrlService - { - private readonly HttpClient _httpClient; - private readonly ILogger _logger; - - public UrlService(HttpClient httpClient, ILogger logger) - { - httpClient.Timeout = TimeSpan.FromSeconds(90); - - var header = new ProductHeaderValue("FilterLists.Agent"); - var userAgent = new ProductInfoHeaderValue(header); - httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent); - - _httpClient = httpClient; - _logger = logger; - } - - public async Task ValidateAsync(Uri u, CancellationToken cancellationToken) - { - var result = new UrlValidationResult(u); - if (!u.IsValidUrl()) - { - result.SetBroken(); - _logger.LogError($"{u.OriginalString}) is not a valid URL."); - return result; - } - - try - { - var response = - await _httpClient.GetAsync(u, HttpCompletionOption.ResponseHeadersRead, cancellationToken); - if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken)) - result.SetSupportsHttps(); - if (response.IsSuccessStatusCode) - return result; - if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400) - { - result.SetRedirectsTo(response.Headers.Location); - } - else - { - result.SetBroken(); - _logger.LogError( - $"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}."); - } - - return result; - } - catch (HttpRequestException ex) - { - result.SetBroken(); - _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); - return result; - } - catch (TaskCanceledException ex) - { - result.SetBroken(); - _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); - return result; - } - catch (InvalidOperationException ex) - { - result.SetBroken(); - _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); - return result; - } - } - - private async Task IsHttpsSupported(Uri url, CancellationToken cancellationToken) - { - var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps}.Uri; - try - { - var response = await _httpClient.GetAsync(httpsUrl, HttpCompletionOption.ResponseHeadersRead, - cancellationToken); - if (response.IsSuccessStatusCode) - return true; - _logger.LogError( - $"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}."); - return response.IsSuccessStatusCode; - } - catch (HttpRequestException ex) - { - _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); - return false; - } - catch (TaskCanceledException ex) - { - _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); - return false; - } - catch (InvalidOperationException ex) - { - _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex); - return false; - } - } - } -} \ No newline at end of file