prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

simplify validator a bit

Browse source
This commit is contained in:
Collin M. Barrett 2019-07-14 19:20:06 -05:00
parent feb18d5a84
commit a1dde46887
5 changed files with 78 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/FilterLists.Agent/Core/Urls/IUrlValidator.cs → src/FilterLists.Agent/Core/Urls/IEntityUrlValidator.cs
View file

@ -3,7 +3,7 @@
namespace FilterLists.Agent.Core.Urls
{
public interface IUrlValidator
public interface IEntityUrlValidator
{
Task<EntityUrl> ValidateAsync(EntityUrl entityUrl, CancellationToken cancellationToken);
}

8
src/FilterLists.Agent/Features/Urls/ValidateUrls.cs
View file

@ -23,11 +23,11 @@ public Command(IEnumerable<EntityUrl> entityUrls)
public class Handler : IRequestHandler<Command, IEnumerable<EntityUrl>>
{
private const int MaxDegreeOfParallelism = 5;
private readonly IUrlValidator _urlValidator;
private readonly IEntityUrlValidator _entityUrlValidator;
public Handler(IUrlValidator urlValidator)
public Handler(IEntityUrlValidator entityUrlValidator)
{
_urlValidator = urlValidator;
_entityUrlValidator = entityUrlValidator;
}
public async Task<IEnumerable<EntityUrl>> Handle(Command request, CancellationToken cancellationToken)
@ -46,7 +46,7 @@ public async Task<IEnumerable<EntityUrl>> Handle(Command request, CancellationTo
private TransformBlock<EntityUrl, EntityUrl> BuildValidator(CancellationToken cancellationToken)
{
return new TransformBlock<EntityUrl, EntityUrl>(
e => _urlValidator.ValidateAsync(e, cancellationToken),
e => _entityUrlValidator.ValidateAsync(e, cancellationToken),
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
);
}

72
src/FilterLists.Agent/Infrastructure/Web/EntityUrlValidator.cs Normal file
View file

@ -0,0 +1,72 @@
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Core.Urls;
namespace FilterLists.Agent.Infrastructure.Web
{
public class EntityUrlValidator : IEntityUrlValidator
{
private readonly HttpClient _httpClient;
public EntityUrlValidator(HttpClient httpClient)
{
httpClient.Timeout = TimeSpan.FromSeconds(90);
var header = new ProductHeaderValue("FilterLists.Agent");
var userAgent = new ProductInfoHeaderValue(header);
httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent);
_httpClient = httpClient;
}
public async Task<EntityUrl> ValidateAsync(EntityUrl entityUrl, CancellationToken cancellationToken)
{
var url = entityUrl.Url;
if (!Uri.IsWellFormedUriString(url.OriginalString, UriKind.Absolute))
{
entityUrl.SetBroken();
return entityUrl;
}
if (url.Scheme == Uri.UriSchemeHttp)
if (await IsHttpsSupported(url, cancellationToken))
entityUrl.SetSupportsHttps();
try
{
using var response = await GetAsync(url, cancellationToken);
if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400)
entityUrl.SetRedirectsTo(response.Headers.Location);
else if (!response.IsSuccessStatusCode)
entityUrl.SetBroken();
}
catch (Exception ex) when (ex is HttpRequestException ||
ex is TaskCanceledException)
{
entityUrl.SetBroken();
}
return entityUrl;
}
private async Task<bool> IsHttpsSupported(Uri url, CancellationToken cancellationToken)
{
var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps, Port = -1}.Uri;
try
{
using var response = await GetAsync(httpsUrl, cancellationToken);
return response.IsSuccessStatusCode;
}
catch (Exception ex) when (ex is HttpRequestException ||
ex is TaskCanceledException)
{
return false;
}
}
private async Task<HttpResponseMessage> GetAsync(Uri url, CancellationToken cancellationToken)
{
return await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
}
}
}

2
src/FilterLists.Agent/Infrastructure/Web/ServiceCollectionExtensions.cs
View file

@ -11,7 +11,7 @@ public static void AddWebServices(this IServiceCollection services)
{
services.AddHttpClient<IListRepository, ListRepository>()
.AddPolicyHandlerFromRegistry("waitAndRetryTooManyRequests");
services.AddHttpClient<IUrlValidator, UrlValidator>().ConfigureHttpMessageHandlerBuilder(b =>
services.AddHttpClient<IEntityUrlValidator, EntityUrlValidator>().ConfigureHttpMessageHandlerBuilder(b =>
{
b.PrimaryHandler = new HttpClientHandler {AllowAutoRedirect = false};
b.Build();

104
src/FilterLists.Agent/Infrastructure/Web/UrlValidator.cs
View file

@ -1,104 +0,0 @@
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Core.Urls;
using Microsoft.Extensions.Logging;
namespace FilterLists.Agent.Infrastructure.Web
{
public class UrlValidator : IUrlValidator
{
private readonly HttpClient _httpClient;
private readonly ILogger<UrlValidator> _logger;
public UrlValidator(HttpClient httpClient, ILogger<UrlValidator> logger)
{
httpClient.Timeout = TimeSpan.FromSeconds(90);
var header = new ProductHeaderValue("FilterLists.Agent");
var userAgent = new ProductInfoHeaderValue(header);
httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent);
_httpClient = httpClient;
_logger = logger;
}
public async Task<EntityUrl> ValidateAsync(EntityUrl entityUrl, CancellationToken cancellationToken)
{
var url = entityUrl.Url;
if (!Uri.IsWellFormedUriString(url.OriginalString, UriKind.Absolute))
{
entityUrl.SetBroken();
_logger.LogError(
$"Url validation for ({url.OriginalString}) failed because it appears to be improperly formatted.");
return entityUrl;
}
try
{
using var response =
await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
if (url.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(url, cancellationToken))
entityUrl.SetSupportsHttps();
if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400)
{
entityUrl.SetRedirectsTo(response.Headers.Location);
}
else if (!response.IsSuccessStatusCode)
{
entityUrl.SetBroken();
_logger.LogError(
$"Url validation for ({url.AbsoluteUri}) failed with status code: {response.StatusCode}.");
}
}
catch (HttpRequestException ex)
{
entityUrl.SetBroken();
_logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex);
}
catch (TaskCanceledException ex)
{
entityUrl.SetBroken();
_logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex);
}
catch (InvalidOperationException ex)
{
entityUrl.SetBroken();
_logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex);
}
return entityUrl;
}
private async Task<bool> IsHttpsSupported(Uri url, CancellationToken cancellationToken)
{
var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps, Port = -1}.Uri;
try
{
using var response = await _httpClient.GetAsync(httpsUrl, HttpCompletionOption.ResponseHeadersRead,
cancellationToken);
if (response.IsSuccessStatusCode)
return true;
_logger.LogInformation(
$"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}.");
return response.IsSuccessStatusCode;
}
catch (HttpRequestException ex)
{
_logger.LogInformation($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
catch (TaskCanceledException ex)
{
_logger.LogInformation($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
catch (InvalidOperationException ex)
{
_logger.LogInformation($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
}
}
}