prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

mv url validation logic to UrlService

Browse source
This commit is contained in:
Collin M. Barrett 2019-07-07 20:56:43 -05:00
parent 7af5e0080e
commit deaf5e550f
3 changed files with 102 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/FilterLists.Agent/Core/Interfaces/Services/IUrlService.cs
View file

@ -1,9 +1,12 @@
using System.Net.Http;
using System;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Features.Urls.Models.ValidationResults;
namespace FilterLists.Agent.Core.Interfaces.Services
{
public interface IUrlService
{
HttpClient HttpClient { get; }
Task<UrlValidationResult> ValidateAsync(Uri u, CancellationToken cancellationToken);
}
}

91
src/FilterLists.Agent/Features/Urls/ValidateUrls.cs
View file

@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
@ -9,7 +8,6 @@
using FilterLists.Agent.Extensions;
using FilterLists.Agent.Features.Urls.Models.ValidationResults;
using MediatR;
using Microsoft.Extensions.Logging;
namespace FilterLists.Agent.Features.Urls
{
@ -28,13 +26,11 @@ public Command(IEnumerable<Uri> urls)
public class Handler : IRequestHandler<Command, List<UrlValidationResult>>
{
private const int MaxDegreeOfParallelism = 5;
private readonly HttpClient _httpClient;
private readonly ILogger<Handler> _logger;
private readonly IUrlService _urlService;
public Handler(IUrlService urlService, ILogger<Handler> logger)
public Handler(IUrlService urlService)
{
_httpClient = urlService.HttpClient;
_logger = logger;
_urlService = urlService;
}
public async Task<List<UrlValidationResult>> Handle(Command request, CancellationToken cancellationToken)
@ -59,89 +55,10 @@ public async Task<List<UrlValidationResult>> Handle(Command request, Cancellatio
private TransformBlock<Uri, UrlValidationResult> BuildValidator(CancellationToken cancellationToken)
{
return new TransformBlock<Uri, UrlValidationResult>(
async u =>
{
var result = new UrlValidationResult(u);
if (!u.IsValidUrl())
{
result.SetBroken();
_logger.LogError($"{u.OriginalString}) is not a valid URL.");
return result;
}
try
{
var response = await _httpClient.GetAsync(u, HttpCompletionOption.ResponseHeadersRead,
cancellationToken);
if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken))
result.SetSupportsHttps();
if (response.IsSuccessStatusCode)
return result;
if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400)
{
result.SetRedirectsTo(response.Headers.Location);
}
else
{
result.SetBroken();
_logger.LogError(
$"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}.");
}
return result;
}
catch (HttpRequestException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
catch (TaskCanceledException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
catch (InvalidOperationException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
},
async u => await _urlService.ValidateAsync(u, cancellationToken),
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
);
}
private async Task<bool> IsHttpsSupported(Uri url, CancellationToken cancellationToken)
{
var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps}.Uri;
try
{
var response = await _httpClient.GetAsync(httpsUrl, HttpCompletionOption.ResponseHeadersRead,
cancellationToken);
if (response.IsSuccessStatusCode)
return true;
_logger.LogError(
$"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}.");
return response.IsSuccessStatusCode;
}
catch (HttpRequestException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
catch (TaskCanceledException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
catch (InvalidOperationException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
}
}
}
}

98
src/FilterLists.Agent/Infrastructure/Services/UrlService.cs
View file

@ -1,25 +1,113 @@
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Core.Interfaces.Services;
using FilterLists.Agent.Extensions;
using FilterLists.Agent.Features.Urls.Models.ValidationResults;
using JetBrains.Annotations;
using Microsoft.Extensions.Logging;
namespace FilterLists.Agent.Infrastructure.Services
{
[UsedImplicitly]
public class UrlService : IUrlService
{
public UrlService(HttpClient client)
private readonly HttpClient _httpClient;
private readonly ILogger<UrlService> _logger;
public UrlService(HttpClient httpClient, ILogger<UrlService> logger)
{
client.Timeout = TimeSpan.FromSeconds(90);
httpClient.Timeout = TimeSpan.FromSeconds(90);
var header = new ProductHeaderValue("FilterLists.Agent");
var userAgent = new ProductInfoHeaderValue(header);
client.DefaultRequestHeaders.UserAgent.Add(userAgent);
httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent);
HttpClient = client;
_httpClient = httpClient;
_logger = logger;
}
public HttpClient HttpClient { get; }
public async Task<UrlValidationResult> ValidateAsync(Uri u, CancellationToken cancellationToken)
{
var result = new UrlValidationResult(u);
if (!u.IsValidUrl())
{
result.SetBroken();
_logger.LogError($"{u.OriginalString}) is not a valid URL.");
return result;
}
try
{
var response =
await _httpClient.GetAsync(u, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken))
result.SetSupportsHttps();
if (response.IsSuccessStatusCode)
return result;
if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400)
{
result.SetRedirectsTo(response.Headers.Location);
}
else
{
result.SetBroken();
_logger.LogError(
$"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}.");
}
return result;
}
catch (HttpRequestException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
catch (TaskCanceledException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
catch (InvalidOperationException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
}
private async Task<bool> IsHttpsSupported(Uri url, CancellationToken cancellationToken)
{
var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps}.Uri;
try
{
var response = await _httpClient.GetAsync(httpsUrl, HttpCompletionOption.ResponseHeadersRead,
cancellationToken);
if (response.IsSuccessStatusCode)
return true;
_logger.LogError(
$"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}.");
return response.IsSuccessStatusCode;
}
catch (HttpRequestException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
catch (TaskCanceledException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
catch (InvalidOperationException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
}
}
}