prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

conmbine IUrlService and IUrlRepository

Browse source
This commit is contained in:
Collin M. Barrett 2019-07-08 13:07:30 -05:00
parent bc4149414b
commit 4202bce222
6 changed files with 114 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/FilterLists.Agent/Core/Interfaces/Repositories/IUrlRepository.cs
View file

@ -1,11 +1,15 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Features.Urls.Models.ValidationResults;
namespace FilterLists.Agent.Core.Interfaces.Repositories
{
public interface IUrlRepository
{
Task<IEnumerable<Uri>> GetAllAsync<TModel>();
Task<UrlValidationResult> ValidateAsync(Uri u, CancellationToken cancellationToken);
}
}

12
src/FilterLists.Agent/Core/Interfaces/Services/IUrlService.cs
View file

@ -1,12 +0,0 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Features.Urls.Models.ValidationResults;
namespace FilterLists.Agent.Core.Interfaces.Services
{
public interface IUrlService
{
Task<UrlValidationResult> ValidateAsync(Uri u, CancellationToken cancellationToken);
}
}

9
src/FilterLists.Agent/Extensions/ServiceCollectionExtensions.cs
View file

@ -4,10 +4,8 @@
using CommandLine;
using FilterLists.Agent.AppSettings;
using FilterLists.Agent.Core.Interfaces.Repositories;
using FilterLists.Agent.Core.Interfaces.Services;
using FilterLists.Agent.Infrastructure.Clients;
using FilterLists.Agent.Infrastructure.Repositories;
using FilterLists.Agent.Infrastructure.Services;
using LibGit2Sharp;
using MediatR;
using Microsoft.Extensions.Configuration;
@ -32,11 +30,10 @@ public static void RegisterAgentServices(this IServiceCollection services)
services.AddMediatR(AppDomain.CurrentDomain.GetAssemblies());
services.AddSingleton<IFilterListsApiClient, FilterListsApiClient>();
services.AddGitHubClient();
services.AddUrlService();
services.AddUrlRepository();
services.AddListRepository();
services.AddArchiveRepository();
services.AddTransient<IListInfoRepository, ListInfoRepository>();
services.AddTransient<IUrlRepository, UrlRepository>();
services.AddTransient<IGitHubIssuesRepository, GitHubIssuesRepository>();
}
@ -86,9 +83,9 @@ private static void AddGitHubClient(this IServiceCollection services)
});
}
private static void AddUrlService(this IServiceCollection services)
private static void AddUrlRepository(this IServiceCollection services)
{
services.AddHttpClient<IUrlService, UrlService>()
services.AddHttpClient<IUrlRepository, UrlRepository>()
.ConfigureHttpMessageHandlerBuilder(b =>
{
b.PrimaryHandler = new HttpClientHandler {AllowAutoRedirect = false};

10
src/FilterLists.Agent/Features/Urls/ValidateUrls.cs
View file

@ -4,7 +4,7 @@
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using FilterLists.Agent.Core.Interfaces.Services;
using FilterLists.Agent.Core.Interfaces.Repositories;
using FilterLists.Agent.Extensions;
using FilterLists.Agent.Features.Urls.Models.ValidationResults;
using MediatR;
@ -26,11 +26,11 @@ public Command(IEnumerable<Uri> urls)
public class Handler : IRequestHandler<Command, List<UrlValidationResult>>
{
private const int MaxDegreeOfParallelism = 5;
private readonly IUrlService _urlService;
private readonly IUrlRepository _repo;
public Handler(IUrlService urlService)
public Handler(IUrlRepository urlRepository)
{
_urlService = urlService;
_repo = urlRepository;
}
public async Task<List<UrlValidationResult>> Handle(Command request, CancellationToken cancellationToken)
@ -55,7 +55,7 @@ public async Task<List<UrlValidationResult>> Handle(Command request, Cancellatio
private TransformBlock<Uri, UrlValidationResult> BuildValidator(CancellationToken cancellationToken)
{
return new TransformBlock<Uri, UrlValidationResult>(
async u => await _urlService.ValidateAsync(u, cancellationToken),
async u => await _repo.ValidateAsync(u, cancellationToken),
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
);
}

103
src/FilterLists.Agent/Infrastructure/Repositories/UrlRepository.cs
View file

@ -2,15 +2,23 @@
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Core.Interfaces.Repositories;
using FilterLists.Agent.Extensions;
using FilterLists.Agent.Features.Urls.Models.DataFileUrls;
using FilterLists.Agent.Features.Urls.Models.ValidationResults;
using FilterLists.Agent.Infrastructure.Clients;
using JetBrains.Annotations;
using Microsoft.Extensions.Localization;
using Microsoft.Extensions.Logging;
using RestSharp;
namespace FilterLists.Agent.Infrastructure.Repositories
{
[UsedImplicitly]
public class UrlRepository : IUrlRepository
{
private static readonly Dictionary<string, string> EntityUrlsEndpoints = new Dictionary<string, string>
@ -23,12 +31,23 @@ public class UrlRepository : IUrlRepository
};
private readonly IFilterListsApiClient _apiClient;
private readonly HttpClient _httpClient;
private readonly IStringLocalizer<UrlRepository> _localizer;
private readonly ILogger<UrlRepository> _logger;
public UrlRepository(IFilterListsApiClient apiClient, IStringLocalizer<UrlRepository> stringLocalizer)
public UrlRepository(IFilterListsApiClient apiClient, HttpClient httpClient,
IStringLocalizer<UrlRepository> stringLocalizer, ILogger<UrlRepository> logger)
{
_apiClient = apiClient;
httpClient.Timeout = TimeSpan.FromSeconds(90);
var header = new ProductHeaderValue("FilterLists.Agent");
var userAgent = new ProductInfoHeaderValue(header);
httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent);
_httpClient = httpClient;
_localizer = stringLocalizer;
_logger = logger;
}
public async Task<IEnumerable<Uri>> GetAllAsync<TModel>()
@ -40,5 +59,87 @@ public async Task<IEnumerable<Uri>> GetAllAsync<TModel>()
return response.SelectMany(r =>
r.GetType().GetProperties().Where(p => p.GetValue(r) != null).Select(p => (Uri)p.GetValue(r)));
}
public async Task<UrlValidationResult> ValidateAsync(Uri u, CancellationToken cancellationToken)
{
var result = new UrlValidationResult(u);
if (!u.IsValidUrl())
{
result.SetBroken();
_logger.LogError($"{u.OriginalString}) is not a valid URL.");
return result;
}
try
{
var response =
await _httpClient.GetAsync(u, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken))
result.SetSupportsHttps();
if (response.IsSuccessStatusCode)
return result;
if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400)
{
result.SetRedirectsTo(response.Headers.Location);
}
else
{
result.SetBroken();
_logger.LogError(
$"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}.");
}
return result;
}
catch (HttpRequestException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
catch (TaskCanceledException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
catch (InvalidOperationException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
}
private async Task<bool> IsHttpsSupported(Uri url, CancellationToken cancellationToken)
{
var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps}.Uri;
try
{
var response = await _httpClient.GetAsync(httpsUrl, HttpCompletionOption.ResponseHeadersRead,
cancellationToken);
if (response.IsSuccessStatusCode)
return true;
_logger.LogError(
$"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}.");
return response.IsSuccessStatusCode;
}
catch (HttpRequestException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
catch (TaskCanceledException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
catch (InvalidOperationException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
}
}
}

113
src/FilterLists.Agent/Infrastructure/Services/UrlService.cs
View file

@ -1,113 +0,0 @@
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Core.Interfaces.Services;
using FilterLists.Agent.Extensions;
using FilterLists.Agent.Features.Urls.Models.ValidationResults;
using JetBrains.Annotations;
using Microsoft.Extensions.Logging;
namespace FilterLists.Agent.Infrastructure.Services
{
[UsedImplicitly]
public class UrlService : IUrlService
{
private readonly HttpClient _httpClient;
private readonly ILogger<UrlService> _logger;
public UrlService(HttpClient httpClient, ILogger<UrlService> logger)
{
httpClient.Timeout = TimeSpan.FromSeconds(90);
var header = new ProductHeaderValue("FilterLists.Agent");
var userAgent = new ProductInfoHeaderValue(header);
httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent);
_httpClient = httpClient;
_logger = logger;
}
public async Task<UrlValidationResult> ValidateAsync(Uri u, CancellationToken cancellationToken)
{
var result = new UrlValidationResult(u);
if (!u.IsValidUrl())
{
result.SetBroken();
_logger.LogError($"{u.OriginalString}) is not a valid URL.");
return result;
}
try
{
var response =
await _httpClient.GetAsync(u, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken))
result.SetSupportsHttps();
if (response.IsSuccessStatusCode)
return result;
if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400)
{
result.SetRedirectsTo(response.Headers.Location);
}
else
{
result.SetBroken();
_logger.LogError(
$"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}.");
}
return result;
}
catch (HttpRequestException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
catch (TaskCanceledException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
catch (InvalidOperationException ex)
{
result.SetBroken();
_logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
return result;
}
}
private async Task<bool> IsHttpsSupported(Uri url, CancellationToken cancellationToken)
{
var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps}.Uri;
try
{
var response = await _httpClient.GetAsync(httpsUrl, HttpCompletionOption.ResponseHeadersRead,
cancellationToken);
if (response.IsSuccessStatusCode)
return true;
_logger.LogError(
$"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}.");
return response.IsSuccessStatusCode;
}
catch (HttpRequestException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
catch (TaskCanceledException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
catch (InvalidOperationException ex)
{
_logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
return false;
}
}
}
}