From a2fb045cc318e8087d998e6cd188f125d9e096a2 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sat, 13 Jul 2019 12:24:54 -0500 Subject: [PATCH] re-architect URL validator --- ...epository.cs => IListViewUrlRepository.cs} | 4 +- .../Core/Lists/{ListUrl.cs => ListViewUrl.cs} | 2 +- src/FilterLists.Agent/Core/Urls/EntityUrl.cs | 42 ++++++++++++++ .../Core/Urls/FilterListsEntity.cs | 11 ++++ .../Core/Urls/IEntityUrlRepository.cs | 11 ++++ .../Core/Urls/IUrlRepository.cs | 12 ---- .../Core/Urls/IUrlValidator.cs | 5 +- .../Core/Urls/UrlValidationResult.cs | 39 ------------- .../Extensions/UriExtensions.cs | 34 ----------- .../Features/Lists/ArchiveLists.cs | 6 +- .../Features/Lists/DownloadLists.cs | 14 ++--- .../Features/Lists/DownloadRawText.cs | 12 ++-- .../Features/Lists/DownloadSevenZip.cs | 10 ++-- .../Features/Lists/DownloadZip.cs | 10 ++-- .../Features/Urls/BuildGitHubIssueBody.cs | 21 ++++--- .../Urls/CreateOrUpdateUrlValidationIssue.cs | 15 +++-- .../Models/{DataFileUrls => }/LicenseUrls.cs | 3 +- .../Models/{DataFileUrls => }/ListUrls.cs | 3 +- .../{DataFileUrls => }/MaintainerUrls.cs | 3 +- .../Models/{DataFileUrls => }/SoftwareUrls.cs | 3 +- .../Models/{DataFileUrls => }/SyntaxUrls.cs | 3 +- .../DataFileUrlValidationResults.cs | 18 ------ .../Features/Urls/ValidateAllUrls.cs | 49 ++++------------ .../Features/Urls/ValidateUrls.cs | 41 ++++++------- .../FilterListsApi/EntityUrlRepository.cs | 57 +++++++++++++++++++ ...Repository.cs => ListViewUrlRepository.cs} | 8 +-- .../ServiceCollectionExtensions.cs | 4 +- .../FilterListsApi/UrlRepository.cs | 46 --------------- .../Infrastructure/Web/UrlValidator.cs | 52 +++++++---------- 29 files changed, 235 insertions(+), 303 deletions(-) rename src/FilterLists.Agent/Core/Lists/{IListUrlRepository.cs => IListViewUrlRepository.cs} (52%) rename src/FilterLists.Agent/Core/Lists/{ListUrl.cs => ListViewUrl.cs} (89%) create mode 100644 src/FilterLists.Agent/Core/Urls/EntityUrl.cs create mode 100644 src/FilterLists.Agent/Core/Urls/FilterListsEntity.cs create mode 100644 src/FilterLists.Agent/Core/Urls/IEntityUrlRepository.cs delete mode 100644 src/FilterLists.Agent/Core/Urls/IUrlRepository.cs delete mode 100644 src/FilterLists.Agent/Core/Urls/UrlValidationResult.cs rename src/FilterLists.Agent/Features/Urls/Models/{DataFileUrls => }/LicenseUrls.cs (62%) rename src/FilterLists.Agent/Features/Urls/Models/{DataFileUrls => }/ListUrls.cs (88%) rename src/FilterLists.Agent/Features/Urls/Models/{DataFileUrls => }/MaintainerUrls.cs (62%) rename src/FilterLists.Agent/Features/Urls/Models/{DataFileUrls => }/SoftwareUrls.cs (69%) rename src/FilterLists.Agent/Features/Urls/Models/{DataFileUrls => }/SyntaxUrls.cs (62%) delete mode 100644 src/FilterLists.Agent/Features/Urls/Models/ValidationResults/DataFileUrlValidationResults.cs create mode 100644 src/FilterLists.Agent/Infrastructure/FilterListsApi/EntityUrlRepository.cs rename src/FilterLists.Agent/Infrastructure/FilterListsApi/{ListUrlRepository.cs => ListViewUrlRepository.cs} (62%) delete mode 100644 src/FilterLists.Agent/Infrastructure/FilterListsApi/UrlRepository.cs diff --git a/src/FilterLists.Agent/Core/Lists/IListUrlRepository.cs b/src/FilterLists.Agent/Core/Lists/IListViewUrlRepository.cs similarity index 52% rename from src/FilterLists.Agent/Core/Lists/IListUrlRepository.cs rename to src/FilterLists.Agent/Core/Lists/IListViewUrlRepository.cs index 5cdf82efd..fd37620ec 100644 --- a/src/FilterLists.Agent/Core/Lists/IListUrlRepository.cs +++ b/src/FilterLists.Agent/Core/Lists/IListViewUrlRepository.cs @@ -4,8 +4,8 @@ namespace FilterLists.Agent.Core.Lists { - public interface IListUrlRepository + public interface IListViewUrlRepository { - Task> GetAllAsync(CancellationToken cancellationToken); + Task> GetAllAsync(CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Core/Lists/ListUrl.cs b/src/FilterLists.Agent/Core/Lists/ListViewUrl.cs similarity index 89% rename from src/FilterLists.Agent/Core/Lists/ListUrl.cs rename to src/FilterLists.Agent/Core/Lists/ListViewUrl.cs index 5dab18bf4..820bb8ae9 100644 --- a/src/FilterLists.Agent/Core/Lists/ListUrl.cs +++ b/src/FilterLists.Agent/Core/Lists/ListViewUrl.cs @@ -4,7 +4,7 @@ namespace FilterLists.Agent.Core.Lists { [UsedImplicitly] - public class ListUrl + public class ListViewUrl { public int Id { get; [UsedImplicitly] private set; } public Uri ViewUrl { get; [UsedImplicitly] private set; } diff --git a/src/FilterLists.Agent/Core/Urls/EntityUrl.cs b/src/FilterLists.Agent/Core/Urls/EntityUrl.cs new file mode 100644 index 000000000..24d09dc4a --- /dev/null +++ b/src/FilterLists.Agent/Core/Urls/EntityUrl.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace FilterLists.Agent.Core.Urls +{ + public class EntityUrl + { + public EntityUrl(FilterListsEntity filterListsEntity, int id, Uri viewUrl) + { + FilterListsEntity = filterListsEntity; + Id = id; + ViewUrl = viewUrl; + } + + public FilterListsEntity FilterListsEntity { get; } + public int Id { get; } + public Uri ViewUrl { get; } + public List ValidationMessages { get; } = new List(); + + public bool IsValid() + { + return !ValidationMessages.Any(); + } + + public void SetBroken() + { + ValidationMessages.Add("This URL might be broken and should maybe be fixed, replaced, or removed."); + } + + public void SetSupportsHttps() + { + ValidationMessages.Add( + "This URL uses the HTTP protocol but appears to be able to support and should probably be changed to HTTPS."); + } + + public void SetRedirectsTo(Uri uri) + { + ValidationMessages.Add($"This URL appears to redirect and should maybe be changed to \"{uri}\"."); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/Core/Urls/FilterListsEntity.cs b/src/FilterLists.Agent/Core/Urls/FilterListsEntity.cs new file mode 100644 index 000000000..f5f0e57dd --- /dev/null +++ b/src/FilterLists.Agent/Core/Urls/FilterListsEntity.cs @@ -0,0 +1,11 @@ +namespace FilterLists.Agent.Core.Urls +{ + public enum FilterListsEntity + { + License, + FilterList, + Maintainer, + Software, + Syntax + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/Core/Urls/IEntityUrlRepository.cs b/src/FilterLists.Agent/Core/Urls/IEntityUrlRepository.cs new file mode 100644 index 000000000..9c99a6eb1 --- /dev/null +++ b/src/FilterLists.Agent/Core/Urls/IEntityUrlRepository.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace FilterLists.Agent.Core.Urls +{ + public interface IEntityUrlRepository + { + Task> GetAllAsync(CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/Core/Urls/IUrlRepository.cs b/src/FilterLists.Agent/Core/Urls/IUrlRepository.cs deleted file mode 100644 index cf69c6b7f..000000000 --- a/src/FilterLists.Agent/Core/Urls/IUrlRepository.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; - -namespace FilterLists.Agent.Core.Urls -{ - public interface IUrlRepository - { - Task> GetAllAsync(CancellationToken cancellationToken) where TModel : class, new(); - } -} \ No newline at end of file diff --git a/src/FilterLists.Agent/Core/Urls/IUrlValidator.cs b/src/FilterLists.Agent/Core/Urls/IUrlValidator.cs index 120534fb3..849ef3851 100644 --- a/src/FilterLists.Agent/Core/Urls/IUrlValidator.cs +++ b/src/FilterLists.Agent/Core/Urls/IUrlValidator.cs @@ -1,11 +1,10 @@ -using System; -using System.Threading; +using System.Threading; using System.Threading.Tasks; namespace FilterLists.Agent.Core.Urls { public interface IUrlValidator { - Task ValidateAsync(Uri u, CancellationToken cancellationToken); + Task ValidateAsync(EntityUrl entityUrl, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Core/Urls/UrlValidationResult.cs b/src/FilterLists.Agent/Core/Urls/UrlValidationResult.cs deleted file mode 100644 index 9e1a7edf2..000000000 --- a/src/FilterLists.Agent/Core/Urls/UrlValidationResult.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; - -namespace FilterLists.Agent.Core.Urls -{ - public class UrlValidationResult - { - public UrlValidationResult(Uri url) - { - Url = url; - } - - public Uri Url { get; } - - public List Messages { get; } = new List(); - - public bool IsValid() - { - return !Messages.Any(); - } - - public void SetBroken() - { - Messages.Add("This URL might be broken and should maybe be fixed, replaced, or removed."); - } - - public void SetSupportsHttps() - { - Messages.Add( - "This URL uses the HTTP protocol but appears to be able to support and should probably be changed to HTTPS."); - } - - public void SetRedirectsTo(Uri uri) - { - Messages.Add($"This URL appears to redirect and should maybe be changed to \"{uri}\"."); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Agent/Extensions/UriExtensions.cs b/src/FilterLists.Agent/Extensions/UriExtensions.cs index ed75427b4..0dd88cb42 100644 --- a/src/FilterLists.Agent/Extensions/UriExtensions.cs +++ b/src/FilterLists.Agent/Extensions/UriExtensions.cs @@ -1,8 +1,5 @@ using System; -using System.Collections.Generic; using System.IO; -using System.Linq; -using FilterLists.Agent.Core.Lists; namespace FilterLists.Agent.Extensions { @@ -12,36 +9,5 @@ public static string GetExtension(this Uri uri) { return Path.GetExtension(uri.AbsolutePath); } - - public static bool IsValidUrl(this Uri uri) - { - var uriString = uri.OriginalString; - if (!Uri.IsWellFormedUriString(uriString, UriKind.Absolute)) - return false; - if (!Uri.TryCreate(uriString, UriKind.Absolute, out var tmp)) - return false; - return tmp.Scheme == Uri.UriSchemeHttp || tmp.Scheme == Uri.UriSchemeHttps; - } - - public static IEnumerable DistributeByHost(this IEnumerable listInfo) - { - var listInfoList = listInfo.ToList(); - var distributedLists = listInfoList.Where(u => u.IsValidUrl()) - .GroupBy(u => u.Host) - .SelectMany((g, gi) => g.Select((u, i) => new {Index = i, GroupIndex = gi, ListInfo = u})) - .OrderBy(a => a.Index) - .ThenBy(a => a.GroupIndex) - .Select(a => a.ListInfo); - return listInfoList.Where(u => !u.IsValidUrl()).Concat(distributedLists); - } - - public static IEnumerable DistributeByHost(this IEnumerable listInfo) - { - return listInfo.GroupBy(l => l.ViewUrl.Host) - .SelectMany((g, gi) => g.Select((l, i) => new {Index = i, GroupIndex = gi, ListInfo = l})) - .OrderBy(a => a.Index) - .ThenBy(a => a.GroupIndex) - .Select(a => a.ListInfo); - } } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Lists/ArchiveLists.cs b/src/FilterLists.Agent/Features/Lists/ArchiveLists.cs index dcb4ce57a..44c157aee 100644 --- a/src/FilterLists.Agent/Features/Lists/ArchiveLists.cs +++ b/src/FilterLists.Agent/Features/Lists/ArchiveLists.cs @@ -14,12 +14,12 @@ public class Command : IRequest public class Handler : AsyncRequestHandler { private readonly IMediator _mediator; - private readonly IListUrlRepository _repo; + private readonly IListViewUrlRepository _repo; - public Handler(IMediator mediator, IListUrlRepository listUrlRepository) + public Handler(IMediator mediator, IListViewUrlRepository listViewUrlRepository) { _mediator = mediator; - _repo = listUrlRepository; + _repo = listViewUrlRepository; } protected override async Task Handle(Command request, CancellationToken cancellationToken) diff --git a/src/FilterLists.Agent/Features/Lists/DownloadLists.cs b/src/FilterLists.Agent/Features/Lists/DownloadLists.cs index 48ae88ffc..084a58035 100644 --- a/src/FilterLists.Agent/Features/Lists/DownloadLists.cs +++ b/src/FilterLists.Agent/Features/Lists/DownloadLists.cs @@ -15,20 +15,20 @@ public static class DownloadLists { public class Command : IRequest { - public Command(IEnumerable listInfo) + public Command(IEnumerable listInfo) { ListInfo = listInfo; } - public IEnumerable ListInfo { get; } + public IEnumerable ListInfo { get; } } public class Handler : AsyncRequestHandler { private const int MaxDegreeOfParallelism = 5; - private static readonly Dictionary> CommandsByExtension - = new Dictionary> + private static readonly Dictionary> CommandsByExtension + = new Dictionary> { {"", l => new DownloadRawText.Command(l)}, {".7z", l => new DownloadSevenZip.Command(l)}, @@ -69,16 +69,16 @@ public Handler(ILogger logger, IMediator mediator) protected override async Task Handle(Command request, CancellationToken cancellationToken) { var downloader = BuildDownloader(cancellationToken); - var orderedListInfo = request.ListInfo.DistributeByHost(); + var orderedListInfo = request.ListInfo; foreach (var listInfo in orderedListInfo) await downloader.SendAsync(listInfo, cancellationToken); downloader.Complete(); await downloader.Completion; } - private ActionBlock BuildDownloader(CancellationToken cancellationToken) + private ActionBlock BuildDownloader(CancellationToken cancellationToken) { - return new ActionBlock( + return new ActionBlock( async l => { var errorMessage = $"Error downloading list {l.Id} from {l.ViewUrl}."; diff --git a/src/FilterLists.Agent/Features/Lists/DownloadRawText.cs b/src/FilterLists.Agent/Features/Lists/DownloadRawText.cs index 6b4cb39e0..33e89441b 100644 --- a/src/FilterLists.Agent/Features/Lists/DownloadRawText.cs +++ b/src/FilterLists.Agent/Features/Lists/DownloadRawText.cs @@ -12,12 +12,12 @@ public static class DownloadRawText { public class Command : IRequest { - public Command(ListUrl listUrl) + public Command(ListViewUrl listViewUrl) { - ListUrl = listUrl; + ListViewUrl = listViewUrl; } - public ListUrl ListUrl { get; } + public ListViewUrl ListViewUrl { get; } } public class Handler : AsyncRequestHandler @@ -34,16 +34,16 @@ public Handler(IListRepository listRepository) protected override async Task Handle(Command request, CancellationToken cancellationToken) { var destinationPath = BuildDestinationPath(request); - using var input = await _repo.GetAsStreamAsync(request.ListUrl.ViewUrl, cancellationToken); + using var input = await _repo.GetAsStreamAsync(request.ListViewUrl.ViewUrl, cancellationToken); using var output = File.OpenWrite(destinationPath); await input.CopyToAsync(output, cancellationToken); } private string BuildDestinationPath(Command request) { - var sourceExtension = request.ListUrl.ViewUrl.GetExtension(); + var sourceExtension = request.ListViewUrl.ViewUrl.GetExtension(); var destinationExtension = _extensionsToRewrite.Contains(sourceExtension) ? ".txt" : sourceExtension; - var destinationPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}{destinationExtension}"); + var destinationPath = Path.Combine(RepoDirectory, $"{request.ListViewUrl.Id}{destinationExtension}"); return destinationPath; } } diff --git a/src/FilterLists.Agent/Features/Lists/DownloadSevenZip.cs b/src/FilterLists.Agent/Features/Lists/DownloadSevenZip.cs index 4ebfc1a5a..4b85187b3 100644 --- a/src/FilterLists.Agent/Features/Lists/DownloadSevenZip.cs +++ b/src/FilterLists.Agent/Features/Lists/DownloadSevenZip.cs @@ -13,12 +13,12 @@ public static class DownloadSevenZip { public class Command : IRequest { - public Command(ListUrl listUrl) + public Command(ListViewUrl listViewUrl) { - ListUrl = listUrl; + ListViewUrl = listViewUrl; } - public ListUrl ListUrl { get; } + public ListViewUrl ListViewUrl { get; } } public class Handler : AsyncRequestHandler @@ -33,8 +33,8 @@ public Handler(IListRepository listRepository) protected override async Task Handle(Command request, CancellationToken cancellationToken) { - var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}"); - using var input = await _repo.GetAsStreamAsync(request.ListUrl.ViewUrl, cancellationToken); + var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListViewUrl.Id}"); + using var input = await _repo.GetAsStreamAsync(request.ListViewUrl.ViewUrl, cancellationToken); using var archive = SevenZipArchive.Open(input); foreach (var entry in archive.Entries) if (!entry.IsDirectory) diff --git a/src/FilterLists.Agent/Features/Lists/DownloadZip.cs b/src/FilterLists.Agent/Features/Lists/DownloadZip.cs index a2498d56c..716a74720 100644 --- a/src/FilterLists.Agent/Features/Lists/DownloadZip.cs +++ b/src/FilterLists.Agent/Features/Lists/DownloadZip.cs @@ -12,12 +12,12 @@ public static class DownloadZip { public class Command : IRequest { - public Command(ListUrl listUrl) + public Command(ListViewUrl listViewUrl) { - ListUrl = listUrl; + ListViewUrl = listViewUrl; } - public ListUrl ListUrl { get; } + public ListViewUrl ListViewUrl { get; } } public class Handler : AsyncRequestHandler @@ -32,8 +32,8 @@ public Handler(IListRepository listRepository) protected override async Task Handle(Command request, CancellationToken cancellationToken) { - var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}"); - using var input = await _repo.GetAsStreamAsync(request.ListUrl.ViewUrl, cancellationToken); + var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListViewUrl.Id}"); + using var input = await _repo.GetAsStreamAsync(request.ListViewUrl.ViewUrl, cancellationToken); using var reader = ReaderFactory.Open(input); while (reader.MoveToNextEntry()) if (!reader.Entry.IsDirectory) diff --git a/src/FilterLists.Agent/Features/Urls/BuildGitHubIssueBody.cs b/src/FilterLists.Agent/Features/Urls/BuildGitHubIssueBody.cs index bd5471cdd..ae87893a2 100644 --- a/src/FilterLists.Agent/Features/Urls/BuildGitHubIssueBody.cs +++ b/src/FilterLists.Agent/Features/Urls/BuildGitHubIssueBody.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; +using System.Linq; using System.Text; -using FilterLists.Agent.Features.Urls.Models.ValidationResults; +using FilterLists.Agent.Core.Urls; using MediatR; namespace FilterLists.Agent.Features.Urls @@ -9,12 +10,12 @@ public static class BuildGitHubIssueBody { public class Command : IRequest { - public Command(IEnumerable dataFileUrlValidationResults) + public Command(IEnumerable invalidEntityUrls) { - DataFileUrlValidationResults = dataFileUrlValidationResults; + InvalidEntityUrls = invalidEntityUrls; } - public IEnumerable DataFileUrlValidationResults { get; } + public IEnumerable InvalidEntityUrls { get; } } public class Handler : RequestHandler @@ -26,16 +27,18 @@ protected override string Handle(Command request) { var body = new StringBuilder(); body.Append(IssueHeader); - foreach (var fileReults in request.DataFileUrlValidationResults) + var entityInvalidUrlGroups = request.InvalidEntityUrls.GroupBy(i => i.FilterListsEntity); + foreach (var entityInvalidUrls in entityInvalidUrlGroups) { body.Append( - $"

{fileReults.DataFileName}

"); + $"

{nameof(entityInvalidUrls.Key)}

"); body.Append("
    "); - foreach (var result in fileReults.Results) + foreach (var invalidUrl in entityInvalidUrls) { - body.Append($"
  • {result.Url.OriginalString}"); + body.Append( + $"
  • {invalidUrl.ViewUrl.OriginalString}"); body.Append("
      "); - foreach (var message in result.Messages) + foreach (var message in invalidUrl.ValidationMessages) body.Append($"
    • {message}
      • "); body.Append("
    "); body.Append("
    • "); diff --git a/src/FilterLists.Agent/Features/Urls/CreateOrUpdateUrlValidationIssue.cs b/src/FilterLists.Agent/Features/Urls/CreateOrUpdateUrlValidationIssue.cs index 1586515c2..3da6ea366 100644 --- a/src/FilterLists.Agent/Features/Urls/CreateOrUpdateUrlValidationIssue.cs +++ b/src/FilterLists.Agent/Features/Urls/CreateOrUpdateUrlValidationIssue.cs @@ -2,8 +2,8 @@ using System.Threading; using System.Threading.Tasks; using FilterLists.Agent.Core.GitHub; +using FilterLists.Agent.Core.Urls; using FilterLists.Agent.Extensions; -using FilterLists.Agent.Features.Urls.Models.ValidationResults; using MediatR; using Octokit; @@ -13,12 +13,12 @@ public static class CreateOrUpdateUrlValidationIssue { public class Command : IRequest { - public Command(IEnumerable dataFileUrlValidationResults) + public Command(IEnumerable invalidEntityUrls) { - DataFileUrlValidationResults = dataFileUrlValidationResults; + InvalidEntityUrls = invalidEntityUrls; } - public IEnumerable DataFileUrlValidationResults { get; } + public IEnumerable InvalidEntityUrls { get; } } public class Handler : AsyncRequestHandler @@ -39,7 +39,7 @@ public Handler(IIssuesRepository issuesRepository, IMediator mediator) protected override async Task Handle(Command request, CancellationToken cancellationToken) { var issue = await GetOpenIssueOrNull() ?? await CreateBaseIssue(); - await UpdateIssue(issue, request.DataFileUrlValidationResults, cancellationToken); + await UpdateIssue(issue, request.InvalidEntityUrls, cancellationToken); } private async Task GetOpenIssueOrNull() @@ -59,15 +59,14 @@ private async Task CreateBaseIssue() return await _repo.CreateIssueAsync(newIssue); } - private async Task UpdateIssue(Issue issue, - IEnumerable dataFileUrlValidationResults, + private async Task UpdateIssue(Issue issue, IEnumerable invalidEntityUrls, CancellationToken cancellationToken) { var updateIssue = issue.ToUpdate(); updateIssue.AddLabel(HelpWantedLabel); updateIssue.AddLabel(AgentBotLabel); updateIssue.AddLabel(DataLabel); - updateIssue.Body = await _mediator.Send(new BuildGitHubIssueBody.Command(dataFileUrlValidationResults), + updateIssue.Body = await _mediator.Send(new BuildGitHubIssueBody.Command(invalidEntityUrls), cancellationToken); await _repo.UpdateIssueAsync(issue.Number, updateIssue); } diff --git a/src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/LicenseUrls.cs b/src/FilterLists.Agent/Features/Urls/Models/LicenseUrls.cs similarity index 62% rename from src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/LicenseUrls.cs rename to src/FilterLists.Agent/Features/Urls/Models/LicenseUrls.cs index 4a909c50e..16aab193a 100644 --- a/src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/LicenseUrls.cs +++ b/src/FilterLists.Agent/Features/Urls/Models/LicenseUrls.cs @@ -1,11 +1,12 @@ using System; using JetBrains.Annotations; -namespace FilterLists.Agent.Features.Urls.Models.DataFileUrls +namespace FilterLists.Agent.Features.Urls.Models { [UsedImplicitly] public class LicenseUrls { + public int Id { get; [UsedImplicitly] private set; } public Uri DescriptionUrl { get; [UsedImplicitly] private set; } } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/ListUrls.cs b/src/FilterLists.Agent/Features/Urls/Models/ListUrls.cs similarity index 88% rename from src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/ListUrls.cs rename to src/FilterLists.Agent/Features/Urls/Models/ListUrls.cs index fa1f5d12e..17f60f328 100644 --- a/src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/ListUrls.cs +++ b/src/FilterLists.Agent/Features/Urls/Models/ListUrls.cs @@ -1,11 +1,12 @@ using System; using JetBrains.Annotations; -namespace FilterLists.Agent.Features.Urls.Models.DataFileUrls +namespace FilterLists.Agent.Features.Urls.Models { [UsedImplicitly] public class ListUrls { + public int Id { get; [UsedImplicitly] private set; } public Uri ChatUrl { get; [UsedImplicitly] private set; } public Uri DescriptionSourceUrl { get; [UsedImplicitly] private set; } public Uri DonateUrl { get; [UsedImplicitly] private set; } diff --git a/src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/MaintainerUrls.cs b/src/FilterLists.Agent/Features/Urls/Models/MaintainerUrls.cs similarity index 62% rename from src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/MaintainerUrls.cs rename to src/FilterLists.Agent/Features/Urls/Models/MaintainerUrls.cs index ad9374751..23ffbae0c 100644 --- a/src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/MaintainerUrls.cs +++ b/src/FilterLists.Agent/Features/Urls/Models/MaintainerUrls.cs @@ -1,11 +1,12 @@ using System; using JetBrains.Annotations; -namespace FilterLists.Agent.Features.Urls.Models.DataFileUrls +namespace FilterLists.Agent.Features.Urls.Models { [UsedImplicitly] public class MaintainerUrls { + public int Id { get; [UsedImplicitly] private set; } public Uri HomeUrl { get; [UsedImplicitly] private set; } } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/SoftwareUrls.cs b/src/FilterLists.Agent/Features/Urls/Models/SoftwareUrls.cs similarity index 69% rename from src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/SoftwareUrls.cs rename to src/FilterLists.Agent/Features/Urls/Models/SoftwareUrls.cs index 7f9ffa595..992707212 100644 --- a/src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/SoftwareUrls.cs +++ b/src/FilterLists.Agent/Features/Urls/Models/SoftwareUrls.cs @@ -1,11 +1,12 @@ using System; using JetBrains.Annotations; -namespace FilterLists.Agent.Features.Urls.Models.DataFileUrls +namespace FilterLists.Agent.Features.Urls.Models { [UsedImplicitly] public class SoftwareUrls { + public int Id { get; [UsedImplicitly] private set; } public Uri DownloadUrl { get; [UsedImplicitly] private set; } public Uri HomeUrl { get; [UsedImplicitly] private set; } } diff --git a/src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/SyntaxUrls.cs b/src/FilterLists.Agent/Features/Urls/Models/SyntaxUrls.cs similarity index 62% rename from src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/SyntaxUrls.cs rename to src/FilterLists.Agent/Features/Urls/Models/SyntaxUrls.cs index 4cc38000b..416277742 100644 --- a/src/FilterLists.Agent/Features/Urls/Models/DataFileUrls/SyntaxUrls.cs +++ b/src/FilterLists.Agent/Features/Urls/Models/SyntaxUrls.cs @@ -1,11 +1,12 @@ using System; using JetBrains.Annotations; -namespace FilterLists.Agent.Features.Urls.Models.DataFileUrls +namespace FilterLists.Agent.Features.Urls.Models { [UsedImplicitly] public class SyntaxUrls { + public int Id { get; [UsedImplicitly] private set; } public Uri DefinitionUrl { get; [UsedImplicitly] private set; } } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Urls/Models/ValidationResults/DataFileUrlValidationResults.cs b/src/FilterLists.Agent/Features/Urls/Models/ValidationResults/DataFileUrlValidationResults.cs deleted file mode 100644 index d65d9ff67..000000000 --- a/src/FilterLists.Agent/Features/Urls/Models/ValidationResults/DataFileUrlValidationResults.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System.Collections.Generic; -using FilterLists.Agent.Core.Urls; - -namespace FilterLists.Agent.Features.Urls.Models.ValidationResults -{ - public class DataFileUrlValidationResults - { - public DataFileUrlValidationResults(string dataFileName, IEnumerable results) - { - DataFileName = dataFileName; - Results = results; - } - - public string DataFileName { get; } - - public IEnumerable Results { get; } - } -} \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Urls/ValidateAllUrls.cs b/src/FilterLists.Agent/Features/Urls/ValidateAllUrls.cs index 90b628aa5..439170640 100644 --- a/src/FilterLists.Agent/Features/Urls/ValidateAllUrls.cs +++ b/src/FilterLists.Agent/Features/Urls/ValidateAllUrls.cs @@ -1,10 +1,8 @@ -using System.Collections.Generic; -using System.Linq; +using System.Linq; using System.Threading; using System.Threading.Tasks; using FilterLists.Agent.Core.Urls; -using FilterLists.Agent.Features.Urls.Models.DataFileUrls; -using FilterLists.Agent.Features.Urls.Models.ValidationResults; +using JetBrains.Annotations; using MediatR; namespace FilterLists.Agent.Features.Urls @@ -15,49 +13,26 @@ public class Command : IRequest { } + [UsedImplicitly] public class Handler : AsyncRequestHandler { private readonly IMediator _mediator; - private readonly IUrlRepository _repo; + private readonly IEntityUrlRepository _repo; - public Handler(IMediator mediator, IUrlRepository urlRepository) + public Handler(IMediator mediator, IEntityUrlRepository entityUrlRepository) { _mediator = mediator; - _repo = urlRepository; + _repo = entityUrlRepository; } protected override async Task Handle(Command request, CancellationToken cancellationToken) { - var results = new List(); - - var licenseUrls = await _repo.GetAllAsync(cancellationToken); - var licenseUrlErrors = await _mediator.Send(new ValidateUrls.Command(licenseUrls), cancellationToken); - if (licenseUrlErrors.Any()) - results.Add(new DataFileUrlValidationResults("License.json", licenseUrlErrors)); - - var listUrls = await _repo.GetAllAsync(cancellationToken); - var listUrlErrors = await _mediator.Send(new ValidateUrls.Command(listUrls), cancellationToken); - if (listUrlErrors.Any()) - results.Add(new DataFileUrlValidationResults("FilterList.json", listUrlErrors)); - - var maintainerUrls = await _repo.GetAllAsync(cancellationToken); - var maintainerUrlErrors = - await _mediator.Send(new ValidateUrls.Command(maintainerUrls), cancellationToken); - if (maintainerUrlErrors.Any()) - results.Add(new DataFileUrlValidationResults("Maintainer.json", maintainerUrlErrors)); - - var softwareUrls = await _repo.GetAllAsync(cancellationToken); - var softwareUrlErrors = await _mediator.Send(new ValidateUrls.Command(softwareUrls), cancellationToken); - if (softwareUrlErrors.Any()) - results.Add(new DataFileUrlValidationResults("Software.json", softwareUrlErrors)); - - var syntaxUrls = await _repo.GetAllAsync(cancellationToken); - var syntaxUrlErrors = await _mediator.Send(new ValidateUrls.Command(syntaxUrls), cancellationToken); - if (syntaxUrlErrors.Any()) - results.Add(new DataFileUrlValidationResults("Syntax.json", syntaxUrlErrors)); - - if (results.Any()) - await _mediator.Send(new CreateOrUpdateUrlValidationIssue.Command(results), cancellationToken); + var entityUrls = await _repo.GetAllAsync(cancellationToken); + var validatedEntityUrls = await _mediator.Send(new ValidateUrls.Command(entityUrls), cancellationToken); + var invalidEntityUrls = validatedEntityUrls.Where(e => !e.IsValid()).ToList(); + if (invalidEntityUrls.Any()) + await _mediator.Send(new CreateOrUpdateUrlValidationIssue.Command(invalidEntityUrls), + cancellationToken); else await _mediator.Send(new CloseUrlValidationIssue.Command(), cancellationToken); } diff --git a/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs b/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs index e4c010433..b79ff30f6 100644 --- a/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs +++ b/src/FilterLists.Agent/Features/Urls/ValidateUrls.cs @@ -1,28 +1,27 @@ -using System; -using System.Collections.Generic; -using System.Linq; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using FilterLists.Agent.Core.Urls; -using FilterLists.Agent.Extensions; +using JetBrains.Annotations; using MediatR; namespace FilterLists.Agent.Features.Urls { public static class ValidateUrls { - public class Command : IRequest> + public class Command : IRequest> { - public Command(IEnumerable urls) + public Command(IEnumerable entityUrls) { - Urls = urls; + EntityUrls = entityUrls; } - public IEnumerable Urls { get; } + public IEnumerable EntityUrls { get; } } - public class Handler : IRequestHandler> + [UsedImplicitly] + public class Handler : IRequestHandler> { private const int MaxDegreeOfParallelism = 5; private readonly IUrlValidator _urlValidator; @@ -32,29 +31,23 @@ public Handler(IUrlValidator urlValidator) _urlValidator = urlValidator; } - public async Task> Handle(Command request, CancellationToken cancellationToken) + public async Task> Handle(Command request, CancellationToken cancellationToken) { var validator = BuildValidator(cancellationToken); - var brokenUrls = new List(); - var distinctUrls = request.Urls.Distinct().DistributeByHost(); - foreach (var url in distinctUrls) - await validator.SendAsync(url, cancellationToken); + foreach (var entityUrl in request.EntityUrls) + await validator.SendAsync(entityUrl, cancellationToken); validator.Complete(); + var validatedEntityUrls = new List(); while (await validator.OutputAvailableAsync(cancellationToken)) - { - var result = await validator.ReceiveAsync(cancellationToken); - if (!result.IsValid()) - brokenUrls.Add(result); - } - + validatedEntityUrls.Add(await validator.ReceiveAsync(cancellationToken)); await validator.Completion; - return brokenUrls; + return validatedEntityUrls; } - private TransformBlock BuildValidator(CancellationToken cancellationToken) + private TransformBlock BuildValidator(CancellationToken cancellationToken) { - return new TransformBlock( - async u => await _urlValidator.ValidateAsync(u, cancellationToken), + return new TransformBlock( + e => _urlValidator.ValidateAsync(e, cancellationToken), new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism} ); } diff --git a/src/FilterLists.Agent/Infrastructure/FilterListsApi/EntityUrlRepository.cs b/src/FilterLists.Agent/Infrastructure/FilterListsApi/EntityUrlRepository.cs new file mode 100644 index 000000000..10e7df6c4 --- /dev/null +++ b/src/FilterLists.Agent/Infrastructure/FilterListsApi/EntityUrlRepository.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FilterLists.Agent.Core.Urls; +using FilterLists.Agent.Features.Urls.Models; +using JetBrains.Annotations; +using RestSharp; + +namespace FilterLists.Agent.Infrastructure.FilterListsApi +{ + [UsedImplicitly] + public class EntityUrlRepository : IEntityUrlRepository + { + private static readonly Dictionary EndpointAndEntityByDto = + new Dictionary + { + {nameof(LicenseUrls), ("licenses", FilterListsEntity.License)}, + {nameof(ListUrls), ("lists", FilterListsEntity.FilterList)}, + {nameof(MaintainerUrls), ("maintainers", FilterListsEntity.Maintainer)}, + {nameof(SoftwareUrls), ("software", FilterListsEntity.Software)}, + {nameof(SyntaxUrls), ("syntaxes", FilterListsEntity.Syntax)} + }; + + private readonly IFilterListsApiClient _apiClient; + + public EntityUrlRepository(IFilterListsApiClient apiClient) + { + _apiClient = apiClient; + } + + public async Task> GetAllAsync(CancellationToken cancellationToken) + { + return (await Task.WhenAll( + Task.Run(() => GetAllForEntity(cancellationToken), cancellationToken), + Task.Run(() => GetAllForEntity(cancellationToken), cancellationToken), + Task.Run(() => GetAllForEntity(cancellationToken), cancellationToken), + Task.Run(() => GetAllForEntity(cancellationToken), cancellationToken), + Task.Run(() => GetAllForEntity(cancellationToken), cancellationToken))) + .SelectMany(r => r); + } + + private async Task> GetAllForEntity(CancellationToken cancellationToken) + { + var request = new RestRequest($"{EndpointAndEntityByDto[typeof(TModel).Name].Item1}/seed"); + var response = await _apiClient.ExecuteAsync>(request, cancellationToken); + return response.SelectMany(e => + { + var propertyInfos = e.GetType().GetProperties(); + var id = (int)propertyInfos.First(p => p.Name == nameof(LicenseUrls.Id)).GetValue(e); + return propertyInfos.Where(p => p.GetValue(e) != null && p.PropertyType == typeof(Uri)).Select(p => + new EntityUrl(EndpointAndEntityByDto[typeof(TModel).Name].Item2, id, (Uri)p.GetValue(e))); + }).GroupBy(e => e.ViewUrl).Select(e => e.First()); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/Infrastructure/FilterListsApi/ListUrlRepository.cs b/src/FilterLists.Agent/Infrastructure/FilterListsApi/ListViewUrlRepository.cs similarity index 62% rename from src/FilterLists.Agent/Infrastructure/FilterListsApi/ListUrlRepository.cs rename to src/FilterLists.Agent/Infrastructure/FilterListsApi/ListViewUrlRepository.cs index 2e5e677e3..c528d2976 100644 --- a/src/FilterLists.Agent/Infrastructure/FilterListsApi/ListUrlRepository.cs +++ b/src/FilterLists.Agent/Infrastructure/FilterListsApi/ListViewUrlRepository.cs @@ -6,19 +6,19 @@ namespace FilterLists.Agent.Infrastructure.FilterListsApi { - public class ListUrlRepository : IListUrlRepository + public class ListViewUrlRepository : IListViewUrlRepository { private readonly IFilterListsApiClient _apiClient; - public ListUrlRepository(IFilterListsApiClient filterListsApiClient) + public ListViewUrlRepository(IFilterListsApiClient filterListsApiClient) { _apiClient = filterListsApiClient; } - public async Task> GetAllAsync(CancellationToken cancellationToken) + public async Task> GetAllAsync(CancellationToken cancellationToken) { var listsRequest = new RestRequest("lists"); - return await _apiClient.ExecuteAsync>(listsRequest, cancellationToken); + return await _apiClient.ExecuteAsync>(listsRequest, cancellationToken); } } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Infrastructure/FilterListsApi/ServiceCollectionExtensions.cs b/src/FilterLists.Agent/Infrastructure/FilterListsApi/ServiceCollectionExtensions.cs index d3b4f1bcb..9caac3c48 100644 --- a/src/FilterLists.Agent/Infrastructure/FilterListsApi/ServiceCollectionExtensions.cs +++ b/src/FilterLists.Agent/Infrastructure/FilterListsApi/ServiceCollectionExtensions.cs @@ -9,8 +9,8 @@ public static class ServiceCollectionExtensions public static void AddFilterListsApiServices(this IServiceCollection services) { services.AddSingleton(); - services.AddTransient(); - services.AddTransient(); + services.AddTransient(); + services.AddTransient(); } } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Infrastructure/FilterListsApi/UrlRepository.cs b/src/FilterLists.Agent/Infrastructure/FilterListsApi/UrlRepository.cs deleted file mode 100644 index ccd7a3667..000000000 --- a/src/FilterLists.Agent/Infrastructure/FilterListsApi/UrlRepository.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using FilterLists.Agent.Core.Urls; -using FilterLists.Agent.Features.Urls.Models.DataFileUrls; -using JetBrains.Annotations; -using Microsoft.Extensions.Localization; -using RestSharp; - -namespace FilterLists.Agent.Infrastructure.FilterListsApi -{ - [UsedImplicitly] - public class UrlRepository : IUrlRepository - { - private static readonly Dictionary EntityUrlsEndpoints = new Dictionary - { - {nameof(LicenseUrls), "licenses"}, - {nameof(ListUrls), "lists"}, - {nameof(MaintainerUrls), "maintainers"}, - {nameof(SoftwareUrls), "software"}, - {nameof(SyntaxUrls), "syntaxes"} - }; - - private readonly IFilterListsApiClient _apiClient; - private readonly IStringLocalizer _localizer; - - public UrlRepository(IFilterListsApiClient apiClient, IStringLocalizer stringLocalizer) - { - _apiClient = apiClient; - _localizer = stringLocalizer; - } - - public async Task> GetAllAsync(CancellationToken cancellationToken) - where TModel : class, new() - { - if (!EntityUrlsEndpoints.ContainsKey(typeof(TModel).Name)) - throw new ArgumentException(_localizer["The type of TModel is not valid."]); - var request = new RestRequest($"{EntityUrlsEndpoints[typeof(TModel).Name]}/seed"); - var response = await _apiClient.ExecuteAsync>(request, cancellationToken); - return response.ToList().SelectMany(r => - r.GetType().GetProperties().Where(p => p.GetValue(r) != null).Select(p => (Uri)p.GetValue(r))); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Agent/Infrastructure/Web/UrlValidator.cs b/src/FilterLists.Agent/Infrastructure/Web/UrlValidator.cs index e239696f7..cc0b00a3a 100644 --- a/src/FilterLists.Agent/Infrastructure/Web/UrlValidator.cs +++ b/src/FilterLists.Agent/Infrastructure/Web/UrlValidator.cs @@ -4,8 +4,6 @@ using System.Threading; using System.Threading.Tasks; using FilterLists.Agent.Core.Urls; -using FilterLists.Agent.Extensions; -using FilterLists.Agent.Infrastructure.FilterListsApi; using JetBrains.Annotations; using Microsoft.Extensions.Logging; @@ -15,9 +13,9 @@ namespace FilterLists.Agent.Infrastructure.Web public class UrlValidator : IUrlValidator { private readonly HttpClient _httpClient; - private readonly ILogger _logger; + private readonly ILogger _logger; - public UrlValidator(HttpClient httpClient, ILogger logger) + public UrlValidator(HttpClient httpClient, ILogger logger) { httpClient.Timeout = TimeSpan.FromSeconds(90); var header = new ProductHeaderValue("FilterLists.Agent"); @@ -28,55 +26,43 @@ public UrlValidator(HttpClient httpClient, ILogger logger) _logger = logger; } - public async Task ValidateAsync(Uri u, CancellationToken cancellationToken) + public async Task ValidateAsync(EntityUrl entityUrl, CancellationToken cancellationToken) { - var result = new UrlValidationResult(u); - if (!u.IsValidUrl()) - { - result.SetBroken(); - _logger.LogError($"{u.OriginalString}) is not a valid URL."); - return result; - } - + var url = entityUrl.ViewUrl; try { using var response = - await _httpClient.GetAsync(u, HttpCompletionOption.ResponseHeadersRead, cancellationToken); - if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken)) - result.SetSupportsHttps(); - if (response.IsSuccessStatusCode) - return result; + await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + if (url.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(url, cancellationToken)) + entityUrl.SetSupportsHttps(); if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400) { - result.SetRedirectsTo(response.Headers.Location); + entityUrl.SetRedirectsTo(response.Headers.Location); } - else + else if (!response.IsSuccessStatusCode) { - result.SetBroken(); + entityUrl.SetBroken(); _logger.LogError( - $"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}."); + $"Url validation for ({url.AbsoluteUri}) failed with status code: {response.StatusCode}."); } - - return result; } catch (HttpRequestException ex) { - result.SetBroken(); - _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); - return result; + entityUrl.SetBroken(); + _logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex); } catch (TaskCanceledException ex) { - result.SetBroken(); - _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); - return result; + entityUrl.SetBroken(); + _logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex); } catch (InvalidOperationException ex) { - result.SetBroken(); - _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex); - return result; + entityUrl.SetBroken(); + _logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex); } + + return entityUrl; } private async Task IsHttpsSupported(Uri url, CancellationToken cancellationToken)