re-architect URL validator

src/FilterLists.Agent/Core/Lists/IListViewUrlRepository.cs

@@ -4,8 +4,8 @@
 
 namespace FilterLists.Agent.Core.Lists
 {
-    public interface IListUrlRepository
+    public interface IListViewUrlRepository
     {
-        Task<IEnumerable<ListUrl>> GetAllAsync(CancellationToken cancellationToken);
+        Task<IEnumerable<ListViewUrl>> GetAllAsync(CancellationToken cancellationToken);
     }
 }

src/FilterLists.Agent/Core/Lists/ListViewUrl.cs

@@ -4,7 +4,7 @@
 namespace FilterLists.Agent.Core.Lists
 {
     [UsedImplicitly]
-    public class ListUrl
+    public class ListViewUrl
     {
         public int Id { get; [UsedImplicitly] private set; }
         public Uri ViewUrl { get; [UsedImplicitly] private set; }

src/FilterLists.Agent/Core/Urls/EntityUrl.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace FilterLists.Agent.Core.Urls
+{
+    public class EntityUrl
+    {
+        public EntityUrl(FilterListsEntity filterListsEntity, int id, Uri viewUrl)
+        {
+            FilterListsEntity = filterListsEntity;
+            Id = id;
+            ViewUrl = viewUrl;
+        }
+
+        public FilterListsEntity FilterListsEntity { get; }
+        public int Id { get; }
+        public Uri ViewUrl { get; }
+        public List<string> ValidationMessages { get; } = new List<string>();
+
+        public bool IsValid()
+        {
+            return !ValidationMessages.Any();
+        }
+
+        public void SetBroken()
+        {
+            ValidationMessages.Add("This URL might be broken and should maybe be fixed, replaced, or removed.");
+        }
+
+        public void SetSupportsHttps()
+        {
+            ValidationMessages.Add(
+                "This URL uses the HTTP protocol but appears to be able to support and should probably be changed to HTTPS.");
+        }
+
+        public void SetRedirectsTo(Uri uri)
+        {
+            ValidationMessages.Add($"This URL appears to redirect and should maybe be changed to \"{uri}\".");
+        }
+    }
+}

src/FilterLists.Agent/Core/Urls/FilterListsEntity.cs

@@ -0,0 +1,11 @@
+namespace FilterLists.Agent.Core.Urls
+{
+    public enum FilterListsEntity
+    {
+        License,
+        FilterList,
+        Maintainer,
+        Software,
+        Syntax
+    }
+}

src/FilterLists.Agent/Core/Urls/IEntityUrlRepository.cs

@@ -0,0 +1,11 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace FilterLists.Agent.Core.Urls
+{
+    public interface IEntityUrlRepository
+    {
+        Task<IEnumerable<EntityUrl>> GetAllAsync(CancellationToken cancellationToken);
+    }
+}

src/FilterLists.Agent/Core/Urls/IUrlRepository.cs

@@ -1,12 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace FilterLists.Agent.Core.Urls
-{
-    public interface IUrlRepository
-    {
-        Task<IEnumerable<Uri>> GetAllAsync<TModel>(CancellationToken cancellationToken) where TModel : class, new();
-    }
-}

src/FilterLists.Agent/Core/Urls/IUrlValidator.cs

@@ -1,11 +1,10 @@
-using System;
-using System.Threading;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace FilterLists.Agent.Core.Urls
 {
     public interface IUrlValidator
     {
-        Task<UrlValidationResult> ValidateAsync(Uri u, CancellationToken cancellationToken);
+        Task<EntityUrl> ValidateAsync(EntityUrl entityUrl, CancellationToken cancellationToken);
     }
 }

src/FilterLists.Agent/Core/Urls/UrlValidationResult.cs

@@ -1,39 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace FilterLists.Agent.Core.Urls
-{
-    public class UrlValidationResult
-    {
-        public UrlValidationResult(Uri url)
-        {
-            Url = url;
-        }
-
-        public Uri Url { get; }
-
-        public List<string> Messages { get; } = new List<string>();
-
-        public bool IsValid()
-        {
-            return !Messages.Any();
-        }
-
-        public void SetBroken()
-        {
-            Messages.Add("This URL might be broken and should maybe be fixed, replaced, or removed.");
-        }
-
-        public void SetSupportsHttps()
-        {
-            Messages.Add(
-                "This URL uses the HTTP protocol but appears to be able to support and should probably be changed to HTTPS.");
-        }
-
-        public void SetRedirectsTo(Uri uri)
-        {
-            Messages.Add($"This URL appears to redirect and should maybe be changed to \"{uri}\".");
-        }
-    }
-}

src/FilterLists.Agent/Extensions/UriExtensions.cs

@@ -1,8 +1,5 @@
 using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
-using FilterLists.Agent.Core.Lists;
 
 namespace FilterLists.Agent.Extensions
 {
@@ -12,36 +9,5 @@ public static string GetExtension(this Uri uri)
         {
             return Path.GetExtension(uri.AbsolutePath);
         }
-
-        public static bool IsValidUrl(this Uri uri)
-        {
-            var uriString = uri.OriginalString;
-            if (!Uri.IsWellFormedUriString(uriString, UriKind.Absolute))
-                return false;
-            if (!Uri.TryCreate(uriString, UriKind.Absolute, out var tmp))
-                return false;
-            return tmp.Scheme == Uri.UriSchemeHttp || tmp.Scheme == Uri.UriSchemeHttps;
-        }
-
-        public static IEnumerable<Uri> DistributeByHost(this IEnumerable<Uri> listInfo)
-        {
-            var listInfoList = listInfo.ToList();
-            var distributedLists = listInfoList.Where(u => u.IsValidUrl())
-                .GroupBy(u => u.Host)
-                .SelectMany((g, gi) => g.Select((u, i) => new {Index = i, GroupIndex = gi, ListInfo = u}))
-                .OrderBy(a => a.Index)
-                .ThenBy(a => a.GroupIndex)
-                .Select(a => a.ListInfo);
-            return listInfoList.Where(u => !u.IsValidUrl()).Concat(distributedLists);
-        }
-
-        public static IEnumerable<ListUrl> DistributeByHost(this IEnumerable<ListUrl> listInfo)
-        {
-            return listInfo.GroupBy(l => l.ViewUrl.Host)
-                .SelectMany((g, gi) => g.Select((l, i) => new {Index = i, GroupIndex = gi, ListInfo = l}))
-                .OrderBy(a => a.Index)
-                .ThenBy(a => a.GroupIndex)
-                .Select(a => a.ListInfo);
-        }
     }
 }

src/FilterLists.Agent/Features/Lists/ArchiveLists.cs

@@ -14,12 +14,12 @@ public class Command : IRequest
         public class Handler : AsyncRequestHandler<Command>
         {
             private readonly IMediator _mediator;
-            private readonly IListUrlRepository _repo;
+            private readonly IListViewUrlRepository _repo;
 
-            public Handler(IMediator mediator, IListUrlRepository listUrlRepository)
+            public Handler(IMediator mediator, IListViewUrlRepository listViewUrlRepository)
             {
                 _mediator = mediator;
-                _repo = listUrlRepository;
+                _repo = listViewUrlRepository;
             }
 
             protected override async Task Handle(Command request, CancellationToken cancellationToken)

src/FilterLists.Agent/Features/Lists/DownloadLists.cs

@@ -15,20 +15,20 @@ public static class DownloadLists
     {
         public class Command : IRequest
         {
-            public Command(IEnumerable<ListUrl> listInfo)
+            public Command(IEnumerable<ListViewUrl> listInfo)
             {
                 ListInfo = listInfo;
             }
 
-            public IEnumerable<ListUrl> ListInfo { get; }
+            public IEnumerable<ListViewUrl> ListInfo { get; }
         }
 
         public class Handler : AsyncRequestHandler<Command>
         {
             private const int MaxDegreeOfParallelism = 5;
 
-            private static readonly Dictionary<string, Func<ListUrl, IRequest>> CommandsByExtension
-                = new Dictionary<string, Func<ListUrl, IRequest>>
+            private static readonly Dictionary<string, Func<ListViewUrl, IRequest>> CommandsByExtension
+                = new Dictionary<string, Func<ListViewUrl, IRequest>>
                 {
                     {"", l => new DownloadRawText.Command(l)},
                     {".7z", l => new DownloadSevenZip.Command(l)},
@@ -69,16 +69,16 @@ public Handler(ILogger<Handler> logger, IMediator mediator)
             protected override async Task Handle(Command request, CancellationToken cancellationToken)
             {
                 var downloader = BuildDownloader(cancellationToken);
-                var orderedListInfo = request.ListInfo.DistributeByHost();
+                var orderedListInfo = request.ListInfo;
                 foreach (var listInfo in orderedListInfo)
                     await downloader.SendAsync(listInfo, cancellationToken);
                 downloader.Complete();
                 await downloader.Completion;
             }
 
-            private ActionBlock<ListUrl> BuildDownloader(CancellationToken cancellationToken)
+            private ActionBlock<ListViewUrl> BuildDownloader(CancellationToken cancellationToken)
             {
-                return new ActionBlock<ListUrl>(
+                return new ActionBlock<ListViewUrl>(
                     async l =>
                     {
                         var errorMessage = $"Error downloading list {l.Id} from {l.ViewUrl}.";

src/FilterLists.Agent/Features/Lists/DownloadRawText.cs

@@ -12,12 +12,12 @@ public static class DownloadRawText
     {
         public class Command : IRequest
         {
-            public Command(ListUrl listUrl)
+            public Command(ListViewUrl listViewUrl)
             {
-                ListUrl = listUrl;
+                ListViewUrl = listViewUrl;
             }
 
-            public ListUrl ListUrl { get; }
+            public ListViewUrl ListViewUrl { get; }
         }
 
         public class Handler : AsyncRequestHandler<Command>
@@ -34,16 +34,16 @@ public Handler(IListRepository listRepository)
             protected override async Task Handle(Command request, CancellationToken cancellationToken)
             {
                 var destinationPath = BuildDestinationPath(request);
-                using var input = await _repo.GetAsStreamAsync(request.ListUrl.ViewUrl, cancellationToken);
+                using var input = await _repo.GetAsStreamAsync(request.ListViewUrl.ViewUrl, cancellationToken);
                 using var output = File.OpenWrite(destinationPath);
                 await input.CopyToAsync(output, cancellationToken);
             }
 
             private string BuildDestinationPath(Command request)
             {
-                var sourceExtension = request.ListUrl.ViewUrl.GetExtension();
+                var sourceExtension = request.ListViewUrl.ViewUrl.GetExtension();
                 var destinationExtension = _extensionsToRewrite.Contains(sourceExtension) ? ".txt" : sourceExtension;
-                var destinationPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}{destinationExtension}");
+                var destinationPath = Path.Combine(RepoDirectory, $"{request.ListViewUrl.Id}{destinationExtension}");
                 return destinationPath;
             }
         }

src/FilterLists.Agent/Features/Lists/DownloadSevenZip.cs

@@ -13,12 +13,12 @@ public static class DownloadSevenZip
     {
         public class Command : IRequest
         {
-            public Command(ListUrl listUrl)
+            public Command(ListViewUrl listViewUrl)
             {
-                ListUrl = listUrl;
+                ListViewUrl = listViewUrl;
             }
 
-            public ListUrl ListUrl { get; }
+            public ListViewUrl ListViewUrl { get; }
         }
 
         public class Handler : AsyncRequestHandler<Command>
@@ -33,8 +33,8 @@ public Handler(IListRepository listRepository)
 
             protected override async Task Handle(Command request, CancellationToken cancellationToken)
             {
-                var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}");
-                using var input = await _repo.GetAsStreamAsync(request.ListUrl.ViewUrl, cancellationToken);
+                var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListViewUrl.Id}");
+                using var input = await _repo.GetAsStreamAsync(request.ListViewUrl.ViewUrl, cancellationToken);
                 using var archive = SevenZipArchive.Open(input);
                 foreach (var entry in archive.Entries)
                     if (!entry.IsDirectory)

src/FilterLists.Agent/Features/Lists/DownloadZip.cs

@@ -12,12 +12,12 @@ public static class DownloadZip
     {
         public class Command : IRequest
         {
-            public Command(ListUrl listUrl)
+            public Command(ListViewUrl listViewUrl)
             {
-                ListUrl = listUrl;
+                ListViewUrl = listViewUrl;
             }
 
-            public ListUrl ListUrl { get; }
+            public ListViewUrl ListViewUrl { get; }
         }
 
         public class Handler : AsyncRequestHandler<Command>
@@ -32,8 +32,8 @@ public Handler(IListRepository listRepository)
 
             protected override async Task Handle(Command request, CancellationToken cancellationToken)
             {
-                var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}");
-                using var input = await _repo.GetAsStreamAsync(request.ListUrl.ViewUrl, cancellationToken);
+                var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListViewUrl.Id}");
+                using var input = await _repo.GetAsStreamAsync(request.ListViewUrl.ViewUrl, cancellationToken);
                 using var reader = ReaderFactory.Open(input);
                 while (reader.MoveToNextEntry())
                     if (!reader.Entry.IsDirectory)

src/FilterLists.Agent/Features/Urls/BuildGitHubIssueBody.cs

@@ -1,6 +1,7 @@
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
-using FilterLists.Agent.Features.Urls.Models.ValidationResults;
+using FilterLists.Agent.Core.Urls;
 using MediatR;
 
 namespace FilterLists.Agent.Features.Urls
@@ -9,12 +10,12 @@ public static class BuildGitHubIssueBody
     {
         public class Command : IRequest<string>
         {
-            public Command(IEnumerable<DataFileUrlValidationResults> dataFileUrlValidationResults)
+            public Command(IEnumerable<EntityUrl> invalidEntityUrls)
             {
-                DataFileUrlValidationResults = dataFileUrlValidationResults;
+                InvalidEntityUrls = invalidEntityUrls;
             }
 
-            public IEnumerable<DataFileUrlValidationResults> DataFileUrlValidationResults { get; }
+            public IEnumerable<EntityUrl> InvalidEntityUrls { get; }
         }
 
         public class Handler : RequestHandler<Command, string>
@@ -26,16 +27,18 @@ protected override string Handle(Command request)
             {
                 var body = new StringBuilder();
                 body.Append(IssueHeader);
-                foreach (var fileReults in request.DataFileUrlValidationResults)
+                var entityInvalidUrlGroups = request.InvalidEntityUrls.GroupBy(i => i.FilterListsEntity);
+                foreach (var entityInvalidUrls in entityInvalidUrlGroups)
                 {
                     body.Append(
-                        $"<h1><a href=\"https://github.com/collinbarrett/FilterLists/blob/master/data/{fileReults.DataFileName}\">{fileReults.DataFileName}</a></h1>");
+                        $"<h1><a href=\"https://github.com/collinbarrett/FilterLists/blob/master/data/{nameof(entityInvalidUrls.Key)}\">{nameof(entityInvalidUrls.Key)}</a></h1>");
                     body.Append("<ul>");
-                    foreach (var result in fileReults.Results)
+                    foreach (var invalidUrl in entityInvalidUrls)
                     {
-                        body.Append($"<li><a href=\"{result.Url.OriginalString}\">{result.Url.OriginalString}</a>");
+                        body.Append(
+                            $"<li><a href=\"{invalidUrl.ViewUrl.OriginalString}\">{invalidUrl.ViewUrl.OriginalString}</a>");
                         body.Append("<ul>");
-                        foreach (var message in result.Messages)
+                        foreach (var message in invalidUrl.ValidationMessages)
                             body.Append($"<li>{message}</li>");
                         body.Append("</ul>");
                         body.Append("</li>");

src/FilterLists.Agent/Features/Urls/CreateOrUpdateUrlValidationIssue.cs

@@ -2,8 +2,8 @@
 using System.Threading;
 using System.Threading.Tasks;
 using FilterLists.Agent.Core.GitHub;
+using FilterLists.Agent.Core.Urls;
 using FilterLists.Agent.Extensions;
-using FilterLists.Agent.Features.Urls.Models.ValidationResults;
 using MediatR;
 using Octokit;
 
@@ -13,12 +13,12 @@ public static class CreateOrUpdateUrlValidationIssue
     {
         public class Command : IRequest
         {
-            public Command(IEnumerable<DataFileUrlValidationResults> dataFileUrlValidationResults)
+            public Command(IEnumerable<EntityUrl> invalidEntityUrls)
             {
-                DataFileUrlValidationResults = dataFileUrlValidationResults;
+                InvalidEntityUrls = invalidEntityUrls;
             }
 
-            public IEnumerable<DataFileUrlValidationResults> DataFileUrlValidationResults { get; }
+            public IEnumerable<EntityUrl> InvalidEntityUrls { get; }
         }
 
         public class Handler : AsyncRequestHandler<Command>
@@ -39,7 +39,7 @@ public Handler(IIssuesRepository issuesRepository, IMediator mediator)
             protected override async Task Handle(Command request, CancellationToken cancellationToken)
             {
                 var issue = await GetOpenIssueOrNull() ?? await CreateBaseIssue();
-                await UpdateIssue(issue, request.DataFileUrlValidationResults, cancellationToken);
+                await UpdateIssue(issue, request.InvalidEntityUrls, cancellationToken);
             }
 
             private async Task<Issue> GetOpenIssueOrNull()
@@ -59,15 +59,14 @@ private async Task<Issue> CreateBaseIssue()
                 return await _repo.CreateIssueAsync(newIssue);
             }
 
-            private async Task UpdateIssue(Issue issue,
-                IEnumerable<DataFileUrlValidationResults> dataFileUrlValidationResults,
+            private async Task UpdateIssue(Issue issue, IEnumerable<EntityUrl> invalidEntityUrls,
                 CancellationToken cancellationToken)
             {
                 var updateIssue = issue.ToUpdate();
                 updateIssue.AddLabel(HelpWantedLabel);
                 updateIssue.AddLabel(AgentBotLabel);
                 updateIssue.AddLabel(DataLabel);
-                updateIssue.Body = await _mediator.Send(new BuildGitHubIssueBody.Command(dataFileUrlValidationResults),
+                updateIssue.Body = await _mediator.Send(new BuildGitHubIssueBody.Command(invalidEntityUrls),
                     cancellationToken);
                 await _repo.UpdateIssueAsync(issue.Number, updateIssue);
             }

src/FilterLists.Agent/Features/Urls/Models/LicenseUrls.cs

@@ -1,11 +1,12 @@
 using System;
 using JetBrains.Annotations;
 
-namespace FilterLists.Agent.Features.Urls.Models.DataFileUrls
+namespace FilterLists.Agent.Features.Urls.Models
 {
     [UsedImplicitly]
     public class LicenseUrls
     {
+        public int Id { get; [UsedImplicitly] private set; }
         public Uri DescriptionUrl { get; [UsedImplicitly] private set; }
     }
 }

src/FilterLists.Agent/Features/Urls/Models/ListUrls.cs

@@ -1,11 +1,12 @@
 using System;
 using JetBrains.Annotations;
 
-namespace FilterLists.Agent.Features.Urls.Models.DataFileUrls
+namespace FilterLists.Agent.Features.Urls.Models
 {
     [UsedImplicitly]
     public class ListUrls
     {
+        public int Id { get; [UsedImplicitly] private set; }
         public Uri ChatUrl { get; [UsedImplicitly] private set; }
         public Uri DescriptionSourceUrl { get; [UsedImplicitly] private set; }
         public Uri DonateUrl { get; [UsedImplicitly] private set; }

src/FilterLists.Agent/Features/Urls/Models/MaintainerUrls.cs

@@ -1,11 +1,12 @@
 using System;
 using JetBrains.Annotations;
 
-namespace FilterLists.Agent.Features.Urls.Models.DataFileUrls
+namespace FilterLists.Agent.Features.Urls.Models
 {
     [UsedImplicitly]
     public class MaintainerUrls
     {
+        public int Id { get; [UsedImplicitly] private set; }
         public Uri HomeUrl { get; [UsedImplicitly] private set; }
     }
 }

src/FilterLists.Agent/Features/Urls/Models/SoftwareUrls.cs

@@ -1,11 +1,12 @@
 using System;
 using JetBrains.Annotations;
 
-namespace FilterLists.Agent.Features.Urls.Models.DataFileUrls
+namespace FilterLists.Agent.Features.Urls.Models
 {
     [UsedImplicitly]
     public class SoftwareUrls
     {
+        public int Id { get; [UsedImplicitly] private set; }
         public Uri DownloadUrl { get; [UsedImplicitly] private set; }
         public Uri HomeUrl { get; [UsedImplicitly] private set; }
     }

src/FilterLists.Agent/Features/Urls/Models/SyntaxUrls.cs

@@ -1,11 +1,12 @@
 using System;
 using JetBrains.Annotations;
 
-namespace FilterLists.Agent.Features.Urls.Models.DataFileUrls
+namespace FilterLists.Agent.Features.Urls.Models
 {
     [UsedImplicitly]
     public class SyntaxUrls
     {
+        public int Id { get; [UsedImplicitly] private set; }
         public Uri DefinitionUrl { get; [UsedImplicitly] private set; }
     }
 }

src/FilterLists.Agent/Features/Urls/Models/ValidationResults/DataFileUrlValidationResults.cs

@@ -1,18 +0,0 @@
-using System.Collections.Generic;
-using FilterLists.Agent.Core.Urls;
-
-namespace FilterLists.Agent.Features.Urls.Models.ValidationResults
-{
-    public class DataFileUrlValidationResults
-    {
-        public DataFileUrlValidationResults(string dataFileName, IEnumerable<UrlValidationResult> results)
-        {
-            DataFileName = dataFileName;
-            Results = results;
-        }
-
-        public string DataFileName { get; }
-
-        public IEnumerable<UrlValidationResult> Results { get; }
-    }
-}

src/FilterLists.Agent/Features/Urls/ValidateAllUrls.cs

@@ -1,10 +1,8 @@
-using System.Collections.Generic;
-using System.Linq;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using FilterLists.Agent.Core.Urls;
-using FilterLists.Agent.Features.Urls.Models.DataFileUrls;
-using FilterLists.Agent.Features.Urls.Models.ValidationResults;
+using JetBrains.Annotations;
 using MediatR;
 
 namespace FilterLists.Agent.Features.Urls
@@ -15,49 +13,26 @@ public class Command : IRequest
         {
         }
 
+        [UsedImplicitly]
         public class Handler : AsyncRequestHandler<Command>
         {
             private readonly IMediator _mediator;
-            private readonly IUrlRepository _repo;
+            private readonly IEntityUrlRepository _repo;
 
-            public Handler(IMediator mediator, IUrlRepository urlRepository)
+            public Handler(IMediator mediator, IEntityUrlRepository entityUrlRepository)
             {
                 _mediator = mediator;
-                _repo = urlRepository;
+                _repo = entityUrlRepository;
             }
 
             protected override async Task Handle(Command request, CancellationToken cancellationToken)
             {
-                var results = new List<DataFileUrlValidationResults>();
-
-                var licenseUrls = await _repo.GetAllAsync<LicenseUrls>(cancellationToken);
-                var licenseUrlErrors = await _mediator.Send(new ValidateUrls.Command(licenseUrls), cancellationToken);
-                if (licenseUrlErrors.Any())
-                    results.Add(new DataFileUrlValidationResults("License.json", licenseUrlErrors));
-
-                var listUrls = await _repo.GetAllAsync<ListUrls>(cancellationToken);
-                var listUrlErrors = await _mediator.Send(new ValidateUrls.Command(listUrls), cancellationToken);
-                if (listUrlErrors.Any())
-                    results.Add(new DataFileUrlValidationResults("FilterList.json", listUrlErrors));
-
-                var maintainerUrls = await _repo.GetAllAsync<MaintainerUrls>(cancellationToken);
-                var maintainerUrlErrors =
-                    await _mediator.Send(new ValidateUrls.Command(maintainerUrls), cancellationToken);
-                if (maintainerUrlErrors.Any())
-                    results.Add(new DataFileUrlValidationResults("Maintainer.json", maintainerUrlErrors));
-
-                var softwareUrls = await _repo.GetAllAsync<SoftwareUrls>(cancellationToken);
-                var softwareUrlErrors = await _mediator.Send(new ValidateUrls.Command(softwareUrls), cancellationToken);
-                if (softwareUrlErrors.Any())
-                    results.Add(new DataFileUrlValidationResults("Software.json", softwareUrlErrors));
-
-                var syntaxUrls = await _repo.GetAllAsync<SyntaxUrls>(cancellationToken);
-                var syntaxUrlErrors = await _mediator.Send(new ValidateUrls.Command(syntaxUrls), cancellationToken);
-                if (syntaxUrlErrors.Any())
-                    results.Add(new DataFileUrlValidationResults("Syntax.json", syntaxUrlErrors));
-
-                if (results.Any())
-                    await _mediator.Send(new CreateOrUpdateUrlValidationIssue.Command(results), cancellationToken);
+                var entityUrls = await _repo.GetAllAsync(cancellationToken);
+                var validatedEntityUrls = await _mediator.Send(new ValidateUrls.Command(entityUrls), cancellationToken);
+                var invalidEntityUrls = validatedEntityUrls.Where(e => !e.IsValid()).ToList();
+                if (invalidEntityUrls.Any())
+                    await _mediator.Send(new CreateOrUpdateUrlValidationIssue.Command(invalidEntityUrls),
+                        cancellationToken);
                 else
                     await _mediator.Send(new CloseUrlValidationIssue.Command(), cancellationToken);
             }

src/FilterLists.Agent/Features/Urls/ValidateUrls.cs

@@ -1,28 +1,27 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
+using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
 using FilterLists.Agent.Core.Urls;
-using FilterLists.Agent.Extensions;
+using JetBrains.Annotations;
 using MediatR;
 
 namespace FilterLists.Agent.Features.Urls
 {
     public static class ValidateUrls
     {
-        public class Command : IRequest<List<UrlValidationResult>>
+        public class Command : IRequest<IEnumerable<EntityUrl>>
         {
-            public Command(IEnumerable<Uri> urls)
+            public Command(IEnumerable<EntityUrl> entityUrls)
             {
-                Urls = urls;
+                EntityUrls = entityUrls;
             }
 
-            public IEnumerable<Uri> Urls { get; }
+            public IEnumerable<EntityUrl> EntityUrls { get; }
         }
 
-        public class Handler : IRequestHandler<Command, List<UrlValidationResult>>
+        [UsedImplicitly]
+        public class Handler : IRequestHandler<Command, IEnumerable<EntityUrl>>
         {
             private const int MaxDegreeOfParallelism = 5;
             private readonly IUrlValidator _urlValidator;
@@ -32,29 +31,23 @@ public Handler(IUrlValidator urlValidator)
                 _urlValidator = urlValidator;
             }
 
-            public async Task<List<UrlValidationResult>> Handle(Command request, CancellationToken cancellationToken)
+            public async Task<IEnumerable<EntityUrl>> Handle(Command request, CancellationToken cancellationToken)
             {
                 var validator = BuildValidator(cancellationToken);
-                var brokenUrls = new List<UrlValidationResult>();
-                var distinctUrls = request.Urls.Distinct().DistributeByHost();
-                foreach (var url in distinctUrls)
-                    await validator.SendAsync(url, cancellationToken);
+                foreach (var entityUrl in request.EntityUrls)
+                    await validator.SendAsync(entityUrl, cancellationToken);
                 validator.Complete();
+                var validatedEntityUrls = new List<EntityUrl>();
                 while (await validator.OutputAvailableAsync(cancellationToken))
-                {
-                    var result = await validator.ReceiveAsync(cancellationToken);
-                    if (!result.IsValid())
-                        brokenUrls.Add(result);
-                }
-
+                    validatedEntityUrls.Add(await validator.ReceiveAsync(cancellationToken));
                 await validator.Completion;
-                return brokenUrls;
+                return validatedEntityUrls;
             }
 
-            private TransformBlock<Uri, UrlValidationResult> BuildValidator(CancellationToken cancellationToken)
+            private TransformBlock<EntityUrl, EntityUrl> BuildValidator(CancellationToken cancellationToken)
             {
-                return new TransformBlock<Uri, UrlValidationResult>(
-                    async u => await _urlValidator.ValidateAsync(u, cancellationToken),
+                return new TransformBlock<EntityUrl, EntityUrl>(
+                    e => _urlValidator.ValidateAsync(e, cancellationToken),
                     new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
                 );
             }

src/FilterLists.Agent/Infrastructure/FilterListsApi/EntityUrlRepository.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using FilterLists.Agent.Core.Urls;
+using FilterLists.Agent.Features.Urls.Models;
+using JetBrains.Annotations;
+using RestSharp;
+
+namespace FilterLists.Agent.Infrastructure.FilterListsApi
+{
+    [UsedImplicitly]
+    public class EntityUrlRepository : IEntityUrlRepository
+    {
+        private static readonly Dictionary<string, (string, FilterListsEntity)> EndpointAndEntityByDto =
+            new Dictionary<string, (string, FilterListsEntity)>
+            {
+                {nameof(LicenseUrls), ("licenses", FilterListsEntity.License)},
+                {nameof(ListUrls), ("lists", FilterListsEntity.FilterList)},
+                {nameof(MaintainerUrls), ("maintainers", FilterListsEntity.Maintainer)},
+                {nameof(SoftwareUrls), ("software", FilterListsEntity.Software)},
+                {nameof(SyntaxUrls), ("syntaxes", FilterListsEntity.Syntax)}
+            };
+
+        private readonly IFilterListsApiClient _apiClient;
+
+        public EntityUrlRepository(IFilterListsApiClient apiClient)
+        {
+            _apiClient = apiClient;
+        }
+
+        public async Task<IEnumerable<EntityUrl>> GetAllAsync(CancellationToken cancellationToken)
+        {
+            return (await Task.WhenAll(
+                    Task.Run(() => GetAllForEntity<LicenseUrls>(cancellationToken), cancellationToken),
+                    Task.Run(() => GetAllForEntity<ListUrls>(cancellationToken), cancellationToken),
+                    Task.Run(() => GetAllForEntity<MaintainerUrls>(cancellationToken), cancellationToken),
+                    Task.Run(() => GetAllForEntity<SoftwareUrls>(cancellationToken), cancellationToken),
+                    Task.Run(() => GetAllForEntity<SyntaxUrls>(cancellationToken), cancellationToken)))
+                .SelectMany(r => r);
+        }
+
+        private async Task<IEnumerable<EntityUrl>> GetAllForEntity<TModel>(CancellationToken cancellationToken)
+        {
+            var request = new RestRequest($"{EndpointAndEntityByDto[typeof(TModel).Name].Item1}/seed");
+            var response = await _apiClient.ExecuteAsync<IEnumerable<TModel>>(request, cancellationToken);
+            return response.SelectMany(e =>
+            {
+                var propertyInfos = e.GetType().GetProperties();
+                var id = (int)propertyInfos.First(p => p.Name == nameof(LicenseUrls.Id)).GetValue(e);
+                return propertyInfos.Where(p => p.GetValue(e) != null && p.PropertyType == typeof(Uri)).Select(p =>
+                    new EntityUrl(EndpointAndEntityByDto[typeof(TModel).Name].Item2, id, (Uri)p.GetValue(e)));
+            }).GroupBy(e => e.ViewUrl).Select(e => e.First());
+        }
+    }
+}

src/FilterLists.Agent/Infrastructure/FilterListsApi/ListViewUrlRepository.cs

@@ -6,19 +6,19 @@
 
 namespace FilterLists.Agent.Infrastructure.FilterListsApi
 {
-    public class ListUrlRepository : IListUrlRepository
+    public class ListViewUrlRepository : IListViewUrlRepository
     {
         private readonly IFilterListsApiClient _apiClient;
 
-        public ListUrlRepository(IFilterListsApiClient filterListsApiClient)
+        public ListViewUrlRepository(IFilterListsApiClient filterListsApiClient)
         {
             _apiClient = filterListsApiClient;
         }
 
-        public async Task<IEnumerable<ListUrl>> GetAllAsync(CancellationToken cancellationToken)
+        public async Task<IEnumerable<ListViewUrl>> GetAllAsync(CancellationToken cancellationToken)
         {
             var listsRequest = new RestRequest("lists");
-            return await _apiClient.ExecuteAsync<IEnumerable<ListUrl>>(listsRequest, cancellationToken);
+            return await _apiClient.ExecuteAsync<IEnumerable<ListViewUrl>>(listsRequest, cancellationToken);
         }
     }
 }

src/FilterLists.Agent/Infrastructure/FilterListsApi/ServiceCollectionExtensions.cs

@@ -9,8 +9,8 @@ public static class ServiceCollectionExtensions
         public static void AddFilterListsApiServices(this IServiceCollection services)
         {
             services.AddSingleton<IFilterListsApiClient, FilterListsApiClient>();
-            services.AddTransient<IListUrlRepository, ListUrlRepository>();
-            services.AddTransient<IUrlRepository, UrlRepository>();
+            services.AddTransient<IListViewUrlRepository, ListViewUrlRepository>();
+            services.AddTransient<IEntityUrlRepository, EntityUrlRepository>();
         }
     }
 }

src/FilterLists.Agent/Infrastructure/FilterListsApi/UrlRepository.cs

@@ -1,46 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using FilterLists.Agent.Core.Urls;
-using FilterLists.Agent.Features.Urls.Models.DataFileUrls;
-using JetBrains.Annotations;
-using Microsoft.Extensions.Localization;
-using RestSharp;
-
-namespace FilterLists.Agent.Infrastructure.FilterListsApi
-{
-    [UsedImplicitly]
-    public class UrlRepository : IUrlRepository
-    {
-        private static readonly Dictionary<string, string> EntityUrlsEndpoints = new Dictionary<string, string>
-        {
-            {nameof(LicenseUrls), "licenses"},
-            {nameof(ListUrls), "lists"},
-            {nameof(MaintainerUrls), "maintainers"},
-            {nameof(SoftwareUrls), "software"},
-            {nameof(SyntaxUrls), "syntaxes"}
-        };
-
-        private readonly IFilterListsApiClient _apiClient;
-        private readonly IStringLocalizer<UrlRepository> _localizer;
-
-        public UrlRepository(IFilterListsApiClient apiClient, IStringLocalizer<UrlRepository> stringLocalizer)
-        {
-            _apiClient = apiClient;
-            _localizer = stringLocalizer;
-        }
-
-        public async Task<IEnumerable<Uri>> GetAllAsync<TModel>(CancellationToken cancellationToken)
-            where TModel : class, new()
-        {
-            if (!EntityUrlsEndpoints.ContainsKey(typeof(TModel).Name))
-                throw new ArgumentException(_localizer["The type of TModel is not valid."]);
-            var request = new RestRequest($"{EntityUrlsEndpoints[typeof(TModel).Name]}/seed");
-            var response = await _apiClient.ExecuteAsync<IEnumerable<TModel>>(request, cancellationToken);
-            return response.ToList().SelectMany(r =>
-                r.GetType().GetProperties().Where(p => p.GetValue(r) != null).Select(p => (Uri)p.GetValue(r)));
-        }
-    }
-}

src/FilterLists.Agent/Infrastructure/Web/UrlValidator.cs

@@ -4,8 +4,6 @@
 using System.Threading;
 using System.Threading.Tasks;
 using FilterLists.Agent.Core.Urls;
-using FilterLists.Agent.Extensions;
-using FilterLists.Agent.Infrastructure.FilterListsApi;
 using JetBrains.Annotations;
 using Microsoft.Extensions.Logging;
 
@@ -15,9 +13,9 @@ namespace FilterLists.Agent.Infrastructure.Web
     public class UrlValidator : IUrlValidator
     {
         private readonly HttpClient _httpClient;
-        private readonly ILogger<UrlRepository> _logger;
+        private readonly ILogger<UrlValidator> _logger;
 
-        public UrlValidator(HttpClient httpClient, ILogger<UrlRepository> logger)
+        public UrlValidator(HttpClient httpClient, ILogger<UrlValidator> logger)
         {
             httpClient.Timeout = TimeSpan.FromSeconds(90);
             var header = new ProductHeaderValue("FilterLists.Agent");
@@ -28,55 +26,43 @@ public UrlValidator(HttpClient httpClient, ILogger<UrlRepository> logger)
             _logger = logger;
         }
 
-        public async Task<UrlValidationResult> ValidateAsync(Uri u, CancellationToken cancellationToken)
+        public async Task<EntityUrl> ValidateAsync(EntityUrl entityUrl, CancellationToken cancellationToken)
         {
-            var result = new UrlValidationResult(u);
-            if (!u.IsValidUrl())
-            {
-                result.SetBroken();
-                _logger.LogError($"{u.OriginalString}) is not a valid URL.");
-                return result;
-            }
-
+            var url = entityUrl.ViewUrl;
             try
             {
                 using var response =
-                    await _httpClient.GetAsync(u, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
-                if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken))
-                    result.SetSupportsHttps();
-                if (response.IsSuccessStatusCode)
-                    return result;
+                    await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+                if (url.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(url, cancellationToken))
+                    entityUrl.SetSupportsHttps();
                 if ((int)response.StatusCode >= 300 && (int)response.StatusCode < 400)
                 {
-                    result.SetRedirectsTo(response.Headers.Location);
+                    entityUrl.SetRedirectsTo(response.Headers.Location);
                 }
-                else
+                else if (!response.IsSuccessStatusCode)
                 {
-                    result.SetBroken();
+                    entityUrl.SetBroken();
                     _logger.LogError(
-                        $"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}.");
+                        $"Url validation for ({url.AbsoluteUri}) failed with status code: {response.StatusCode}.");
                 }
-
-                return result;
             }
             catch (HttpRequestException ex)
             {
-                result.SetBroken();
-                _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
-                return result;
+                entityUrl.SetBroken();
+                _logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex);
             }
             catch (TaskCanceledException ex)
             {
-                result.SetBroken();
-                _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
-                return result;
+                entityUrl.SetBroken();
+                _logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex);
             }
             catch (InvalidOperationException ex)
             {
-                result.SetBroken();
-                _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
-                return result;
+                entityUrl.SetBroken();
+                _logger.LogError($"Url validation for ({url.AbsoluteUri}) failed.", ex);
             }
+
+            return entityUrl;
         }
 
         private async Task<bool> IsHttpsSupported(Uri url, CancellationToken cancellationToken)