split out IUrlValidator and IUrlRepository

src/FilterLists.Agent/Core/Lists/IListRepository.cs

@@ -3,7 +3,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 
-namespace FilterLists.Agent.Core.List
+namespace FilterLists.Agent.Core.Lists
 {
     public interface IListRepository
     {

src/FilterLists.Agent/Core/Lists/IListUrlRepository.cs

@@ -2,7 +2,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 
-namespace FilterLists.Agent.Core.List
+namespace FilterLists.Agent.Core.Lists
 {
     public interface IListUrlRepository
     {

src/FilterLists.Agent/Core/Lists/ListUrl.cs

@@ -1,7 +1,7 @@
 using System;
 using JetBrains.Annotations;
 
-namespace FilterLists.Agent.Core.List
+namespace FilterLists.Agent.Core.Lists
 {
     [UsedImplicitly]
     public class ListUrl

src/FilterLists.Agent/Core/Urls/IUrlRepository.cs

@@ -2,14 +2,11 @@
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
-using FilterLists.Agent.Features.Urls.Models.ValidationResults;
 
-namespace FilterLists.Agent.Core
+namespace FilterLists.Agent.Core.Urls
 {
     public interface IUrlRepository
     {
         Task<IEnumerable<Uri>> GetAllAsync<TModel>(CancellationToken cancellationToken);
-
-        Task<UrlValidationResult> ValidateAsync(Uri u, CancellationToken cancellationToken);
     }
 }

src/FilterLists.Agent/Core/Urls/IUrlValidator.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace FilterLists.Agent.Core.Urls
+{
+    public interface IUrlValidator
+    {
+        Task<UrlValidationResult> ValidateAsync(Uri u, CancellationToken cancellationToken);
+    }
+}

src/FilterLists.Agent/Core/Urls/UrlValidationResult.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 
-namespace FilterLists.Agent.Features.Urls.Models.ValidationResults
+namespace FilterLists.Agent.Core.Urls
 {
     public class UrlValidationResult
     {

src/FilterLists.Agent/Extensions/UriExtensions.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using FilterLists.Agent.Core.List;
+using FilterLists.Agent.Core.Lists;
 
 namespace FilterLists.Agent.Extensions
 {

src/FilterLists.Agent/Features/Lists/ArchiveLists.cs

@@ -1,6 +1,6 @@
 using System.Threading;
 using System.Threading.Tasks;
-using FilterLists.Agent.Core.List;
+using FilterLists.Agent.Core.Lists;
 using MediatR;
 
 namespace FilterLists.Agent.Features.Lists

src/FilterLists.Agent/Features/Lists/DownloadLists.cs

@@ -4,7 +4,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
-using FilterLists.Agent.Core.List;
+using FilterLists.Agent.Core.Lists;
 using FilterLists.Agent.Extensions;
 using MediatR;
 using Microsoft.Extensions.Logging;

src/FilterLists.Agent/Features/Lists/DownloadRawText.cs

@@ -2,7 +2,7 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using FilterLists.Agent.Core.List;
+using FilterLists.Agent.Core.Lists;
 using FilterLists.Agent.Extensions;
 using MediatR;
 

src/FilterLists.Agent/Features/Lists/DownloadSevenZip.cs

@@ -1,7 +1,7 @@
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
-using FilterLists.Agent.Core.List;
+using FilterLists.Agent.Core.Lists;
 using MediatR;
 using SharpCompress.Archives;
 using SharpCompress.Archives.SevenZip;

src/FilterLists.Agent/Features/Lists/DownloadZip.cs

@@ -1,7 +1,7 @@
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
-using FilterLists.Agent.Core.List;
+using FilterLists.Agent.Core.Lists;
 using MediatR;
 using SharpCompress.Common;
 using SharpCompress.Readers;

src/FilterLists.Agent/Features/Urls/Models/ValidationResults/DataFileUrlValidationResults.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using FilterLists.Agent.Core.Urls;
 
 namespace FilterLists.Agent.Features.Urls.Models.ValidationResults
 {

src/FilterLists.Agent/Features/Urls/ValidateAllUrls.cs

@@ -2,7 +2,7 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using FilterLists.Agent.Core;
+using FilterLists.Agent.Core.Urls;
 using FilterLists.Agent.Features.Urls.Models.DataFileUrls;
 using FilterLists.Agent.Features.Urls.Models.ValidationResults;
 using MediatR;

src/FilterLists.Agent/Features/Urls/ValidateUrls.cs

@@ -4,9 +4,8 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
-using FilterLists.Agent.Core;
+using FilterLists.Agent.Core.Urls;
 using FilterLists.Agent.Extensions;
-using FilterLists.Agent.Features.Urls.Models.ValidationResults;
 using MediatR;
 
 namespace FilterLists.Agent.Features.Urls
@@ -26,11 +25,11 @@ public Command(IEnumerable<Uri> urls)
         public class Handler : IRequestHandler<Command, List<UrlValidationResult>>
         {
             private const int MaxDegreeOfParallelism = 5;
-            private readonly IUrlRepository _repo;
+            private readonly IUrlValidator _urlValidator;
 
-            public Handler(IUrlRepository urlRepository)
+            public Handler(IUrlValidator urlValidator)
             {
-                _repo = urlRepository;
+                _urlValidator = urlValidator;
             }
 
             public async Task<List<UrlValidationResult>> Handle(Command request, CancellationToken cancellationToken)
@@ -55,7 +54,7 @@ public async Task<List<UrlValidationResult>> Handle(Command request, Cancellatio
             private TransformBlock<Uri, UrlValidationResult> BuildValidator(CancellationToken cancellationToken)
             {
                 return new TransformBlock<Uri, UrlValidationResult>(
-                    async u => await _repo.ValidateAsync(u, cancellationToken),
+                    async u => await _urlValidator.ValidateAsync(u, cancellationToken),
                     new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
                 );
             }

src/FilterLists.Agent/Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs

@@ -1,9 +1,6 @@
 using System;
-using System.Net;
-using System.Net.Http;
 using CommandLine;
 using FilterLists.Agent.AppSettings;
-using FilterLists.Agent.Core;
 using FilterLists.Agent.Infrastructure.Disk;
 using FilterLists.Agent.Infrastructure.FilterListsApi;
 using FilterLists.Agent.Infrastructure.GitHub;
@@ -13,7 +10,6 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
-using Polly;
 using Serilog;
 
 namespace FilterLists.Agent.Infrastructure.DependencyInjection
@@ -27,11 +23,10 @@ public static void ConfigureServices(this IServiceCollection services)
             services.AddLocalization();
             services.AddTransient<Parser>();
             services.AddMediatR(AppDomain.CurrentDomain.GetAssemblies());
+            services.AddDiskResources();
             services.AddFilterListsApiResources();
             services.AddGitHubResources();
-            services.AddUrlRepository();
-            services.AddListRepository();
-            services.AddArchiveRepository();
+            services.AddWebResources();
         }
 
         private static void AddConfiguration(this IServiceCollection services)
@@ -76,18 +71,5 @@ private static void AddLoggingCustom(this IServiceCollection services)
                     .CreateLogger());
             });
         }
-
-        private static void AddUrlRepository(this IServiceCollection services)
-        {
-            services.AddHttpClient<IUrlRepository, UrlRepository>()
-                .ConfigureHttpMessageHandlerBuilder(b =>
-                {
-                    b.PrimaryHandler = new HttpClientHandler {AllowAutoRedirect = false};
-                    b.Build();
-                })
-                .AddTransientHttpErrorPolicy(b =>
-                    b.OrResult(r => r.StatusCode == HttpStatusCode.TooManyRequests)
-                        .WaitAndRetryAsync(5, i => i * TimeSpan.FromSeconds(3)));
-        }
     }
 }

src/FilterLists.Agent/Infrastructure/Disk/ServiceCollectionExtensions.cs

@@ -7,7 +7,7 @@ namespace FilterLists.Agent.Infrastructure.Disk
 {
     public static class ServiceCollectionExtensions
     {
-        public static void AddArchiveRepository(this IServiceCollection services)
+        public static void AddDiskResources(this IServiceCollection services)
         {
             services.AddTransient<IRepository, Repository>(s =>
             {

src/FilterLists.Agent/Infrastructure/FilterListsApi/FilterListsApiClient.cs

@@ -8,6 +8,11 @@
 
 namespace FilterLists.Agent.Infrastructure.FilterListsApi
 {
+    public interface IFilterListsApiClient
+    {
+        Task<TResponse> ExecuteAsync<TResponse>(IRestRequest restRequest, CancellationToken cancellationToken);
+    }
+
     public class FilterListsApiClient : IFilterListsApiClient
     {
         private readonly IStringLocalizer<FilterListsApiClient> _localizer;

src/FilterLists.Agent/Infrastructure/FilterListsApi/IFilterListsApiClient.cs

@@ -1,11 +0,0 @@
-using System.Threading;
-using System.Threading.Tasks;
-using RestSharp;
-
-namespace FilterLists.Agent.Infrastructure.FilterListsApi
-{
-    public interface IFilterListsApiClient
-    {
-        Task<TResponse> ExecuteAsync<TResponse>(IRestRequest restRequest, CancellationToken cancellationToken);
-    }
-}

src/FilterLists.Agent/Infrastructure/FilterListsApi/ListUrlRepository.cs

@@ -1,7 +1,7 @@
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
-using FilterLists.Agent.Core.List;
+using FilterLists.Agent.Core.Lists;
 using RestSharp;
 
 namespace FilterLists.Agent.Infrastructure.FilterListsApi

src/FilterLists.Agent/Infrastructure/FilterListsApi/ServiceCollectionExtensions.cs

@@ -1,4 +1,5 @@
-using FilterLists.Agent.Core.List;
+using FilterLists.Agent.Core.Lists;
+using FilterLists.Agent.Core.Urls;
 using Microsoft.Extensions.DependencyInjection;
 
 namespace FilterLists.Agent.Infrastructure.FilterListsApi
@@ -9,6 +10,7 @@ public static void AddFilterListsApiResources(this IServiceCollection services)
         {
             services.AddSingleton<IFilterListsApiClient, FilterListsApiClient>();
             services.AddTransient<IListUrlRepository, ListUrlRepository>();
+            services.AddTransient<IUrlRepository, UrlRepository>();
         }
     }
 }

src/FilterLists.Agent/Infrastructure/FilterListsApi/UrlRepository.cs

@@ -0,0 +1,45 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using FilterLists.Agent.Core.Urls;
+using FilterLists.Agent.Features.Urls.Models.DataFileUrls;
+using JetBrains.Annotations;
+using Microsoft.Extensions.Localization;
+using RestSharp;
+
+namespace FilterLists.Agent.Infrastructure.FilterListsApi
+{
+    [UsedImplicitly]
+    public class UrlRepository : IUrlRepository
+    {
+        private static readonly Dictionary<string, string> EntityUrlsEndpoints = new Dictionary<string, string>
+        {
+            {nameof(LicenseUrls), "licenses"},
+            {nameof(ListUrls), "lists"},
+            {nameof(MaintainerUrls), "maintainers"},
+            {nameof(SoftwareUrls), "software"},
+            {nameof(SyntaxUrls), "syntaxes"}
+        };
+
+        private readonly IFilterListsApiClient _apiClient;
+        private readonly IStringLocalizer<UrlRepository> _localizer;
+
+        public UrlRepository(IFilterListsApiClient apiClient, IStringLocalizer<UrlRepository> stringLocalizer)
+        {
+            _apiClient = apiClient;
+            _localizer = stringLocalizer;
+        }
+
+        public async Task<IEnumerable<Uri>> GetAllAsync<TModel>(CancellationToken cancellationToken)
+        {
+            if (!EntityUrlsEndpoints.ContainsKey(typeof(TModel).Name))
+                throw new ArgumentException(_localizer["The type of TModel is not valid."]);
+            var request = new RestRequest($"{EntityUrlsEndpoints[typeof(TModel).Name]}/seed");
+            var response = await _apiClient.ExecuteAsync<IEnumerable<TModel>>(request, cancellationToken);
+            return response.ToList().SelectMany(r =>
+                r.GetType().GetProperties().Where(p => p.GetValue(r) != null).Select(p => (Uri)p.GetValue(r)));
+        }
+    }
+}

src/FilterLists.Agent/Infrastructure/Web/ListRepository.cs

@@ -5,7 +5,7 @@
 using System.Net.Http.Headers;
 using System.Threading;
 using System.Threading.Tasks;
-using FilterLists.Agent.Core.List;
+using FilterLists.Agent.Core.Lists;
 using JetBrains.Annotations;
 using Microsoft.Extensions.Logging;
 

src/FilterLists.Agent/Infrastructure/Web/ServiceCollectionExtensions.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Net;
-using FilterLists.Agent.Core.List;
+using System.Net.Http;
+using FilterLists.Agent.Core.Lists;
+using FilterLists.Agent.Core.Urls;
 using Microsoft.Extensions.DependencyInjection;
 using Polly;
 
@@ -8,11 +10,20 @@ namespace FilterLists.Agent.Infrastructure.Web
 {
     public static class ServiceCollectionExtensions
     {
-        public static void AddListRepository(this IServiceCollection services)
+        public static void AddWebResources(this IServiceCollection services)
         {
             services.AddHttpClient<IListRepository, ListRepository>().AddTransientHttpErrorPolicy(b =>
                 b.OrResult(r => r.StatusCode == HttpStatusCode.TooManyRequests)
                     .WaitAndRetryAsync(5, i => i * TimeSpan.FromSeconds(3)));
+            services.AddHttpClient<IUrlValidator, UrlValidator>()
+                .ConfigureHttpMessageHandlerBuilder(b =>
+                {
+                    b.PrimaryHandler = new HttpClientHandler {AllowAutoRedirect = false};
+                    b.Build();
+                })
+                .AddTransientHttpErrorPolicy(b =>
+                    b.OrResult(r => r.StatusCode == HttpStatusCode.TooManyRequests)
+                        .WaitAndRetryAsync(5, i => i * TimeSpan.FromSeconds(3)));
         }
     }
 }

src/FilterLists.Agent/Infrastructure/Web/UrlValidator.cs

@@ -1,68 +1,33 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Threading;
 using System.Threading.Tasks;
-using FilterLists.Agent.AppSettings;
-using FilterLists.Agent.Core;
+using FilterLists.Agent.Core.Urls;
 using FilterLists.Agent.Extensions;
-using FilterLists.Agent.Features.Urls.Models.DataFileUrls;
-using FilterLists.Agent.Features.Urls.Models.ValidationResults;
+using FilterLists.Agent.Infrastructure.FilterListsApi;
 using JetBrains.Annotations;
-using Microsoft.Extensions.Localization;
 using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-using RestSharp;
 
-namespace FilterLists.Agent.Infrastructure
+namespace FilterLists.Agent.Infrastructure.Web
 {
     [UsedImplicitly]
-    public class UrlRepository : IUrlRepository
+    public class UrlValidator : IUrlValidator
     {
-        private static readonly Dictionary<string, string> EntityUrlsEndpoints = new Dictionary<string, string>
-        {
-            {nameof(LicenseUrls), "licenses"},
-            {nameof(ListUrls), "lists"},
-            {nameof(MaintainerUrls), "maintainers"},
-            {nameof(SoftwareUrls), "software"},
-            {nameof(SyntaxUrls), "syntaxes"}
-        };
-
-        private readonly IRestClient _apiClient;
         private readonly HttpClient _httpClient;
-        private readonly IStringLocalizer<UrlRepository> _localizer;
         private readonly ILogger<UrlRepository> _logger;
 
-        public UrlRepository(IOptions<FilterListsApiSettings> filterListsApiOptions, HttpClient httpClient,
-            IStringLocalizer<UrlRepository> stringLocalizer, ILogger<UrlRepository> logger)
+        public UrlValidator(HttpClient httpClient, ILogger<UrlRepository> logger)
         {
-            _apiClient = new RestClient(filterListsApiOptions.Value.BaseUrl) {UserAgent = "FilterLists.Agent"};
-
             httpClient.Timeout = TimeSpan.FromSeconds(90);
             var header = new ProductHeaderValue("FilterLists.Agent");
             var userAgent = new ProductInfoHeaderValue(header);
             httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent);
             _httpClient = httpClient;
 
-            _localizer = stringLocalizer;
             _logger = logger;
         }
 
-        public async Task<IEnumerable<Uri>> GetAllAsync<TModel>(CancellationToken cancellationToken)
-        {
-            if (!EntityUrlsEndpoints.ContainsKey(typeof(TModel).Name))
-                throw new ArgumentException(_localizer["The type of TModel is not valid."]);
-            var request = new RestRequest($"{EntityUrlsEndpoints[typeof(TModel).Name]}/seed");
-            var response = await _apiClient.ExecuteTaskAsync<IEnumerable<TModel>>(request, cancellationToken);
-            if (response.ErrorException != null)
-                _logger.LogError(response.ErrorException,
-                    _localizer["Error retrieving response from the FilterLists API."]);
-            return response.Data.SelectMany(r =>
-                r.GetType().GetProperties().Where(p => p.GetValue(r) != null).Select(p => (Uri)p.GetValue(r)));
-        }
-
         public async Task<UrlValidationResult> ValidateAsync(Uri u, CancellationToken cancellationToken)
         {
             var result = new UrlValidationResult(u);