further support for url validation

src/FilterLists.Agent/Features/Urls/ValidateAllUrls.cs

@@ -0,0 +1,39 @@
+using System.Threading;
+using System.Threading.Tasks;
+using FilterLists.Agent.Core.Entities;
+using FilterLists.Agent.Core.Interfaces;
+using MediatR;
+
+namespace FilterLists.Agent.Features.Urls
+{
+    public static class ValidateAllUrls
+    {
+        public class Command : IRequest
+        {
+        }
+
+        public class Handler : AsyncRequestHandler<Command>
+        {
+            private readonly IMediator _mediator;
+            private readonly IUrlsRepository _repo;
+
+            public Handler(IMediator mediator, IUrlsRepository urlsRepository)
+            {
+                _mediator = mediator;
+                _repo = urlsRepository;
+            }
+
+            protected override async Task Handle(Command request, CancellationToken cancellationToken)
+            {
+                var maintainerUrls = await _repo.GetAllAsync<MaintainerUrls>();
+                var errors = await _mediator.Send(new ValidateUrls.Command(maintainerUrls), cancellationToken);
+
+                //TODO: validate if http can be changed to https
+
+                //TODO: iterate through each IEntityUrl impl
+
+                //TODO: create or edit GitHub issue with results
+            }
+        }
+    }
+}

src/FilterLists.Agent/Features/Urls/ValidateUrls.cs

@@ -1,33 +1,84 @@
 using System;
+using System.Collections.Generic;
+using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
-using FilterLists.Agent.Core.Entities;
-using FilterLists.Agent.Core.Interfaces;
+using System.Threading.Tasks.Dataflow;
+using FilterLists.Agent.Infrastructure.Clients;
 using MediatR;
+using Microsoft.Extensions.Logging;
 
 namespace FilterLists.Agent.Features.Urls
 {
     public static class ValidateUrls
     {
-        public class Command : IRequest
+        public class Command : IRequest<IEnumerable<Uri>>
         {
-        }
-
-        public class Handler : AsyncRequestHandler<Command>
-        {
-            private readonly IMediator _mediator;
-            private readonly IUrlsRepository _repo;
-
-            public Handler(IMediator mediator, IUrlsRepository urlsRepository)
+            public Command(IEnumerable<Uri> urls)
             {
-                _mediator = mediator;
-                _repo = urlsRepository;
+                Urls = urls;
             }
 
-            protected override async Task Handle(Command request, CancellationToken cancellationToken)
+            public IEnumerable<Uri> Urls { get; }
+        }
+
+        public class Handler : IRequestHandler<Command, IEnumerable<Uri>>
+        {
+            private const int MaxDegreeOfParallelism = 5;
+            private readonly HttpClient _httpClient;
+            private readonly ILogger<Handler> _logger;
+
+            public Handler(AgentHttpClient agentHttpClient, ILogger<Handler> logger)
             {
-                var listUrls = await _repo.GetAllAsync<ListUrls>();
-                throw new NotImplementedException();
+                _httpClient = agentHttpClient.Client;
+                _logger = logger;
+            }
+
+            public async Task<IEnumerable<Uri>> Handle(Command request, CancellationToken cancellationToken)
+            {
+                var validator = BuildValidator(cancellationToken);
+                var brokenUrls = new List<Uri>();
+                foreach (var url in request.Urls)
+                    await validator.SendAsync(url, cancellationToken);
+                validator.Complete();
+                while (await validator.OutputAvailableAsync(cancellationToken))
+                {
+                    var (url, result) = await validator.ReceiveAsync(cancellationToken);
+                    if (!result)
+                        brokenUrls.Add(url);
+                }
+
+                await validator.Completion;
+                return brokenUrls;
+            }
+
+            private TransformBlock<Uri, (Uri, bool)> BuildValidator(CancellationToken cancellationToken)
+            {
+                return new TransformBlock<Uri, (Uri, bool)>(
+                    async u =>
+                    {
+                        var errorMessage = $"The following URL is broken: {u}.";
+                        try
+                        {
+                            var response = await _httpClient.GetAsync(u, cancellationToken);
+                            if (response.IsSuccessStatusCode)
+                                return (u, true);
+                            _logger.LogError($"{errorMessage} {response.StatusCode}");
+                            return (u, false);
+                        }
+                        catch (HttpRequestException ex)
+                        {
+                            _logger.LogError(ex, errorMessage);
+                            return (u, false);
+                        }
+                        catch (TaskCanceledException ex)
+                        {
+                            _logger.LogError(ex, errorMessage);
+                            return (u, false);
+                        }
+                    },
+                    new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
+                );
             }
         }
     }

src/FilterLists.Agent/Program.cs

@@ -17,7 +17,7 @@ public static async Task Main()
             BuildServiceProvider();
             var mediator = _serviceProvider.GetService<IMediator>();
             await mediator.Send(new CaptureLists.Command());
-            await mediator.Send(new ValidateUrls.Command());
+            await mediator.Send(new ValidateAllUrls.Command());
         }
 
         private static void BuildServiceProvider()