diff --git a/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs b/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs index 5e2750c27..0191a1dc1 100644 --- a/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs +++ b/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; -using System.Linq; using System.Threading; using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; using FilterLists.Agent.Entities; using FilterLists.Agent.Infrastructure; using MediatR; @@ -17,6 +17,7 @@ public class Command : IRequest public class Handler : AsyncRequestHandler { + private const int MaxDegreeOfParallelism = 50; private readonly IFilterListsApiClient _apiClient; private readonly IMediator _mediator; @@ -27,10 +28,26 @@ public Handler(IFilterListsApiClient apiClient, IMediator mediator) } protected override async Task Handle(Command request, CancellationToken cancellationToken) + { + var lists = await GetListInfo(); + await DownloadLists(lists, cancellationToken); + } + + private async Task> GetListInfo() { var listsRequest = new RestRequest("lists"); - var lists = await _apiClient.ExecuteAsync>(listsRequest); - await Task.WhenAll(lists.Select(l => _mediator.Send(new CaptureList.Command(l), cancellationToken))); + return await _apiClient.ExecuteAsync>(listsRequest); + } + + private async Task DownloadLists(IEnumerable lists, CancellationToken cancellationToken) + { + var downloader = new TransformBlock( + l => _mediator.Send(new DownloadList.Command(l), cancellationToken), + new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}); + foreach (var list in lists) + await downloader.SendAsync(list, cancellationToken); + downloader.Complete(); + await downloader.Completion; } } } diff --git a/src/FilterLists.Agent/ListArchiver/CaptureList.cs b/src/FilterLists.Agent/ListArchiver/DownloadList.cs similarity index 89% rename from src/FilterLists.Agent/ListArchiver/CaptureList.cs rename to src/FilterLists.Agent/ListArchiver/DownloadList.cs index 389406728..aae7d53e0 100644 --- a/src/FilterLists.Agent/ListArchiver/CaptureList.cs +++ b/src/FilterLists.Agent/ListArchiver/DownloadList.cs @@ -1,4 +1,5 @@ -using System.IO; +using System.Diagnostics; +using System.IO; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -7,7 +8,7 @@ namespace FilterLists.Agent.ListArchiver { - public static class CaptureList + public static class DownloadList { public class Command : IRequest { @@ -31,6 +32,8 @@ public Handler(HttpClient httpClient) protected override async Task Handle(Command request, CancellationToken cancellationToken) { if (Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath) == ".txt") + { + Debug.WriteLine($"Downloading list {request.ListInfo.Id}..."); try { using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken)) @@ -47,6 +50,7 @@ protected override async Task Handle(Command request, CancellationToken cancella catch (HttpRequestException) { } + } } } } diff --git a/src/FilterLists.Agent/Program.cs b/src/FilterLists.Agent/Program.cs index 2c8f68a38..02c48e4e1 100644 --- a/src/FilterLists.Agent/Program.cs +++ b/src/FilterLists.Agent/Program.cs @@ -8,7 +8,6 @@ using MediatR; using Microsoft.Extensions.DependencyInjection; -//TODO: foreach list, download and persist raw list to disk with standardized name and overwriting the previous version //TODO: git add . //TODO: git commit //TODO: foreach list, upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update