prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

use TPL dataflow to batch parallel downloads

Browse source
This commit is contained in:
Collin M. Barrett 2019-06-27 08:28:39 -05:00
parent 190e3affab
commit 7f37515cfa
3 changed files with 26 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs
View file

@ -1,7 +1,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using FilterLists.Agent.Entities;
using FilterLists.Agent.Infrastructure;
using MediatR;
@ -17,6 +17,7 @@ public class Command : IRequest
public class Handler : AsyncRequestHandler<Command>
{
private const int MaxDegreeOfParallelism = 50;
private readonly IFilterListsApiClient _apiClient;
private readonly IMediator _mediator;
@ -27,10 +28,26 @@ public Handler(IFilterListsApiClient apiClient, IMediator mediator)
}
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
var lists = await GetListInfo();
await DownloadLists(lists, cancellationToken);
}
private async Task<IEnumerable<ListInfo>> GetListInfo()
{
var listsRequest = new RestRequest("lists");
var lists = await _apiClient.ExecuteAsync<IEnumerable<ListInfo>>(listsRequest);
await Task.WhenAll(lists.Select(l => _mediator.Send(new CaptureList.Command(l), cancellationToken)));
return await _apiClient.ExecuteAsync<IEnumerable<ListInfo>>(listsRequest);
}
private async Task DownloadLists(IEnumerable<ListInfo> lists, CancellationToken cancellationToken)
{
var downloader = new TransformBlock<ListInfo, Task>(
l => _mediator.Send(new DownloadList.Command(l), cancellationToken),
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism});
foreach (var list in lists)
await downloader.SendAsync(list, cancellationToken);
downloader.Complete();
await downloader.Completion;
}
}
}

8
src/FilterLists.Agent/ListArchiver/CaptureList.cs → src/FilterLists.Agent/ListArchiver/DownloadList.cs
View file

@ -1,4 +1,5 @@
using System.IO;
using System.Diagnostics;
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
@ -7,7 +8,7 @@
namespace FilterLists.Agent.ListArchiver
{
public static class CaptureList
public static class DownloadList
{
public class Command : IRequest
{
@ -31,6 +32,8 @@ public Handler(HttpClient httpClient)
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
if (Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath) == ".txt")
{
Debug.WriteLine($"Downloading list {request.ListInfo.Id}...");
try
{
using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken))
@ -47,6 +50,7 @@ protected override async Task Handle(Command request, CancellationToken cancella
catch (HttpRequestException)
{
}
}
}
}
}

1
src/FilterLists.Agent/Program.cs
View file

@ -8,7 +8,6 @@
using MediatR;
using Microsoft.Extensions.DependencyInjection;
//TODO: foreach list, download and persist raw list to disk with standardized name and overwriting the previous version
//TODO: git add .
//TODO: git commit
//TODO: foreach list, upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update