prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

initial support for parallel archive downloads

Browse source
This commit is contained in:
Collin M. Barrett 2019-06-26 21:19:34 -05:00
parent d93aced937
commit 7060bf2c11
3 changed files with 24 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
docker-compose.yml
View file

@ -62,6 +62,8 @@ services:
context: .
dockerfile: src/FilterLists.Agent/Dockerfile
target: final
volumes:
- archives:/app/archives
networks:
reverse-proxy:
@ -69,3 +71,4 @@ networks:
volumes:
data:
archives:

4
src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Entities;
@ -29,8 +30,7 @@ protected override async Task Handle(Command request, CancellationToken cancella
{
var listsRequest = new RestRequest("lists");
var lists = await _apiClient.ExecuteAsync<IEnumerable<ListInfo>>(listsRequest);
foreach (var list in lists)
await _mediator.Send(new CaptureList.Command(list), cancellationToken);
await Task.WhenAll(lists.Select(l => _mediator.Send(new CaptureList.Command(l), cancellationToken)));
}
}
}

20
src/FilterLists.Agent/ListArchiver/CaptureList.cs
View file

@ -1,4 +1,5 @@
using System.Net.Http;
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Entities;
@ -29,6 +30,23 @@ public Handler(HttpClient httpClient)
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
if (Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath) == ".txt")
try
{
using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken))
{
if (result.IsSuccessStatusCode)
using (Stream output =
File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt")))
using (var input = await result.Content.ReadAsStreamAsync())
{
input.CopyTo(output);
}
}
}
catch (HttpRequestException)
{
}
}
}
}