From 63e2d78b1d016510dea95fa3505e6f6f3bdca756 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Thu, 27 Jun 2019 12:25:01 -0500 Subject: [PATCH] wip - download differently by file extension --- .../ListArchiver/CaptureAllLists.cs | 17 +--- .../ListArchiver/DownloadList.cs | 99 +++++++++---------- .../ListArchiver/DownloadLists.cs | 44 +++++++++ .../DownloadTxt.cs | 58 +++++++++++ src/FilterLists.Agent/Program.cs | 4 - 5 files changed, 151 insertions(+), 71 deletions(-) create mode 100644 src/FilterLists.Agent/ListArchiver/DownloadLists.cs create mode 100644 src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs diff --git a/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs b/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs index 0191a1dc1..eb7d61877 100644 --- a/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs +++ b/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs @@ -1,7 +1,6 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using System.Threading.Tasks.Dataflow; using FilterLists.Agent.Entities; using FilterLists.Agent.Infrastructure; using MediatR; @@ -17,7 +16,6 @@ public class Command : IRequest public class Handler : AsyncRequestHandler { - private const int MaxDegreeOfParallelism = 50; private readonly IFilterListsApiClient _apiClient; private readonly IMediator _mediator; @@ -30,7 +28,9 @@ public Handler(IFilterListsApiClient apiClient, IMediator mediator) protected override async Task Handle(Command request, CancellationToken cancellationToken) { var lists = await GetListInfo(); - await DownloadLists(lists, cancellationToken); + await _mediator.Send(new DownloadLists.Command(lists), cancellationToken); + //TODO: git add . + //TODO: git commit } private async Task> GetListInfo() @@ -38,17 +38,6 @@ private async Task> GetListInfo() var listsRequest = new RestRequest("lists"); return await _apiClient.ExecuteAsync>(listsRequest); } - - private async Task DownloadLists(IEnumerable lists, CancellationToken cancellationToken) - { - var downloader = new TransformBlock( - l => _mediator.Send(new DownloadList.Command(l), cancellationToken), - new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}); - foreach (var list in lists) - await downloader.SendAsync(list, cancellationToken); - downloader.Complete(); - await downloader.Completion; - } } } } \ No newline at end of file diff --git a/src/FilterLists.Agent/ListArchiver/DownloadList.cs b/src/FilterLists.Agent/ListArchiver/DownloadList.cs index 592aedde4..19ab599ff 100644 --- a/src/FilterLists.Agent/ListArchiver/DownloadList.cs +++ b/src/FilterLists.Agent/ListArchiver/DownloadList.cs @@ -1,40 +1,12 @@ -using System.Diagnostics; +using System; +using System.Collections.Generic; using System.IO; -using System.Net.Http; using System.Threading; using System.Threading.Tasks; using FilterLists.Agent.Entities; +using FilterLists.Agent.ListArchiver.DownloadRequestsByFileExtension; using MediatR; -/* -TODO: support downloading all of the following extensions -"" (no extension) -7z -acl -action -all -aspx -bat -blacklist -conf -csv -dat -deny -host -hosts -ips -ipset -json -list -lsrules -netset -p2p -php -tpl -txt -zip -*/ - namespace FilterLists.Agent.ListArchiver { public static class DownloadList @@ -51,35 +23,56 @@ public Command(ListInfo listInfo) public class Handler : AsyncRequestHandler { - private readonly HttpClient _httpClient; + private static readonly Dictionary> DownloadRequestsByFileExtension + = new Dictionary> + { + {"", l => new DownloadTxt.Command(l)}, + {".7z", l => new DownloadTxt.Command(l)}, + {".acl", l => new DownloadTxt.Command(l)}, + {".action", l => new DownloadTxt.Command(l)}, + {".all", l => new DownloadTxt.Command(l)}, + {".aspx", l => new DownloadTxt.Command(l)}, + {".bat", l => new DownloadTxt.Command(l)}, + {".blacklist", l => new DownloadTxt.Command(l)}, + {".conf", l => new DownloadTxt.Command(l)}, + {".csv", l => new DownloadTxt.Command(l)}, + {".dat", l => new DownloadTxt.Command(l)}, + {".deny", l => new DownloadTxt.Command(l)}, + {".host", l => new DownloadTxt.Command(l)}, + {".hosts", l => new DownloadTxt.Command(l)}, + {".ips", l => new DownloadTxt.Command(l)}, + {".ipset", l => new DownloadTxt.Command(l)}, + {".json", l => new DownloadTxt.Command(l)}, + {".list", l => new DownloadTxt.Command(l)}, + {".lsrules", l => new DownloadTxt.Command(l)}, + {".netset", l => new DownloadTxt.Command(l)}, + {".p2p", l => new DownloadTxt.Command(l)}, + {".php", l => new DownloadTxt.Command(l)}, + {".tpl", l => new DownloadTxt.Command(l)}, + {".txt", l => new DownloadTxt.Command(l)}, + {".zip", l => new DownloadTxt.Command(l)} + }; - public Handler(HttpClient httpClient) + private readonly IMediator _mediator; + + public Handler(IMediator mediator) { - _httpClient = httpClient; + _mediator = mediator; } protected override async Task Handle(Command request, CancellationToken cancellationToken) { - if (!Path.HasExtension(request.ListInfo.ViewUrl.AbsolutePath) || - Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath) == ".txt") + try { - Debug.WriteLine($"Downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}..."); - try - { - using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken)) - { - if (result.IsSuccessStatusCode) - using (Stream output = - File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt"))) - using (var input = await result.Content.ReadAsStreamAsync()) - { - input.CopyTo(output); - } - } - } - catch (HttpRequestException) - { - } + var extension = Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath); + if (DownloadRequestsByFileExtension.ContainsKey(extension)) + await _mediator.Send(DownloadRequestsByFileExtension[extension].Invoke(request.ListInfo), + cancellationToken); + //TODO: upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update + } + catch (ArgumentException) + { + //TODO: log } } } diff --git a/src/FilterLists.Agent/ListArchiver/DownloadLists.cs b/src/FilterLists.Agent/ListArchiver/DownloadLists.cs new file mode 100644 index 000000000..ce1fd0b9c --- /dev/null +++ b/src/FilterLists.Agent/ListArchiver/DownloadLists.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using FilterLists.Agent.Entities; +using MediatR; + +namespace FilterLists.Agent.ListArchiver +{ + public static class DownloadLists + { + public class Command : IRequest + { + public Command(IEnumerable lists) + { + Lists = lists; + } + + public IEnumerable Lists { get; } + } + + public class Handler : AsyncRequestHandler + { + private const int MaxDegreeOfParallelism = 50; + private readonly IMediator _mediator; + + public Handler(IMediator mediator) + { + _mediator = mediator; + } + + protected override async Task Handle(Command request, CancellationToken cancellationToken) + { + var downloader = new TransformBlock( + l => _mediator.Send(new DownloadList.Command(l), cancellationToken), + new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}); + foreach (var list in request.Lists) + await downloader.SendAsync(list, cancellationToken); + downloader.Complete(); + await downloader.Completion; + } + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs b/src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs new file mode 100644 index 000000000..de2f542e2 --- /dev/null +++ b/src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs @@ -0,0 +1,58 @@ +using System.Diagnostics; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using FilterLists.Agent.Entities; +using MediatR; + +namespace FilterLists.Agent.ListArchiver.DownloadRequestsByFileExtension +{ + public static class DownloadTxt + { + public class Command : IRequest + { + public Command(ListInfo listInfo) + { + ListInfo = listInfo; + } + + public ListInfo ListInfo { get; } + } + + public class Handler : AsyncRequestHandler + { + private readonly HttpClient _httpClient; + + public Handler(HttpClient httpClient) + { + _httpClient = httpClient; + } + + protected override async Task Handle(Command request, CancellationToken cancellationToken) + { + if (!Path.HasExtension(request.ListInfo.ViewUrl.AbsolutePath) || + Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath) == ".txt") + { + Debug.WriteLine($"Downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}..."); + try + { + using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken)) + { + if (result.IsSuccessStatusCode) + using (Stream output = + File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt"))) + using (var input = await result.Content.ReadAsStreamAsync()) + { + input.CopyTo(output); + } + } + } + catch (HttpRequestException) + { + } + } + } + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/Program.cs b/src/FilterLists.Agent/Program.cs index 02c48e4e1..6701f4cc5 100644 --- a/src/FilterLists.Agent/Program.cs +++ b/src/FilterLists.Agent/Program.cs @@ -8,10 +8,6 @@ using MediatR; using Microsoft.Extensions.DependencyInjection; -//TODO: git add . -//TODO: git commit -//TODO: foreach list, upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update - namespace FilterLists.Agent { public static class Program