From 5ab0c89235ad8d7e9f080a30f53b6abae73ab3f7 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Thu, 27 Jun 2019 12:40:18 -0500 Subject: [PATCH] misc cleanup --- .../{CaptureAllLists.cs => CaptureLists.cs} | 2 +- .../ListArchiver/DownloadList.cs | 51 ++++++++++--------- .../DownloadTxt.cs | 31 +++++------ src/FilterLists.Agent/Program.cs | 2 +- 4 files changed, 44 insertions(+), 42 deletions(-) rename src/FilterLists.Agent/ListArchiver/{CaptureAllLists.cs => CaptureLists.cs} (96%) diff --git a/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs b/src/FilterLists.Agent/ListArchiver/CaptureLists.cs similarity index 96% rename from src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs rename to src/FilterLists.Agent/ListArchiver/CaptureLists.cs index eb7d61877..404414379 100644 --- a/src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs +++ b/src/FilterLists.Agent/ListArchiver/CaptureLists.cs @@ -8,7 +8,7 @@ namespace FilterLists.Agent.ListArchiver { - public static class CaptureAllLists + public static class CaptureLists { public class Command : IRequest { diff --git a/src/FilterLists.Agent/ListArchiver/DownloadList.cs b/src/FilterLists.Agent/ListArchiver/DownloadList.cs index 19ab599ff..f6567fc00 100644 --- a/src/FilterLists.Agent/ListArchiver/DownloadList.cs +++ b/src/FilterLists.Agent/ListArchiver/DownloadList.cs @@ -27,30 +27,30 @@ private static readonly Dictionary> DownloadReq = new Dictionary> { {"", l => new DownloadTxt.Command(l)}, - {".7z", l => new DownloadTxt.Command(l)}, - {".acl", l => new DownloadTxt.Command(l)}, - {".action", l => new DownloadTxt.Command(l)}, - {".all", l => new DownloadTxt.Command(l)}, - {".aspx", l => new DownloadTxt.Command(l)}, - {".bat", l => new DownloadTxt.Command(l)}, - {".blacklist", l => new DownloadTxt.Command(l)}, - {".conf", l => new DownloadTxt.Command(l)}, - {".csv", l => new DownloadTxt.Command(l)}, - {".dat", l => new DownloadTxt.Command(l)}, - {".deny", l => new DownloadTxt.Command(l)}, - {".host", l => new DownloadTxt.Command(l)}, - {".hosts", l => new DownloadTxt.Command(l)}, - {".ips", l => new DownloadTxt.Command(l)}, - {".ipset", l => new DownloadTxt.Command(l)}, - {".json", l => new DownloadTxt.Command(l)}, - {".list", l => new DownloadTxt.Command(l)}, - {".lsrules", l => new DownloadTxt.Command(l)}, - {".netset", l => new DownloadTxt.Command(l)}, - {".p2p", l => new DownloadTxt.Command(l)}, - {".php", l => new DownloadTxt.Command(l)}, - {".tpl", l => new DownloadTxt.Command(l)}, + {".7z", l => throw new NotImplementedException()}, + {".acl", l => throw new NotImplementedException()}, + {".action", l => throw new NotImplementedException()}, + {".all", l => throw new NotImplementedException()}, + {".aspx", l => throw new NotImplementedException()}, + {".bat", l => throw new NotImplementedException()}, + {".blacklist", l => throw new NotImplementedException()}, + {".conf", l => throw new NotImplementedException()}, + {".csv", l => throw new NotImplementedException()}, + {".dat", l => throw new NotImplementedException()}, + {".deny", l => throw new NotImplementedException()}, + {".host", l => throw new NotImplementedException()}, + {".hosts", l => throw new NotImplementedException()}, + {".ips", l => throw new NotImplementedException()}, + {".ipset", l => throw new NotImplementedException()}, + {".json", l => throw new NotImplementedException()}, + {".list", l => throw new NotImplementedException()}, + {".lsrules", l => throw new NotImplementedException()}, + {".netset", l => throw new NotImplementedException()}, + {".p2p", l => throw new NotImplementedException()}, + {".php", l => throw new NotImplementedException()}, + {".tpl", l => throw new NotImplementedException()}, {".txt", l => new DownloadTxt.Command(l)}, - {".zip", l => new DownloadTxt.Command(l)} + {".zip", l => throw new NotImplementedException()} }; private readonly IMediator _mediator; @@ -68,8 +68,13 @@ protected override async Task Handle(Command request, CancellationToken cancella if (DownloadRequestsByFileExtension.ContainsKey(extension)) await _mediator.Send(DownloadRequestsByFileExtension[extension].Invoke(request.ListInfo), cancellationToken); + //TODO: handle and/or log unrecognized extension //TODO: upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update } + catch (NotImplementedException) + { + //TODO: log + } catch (ArgumentException) { //TODO: log diff --git a/src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs b/src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs index de2f542e2..1e5f0bbdc 100644 --- a/src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs +++ b/src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs @@ -31,27 +31,24 @@ public Handler(HttpClient httpClient) protected override async Task Handle(Command request, CancellationToken cancellationToken) { - if (!Path.HasExtension(request.ListInfo.ViewUrl.AbsolutePath) || - Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath) == ".txt") + Debug.WriteLine($"Downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}..."); + try { - Debug.WriteLine($"Downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}..."); - try - { - using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken)) - { - if (result.IsSuccessStatusCode) - using (Stream output = - File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt"))) - using (var input = await result.Content.ReadAsStreamAsync()) - { - input.CopyTo(output); - } - } - } - catch (HttpRequestException) + using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken)) { + if (result.IsSuccessStatusCode) + using (Stream output = + File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt"))) + using (var input = await result.Content.ReadAsStreamAsync()) + { + input.CopyTo(output); + } } } + catch (HttpRequestException) + { + //TODO: log + } } } } diff --git a/src/FilterLists.Agent/Program.cs b/src/FilterLists.Agent/Program.cs index 6701f4cc5..af6df94d6 100644 --- a/src/FilterLists.Agent/Program.cs +++ b/src/FilterLists.Agent/Program.cs @@ -19,7 +19,7 @@ public static async Task Main() RegisterServices(); var mediator = _serviceProvider.GetService(); - await mediator.Send(new CaptureAllLists.Command()); + await mediator.Send(new CaptureLists.Command()); ((IDisposable) _serviceProvider).Dispose(); }