From 8167b4f3a6f93484b0195489e5279172f5b0fad1 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sun, 30 Jun 2019 19:43:20 -0500 Subject: [PATCH] wip refactor DownloadList flow --- .../Extensions/UriExtensions.cs | 13 +++ .../Features/Archiver/DownloadList.cs | 100 ------------------ .../Features/Archiver/DownloadLists.cs | 44 +++++++- .../Features/Archiver/DownloadRawText.cs | 51 +++++++++ .../Features/Archiver/FileType.cs | 9 -- .../Features/Archiver/StreamExtensions.cs | 88 +++++++-------- 6 files changed, 150 insertions(+), 155 deletions(-) create mode 100644 src/FilterLists.Agent/Extensions/UriExtensions.cs delete mode 100644 src/FilterLists.Agent/Features/Archiver/DownloadList.cs create mode 100644 src/FilterLists.Agent/Features/Archiver/DownloadRawText.cs delete mode 100644 src/FilterLists.Agent/Features/Archiver/FileType.cs diff --git a/src/FilterLists.Agent/Extensions/UriExtensions.cs b/src/FilterLists.Agent/Extensions/UriExtensions.cs new file mode 100644 index 000000000..0dd88cb42 --- /dev/null +++ b/src/FilterLists.Agent/Extensions/UriExtensions.cs @@ -0,0 +1,13 @@ +using System; +using System.IO; + +namespace FilterLists.Agent.Extensions +{ + public static class UriExtensions + { + public static string GetExtension(this Uri uri) + { + return Path.GetExtension(uri.AbsolutePath); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Archiver/DownloadList.cs b/src/FilterLists.Agent/Features/Archiver/DownloadList.cs deleted file mode 100644 index 29a8cc053..000000000 --- a/src/FilterLists.Agent/Features/Archiver/DownloadList.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using FilterLists.Agent.Core.Entities; -using FilterLists.Agent.Infrastructure.Clients; -using MediatR; - -namespace FilterLists.Agent.Features.Archiver -{ - public static class DownloadList - { - public class Command : IRequest - { - public Command(ListInfo listInfo) - { - ListInfo = listInfo; - } - - public ListInfo ListInfo { get; } - } - - public class Handler : AsyncRequestHandler - { - private static readonly Dictionary DownloadRequestsByFileExtension - = new Dictionary - { - {"", FileType.RawText}, - {".7z", FileType.NonForwardOnlyCompressed}, - {".acl", FileType.RawText}, - {".action", FileType.RawText}, - {".all", FileType.RawText}, - {".aspx", FileType.RawText}, - {".bat", FileType.RawText}, - {".blacklist", FileType.RawText}, - {".conf", FileType.RawText}, - {".csv", FileType.RawText}, - {".dat", FileType.RawText}, - {".deny", FileType.RawText}, - {".host", FileType.RawText}, - {".hosts", FileType.RawText}, - {".ips", FileType.RawText}, - {".ipset", FileType.RawText}, - {".json", FileType.RawText}, - {".list", FileType.RawText}, - {".lsrules", FileType.RawText}, - {".netset", FileType.RawText}, - {".p2p", FileType.RawText}, - {".php", FileType.RawText}, - {".tpl", FileType.RawText}, - {".txt", FileType.RawText}, - {".zip", FileType.ForwardOnlyCompressed} - }; - - private readonly HttpClient _httpClient; - - public Handler(AgentHttpClient httpClient) - { - _httpClient = httpClient.Client; - } - - protected override async Task Handle(Command request, CancellationToken cancellationToken) - { - using (var response = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken)) - { - if (response.IsSuccessStatusCode) - using (var input = await response.Content.ReadAsStreamAsync()) - { - var sourceExtension = Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath); - string destinationExtension; - if (string.IsNullOrEmpty(sourceExtension) || sourceExtension == ".zip" || sourceExtension == ".7z") - destinationExtension = ".txt"; - else - destinationExtension = sourceExtension; - - using (var output = File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}{destinationExtension}"))) - { - switch (DownloadRequestsByFileExtension[sourceExtension]) - { - case FileType.RawText: - await input.CopyToAsync(output, cancellationToken); - break; - case FileType.ForwardOnlyCompressed: - await input.CopyToWithCompressedReaderApi(output, cancellationToken); - break; - case FileType.NonForwardOnlyCompressed: - input.CopyToWithCompressedArchiveApi(output); - break; - default: - throw new NotImplementedException(); - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Archiver/DownloadLists.cs b/src/FilterLists.Agent/Features/Archiver/DownloadLists.cs index 4d5ba7eac..1a2438453 100644 --- a/src/FilterLists.Agent/Features/Archiver/DownloadLists.cs +++ b/src/FilterLists.Agent/Features/Archiver/DownloadLists.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; @@ -23,6 +24,37 @@ public Command(IEnumerable listInfo) public class Handler : AsyncRequestHandler { private const int MaxDegreeOfParallelism = 5; + + private static readonly Dictionary> CommandsByExtension + = new Dictionary> + { + {"", l => new DownloadRawText.Command(l)}, + //{".7z", l => new DownloadRawText.Command(l)}, + {".acl", l => new DownloadRawText.Command(l)}, + {".action", l => new DownloadRawText.Command(l)}, + {".all", l => new DownloadRawText.Command(l)}, + {".aspx", l => new DownloadRawText.Command(l)}, + {".bat", l => new DownloadRawText.Command(l)}, + {".blacklist", l => new DownloadRawText.Command(l)}, + {".conf", l => new DownloadRawText.Command(l)}, + {".csv", l => new DownloadRawText.Command(l)}, + {".dat", l => new DownloadRawText.Command(l)}, + {".deny", l => new DownloadRawText.Command(l)}, + {".host", l => new DownloadRawText.Command(l)}, + {".hosts", l => new DownloadRawText.Command(l)}, + {".ips", l => new DownloadRawText.Command(l)}, + {".ipset", l => new DownloadRawText.Command(l)}, + {".json", l => new DownloadRawText.Command(l)}, + {".list", l => new DownloadRawText.Command(l)}, + {".lsrules", l => new DownloadRawText.Command(l)}, + {".netset", l => new DownloadRawText.Command(l)}, + {".p2p", l => new DownloadRawText.Command(l)}, + {".php", l => new DownloadRawText.Command(l)}, + {".tpl", l => new DownloadRawText.Command(l)}, + {".txt", l => new DownloadRawText.Command(l)}, + //{".zip", l => new DownloadRawText.Command(l)} + }; + private readonly IMediator _mediator; public Handler(IMediator mediator) @@ -33,7 +65,15 @@ public Handler(IMediator mediator) protected override async Task Handle(Command request, CancellationToken cancellationToken) { var downloader = new ActionBlock( - async l => await _mediator.Send(new DownloadList.Command(l), cancellationToken), + async l => + { + var extension = l.ViewUrl.GetExtension(); + if (CommandsByExtension.ContainsKey(extension)) + { + var command = CommandsByExtension[extension].Invoke(l); + await _mediator.Send(command, cancellationToken); + } + }, new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism} ); var orderedListInfo = request.ListInfo.DistributeByHost(); diff --git a/src/FilterLists.Agent/Features/Archiver/DownloadRawText.cs b/src/FilterLists.Agent/Features/Archiver/DownloadRawText.cs new file mode 100644 index 000000000..223d7dc12 --- /dev/null +++ b/src/FilterLists.Agent/Features/Archiver/DownloadRawText.cs @@ -0,0 +1,51 @@ +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using FilterLists.Agent.Core.Entities; +using FilterLists.Agent.Extensions; +using FilterLists.Agent.Infrastructure.Clients; +using MediatR; + +namespace FilterLists.Agent.Features.Archiver +{ + public static class DownloadRawText + { + public class Command : IRequest + { + public Command(ListInfo listInfo) + { + ListInfo = listInfo; + } + + public ListInfo ListInfo { get; } + } + + public class Handler : AsyncRequestHandler + { + private const string RepoDirectory = @"archives"; + private readonly HttpClient _httpClient; + + public Handler(AgentHttpClient httpClient) + { + _httpClient = httpClient.Client; + } + + protected override async Task Handle(Command request, CancellationToken cancellationToken) + { + var sourceExtension = request.ListInfo.ViewUrl.GetExtension(); + var destinationExtension = string.IsNullOrEmpty(sourceExtension) ? ".txt" : sourceExtension; + var destinationPath = Path.Combine(RepoDirectory, $"{request.ListInfo.Id}{destinationExtension}"); + using (var response = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken)) + { + if (response.IsSuccessStatusCode) + using (var input = await response.Content.ReadAsStreamAsync()) + using (var output = File.OpenWrite(destinationPath)) + { + await input.CopyToAsync(output, cancellationToken); + } + } + } + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Archiver/FileType.cs b/src/FilterLists.Agent/Features/Archiver/FileType.cs deleted file mode 100644 index f9f3d564e..000000000 --- a/src/FilterLists.Agent/Features/Archiver/FileType.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace FilterLists.Agent.Features.Archiver -{ - public enum FileType - { - RawText, - ForwardOnlyCompressed, - NonForwardOnlyCompressed - } -} \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Archiver/StreamExtensions.cs b/src/FilterLists.Agent/Features/Archiver/StreamExtensions.cs index d05e1cd57..cbebe8a17 100644 --- a/src/FilterLists.Agent/Features/Archiver/StreamExtensions.cs +++ b/src/FilterLists.Agent/Features/Archiver/StreamExtensions.cs @@ -1,46 +1,46 @@ -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using SharpCompress.Archives; -using SharpCompress.Archives.SevenZip; -using SharpCompress.Readers; +//using System.IO; +//using System.Threading; +//using System.Threading.Tasks; +//using SharpCompress.Archives; +//using SharpCompress.Archives.SevenZip; +//using SharpCompress.Readers; -namespace FilterLists.Agent.Features.Archiver -{ - public static class StreamExtensions - { - /// - /// Downloads compressed streams that support forward-only reading. - /// e.g. Zip, GZip, BZip2, Tar, Rar, LZip, XZ - /// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md - /// - public static async Task CopyToWithCompressedReaderApi(this Stream input, Stream output, - CancellationToken cancellationToken) - { - using (var reader = ReaderFactory.Open(input)) - { - while (reader.MoveToNextEntry()) - if (!reader.Entry.IsDirectory) - using (var entryStream = reader.OpenEntryStream()) - { - await entryStream.CopyToAsync(output, cancellationToken); - } - } - } +//namespace FilterLists.Agent.Features.Archiver +//{ +// public static class StreamExtensions +// { +// /// +// /// Downloads compressed streams that support forward-only reading. +// /// e.g. Zip, GZip, BZip2, Tar, Rar, LZip, XZ +// /// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md +// /// +// public static async Task CopyToWithCompressedReaderApi(this Stream input, Stream output, +// CancellationToken cancellationToken) +// { +// using (var reader = ReaderFactory.Open(input)) +// { +// while (reader.MoveToNextEntry()) +// if (!reader.Entry.IsDirectory) +// using (var entryStream = reader.OpenEntryStream()) +// { +// await entryStream.CopyToAsync(output, cancellationToken); +// } +// } +// } - /// - /// Downloads compressed streams that only support the Archive API. - /// e.g. 7zip - /// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md - /// - public static void CopyToWithCompressedArchiveApi(this Stream input, Stream output) - { - using (var archive = SevenZipArchive.Open(input)) - { - foreach (var entry in archive.Entries) - if (!entry.IsDirectory) - entry.WriteTo(output); - } - } - } -} \ No newline at end of file +// /// +// /// Downloads compressed streams that only support the Archive API. +// /// e.g. 7zip +// /// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md +// /// +// public static void CopyToWithCompressedArchiveApi(this Stream input, Stream output) +// { +// using (var archive = SevenZipArchive.Open(input)) +// { +// foreach (var entry in archive.Entries) +// if (!entry.IsDirectory) +// entry.WriteTo(output); +// } +// } +// } +//} \ No newline at end of file