diff --git a/src/FilterLists.Agent/FilterLists.Agent.csproj b/src/FilterLists.Agent/FilterLists.Agent.csproj index efd7d2e64..a0df16f14 100644 --- a/src/FilterLists.Agent/FilterLists.Agent.csproj +++ b/src/FilterLists.Agent/FilterLists.Agent.csproj @@ -29,7 +29,6 @@ - @@ -37,8 +36,6 @@ - - @@ -46,12 +43,7 @@ - - - - - PreserveNewest - + \ No newline at end of file diff --git a/src/FilterLists.Agent/ListArchiver/DownloadList.cs b/src/FilterLists.Agent/ListArchiver/DownloadList.cs index b6244fd20..4cb10fa04 100644 --- a/src/FilterLists.Agent/ListArchiver/DownloadList.cs +++ b/src/FilterLists.Agent/ListArchiver/DownloadList.cs @@ -1,12 +1,12 @@ using System; using System.Collections.Generic; using System.IO; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using FilterLists.Agent.Entities; -using FilterLists.Agent.ListArchiver.DownloadRequestsByFileExtension; +using FilterLists.Agent.Infrastructure; using MediatR; -using Microsoft.Extensions.Logging; //TODO: upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update @@ -26,43 +26,41 @@ public Command(ListInfo listInfo) public class Handler : AsyncRequestHandler { - private static readonly Dictionary> DownloadRequestsByFileExtension - = new Dictionary> + private static readonly Dictionary DownloadRequestsByFileExtension + = new Dictionary { - {"", l => new DownloadTxt.Command(l)}, - //{".7z", l => throw new NotImplementedException()}, - //{".acl", l => throw new NotImplementedException()}, - //{".action", l => throw new NotImplementedException()}, - //{".all", l => throw new NotImplementedException()}, - //{".aspx", l => throw new NotImplementedException()}, - //{".bat", l => throw new NotImplementedException()}, - //{".blacklist", l => throw new NotImplementedException()}, - //{".conf", l => throw new NotImplementedException()}, - //{".csv", l => throw new NotImplementedException()}, - //{".dat", l => throw new NotImplementedException()}, - //{".deny", l => throw new NotImplementedException()}, - //{".host", l => throw new NotImplementedException()}, - //{".hosts", l => throw new NotImplementedException()}, - //{".ips", l => throw new NotImplementedException()}, - //{".ipset", l => throw new NotImplementedException()}, - //{".json", l => throw new NotImplementedException()}, - //{".list", l => throw new NotImplementedException()}, - //{".lsrules", l => throw new NotImplementedException()}, - //{".netset", l => throw new NotImplementedException()}, - //{".p2p", l => throw new NotImplementedException()}, - //{".php", l => throw new NotImplementedException()}, - //{".tpl", l => throw new NotImplementedException()}, - {".txt", l => new DownloadTxt.Command(l)} - //{".zip", l => throw new NotImplementedException()} + {"", FileType.RawText}, + {".7z", FileType.NonForwardOnlyCompressed}, + {".acl", FileType.RawText}, + {".action", FileType.RawText}, + {".all", FileType.RawText}, + {".aspx", FileType.RawText}, + {".bat", FileType.RawText}, + {".blacklist", FileType.RawText}, + {".conf", FileType.RawText}, + {".csv", FileType.RawText}, + {".dat", FileType.RawText}, + {".deny", FileType.RawText}, + {".host", FileType.RawText}, + {".hosts", FileType.RawText}, + {".ips", FileType.RawText}, + {".ipset", FileType.RawText}, + {".json", FileType.RawText}, + {".list", FileType.RawText}, + {".lsrules", FileType.RawText}, + {".netset", FileType.RawText}, + {".p2p", FileType.RawText}, + {".php", FileType.RawText}, + {".tpl", FileType.RawText}, + {".txt", FileType.RawText}, + {".zip", FileType.ForwardOnlyCompressed} }; - private readonly ILogger _logger; - private readonly IMediator _mediator; + private readonly HttpClient _httpClient; - public Handler(ILogger logger, IMediator mediator) + public Handler(AgentHttpClient httpClient) { - _logger = logger; - _mediator = mediator; + _httpClient = httpClient.Client; } protected override async Task Handle(Command request, CancellationToken cancellationToken) @@ -72,18 +70,38 @@ protected override async Task Handle(Command request, CancellationToken cancella private async Task DownloadByFileExtension(Command request, CancellationToken cancellationToken) { - var extension = Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath); - if (DownloadRequestsByFileExtension.ContainsKey(extension)) + using (var response = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken)) { - _logger.LogInformation( - $"Downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}..."); - await _mediator.Send(DownloadRequestsByFileExtension[extension].Invoke(request.ListInfo), - cancellationToken); - } - else - { - _logger.LogWarning( - $"File extension not supported for list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}."); + if (response.IsSuccessStatusCode) + using (var input = await response.Content.ReadAsStreamAsync()) + { + var sourceExtension = Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath); + string destinationExtension; + if (string.IsNullOrEmpty(sourceExtension) || sourceExtension == ".zip" || + sourceExtension == ".7z") + destinationExtension = ".txt"; + else + destinationExtension = sourceExtension; + + using (var output = File.OpenWrite(Path.Combine("archives", + $"{request.ListInfo.Id}{destinationExtension}"))) + { + switch (DownloadRequestsByFileExtension[sourceExtension]) + { + case FileType.RawText: + await input.CopyToAsync(output, cancellationToken); + break; + case FileType.ForwardOnlyCompressed: + await input.CopyToWithCompressedReaderApi(output, cancellationToken); + break; + case FileType.NonForwardOnlyCompressed: + await input.CopyToWithCompressedArchiveApi(output, cancellationToken); + break; + default: + throw new NotImplementedException(); + } + } + } } } } diff --git a/src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs b/src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs deleted file mode 100644 index 1ee6046fd..000000000 --- a/src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System.IO; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using FilterLists.Agent.Entities; -using FilterLists.Agent.Infrastructure; -using MediatR; -using Microsoft.Extensions.Logging; - -namespace FilterLists.Agent.ListArchiver.DownloadRequestsByFileExtension -{ - public static class DownloadTxt - { - public class Command : IRequest - { - public Command(ListInfo listInfo) - { - ListInfo = listInfo; - } - - public ListInfo ListInfo { get; } - } - - public class Handler : AsyncRequestHandler - { - private readonly HttpClient _httpClient; - private readonly ILogger _logger; - - public Handler(AgentHttpClient httpClient, ILogger logger) - { - _httpClient = httpClient.Client; - _logger = logger; - } - - protected override async Task Handle(Command request, CancellationToken cancellationToken) - { - try - { - using (var response = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken)) - { - if (response.IsSuccessStatusCode) - using (Stream output = - File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt"))) - using (var input = await response.Content.ReadAsStreamAsync()) - { - input.CopyTo(output); - } - } - } - catch (HttpRequestException ex) - { - _logger.LogError(ex, - $"Error downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}."); - } - catch (TaskCanceledException ex) - { - _logger.LogError(ex, - $"Error downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}."); - } - } - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Agent/ListArchiver/FileType.cs b/src/FilterLists.Agent/ListArchiver/FileType.cs new file mode 100644 index 000000000..8872596d9 --- /dev/null +++ b/src/FilterLists.Agent/ListArchiver/FileType.cs @@ -0,0 +1,9 @@ +namespace FilterLists.Agent.ListArchiver +{ + public enum FileType + { + RawText, + ForwardOnlyCompressed, + NonForwardOnlyCompressed + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/ListArchiver/StreamExtensions.cs b/src/FilterLists.Agent/ListArchiver/StreamExtensions.cs new file mode 100644 index 000000000..17cd9280c --- /dev/null +++ b/src/FilterLists.Agent/ListArchiver/StreamExtensions.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Common; +using SharpCompress.Readers; + +namespace FilterLists.Agent.ListArchiver +{ + public static class StreamExtensions + { + /// + /// Downloads compressed streams that support forward-only reading. + /// e.g. Zip, GZip, BZip2, Tar, Rar, LZip, XZ + /// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md + /// + public static async Task CopyToWithCompressedReaderApi(this Stream input, Stream output, + CancellationToken cancellationToken) + { + using (var reader = ReaderFactory.Open(input)) + { + while (reader.MoveToNextEntry()) + if (!reader.Entry.IsDirectory) + using (var entryStream = reader.OpenEntryStream()) + { + await entryStream.CopyToAsync(output, cancellationToken); + } + } + } + + /// + /// Downloads compressed streams that only support the Archive API. + /// e.g. 7zip + /// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md + /// + public static async Task CopyToWithCompressedArchiveApi(this Stream input, Stream output, + CancellationToken cancellationToken) + { + //var archive = ArchiveFactory.Open(@"C:\Code\sharpcompress\TestArchives\sharpcompress.zip"); + //foreach (var entry in archive.Entries) + // if (!entry.IsDirectory) + // { + // Console.WriteLine(entry.Key); + // entry.WriteToDirectory(@"C:\temp", + // new ExtractionOptions {ExtractFullPath = true, Overwrite = true}); + // } + + throw new NotImplementedException(); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/Program.cs b/src/FilterLists.Agent/Program.cs index 5eaa345f4..85e6f95ec 100644 --- a/src/FilterLists.Agent/Program.cs +++ b/src/FilterLists.Agent/Program.cs @@ -1,5 +1,4 @@ using System; -using System.IO; using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; @@ -35,7 +34,7 @@ private static void RegisterServices() serviceCollection.AddLogging(b => { b.AddConsole(); - b.AddApplicationInsights(configuration["ApplicationInsights:InstrumentationKey"]); + b.AddApplicationInsights(configuration["ApplicationInsights:InstrumentationKey"] ?? ""); }); serviceCollection.AddMediatR(typeof(Program).Assembly); serviceCollection.AddHttpClient(); @@ -50,9 +49,7 @@ private static void RegisterServices() private static IConfigurationRoot GetConfiguration() { return new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) .AddEnvironmentVariables() - .AddJsonFile("appsettings.json", true, false) .Build(); } }