mirror of
https://github.com/collinbarrett/FilterLists.git
synced 2026-03-11 09:04:27 +00:00
wip supporting all file types
This commit is contained in:
Collin M. Barrett
parent
9b6a3b9b57
commit
5f15f72dc7
6 changed files with 126 additions and 121 deletions
|
|
@ -29,7 +29,6 @@
|
|||
<ItemGroup>
|
||||
<PackageReference Include="Autofac" Version="4.9.2" />
|
||||
<PackageReference Include="Autofac.Extensions.DependencyInjection" Version="4.4.0" />
|
||||
<PackageReference Include="JetBrains.Annotations" Version="2019.1.1" />
|
||||
<PackageReference Include="LibGit2Sharp" Version="0.26.0" />
|
||||
<PackageReference Include="MediatR" Version="7.0.0" />
|
||||
<PackageReference Include="MediatR.Extensions.Microsoft.DependencyInjection" Version="7.0.0" />
|
||||
|
|
@ -37,8 +36,6 @@
|
|||
<PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="2.9.3" />
|
||||
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.2.4" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" Version="2.2.0" />
|
||||
|
|
@ -46,12 +43,7 @@
|
|||
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.7.11" />
|
||||
<PackageReference Include="RestSharp" Version="106.6.9" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Update="appsettings.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<PackageReference Include="SharpCompress" Version="0.23.0" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FilterLists.Agent.Entities;
|
||||
using FilterLists.Agent.ListArchiver.DownloadRequestsByFileExtension;
|
||||
using FilterLists.Agent.Infrastructure;
|
||||
using MediatR;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
//TODO: upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update
|
||||
|
||||
|
|
@ -26,43 +26,41 @@ public Command(ListInfo listInfo)
|
|||
|
||||
public class Handler : AsyncRequestHandler<Command>
|
||||
{
|
||||
private static readonly Dictionary<string, Func<ListInfo, IRequest>> DownloadRequestsByFileExtension
|
||||
= new Dictionary<string, Func<ListInfo, IRequest>>
|
||||
private static readonly Dictionary<string, FileType> DownloadRequestsByFileExtension
|
||||
= new Dictionary<string, FileType>
|
||||
{
|
||||
{"", l => new DownloadTxt.Command(l)},
|
||||
//{".7z", l => throw new NotImplementedException()},
|
||||
//{".acl", l => throw new NotImplementedException()},
|
||||
//{".action", l => throw new NotImplementedException()},
|
||||
//{".all", l => throw new NotImplementedException()},
|
||||
//{".aspx", l => throw new NotImplementedException()},
|
||||
//{".bat", l => throw new NotImplementedException()},
|
||||
//{".blacklist", l => throw new NotImplementedException()},
|
||||
//{".conf", l => throw new NotImplementedException()},
|
||||
//{".csv", l => throw new NotImplementedException()},
|
||||
//{".dat", l => throw new NotImplementedException()},
|
||||
//{".deny", l => throw new NotImplementedException()},
|
||||
//{".host", l => throw new NotImplementedException()},
|
||||
//{".hosts", l => throw new NotImplementedException()},
|
||||
//{".ips", l => throw new NotImplementedException()},
|
||||
//{".ipset", l => throw new NotImplementedException()},
|
||||
//{".json", l => throw new NotImplementedException()},
|
||||
//{".list", l => throw new NotImplementedException()},
|
||||
//{".lsrules", l => throw new NotImplementedException()},
|
||||
//{".netset", l => throw new NotImplementedException()},
|
||||
//{".p2p", l => throw new NotImplementedException()},
|
||||
//{".php", l => throw new NotImplementedException()},
|
||||
//{".tpl", l => throw new NotImplementedException()},
|
||||
{".txt", l => new DownloadTxt.Command(l)}
|
||||
//{".zip", l => throw new NotImplementedException()}
|
||||
{"", FileType.RawText},
|
||||
{".7z", FileType.NonForwardOnlyCompressed},
|
||||
{".acl", FileType.RawText},
|
||||
{".action", FileType.RawText},
|
||||
{".all", FileType.RawText},
|
||||
{".aspx", FileType.RawText},
|
||||
{".bat", FileType.RawText},
|
||||
{".blacklist", FileType.RawText},
|
||||
{".conf", FileType.RawText},
|
||||
{".csv", FileType.RawText},
|
||||
{".dat", FileType.RawText},
|
||||
{".deny", FileType.RawText},
|
||||
{".host", FileType.RawText},
|
||||
{".hosts", FileType.RawText},
|
||||
{".ips", FileType.RawText},
|
||||
{".ipset", FileType.RawText},
|
||||
{".json", FileType.RawText},
|
||||
{".list", FileType.RawText},
|
||||
{".lsrules", FileType.RawText},
|
||||
{".netset", FileType.RawText},
|
||||
{".p2p", FileType.RawText},
|
||||
{".php", FileType.RawText},
|
||||
{".tpl", FileType.RawText},
|
||||
{".txt", FileType.RawText},
|
||||
{".zip", FileType.ForwardOnlyCompressed}
|
||||
};
|
||||
|
||||
private readonly ILogger<Handler> _logger;
|
||||
private readonly IMediator _mediator;
|
||||
private readonly HttpClient _httpClient;
|
||||
|
||||
public Handler(ILogger<Handler> logger, IMediator mediator)
|
||||
public Handler(AgentHttpClient httpClient)
|
||||
{
|
||||
_logger = logger;
|
||||
_mediator = mediator;
|
||||
_httpClient = httpClient.Client;
|
||||
}
|
||||
|
||||
protected override async Task Handle(Command request, CancellationToken cancellationToken)
|
||||
|
|
@ -72,18 +70,38 @@ protected override async Task Handle(Command request, CancellationToken cancella
|
|||
|
||||
private async Task DownloadByFileExtension(Command request, CancellationToken cancellationToken)
|
||||
{
|
||||
var extension = Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath);
|
||||
if (DownloadRequestsByFileExtension.ContainsKey(extension))
|
||||
using (var response = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken))
|
||||
{
|
||||
_logger.LogInformation(
|
||||
$"Downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}...");
|
||||
await _mediator.Send(DownloadRequestsByFileExtension[extension].Invoke(request.ListInfo),
|
||||
cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning(
|
||||
$"File extension not supported for list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}.");
|
||||
if (response.IsSuccessStatusCode)
|
||||
using (var input = await response.Content.ReadAsStreamAsync())
|
||||
{
|
||||
var sourceExtension = Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath);
|
||||
string destinationExtension;
|
||||
if (string.IsNullOrEmpty(sourceExtension) || sourceExtension == ".zip" ||
|
||||
sourceExtension == ".7z")
|
||||
destinationExtension = ".txt";
|
||||
else
|
||||
destinationExtension = sourceExtension;
|
||||
|
||||
using (var output = File.OpenWrite(Path.Combine("archives",
|
||||
$"{request.ListInfo.Id}{destinationExtension}")))
|
||||
{
|
||||
switch (DownloadRequestsByFileExtension[sourceExtension])
|
||||
{
|
||||
case FileType.RawText:
|
||||
await input.CopyToAsync(output, cancellationToken);
|
||||
break;
|
||||
case FileType.ForwardOnlyCompressed:
|
||||
await input.CopyToWithCompressedReaderApi(output, cancellationToken);
|
||||
break;
|
||||
case FileType.NonForwardOnlyCompressed:
|
||||
await input.CopyToWithCompressedArchiveApi(output, cancellationToken);
|
||||
break;
|
||||
default:
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,63 +0,0 @@
|
|||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FilterLists.Agent.Entities;
|
||||
using FilterLists.Agent.Infrastructure;
|
||||
using MediatR;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace FilterLists.Agent.ListArchiver.DownloadRequestsByFileExtension
|
||||
{
|
||||
public static class DownloadTxt
|
||||
{
|
||||
public class Command : IRequest
|
||||
{
|
||||
public Command(ListInfo listInfo)
|
||||
{
|
||||
ListInfo = listInfo;
|
||||
}
|
||||
|
||||
public ListInfo ListInfo { get; }
|
||||
}
|
||||
|
||||
public class Handler : AsyncRequestHandler<Command>
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly ILogger<Handler> _logger;
|
||||
|
||||
public Handler(AgentHttpClient httpClient, ILogger<Handler> logger)
|
||||
{
|
||||
_httpClient = httpClient.Client;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task Handle(Command request, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
using (var response = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken))
|
||||
{
|
||||
if (response.IsSuccessStatusCode)
|
||||
using (Stream output =
|
||||
File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt")))
|
||||
using (var input = await response.Content.ReadAsStreamAsync())
|
||||
{
|
||||
input.CopyTo(output);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
$"Error downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}.");
|
||||
}
|
||||
catch (TaskCanceledException ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
$"Error downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
9
src/FilterLists.Agent/ListArchiver/FileType.cs
Normal file
9
src/FilterLists.Agent/ListArchiver/FileType.cs
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
namespace FilterLists.Agent.ListArchiver
|
||||
{
|
||||
public enum FileType
|
||||
{
|
||||
RawText,
|
||||
ForwardOnlyCompressed,
|
||||
NonForwardOnlyCompressed
|
||||
}
|
||||
}
|
||||
52
src/FilterLists.Agent/ListArchiver/StreamExtensions.cs
Normal file
52
src/FilterLists.Agent/ListArchiver/StreamExtensions.cs
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Archives;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Readers;
|
||||
|
||||
namespace FilterLists.Agent.ListArchiver
|
||||
{
|
||||
public static class StreamExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Downloads compressed streams that support forward-only reading.
|
||||
/// e.g. Zip, GZip, BZip2, Tar, Rar, LZip, XZ
|
||||
/// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md
|
||||
/// </summary>
|
||||
public static async Task CopyToWithCompressedReaderApi(this Stream input, Stream output,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using (var reader = ReaderFactory.Open(input))
|
||||
{
|
||||
while (reader.MoveToNextEntry())
|
||||
if (!reader.Entry.IsDirectory)
|
||||
using (var entryStream = reader.OpenEntryStream())
|
||||
{
|
||||
await entryStream.CopyToAsync(output, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads compressed streams that only support the Archive API.
|
||||
/// e.g. 7zip
|
||||
/// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md
|
||||
/// </summary>
|
||||
public static async Task CopyToWithCompressedArchiveApi(this Stream input, Stream output,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
//var archive = ArchiveFactory.Open(@"C:\Code\sharpcompress\TestArchives\sharpcompress.zip");
|
||||
//foreach (var entry in archive.Entries)
|
||||
// if (!entry.IsDirectory)
|
||||
// {
|
||||
// Console.WriteLine(entry.Key);
|
||||
// entry.WriteToDirectory(@"C:\temp",
|
||||
// new ExtractionOptions {ExtractFullPath = true, Overwrite = true});
|
||||
// }
|
||||
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
using System;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using Autofac;
|
||||
using Autofac.Extensions.DependencyInjection;
|
||||
|
|
@ -35,7 +34,7 @@ private static void RegisterServices()
|
|||
serviceCollection.AddLogging(b =>
|
||||
{
|
||||
b.AddConsole();
|
||||
b.AddApplicationInsights(configuration["ApplicationInsights:InstrumentationKey"]);
|
||||
b.AddApplicationInsights(configuration["ApplicationInsights:InstrumentationKey"] ?? "");
|
||||
});
|
||||
serviceCollection.AddMediatR(typeof(Program).Assembly);
|
||||
serviceCollection.AddHttpClient<AgentHttpClient>();
|
||||
|
|
@ -50,9 +49,7 @@ private static void RegisterServices()
|
|||
private static IConfigurationRoot GetConfiguration()
|
||||
{
|
||||
return new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory())
|
||||
.AddEnvironmentVariables()
|
||||
.AddJsonFile("appsettings.json", true, false)
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue