prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip refactor DownloadList flow

Browse source
This commit is contained in:
Collin M. Barrett 2019-06-30 19:43:20 -05:00
parent a05fcf0e41
commit 8167b4f3a6
6 changed files with 150 additions and 155 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
src/FilterLists.Agent/Extensions/UriExtensions.cs Normal file
View file

@ -0,0 +1,13 @@
using System;
using System.IO;
namespace FilterLists.Agent.Extensions
{
public static class UriExtensions
{
public static string GetExtension(this Uri uri)
{
return Path.GetExtension(uri.AbsolutePath);
}
}
}

100
src/FilterLists.Agent/Features/Archiver/DownloadList.cs
View file

@ -1,100 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Core.Entities;
using FilterLists.Agent.Infrastructure.Clients;
using MediatR;
namespace FilterLists.Agent.Features.Archiver
{
public static class DownloadList
{
public class Command : IRequest
{
public Command(ListInfo listInfo)
{
ListInfo = listInfo;
}
public ListInfo ListInfo { get; }
}
public class Handler : AsyncRequestHandler<Command>
{
private static readonly Dictionary<string, FileType> DownloadRequestsByFileExtension
= new Dictionary<string, FileType>
{
{"", FileType.RawText},
{".7z", FileType.NonForwardOnlyCompressed},
{".acl", FileType.RawText},
{".action", FileType.RawText},
{".all", FileType.RawText},
{".aspx", FileType.RawText},
{".bat", FileType.RawText},
{".blacklist", FileType.RawText},
{".conf", FileType.RawText},
{".csv", FileType.RawText},
{".dat", FileType.RawText},
{".deny", FileType.RawText},
{".host", FileType.RawText},
{".hosts", FileType.RawText},
{".ips", FileType.RawText},
{".ipset", FileType.RawText},
{".json", FileType.RawText},
{".list", FileType.RawText},
{".lsrules", FileType.RawText},
{".netset", FileType.RawText},
{".p2p", FileType.RawText},
{".php", FileType.RawText},
{".tpl", FileType.RawText},
{".txt", FileType.RawText},
{".zip", FileType.ForwardOnlyCompressed}
};
private readonly HttpClient _httpClient;
public Handler(AgentHttpClient httpClient)
{
_httpClient = httpClient.Client;
}
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
using (var response = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken))
{
if (response.IsSuccessStatusCode)
using (var input = await response.Content.ReadAsStreamAsync())
{
var sourceExtension = Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath);
string destinationExtension;
if (string.IsNullOrEmpty(sourceExtension) || sourceExtension == ".zip" || sourceExtension == ".7z")
destinationExtension = ".txt";
else
destinationExtension = sourceExtension;
using (var output = File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}{destinationExtension}")))
{
switch (DownloadRequestsByFileExtension[sourceExtension])
{
case FileType.RawText:
await input.CopyToAsync(output, cancellationToken);
break;
case FileType.ForwardOnlyCompressed:
await input.CopyToWithCompressedReaderApi(output, cancellationToken);
break;
case FileType.NonForwardOnlyCompressed:
input.CopyToWithCompressedArchiveApi(output);
break;
default:
throw new NotImplementedException();
}
}
}
}
}
}
}
}

44
src/FilterLists.Agent/Features/Archiver/DownloadLists.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
@ -23,6 +24,37 @@ public Command(IEnumerable<ListInfo> listInfo)
public class Handler : AsyncRequestHandler<Command>
{
private const int MaxDegreeOfParallelism = 5;
private static readonly Dictionary<string, Func<ListInfo, IRequest>> CommandsByExtension
= new Dictionary<string, Func<ListInfo, IRequest>>
{
{"", l => new DownloadRawText.Command(l)},
//{".7z", l => new DownloadRawText.Command(l)},
{".acl", l => new DownloadRawText.Command(l)},
{".action", l => new DownloadRawText.Command(l)},
{".all", l => new DownloadRawText.Command(l)},
{".aspx", l => new DownloadRawText.Command(l)},
{".bat", l => new DownloadRawText.Command(l)},
{".blacklist", l => new DownloadRawText.Command(l)},
{".conf", l => new DownloadRawText.Command(l)},
{".csv", l => new DownloadRawText.Command(l)},
{".dat", l => new DownloadRawText.Command(l)},
{".deny", l => new DownloadRawText.Command(l)},
{".host", l => new DownloadRawText.Command(l)},
{".hosts", l => new DownloadRawText.Command(l)},
{".ips", l => new DownloadRawText.Command(l)},
{".ipset", l => new DownloadRawText.Command(l)},
{".json", l => new DownloadRawText.Command(l)},
{".list", l => new DownloadRawText.Command(l)},
{".lsrules", l => new DownloadRawText.Command(l)},
{".netset", l => new DownloadRawText.Command(l)},
{".p2p", l => new DownloadRawText.Command(l)},
{".php", l => new DownloadRawText.Command(l)},
{".tpl", l => new DownloadRawText.Command(l)},
{".txt", l => new DownloadRawText.Command(l)},
//{".zip", l => new DownloadRawText.Command(l)}
};
private readonly IMediator _mediator;
public Handler(IMediator mediator)
@ -33,7 +65,15 @@ public Handler(IMediator mediator)
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
var downloader = new ActionBlock<ListInfo>(
async l => await _mediator.Send(new DownloadList.Command(l), cancellationToken),
async l =>
{
var extension = l.ViewUrl.GetExtension();
if (CommandsByExtension.ContainsKey(extension))
{
var command = CommandsByExtension[extension].Invoke(l);
await _mediator.Send(command, cancellationToken);
}
},
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
);
var orderedListInfo = request.ListInfo.DistributeByHost();

51
src/FilterLists.Agent/Features/Archiver/DownloadRawText.cs Normal file
View file

@ -0,0 +1,51 @@
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Core.Entities;
using FilterLists.Agent.Extensions;
using FilterLists.Agent.Infrastructure.Clients;
using MediatR;
namespace FilterLists.Agent.Features.Archiver
{
public static class DownloadRawText
{
public class Command : IRequest
{
public Command(ListInfo listInfo)
{
ListInfo = listInfo;
}
public ListInfo ListInfo { get; }
}
public class Handler : AsyncRequestHandler<Command>
{
private const string RepoDirectory = @"archives";
private readonly HttpClient _httpClient;
public Handler(AgentHttpClient httpClient)
{
_httpClient = httpClient.Client;
}
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
var sourceExtension = request.ListInfo.ViewUrl.GetExtension();
var destinationExtension = string.IsNullOrEmpty(sourceExtension) ? ".txt" : sourceExtension;
var destinationPath = Path.Combine(RepoDirectory, $"{request.ListInfo.Id}{destinationExtension}");
using (var response = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken))
{
if (response.IsSuccessStatusCode)
using (var input = await response.Content.ReadAsStreamAsync())
using (var output = File.OpenWrite(destinationPath))
{
await input.CopyToAsync(output, cancellationToken);
}
}
}
}
}
}

9
src/FilterLists.Agent/Features/Archiver/FileType.cs
View file

@ -1,9 +0,0 @@
namespace FilterLists.Agent.Features.Archiver
{
public enum FileType
{
RawText,
ForwardOnlyCompressed,
NonForwardOnlyCompressed
}
}

88
src/FilterLists.Agent/Features/Archiver/StreamExtensions.cs
View file

@ -1,46 +1,46 @@
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Archives.SevenZip;
using SharpCompress.Readers;
//using System.IO;
//using System.Threading;
//using System.Threading.Tasks;
//using SharpCompress.Archives;
//using SharpCompress.Archives.SevenZip;
//using SharpCompress.Readers;
namespace FilterLists.Agent.Features.Archiver
{
public static class StreamExtensions
{
/// <summary>
/// Downloads compressed streams that support forward-only reading.
/// e.g. Zip, GZip, BZip2, Tar, Rar, LZip, XZ
/// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md
/// </summary>
public static async Task CopyToWithCompressedReaderApi(this Stream input, Stream output,
CancellationToken cancellationToken)
{
using (var reader = ReaderFactory.Open(input))
{
while (reader.MoveToNextEntry())
if (!reader.Entry.IsDirectory)
using (var entryStream = reader.OpenEntryStream())
{
await entryStream.CopyToAsync(output, cancellationToken);
}
}
}
//namespace FilterLists.Agent.Features.Archiver
//{
// public static class StreamExtensions
// {
// /// <summary>
// /// Downloads compressed streams that support forward-only reading.
// /// e.g. Zip, GZip, BZip2, Tar, Rar, LZip, XZ
// /// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md
// /// </summary>
// public static async Task CopyToWithCompressedReaderApi(this Stream input, Stream output,
// CancellationToken cancellationToken)
// {
// using (var reader = ReaderFactory.Open(input))
// {
// while (reader.MoveToNextEntry())
// if (!reader.Entry.IsDirectory)
// using (var entryStream = reader.OpenEntryStream())
// {
// await entryStream.CopyToAsync(output, cancellationToken);
// }
// }
// }
/// <summary>
/// Downloads compressed streams that only support the Archive API.
/// e.g. 7zip
/// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md
/// </summary>
public static void CopyToWithCompressedArchiveApi(this Stream input, Stream output)
{
using (var archive = SevenZipArchive.Open(input))
{
foreach (var entry in archive.Entries)
if (!entry.IsDirectory)
entry.WriteTo(output);
}
}
}
}
// /// <summary>
// /// Downloads compressed streams that only support the Archive API.
// /// e.g. 7zip
// /// https://github.com/adamhathcock/sharpcompress/blob/master/FORMATS.md
// /// </summary>
// public static void CopyToWithCompressedArchiveApi(this Stream input, Stream output)
// {
// using (var archive = SevenZipArchive.Open(input))
// {
// foreach (var entry in archive.Entries)
// if (!entry.IsDirectory)
// entry.WriteTo(output);
// }
// }
// }
//}