prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

misc cleanup

Browse source
This commit is contained in:
Collin M. Barrett 2019-06-27 12:40:18 -05:00
parent bd9ffc37cd
commit f44d6dd355
4 changed files with 44 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs → src/FilterLists.Agent/ListArchiver/CaptureLists.cs
View file

@ -8,7 +8,7 @@
namespace FilterLists.Agent.ListArchiver
{
public static class CaptureAllLists
public static class CaptureLists
{
public class Command : IRequest
{

51
src/FilterLists.Agent/ListArchiver/DownloadList.cs
View file

@ -27,30 +27,30 @@ private static readonly Dictionary<string, Func<ListInfo, IRequest>> DownloadReq
= new Dictionary<string, Func<ListInfo, IRequest>>
{
{"", l => new DownloadTxt.Command(l)},
{".7z", l => new DownloadTxt.Command(l)},
{".acl", l => new DownloadTxt.Command(l)},
{".action", l => new DownloadTxt.Command(l)},
{".all", l => new DownloadTxt.Command(l)},
{".aspx", l => new DownloadTxt.Command(l)},
{".bat", l => new DownloadTxt.Command(l)},
{".blacklist", l => new DownloadTxt.Command(l)},
{".conf", l => new DownloadTxt.Command(l)},
{".csv", l => new DownloadTxt.Command(l)},
{".dat", l => new DownloadTxt.Command(l)},
{".deny", l => new DownloadTxt.Command(l)},
{".host", l => new DownloadTxt.Command(l)},
{".hosts", l => new DownloadTxt.Command(l)},
{".ips", l => new DownloadTxt.Command(l)},
{".ipset", l => new DownloadTxt.Command(l)},
{".json", l => new DownloadTxt.Command(l)},
{".list", l => new DownloadTxt.Command(l)},
{".lsrules", l => new DownloadTxt.Command(l)},
{".netset", l => new DownloadTxt.Command(l)},
{".p2p", l => new DownloadTxt.Command(l)},
{".php", l => new DownloadTxt.Command(l)},
{".tpl", l => new DownloadTxt.Command(l)},
{".7z", l => throw new NotImplementedException()},
{".acl", l => throw new NotImplementedException()},
{".action", l => throw new NotImplementedException()},
{".all", l => throw new NotImplementedException()},
{".aspx", l => throw new NotImplementedException()},
{".bat", l => throw new NotImplementedException()},
{".blacklist", l => throw new NotImplementedException()},
{".conf", l => throw new NotImplementedException()},
{".csv", l => throw new NotImplementedException()},
{".dat", l => throw new NotImplementedException()},
{".deny", l => throw new NotImplementedException()},
{".host", l => throw new NotImplementedException()},
{".hosts", l => throw new NotImplementedException()},
{".ips", l => throw new NotImplementedException()},
{".ipset", l => throw new NotImplementedException()},
{".json", l => throw new NotImplementedException()},
{".list", l => throw new NotImplementedException()},
{".lsrules", l => throw new NotImplementedException()},
{".netset", l => throw new NotImplementedException()},
{".p2p", l => throw new NotImplementedException()},
{".php", l => throw new NotImplementedException()},
{".tpl", l => throw new NotImplementedException()},
{".txt", l => new DownloadTxt.Command(l)},
{".zip", l => new DownloadTxt.Command(l)}
{".zip", l => throw new NotImplementedException()}
};
private readonly IMediator _mediator;
@ -68,8 +68,13 @@ protected override async Task Handle(Command request, CancellationToken cancella
if (DownloadRequestsByFileExtension.ContainsKey(extension))
await _mediator.Send(DownloadRequestsByFileExtension[extension].Invoke(request.ListInfo),
cancellationToken);
//TODO: handle and/or log unrecognized extension
//TODO: upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update
}
catch (NotImplementedException)
{
//TODO: log
}
catch (ArgumentException)
{
//TODO: log

31
src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs
View file

@ -31,27 +31,24 @@ public Handler(HttpClient httpClient)
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
if (!Path.HasExtension(request.ListInfo.ViewUrl.AbsolutePath) ||
Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath) == ".txt")
Debug.WriteLine($"Downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}...");
try
{
Debug.WriteLine($"Downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}...");
try
{
using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken))
{
if (result.IsSuccessStatusCode)
using (Stream output =
File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt")))
using (var input = await result.Content.ReadAsStreamAsync())
{
input.CopyTo(output);
}
}
}
catch (HttpRequestException)
using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken))
{
if (result.IsSuccessStatusCode)
using (Stream output =
File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt")))
using (var input = await result.Content.ReadAsStreamAsync())
{
input.CopyTo(output);
}
}
}
catch (HttpRequestException)
{
//TODO: log
}
}
}
}

2
src/FilterLists.Agent/Program.cs
View file

@ -19,7 +19,7 @@ public static async Task Main()
RegisterServices();
var mediator = _serviceProvider.GetService<IMediator>();
await mediator.Send(new CaptureAllLists.Command());
await mediator.Send(new CaptureLists.Command());
((IDisposable) _serviceProvider).Dispose();
}