prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip - download differently by file extension

Browse source
This commit is contained in:
Collin M. Barrett 2019-06-27 12:25:01 -05:00
parent 0b0eb9237c
commit 63e2d78b1d
5 changed files with 151 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/FilterLists.Agent/ListArchiver/CaptureAllLists.cs
View file

@ -1,7 +1,6 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using FilterLists.Agent.Entities;
using FilterLists.Agent.Infrastructure;
using MediatR;
@ -17,7 +16,6 @@ public class Command : IRequest
public class Handler : AsyncRequestHandler<Command>
{
private const int MaxDegreeOfParallelism = 50;
private readonly IFilterListsApiClient _apiClient;
private readonly IMediator _mediator;
@ -30,7 +28,9 @@ public Handler(IFilterListsApiClient apiClient, IMediator mediator)
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
var lists = await GetListInfo();
await DownloadLists(lists, cancellationToken);
await _mediator.Send(new DownloadLists.Command(lists), cancellationToken);
//TODO: git add .
//TODO: git commit
}
private async Task<IEnumerable<ListInfo>> GetListInfo()
@ -38,17 +38,6 @@ private async Task<IEnumerable<ListInfo>> GetListInfo()
var listsRequest = new RestRequest("lists");
return await _apiClient.ExecuteAsync<IEnumerable<ListInfo>>(listsRequest);
}
private async Task DownloadLists(IEnumerable<ListInfo> lists, CancellationToken cancellationToken)
{
var downloader = new TransformBlock<ListInfo, Task>(
l => _mediator.Send(new DownloadList.Command(l), cancellationToken),
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism});
foreach (var list in lists)
await downloader.SendAsync(list, cancellationToken);
downloader.Complete();
await downloader.Completion;
}
}
}
}

99
src/FilterLists.Agent/ListArchiver/DownloadList.cs
View file

@ -1,40 +1,12 @@
using System.Diagnostics;
using System;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Entities;
using FilterLists.Agent.ListArchiver.DownloadRequestsByFileExtension;
using MediatR;
/*
TODO: support downloading all of the following extensions
"" (no extension)
7z
acl
action
all
aspx
bat
blacklist
conf
csv
dat
deny
host
hosts
ips
ipset
json
list
lsrules
netset
p2p
php
tpl
txt
zip
*/
namespace FilterLists.Agent.ListArchiver
{
public static class DownloadList
@ -51,35 +23,56 @@ public Command(ListInfo listInfo)
public class Handler : AsyncRequestHandler<Command>
{
private readonly HttpClient _httpClient;
private static readonly Dictionary<string, Func<ListInfo, IRequest>> DownloadRequestsByFileExtension
= new Dictionary<string, Func<ListInfo, IRequest>>
{
{"", l => new DownloadTxt.Command(l)},
{".7z", l => new DownloadTxt.Command(l)},
{".acl", l => new DownloadTxt.Command(l)},
{".action", l => new DownloadTxt.Command(l)},
{".all", l => new DownloadTxt.Command(l)},
{".aspx", l => new DownloadTxt.Command(l)},
{".bat", l => new DownloadTxt.Command(l)},
{".blacklist", l => new DownloadTxt.Command(l)},
{".conf", l => new DownloadTxt.Command(l)},
{".csv", l => new DownloadTxt.Command(l)},
{".dat", l => new DownloadTxt.Command(l)},
{".deny", l => new DownloadTxt.Command(l)},
{".host", l => new DownloadTxt.Command(l)},
{".hosts", l => new DownloadTxt.Command(l)},
{".ips", l => new DownloadTxt.Command(l)},
{".ipset", l => new DownloadTxt.Command(l)},
{".json", l => new DownloadTxt.Command(l)},
{".list", l => new DownloadTxt.Command(l)},
{".lsrules", l => new DownloadTxt.Command(l)},
{".netset", l => new DownloadTxt.Command(l)},
{".p2p", l => new DownloadTxt.Command(l)},
{".php", l => new DownloadTxt.Command(l)},
{".tpl", l => new DownloadTxt.Command(l)},
{".txt", l => new DownloadTxt.Command(l)},
{".zip", l => new DownloadTxt.Command(l)}
};
public Handler(HttpClient httpClient)
private readonly IMediator _mediator;
public Handler(IMediator mediator)
{
_httpClient = httpClient;
_mediator = mediator;
}
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
if (!Path.HasExtension(request.ListInfo.ViewUrl.AbsolutePath) ||
Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath) == ".txt")
try
{
Debug.WriteLine($"Downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}...");
try
{
using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken))
{
if (result.IsSuccessStatusCode)
using (Stream output =
File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt")))
using (var input = await result.Content.ReadAsStreamAsync())
{
input.CopyTo(output);
}
}
}
catch (HttpRequestException)
{
}
var extension = Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath);
if (DownloadRequestsByFileExtension.ContainsKey(extension))
await _mediator.Send(DownloadRequestsByFileExtension[extension].Invoke(request.ListInfo),
cancellationToken);
//TODO: upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update
}
catch (ArgumentException)
{
//TODO: log
}
}
}

44
src/FilterLists.Agent/ListArchiver/DownloadLists.cs Normal file
View file

@ -0,0 +1,44 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using FilterLists.Agent.Entities;
using MediatR;
namespace FilterLists.Agent.ListArchiver
{
public static class DownloadLists
{
public class Command : IRequest
{
public Command(IEnumerable<ListInfo> lists)
{
Lists = lists;
}
public IEnumerable<ListInfo> Lists { get; }
}
public class Handler : AsyncRequestHandler<Command>
{
private const int MaxDegreeOfParallelism = 50;
private readonly IMediator _mediator;
public Handler(IMediator mediator)
{
_mediator = mediator;
}
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
var downloader = new TransformBlock<ListInfo, Task>(
l => _mediator.Send(new DownloadList.Command(l), cancellationToken),
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism});
foreach (var list in request.Lists)
await downloader.SendAsync(list, cancellationToken);
downloader.Complete();
await downloader.Completion;
}
}
}
}

58
src/FilterLists.Agent/ListArchiver/DownloadRequestsByFileExtension/DownloadTxt.cs Normal file
View file

@ -0,0 +1,58 @@
using System.Diagnostics;
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Entities;
using MediatR;
namespace FilterLists.Agent.ListArchiver.DownloadRequestsByFileExtension
{
public static class DownloadTxt
{
public class Command : IRequest
{
public Command(ListInfo listInfo)
{
ListInfo = listInfo;
}
public ListInfo ListInfo { get; }
}
public class Handler : AsyncRequestHandler<Command>
{
private readonly HttpClient _httpClient;
public Handler(HttpClient httpClient)
{
_httpClient = httpClient;
}
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
if (!Path.HasExtension(request.ListInfo.ViewUrl.AbsolutePath) ||
Path.GetExtension(request.ListInfo.ViewUrl.AbsolutePath) == ".txt")
{
Debug.WriteLine($"Downloading list {request.ListInfo.Id} from {request.ListInfo.ViewUrl}...");
try
{
using (var result = await _httpClient.GetAsync(request.ListInfo.ViewUrl, cancellationToken))
{
if (result.IsSuccessStatusCode)
using (Stream output =
File.OpenWrite(Path.Combine("archives", $"{request.ListInfo.Id}.txt")))
using (var input = await result.Content.ReadAsStreamAsync())
{
input.CopyTo(output);
}
}
}
catch (HttpRequestException)
{
}
}
}
}
}
}

4
src/FilterLists.Agent/Program.cs
View file

@ -8,10 +8,6 @@
using MediatR;
using Microsoft.Extensions.DependencyInjection;
//TODO: git add .
//TODO: git commit
//TODO: foreach list, upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update
namespace FilterLists.Agent
{
public static class Program