prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

re-add exception handling/logging

Browse source
This commit is contained in:
Collin M. Barrett 2019-06-30 19:59:11 -05:00
parent 8167b4f3a6
commit fe826a8d99
1 changed files with 35 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
src/FilterLists.Agent/Features/Archiver/DownloadLists.cs
View file

@ -1,11 +1,13 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using FilterLists.Agent.Core.Entities;
using FilterLists.Agent.Extensions;
using MediatR;
using Microsoft.Extensions.Logging;
namespace FilterLists.Agent.Features.Archiver
{
@ -51,37 +53,56 @@ private static readonly Dictionary<string, Func<ListInfo, IRequest>> CommandsByE
{".p2p", l => new DownloadRawText.Command(l)},
{".php", l => new DownloadRawText.Command(l)},
{".tpl", l => new DownloadRawText.Command(l)},
{".txt", l => new DownloadRawText.Command(l)},
{".txt", l => new DownloadRawText.Command(l)}
//{".zip", l => new DownloadRawText.Command(l)}
};
private readonly ILogger _logger;
private readonly IMediator _mediator;
public Handler(IMediator mediator)
public Handler(ILogger logger, IMediator mediator)
{
_logger = logger;
_mediator = mediator;
}
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
var downloader = new ActionBlock<ListInfo>(
async l =>
{
var extension = l.ViewUrl.GetExtension();
if (CommandsByExtension.ContainsKey(extension))
{
var command = CommandsByExtension[extension].Invoke(l);
await _mediator.Send(command, cancellationToken);
}
},
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
);
var downloader = BuildDownloader(cancellationToken);
var orderedListInfo = request.ListInfo.DistributeByHost();
foreach (var listInfo in orderedListInfo)
await downloader.SendAsync(listInfo, cancellationToken);
downloader.Complete();
await downloader.Completion;
}
private ActionBlock<ListInfo> BuildDownloader(CancellationToken cancellationToken)
{
return new ActionBlock<ListInfo>(
async l =>
{
var errorMessage = $"Error downloading list {l.Id} from {l.ViewUrl}.";
try
{
var extension = l.ViewUrl.GetExtension();
if (CommandsByExtension.ContainsKey(extension))
{
var command = CommandsByExtension[extension].Invoke(l);
await _mediator.Send(command, cancellationToken);
}
}
catch (HttpRequestException ex)
{
_logger.LogError(ex, errorMessage);
}
catch (TaskCanceledException ex)
{
_logger.LogError(ex, errorMessage);
}
},
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
);
}
}
}
}