prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

shard lists to download by host

Browse source
This commit is contained in:
Collin M. Barrett 2019-06-28 19:27:24 -05:00
parent fb69ce76e0
commit e193b20442
1 changed files with 14 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/FilterLists.Agent/ListArchiver/DownloadLists.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
@ -21,10 +22,7 @@ public Command(IEnumerable<ListInfo> listInfo)
public class Handler : AsyncRequestHandler<Command>
{
//TODO: sort lists and use domain sharding for concurrent downloads
//TODO: manual/auto-tune degrees of parallelism
private const int MaxDegreeOfParallelism = 5;
private const int MaxDegreeOfParallelism = 20; //TODO: tune
private readonly IMediator _mediator;
public Handler(IMediator mediator)
@ -38,11 +36,22 @@ protected override async Task Handle(Command request, CancellationToken cancella
async l => await _mediator.Send(new DownloadList.Command(l), cancellationToken),
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
);
foreach (var list in request.ListInfo)
var orderedListInfo = ShardByHost(request.ListInfo);
foreach (var list in orderedListInfo)
await downloader.SendAsync(list, cancellationToken);
downloader.Complete();
await downloader.Completion;
}
//https://keestalkstech.com/2017/10/linq-round-robin-ordering-based-segments/
private static IEnumerable<ListInfo> ShardByHost(IEnumerable<ListInfo> listInfo)
{
return listInfo.GroupBy(l => l.ViewUrl.Host)
.SelectMany((g, gi) => g.Select((l, li) => new {Index = li, GroupIndex = gi, Value = l}))
.OrderBy(u => u.Index)
.ThenBy(u => u.GroupIndex)
.Select(u => u.Value);
}
}
}
}