prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

extract extension method

Browse source
This commit is contained in:
Collin M. Barrett 2019-06-30 17:59:47 -05:00
parent a9ea461d47
commit a05fcf0e41
3 changed files with 22 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/FilterLists.Agent/Extensions/ListInfoExtensions.cs Normal file
View file

@ -0,0 +1,18 @@
using System.Collections.Generic;
using System.Linq;
using FilterLists.Agent.Core.Entities;
namespace FilterLists.Agent.Extensions
{
public static class ListInfoExtensions
{
public static IEnumerable<ListInfo> DistributeByHost(this IEnumerable<ListInfo> listInfo)
{
return listInfo.GroupBy(l => l.ViewUrl.Host)
.SelectMany((g, gi) => g.Select((l, i) => new {Index = i, GroupIndex = gi, ListInfo = l}))
.OrderBy(a => a.Index)
.ThenBy(a => a.GroupIndex)
.Select(a => a.ListInfo);
}
}
}

2
src/FilterLists.Agent/Features/Archiver/DownloadList.cs
View file

@ -8,8 +8,6 @@
using FilterLists.Agent.Infrastructure.Clients;
using MediatR;
//TODO: upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update
namespace FilterLists.Agent.Features.Archiver
{
public static class DownloadList

17
src/FilterLists.Agent/Features/Archiver/DownloadLists.cs
View file

@ -1,9 +1,9 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using FilterLists.Agent.Core.Entities;
using FilterLists.Agent.Extensions;
using MediatR;
namespace FilterLists.Agent.Features.Archiver
@ -36,21 +36,12 @@ protected override async Task Handle(Command request, CancellationToken cancella
async l => await _mediator.Send(new DownloadList.Command(l), cancellationToken),
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
);
var orderedListInfo = ShardByHost(request.ListInfo);
foreach (var list in orderedListInfo)
await downloader.SendAsync(list, cancellationToken);
var orderedListInfo = request.ListInfo.DistributeByHost();
foreach (var listInfo in orderedListInfo)
await downloader.SendAsync(listInfo, cancellationToken);
downloader.Complete();
await downloader.Completion;
}
private static IEnumerable<ListInfo> ShardByHost(IEnumerable<ListInfo> listInfo)
{
return listInfo.GroupBy(l => l.ViewUrl.Host)
.SelectMany((g, gi) => g.Select((l, i) => new {Index = i, GroupIndex = gi, ListInfo = l}))
.OrderBy(a => a.Index)
.ThenBy(a => a.GroupIndex)
.Select(a => a.ListInfo);
}
}
}
}