diff --git a/src/FilterLists.Agent/Extensions/ListInfoExtensions.cs b/src/FilterLists.Agent/Extensions/ListInfoExtensions.cs new file mode 100644 index 000000000..9b17511b5 --- /dev/null +++ b/src/FilterLists.Agent/Extensions/ListInfoExtensions.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using System.Linq; +using FilterLists.Agent.Core.Entities; + +namespace FilterLists.Agent.Extensions +{ + public static class ListInfoExtensions + { + public static IEnumerable DistributeByHost(this IEnumerable listInfo) + { + return listInfo.GroupBy(l => l.ViewUrl.Host) + .SelectMany((g, gi) => g.Select((l, i) => new {Index = i, GroupIndex = gi, ListInfo = l})) + .OrderBy(a => a.Index) + .ThenBy(a => a.GroupIndex) + .Select(a => a.ListInfo); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Archiver/DownloadList.cs b/src/FilterLists.Agent/Features/Archiver/DownloadList.cs index 43681c7a5..29a8cc053 100644 --- a/src/FilterLists.Agent/Features/Archiver/DownloadList.cs +++ b/src/FilterLists.Agent/Features/Archiver/DownloadList.cs @@ -8,8 +8,6 @@ using FilterLists.Agent.Infrastructure.Clients; using MediatR; -//TODO: upsert into MariaDB Rules table https://stackoverflow.com/questions/15271202/mysql-load-data-infile-with-on-duplicate-key-update - namespace FilterLists.Agent.Features.Archiver { public static class DownloadList diff --git a/src/FilterLists.Agent/Features/Archiver/DownloadLists.cs b/src/FilterLists.Agent/Features/Archiver/DownloadLists.cs index b06357d2b..4d5ba7eac 100644 --- a/src/FilterLists.Agent/Features/Archiver/DownloadLists.cs +++ b/src/FilterLists.Agent/Features/Archiver/DownloadLists.cs @@ -1,9 +1,9 @@ using System.Collections.Generic; -using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using FilterLists.Agent.Core.Entities; +using FilterLists.Agent.Extensions; using MediatR; namespace FilterLists.Agent.Features.Archiver @@ -36,21 +36,12 @@ protected override async Task Handle(Command request, CancellationToken cancella async l => await _mediator.Send(new DownloadList.Command(l), cancellationToken), new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism} ); - var orderedListInfo = ShardByHost(request.ListInfo); - foreach (var list in orderedListInfo) - await downloader.SendAsync(list, cancellationToken); + var orderedListInfo = request.ListInfo.DistributeByHost(); + foreach (var listInfo in orderedListInfo) + await downloader.SendAsync(listInfo, cancellationToken); downloader.Complete(); await downloader.Completion; } - - private static IEnumerable ShardByHost(IEnumerable listInfo) - { - return listInfo.GroupBy(l => l.ViewUrl.Host) - .SelectMany((g, gi) => g.Select((l, i) => new {Index = i, GroupIndex = gi, ListInfo = l})) - .OrderBy(a => a.Index) - .ThenBy(a => a.GroupIndex) - .Select(a => a.ListInfo); - } } } } \ No newline at end of file