prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

distribute URLs by host

Browse source
This commit is contained in:
Collin M. Barrett 2019-07-02 20:07:25 -05:00
parent 2aaa32aa4e
commit 1a41cf6a62
3 changed files with 25 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/FilterLists.Agent/Extensions/ListInfoExtensions.cs
View file

@ -1,18 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using FilterLists.Agent.Core.Entities;
namespace FilterLists.Agent.Extensions
{
public static class ListInfoExtensions
{
public static IEnumerable<ListInfo> DistributeByHost(this IEnumerable<ListInfo> listInfo)
{
return listInfo.GroupBy(l => l.ViewUrl.Host)
.SelectMany((g, gi) => g.Select((l, i) => new {Index = i, GroupIndex = gi, ListInfo = l}))
.OrderBy(a => a.Index)
.ThenBy(a => a.GroupIndex)
.Select(a => a.ListInfo);
}
}
}

24
src/FilterLists.Agent/Extensions/UriExtensions.cs
View file

@ -1,5 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using FilterLists.Agent.Core.Entities;
namespace FilterLists.Agent.Extensions
{
@ -19,5 +22,26 @@ public static bool IsValidUrl(this Uri uri)
return false;
return tmp.Scheme == Uri.UriSchemeHttp || tmp.Scheme == Uri.UriSchemeHttps;
}
public static IEnumerable<Uri> DistributeByHost(this IEnumerable<Uri> listInfo)
{
var listInfoList = listInfo.ToList();
var distributedLists = listInfoList.Where(u => u.IsValidUrl())
.GroupBy(u => u.Host)
.SelectMany((g, gi) => g.Select((u, i) => new {Index = i, GroupIndex = gi, ListInfo = u}))
.OrderBy(a => a.Index)
.ThenBy(a => a.GroupIndex)
.Select(a => a.ListInfo);
return listInfoList.Where(u => !u.IsValidUrl()).Concat(distributedLists);
}
public static IEnumerable<ListInfo> DistributeByHost(this IEnumerable<ListInfo> listInfo)
{
return listInfo.GroupBy(l => l.ViewUrl.Host)
.SelectMany((g, gi) => g.Select((l, i) => new {Index = i, GroupIndex = gi, ListInfo = l}))
.OrderBy(a => a.Index)
.ThenBy(a => a.GroupIndex)
.Select(a => a.ListInfo);
}
}
}

2
src/FilterLists.Agent/Features/Urls/ValidateUrls.cs
View file

@ -42,7 +42,7 @@ public async Task<List<UrlValidationResult>> Handle(Command request, Cancellatio
{
var validator = BuildValidator(cancellationToken);
var brokenUrls = new List<UrlValidationResult>();
var distinctUrls = request.Urls.Distinct();
var distinctUrls = request.Urls.Distinct().DistributeByHost();
foreach (var url in distinctUrls)
await validator.SendAsync(url, cancellationToken);
validator.Complete();