prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

only validate distinct URLs per file

Browse source
This commit is contained in:
Collin M. Barrett 2019-07-02 19:55:58 -05:00
parent bdd2bd9b3b
commit 2aaa32aa4e
1 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/FilterLists.Agent/Features/Urls/ValidateUrls.cs
View file

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading;
@ -41,7 +42,8 @@ public async Task<List<UrlValidationResult>> Handle(Command request, Cancellatio
{
var validator = BuildValidator(cancellationToken);
var brokenUrls = new List<UrlValidationResult>();
foreach (var url in request.Urls)
var distinctUrls = request.Urls.Distinct();
foreach (var url in distinctUrls)
await validator.SendAsync(url, cancellationToken);
validator.Complete();
while (await validator.OutputAvailableAsync(cancellationToken))