prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update algo for getting lists to scrape

Browse source
This commit is contained in:
Collin Barrett 2018-02-03 07:07:49 -06:00
parent f651a33ad8
commit fcfdd2c0ac
1 changed files with 26 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
src/FilterLists.Services/ScrapeService/ScrapeService.cs
View file

@ -28,8 +28,32 @@ public async Task ScrapeAsync(int batchSize)
private async Task<List<FilterListViewUrlDto>> GetNextListsToScrape(int batchSize)
{
//TODO: order by least recently scraped
return await dbContext.FilterLists.Take(batchSize).ProjectTo<FilterListViewUrlDto>().ToListAsync();
var neverScraped = await GetNeverScrapedLists(batchSize);
if (neverScraped.Count == batchSize)
return neverScraped;
var leastRecentlyScraped = await GetLeastRecentlyScrapedLists(batchSize - neverScraped.Count);
return neverScraped.Concat(leastRecentlyScraped).ToList();
}
private async Task<List<FilterListViewUrlDto>> GetNeverScrapedLists(int batchSize)
{
return await dbContext.FilterLists
.Where(x => x.Scrapes.Count == 0)
.Take(batchSize)
.ProjectTo<FilterListViewUrlDto>()
.ToListAsync();
}
private async Task<List<FilterListViewUrlDto>> GetLeastRecentlyScrapedLists(int batchSize)
{
return await dbContext.Scrapes
.GroupBy(x => x.FilterList)
.Select(x => x.OrderByDescending(y => y.CreatedDateUtc).First())
.OrderBy(x => x.CreatedDateUtc)
.Select(x => x.FilterList)
.Take(batchSize)
.ProjectTo<FilterListViewUrlDto>()
.ToListAsync();
}
private async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)