From fcfdd2c0aca5e8cc56cd2446e867c69439c5c008 Mon Sep 17 00:00:00 2001
From: Collin Barrett <collinbarrett@users.noreply.github.com>
Date: Sat, 3 Feb 2018 07:07:49 -0600
Subject: [PATCH] update algo for getting lists to scrape

---
 .../ScrapeService/ScrapeService.cs            | 28 +++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/src/FilterLists.Services/ScrapeService/ScrapeService.cs b/src/FilterLists.Services/ScrapeService/ScrapeService.cs
index 903677908..8a5f77365 100644
--- a/src/FilterLists.Services/ScrapeService/ScrapeService.cs
+++ b/src/FilterLists.Services/ScrapeService/ScrapeService.cs
@@ -28,8 +28,32 @@ public async Task ScrapeAsync(int batchSize)
 
         private async Task<List<FilterListViewUrlDto>> GetNextListsToScrape(int batchSize)
         {
-            //TODO: order by least recently scraped
-            return await dbContext.FilterLists.Take(batchSize).ProjectTo<FilterListViewUrlDto>().ToListAsync();
+            var neverScraped = await GetNeverScrapedLists(batchSize);
+            if (neverScraped.Count == batchSize)
+                return neverScraped;
+            var leastRecentlyScraped = await GetLeastRecentlyScrapedLists(batchSize - neverScraped.Count);
+            return neverScraped.Concat(leastRecentlyScraped).ToList();
+        }
+
+        private async Task<List<FilterListViewUrlDto>> GetNeverScrapedLists(int batchSize)
+        {
+            return await dbContext.FilterLists
+                .Where(x => x.Scrapes.Count == 0)
+                .Take(batchSize)
+                .ProjectTo<FilterListViewUrlDto>()
+                .ToListAsync();
+        }
+
+        private async Task<List<FilterListViewUrlDto>> GetLeastRecentlyScrapedLists(int batchSize)
+        {
+            return await dbContext.Scrapes
+                .GroupBy(x => x.FilterList)
+                .Select(x => x.OrderByDescending(y => y.CreatedDateUtc).First())
+                .OrderBy(x => x.CreatedDateUtc)
+                .Select(x => x.FilterList)
+                .Take(batchSize)
+                .ProjectTo<FilterListViewUrlDto>()
+                .ToListAsync();
         }
 
         private async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)