work on ScrapeService, use Async suffix convention

src/FilterLists.Api/V1/Controllers/ForksController.cs

@@ -18,7 +18,7 @@ public ForksController(SeedService seedService)
         [HttpGet]
         public async Task<IActionResult> Seed()
         {
-            return Json(await seedService.GetAll<Fork, ForkSeedDto>());
+            return Json(await seedService.GetAllAsync<Fork, ForkSeedDto>());
         }
     }
 }

src/FilterLists.Api/V1/Controllers/LanguagesController.cs

@@ -18,7 +18,7 @@ public LanguagesController(SeedService seedService)
         [HttpGet]
         public async Task<IActionResult> Seed()
         {
-            return Json(await seedService.GetAll<Language, LanguageSeedDto>());
+            return Json(await seedService.GetAllAsync<Language, LanguageSeedDto>());
         }
     }
 }

src/FilterLists.Api/V1/Controllers/LicensesController.cs

@@ -18,7 +18,7 @@ public LicensesController(SeedService seedService)
         [HttpGet]
         public async Task<IActionResult> Seed()
         {
-            return Json(await seedService.GetAll<License, LicenseSeedDto>());
+            return Json(await seedService.GetAllAsync<License, LicenseSeedDto>());
         }
     }
 }

src/FilterLists.Api/V1/Controllers/ListsController.cs

@@ -20,13 +20,13 @@ public ListsController(SeedService seedService, FilterListService filterListServ
         [HttpGet]
         public async Task<IActionResult> Index()
         {
-            return Json(await filterListService.GetAllSummaries());
+            return Json(await filterListService.GetAllSummariesAsync());
         }
 
         [HttpGet]
         public async Task<IActionResult> Seed()
         {
-            return Json(await seedService.GetAll<FilterList, FilterListSeedDto>());
+            return Json(await seedService.GetAllAsync<FilterList, FilterListSeedDto>());
         }
     }
 }

src/FilterLists.Api/V1/Controllers/ListsLanguagesController.cs

@@ -18,7 +18,7 @@ public ListsLanguagesController(SeedService seedService)
         [HttpGet]
         public async Task<IActionResult> Seed()
         {
-            return Json(await seedService.GetAll<FilterListLanguage, FilterListLanguageSeedDto>());
+            return Json(await seedService.GetAllAsync<FilterListLanguage, FilterListLanguageSeedDto>());
         }
     }
 }

src/FilterLists.Api/V1/Controllers/ListsMaintainersController.cs

@@ -18,7 +18,7 @@ public ListsMaintainersController(SeedService seedService)
         [HttpGet]
         public async Task<IActionResult> Seed()
         {
-            return Json(await seedService.GetAll<FilterListMaintainer, FilterListMaintainerSeedDto>());
+            return Json(await seedService.GetAllAsync<FilterListMaintainer, FilterListMaintainerSeedDto>());
         }
     }
 }

src/FilterLists.Api/V1/Controllers/MaintainersController.cs

@@ -18,7 +18,7 @@ public MaintainersController(SeedService seedService)
         [HttpGet]
         public async Task<IActionResult> Seed()
         {
-            return Json(await seedService.GetAll<Maintainer, MaintainerSeedDto>());
+            return Json(await seedService.GetAllAsync<Maintainer, MaintainerSeedDto>());
         }
     }
 }

src/FilterLists.Api/V1/Controllers/MergesController.cs

@@ -18,7 +18,7 @@ public MergesController(SeedService seedService)
         [HttpGet]
         public async Task<IActionResult> Seed()
         {
-            return Json(await seedService.GetAll<Merge, MergeSeedDto>());
+            return Json(await seedService.GetAllAsync<Merge, MergeSeedDto>());
         }
     }
 }

src/FilterLists.Api/V1/Controllers/ScrapeController.cs

@@ -17,7 +17,7 @@ public ScrapeController(ScrapeService scrapeService)
         public async Task<IActionResult> Index()
         {
 #if DEBUG
-            await scrapeService.Scrape(5);
+            await scrapeService.ScrapeAsync(5);
             return Ok();
 #else
             return StatusCode(403);

src/FilterLists.Api/V1/Controllers/SoftwareController.cs

@@ -18,7 +18,7 @@ public SoftwareController(SeedService seedService)
         [HttpGet]
         public async Task<IActionResult> Seed()
         {
-            return Json(await seedService.GetAll<Software, SoftwareSeedDto>());
+            return Json(await seedService.GetAllAsync<Software, SoftwareSeedDto>());
         }
     }
 }

src/FilterLists.Api/V1/Controllers/SoftwareSyntaxesController.cs

@@ -18,7 +18,7 @@ public SoftwareSyntaxesController(SeedService seedService)
         [HttpGet]
         public async Task<IActionResult> Seed()
         {
-            return Json(await seedService.GetAll<SoftwareSyntax, SoftwareSyntaxSeedDto>());
+            return Json(await seedService.GetAllAsync<SoftwareSyntax, SoftwareSyntaxSeedDto>());
         }
     }
 }

src/FilterLists.Api/V1/Controllers/SyntaxesController.cs

@@ -18,7 +18,7 @@ public SyntaxesController(SeedService seedService)
         [HttpGet]
         public async Task<IActionResult> Seed()
         {
-            return Json(await seedService.GetAll<Syntax, SyntaxSeedDto>());
+            return Json(await seedService.GetAllAsync<Syntax, SyntaxSeedDto>());
         }
     }
 }

src/FilterLists.Services/Services/FilterListService.cs

@@ -17,7 +17,7 @@ public FilterListService(FilterListsDbContext filterListsDbContext)
             this.filterListsDbContext = filterListsDbContext;
         }
 
-        public async Task<IEnumerable<FilterListSummaryDto>> GetAllSummaries()
+        public async Task<IEnumerable<FilterListSummaryDto>> GetAllSummariesAsync()
         {
             return await filterListsDbContext.Set<FilterList>().AsNoTracking().ProjectTo<FilterListSummaryDto>()
                 .ToListAsync();

src/FilterLists.Services/Services/ScrapeService.cs

@@ -3,8 +3,10 @@
 using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
+using AutoMapper.QueryableExtensions;
 using FilterLists.Data;
 using FilterLists.Data.Entities;
+using Microsoft.EntityFrameworkCore;
 
 namespace FilterLists.Services.Services
 {
@@ -18,24 +20,25 @@ public ScrapeService(FilterListsDbContext filterListsDbContext)
         }
 
         //TODO: call via scheduled job
-        public async Task Scrape(int numberLists)
+        public async Task ScrapeAsync(int batchSize)
         {
-            var lists = filterListsDbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(numberLists)
-                .Select(x => new FilterListDto {Id = x.Id, ViewUrl = x.ViewUrl});
+            var lists = await GetNextFilterListDtosToScrape(batchSize);
             var snapshots = await GetSnapshots(lists);
             await AddOrUpdateRules(snapshots);
-            //TODO: update FilterList.ScrapedDateUtc
+        }
+
+        private async Task<IEnumerable<FilterListDto>> GetNextFilterListDtosToScrape(int batchSize)
+        {
+            return await filterListsDbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(batchSize)
+                .ProjectTo<FilterListDto>().ToListAsync();
         }
 
         private static async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListDto> lists)
         {
-            var snapshots = new List<Snapshot>();
-            foreach (var list in lists)
-                snapshots.Add(new Snapshot {Content = await GetContent(list.ViewUrl), FilterListId = list.Id});
-            return snapshots;
+            return await Task.WhenAll(lists.Select(async list =>
+                new Snapshot {Content = await GetContent(list.ViewUrl), FilterListId = list.Id}));
         }
 
-        //TODO: move to string (URL) extension method
         private static async Task<string> GetContent(string url)
         {
             try
@@ -49,51 +52,51 @@ private static async Task<string> GetContent(string url)
             }
             catch (Exception)
             {
-                //TODO: log exception for analysis
+                //TODO: log exception
                 return null;
             }
 
-            //TODO: log httpResponseMessage.StatusCode for analysis
+            //TODO: log httpResponseMessage.StatusCode
             return null;
         }
 
         private async Task AddOrUpdateRules(IEnumerable<Snapshot> snapshots)
         {
-            foreach (var snapshot in snapshots)
-                await AddOrUpdateRules(snapshot);
+            await Task.WhenAll(snapshots.Select(async snapshot => await AddOrUpdateRules(snapshot)));
         }
 
         //TODO: finish and validate
         private async Task AddOrUpdateRules(Snapshot snapshot)
         {
-            var cachedRules = filterListsDbContext.FilterListRules
-                .Where(x => x.FilterListId == snapshot.FilterListId)
-                .Select(x => x.Rule);
-
-            var currentRulesRaw =
+            // add new Rules
+            var snapshotRulesRaw =
                 snapshot.Content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
+            var preExistingSnapshotRules = filterListsDbContext.Rules.Where(x => snapshotRulesRaw.Contains(x.Raw));
+            var newSnapshotRulesRaw = snapshotRulesRaw.Except(preExistingSnapshotRules.Select(x => x.Raw));
+            var newSnapshotRules =
+                newSnapshotRulesRaw.Select(newSnapshotRuleRaw => new Rule {Raw = newSnapshotRuleRaw});
+            filterListsDbContext.Rules.AddRange(newSnapshotRules);
 
-            var existingCurrentRules = filterListsDbContext.Rules.Where(x => currentRulesRaw.Contains(x.Raw));
+            // remove deleted FilterListRules
+            var preExistingFilterListRules =
+                filterListsDbContext.FilterListRules.Where(x => x.FilterListId == snapshot.FilterListId);
+            var deletedFilterListRules =
+                preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId));
+            filterListsDbContext.FilterListRules.RemoveRange(deletedFilterListRules);
 
-            var newCurrentRulesRaw = currentRulesRaw.Except(existingCurrentRules.Select(x => x.Raw));
+            // add new FilterListRules
 
-            var newCurrentRules = newCurrentRulesRaw.Select(newCurrentRuleRaw => new Rule {Raw = newCurrentRuleRaw});
 
-            var deletedRules = cachedRules.Except(existingCurrentRules).ToList();
-
-            filterListsDbContext.FilterListRules.RemoveRange(filterListsDbContext.FilterListRules
-                .Where(x => deletedRules.Select(y => y.Id).Contains(x.RuleId))
-                .Where(x => x.FilterListId == snapshot.FilterListId));
-
-            //TODO: consider never removing FilterListsRules but rather marking a flag as deprecated to expose when rule was removed
-
-            if (newCurrentRules.Any() || deletedRules.Any())
+            // update UpdatedDateUtc
+            if (newSnapshotRulesRaw.Any() || deletedFilterListRules.Any())
             {
                 var list = filterListsDbContext.FilterLists.FindAsync(snapshot.FilterListId).Result;
                 list.UpdatedDateUtc = DateTime.UtcNow;
                 filterListsDbContext.FilterLists.Update(list);
             }
 
+            //TODO: update FilterList.ScrapedDateUtc
+
             await filterListsDbContext.SaveChangesAsync();
         }
 

src/FilterLists.Services/Services/SeedService.cs

@@ -15,7 +15,7 @@ public SeedService(FilterListsDbContext filterListsDbContext)
             this.filterListsDbContext = filterListsDbContext;
         }
 
-        public async Task<IEnumerable<TSeedDto>> GetAll<TEntity, TSeedDto>() where TEntity : class
+        public async Task<IEnumerable<TSeedDto>> GetAllAsync<TEntity, TSeedDto>() where TEntity : class
         {
             return await filterListsDbContext.Set<TEntity>().AsNoTracking().ProjectTo<TSeedDto>().ToArrayAsync();
         }