From 33adbe16a44fc34ac9a317d8de4964b39194bab4 Mon Sep 17 00:00:00 2001 From: Collin Barrett Date: Sun, 28 Jan 2018 19:26:21 -0600 Subject: [PATCH] work on ScrapeService, use Async suffix convention --- .../V1/Controllers/ForksController.cs | 2 +- .../V1/Controllers/LanguagesController.cs | 2 +- .../V1/Controllers/LicensesController.cs | 2 +- .../V1/Controllers/ListsController.cs | 4 +- .../Controllers/ListsLanguagesController.cs | 2 +- .../Controllers/ListsMaintainersController.cs | 2 +- .../V1/Controllers/MaintainersController.cs | 2 +- .../V1/Controllers/MergesController.cs | 2 +- .../V1/Controllers/ScrapeController.cs | 2 +- .../V1/Controllers/SoftwareController.cs | 2 +- .../Controllers/SoftwareSyntaxesController.cs | 2 +- .../V1/Controllers/SyntaxesController.cs | 2 +- .../Services/FilterListService.cs | 2 +- .../Services/ScrapeService.cs | 63 ++++++++++--------- .../Services/SeedService.cs | 2 +- 15 files changed, 48 insertions(+), 45 deletions(-) diff --git a/src/FilterLists.Api/V1/Controllers/ForksController.cs b/src/FilterLists.Api/V1/Controllers/ForksController.cs index 658c22ab0..d8516274c 100644 --- a/src/FilterLists.Api/V1/Controllers/ForksController.cs +++ b/src/FilterLists.Api/V1/Controllers/ForksController.cs @@ -18,7 +18,7 @@ public ForksController(SeedService seedService) [HttpGet] public async Task Seed() { - return Json(await seedService.GetAll()); + return Json(await seedService.GetAllAsync()); } } } \ No newline at end of file diff --git a/src/FilterLists.Api/V1/Controllers/LanguagesController.cs b/src/FilterLists.Api/V1/Controllers/LanguagesController.cs index c9bb163d5..8aecc619b 100644 --- a/src/FilterLists.Api/V1/Controllers/LanguagesController.cs +++ b/src/FilterLists.Api/V1/Controllers/LanguagesController.cs @@ -18,7 +18,7 @@ public LanguagesController(SeedService seedService) [HttpGet] public async Task Seed() { - return Json(await seedService.GetAll()); + return Json(await seedService.GetAllAsync()); } } } \ No newline at end of file diff --git a/src/FilterLists.Api/V1/Controllers/LicensesController.cs b/src/FilterLists.Api/V1/Controllers/LicensesController.cs index 7ce150e74..86791de2a 100644 --- a/src/FilterLists.Api/V1/Controllers/LicensesController.cs +++ b/src/FilterLists.Api/V1/Controllers/LicensesController.cs @@ -18,7 +18,7 @@ public LicensesController(SeedService seedService) [HttpGet] public async Task Seed() { - return Json(await seedService.GetAll()); + return Json(await seedService.GetAllAsync()); } } } \ No newline at end of file diff --git a/src/FilterLists.Api/V1/Controllers/ListsController.cs b/src/FilterLists.Api/V1/Controllers/ListsController.cs index 066490f89..7e1c14bf4 100644 --- a/src/FilterLists.Api/V1/Controllers/ListsController.cs +++ b/src/FilterLists.Api/V1/Controllers/ListsController.cs @@ -20,13 +20,13 @@ public ListsController(SeedService seedService, FilterListService filterListServ [HttpGet] public async Task Index() { - return Json(await filterListService.GetAllSummaries()); + return Json(await filterListService.GetAllSummariesAsync()); } [HttpGet] public async Task Seed() { - return Json(await seedService.GetAll()); + return Json(await seedService.GetAllAsync()); } } } \ No newline at end of file diff --git a/src/FilterLists.Api/V1/Controllers/ListsLanguagesController.cs b/src/FilterLists.Api/V1/Controllers/ListsLanguagesController.cs index bc4e43773..1db69d365 100644 --- a/src/FilterLists.Api/V1/Controllers/ListsLanguagesController.cs +++ b/src/FilterLists.Api/V1/Controllers/ListsLanguagesController.cs @@ -18,7 +18,7 @@ public ListsLanguagesController(SeedService seedService) [HttpGet] public async Task Seed() { - return Json(await seedService.GetAll()); + return Json(await seedService.GetAllAsync()); } } } \ No newline at end of file diff --git a/src/FilterLists.Api/V1/Controllers/ListsMaintainersController.cs b/src/FilterLists.Api/V1/Controllers/ListsMaintainersController.cs index 06f6d945b..8f99ba821 100644 --- a/src/FilterLists.Api/V1/Controllers/ListsMaintainersController.cs +++ b/src/FilterLists.Api/V1/Controllers/ListsMaintainersController.cs @@ -18,7 +18,7 @@ public ListsMaintainersController(SeedService seedService) [HttpGet] public async Task Seed() { - return Json(await seedService.GetAll()); + return Json(await seedService.GetAllAsync()); } } } \ No newline at end of file diff --git a/src/FilterLists.Api/V1/Controllers/MaintainersController.cs b/src/FilterLists.Api/V1/Controllers/MaintainersController.cs index bf2fd4df0..fcb2a3bc9 100644 --- a/src/FilterLists.Api/V1/Controllers/MaintainersController.cs +++ b/src/FilterLists.Api/V1/Controllers/MaintainersController.cs @@ -18,7 +18,7 @@ public MaintainersController(SeedService seedService) [HttpGet] public async Task Seed() { - return Json(await seedService.GetAll()); + return Json(await seedService.GetAllAsync()); } } } \ No newline at end of file diff --git a/src/FilterLists.Api/V1/Controllers/MergesController.cs b/src/FilterLists.Api/V1/Controllers/MergesController.cs index 1a845c53a..926f289aa 100644 --- a/src/FilterLists.Api/V1/Controllers/MergesController.cs +++ b/src/FilterLists.Api/V1/Controllers/MergesController.cs @@ -18,7 +18,7 @@ public MergesController(SeedService seedService) [HttpGet] public async Task Seed() { - return Json(await seedService.GetAll()); + return Json(await seedService.GetAllAsync()); } } } \ No newline at end of file diff --git a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs index 5f39a35ec..f576980f6 100644 --- a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs +++ b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs @@ -17,7 +17,7 @@ public ScrapeController(ScrapeService scrapeService) public async Task Index() { #if DEBUG - await scrapeService.Scrape(5); + await scrapeService.ScrapeAsync(5); return Ok(); #else return StatusCode(403); diff --git a/src/FilterLists.Api/V1/Controllers/SoftwareController.cs b/src/FilterLists.Api/V1/Controllers/SoftwareController.cs index 701d40051..fc03693ff 100644 --- a/src/FilterLists.Api/V1/Controllers/SoftwareController.cs +++ b/src/FilterLists.Api/V1/Controllers/SoftwareController.cs @@ -18,7 +18,7 @@ public SoftwareController(SeedService seedService) [HttpGet] public async Task Seed() { - return Json(await seedService.GetAll()); + return Json(await seedService.GetAllAsync()); } } } \ No newline at end of file diff --git a/src/FilterLists.Api/V1/Controllers/SoftwareSyntaxesController.cs b/src/FilterLists.Api/V1/Controllers/SoftwareSyntaxesController.cs index 4e6ea543e..0d96758f4 100644 --- a/src/FilterLists.Api/V1/Controllers/SoftwareSyntaxesController.cs +++ b/src/FilterLists.Api/V1/Controllers/SoftwareSyntaxesController.cs @@ -18,7 +18,7 @@ public SoftwareSyntaxesController(SeedService seedService) [HttpGet] public async Task Seed() { - return Json(await seedService.GetAll()); + return Json(await seedService.GetAllAsync()); } } } \ No newline at end of file diff --git a/src/FilterLists.Api/V1/Controllers/SyntaxesController.cs b/src/FilterLists.Api/V1/Controllers/SyntaxesController.cs index dc3fbd4b3..72211a963 100644 --- a/src/FilterLists.Api/V1/Controllers/SyntaxesController.cs +++ b/src/FilterLists.Api/V1/Controllers/SyntaxesController.cs @@ -18,7 +18,7 @@ public SyntaxesController(SeedService seedService) [HttpGet] public async Task Seed() { - return Json(await seedService.GetAll()); + return Json(await seedService.GetAllAsync()); } } } \ No newline at end of file diff --git a/src/FilterLists.Services/Services/FilterListService.cs b/src/FilterLists.Services/Services/FilterListService.cs index eee3c3e45..76301e607 100644 --- a/src/FilterLists.Services/Services/FilterListService.cs +++ b/src/FilterLists.Services/Services/FilterListService.cs @@ -17,7 +17,7 @@ public FilterListService(FilterListsDbContext filterListsDbContext) this.filterListsDbContext = filterListsDbContext; } - public async Task> GetAllSummaries() + public async Task> GetAllSummariesAsync() { return await filterListsDbContext.Set().AsNoTracking().ProjectTo() .ToListAsync(); diff --git a/src/FilterLists.Services/Services/ScrapeService.cs b/src/FilterLists.Services/Services/ScrapeService.cs index 92e2f77df..896e75fa2 100644 --- a/src/FilterLists.Services/Services/ScrapeService.cs +++ b/src/FilterLists.Services/Services/ScrapeService.cs @@ -3,8 +3,10 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; +using AutoMapper.QueryableExtensions; using FilterLists.Data; using FilterLists.Data.Entities; +using Microsoft.EntityFrameworkCore; namespace FilterLists.Services.Services { @@ -18,24 +20,25 @@ public ScrapeService(FilterListsDbContext filterListsDbContext) } //TODO: call via scheduled job - public async Task Scrape(int numberLists) + public async Task ScrapeAsync(int batchSize) { - var lists = filterListsDbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(numberLists) - .Select(x => new FilterListDto {Id = x.Id, ViewUrl = x.ViewUrl}); + var lists = await GetNextFilterListDtosToScrape(batchSize); var snapshots = await GetSnapshots(lists); await AddOrUpdateRules(snapshots); - //TODO: update FilterList.ScrapedDateUtc + } + + private async Task> GetNextFilterListDtosToScrape(int batchSize) + { + return await filterListsDbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(batchSize) + .ProjectTo().ToListAsync(); } private static async Task> GetSnapshots(IEnumerable lists) { - var snapshots = new List(); - foreach (var list in lists) - snapshots.Add(new Snapshot {Content = await GetContent(list.ViewUrl), FilterListId = list.Id}); - return snapshots; + return await Task.WhenAll(lists.Select(async list => + new Snapshot {Content = await GetContent(list.ViewUrl), FilterListId = list.Id})); } - //TODO: move to string (URL) extension method private static async Task GetContent(string url) { try @@ -49,51 +52,51 @@ private static async Task GetContent(string url) } catch (Exception) { - //TODO: log exception for analysis + //TODO: log exception return null; } - //TODO: log httpResponseMessage.StatusCode for analysis + //TODO: log httpResponseMessage.StatusCode return null; } private async Task AddOrUpdateRules(IEnumerable snapshots) { - foreach (var snapshot in snapshots) - await AddOrUpdateRules(snapshot); + await Task.WhenAll(snapshots.Select(async snapshot => await AddOrUpdateRules(snapshot))); } //TODO: finish and validate private async Task AddOrUpdateRules(Snapshot snapshot) { - var cachedRules = filterListsDbContext.FilterListRules - .Where(x => x.FilterListId == snapshot.FilterListId) - .Select(x => x.Rule); - - var currentRulesRaw = + // add new Rules + var snapshotRulesRaw = snapshot.Content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); + var preExistingSnapshotRules = filterListsDbContext.Rules.Where(x => snapshotRulesRaw.Contains(x.Raw)); + var newSnapshotRulesRaw = snapshotRulesRaw.Except(preExistingSnapshotRules.Select(x => x.Raw)); + var newSnapshotRules = + newSnapshotRulesRaw.Select(newSnapshotRuleRaw => new Rule {Raw = newSnapshotRuleRaw}); + filterListsDbContext.Rules.AddRange(newSnapshotRules); - var existingCurrentRules = filterListsDbContext.Rules.Where(x => currentRulesRaw.Contains(x.Raw)); + // remove deleted FilterListRules + var preExistingFilterListRules = + filterListsDbContext.FilterListRules.Where(x => x.FilterListId == snapshot.FilterListId); + var deletedFilterListRules = + preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId)); + filterListsDbContext.FilterListRules.RemoveRange(deletedFilterListRules); - var newCurrentRulesRaw = currentRulesRaw.Except(existingCurrentRules.Select(x => x.Raw)); + // add new FilterListRules - var newCurrentRules = newCurrentRulesRaw.Select(newCurrentRuleRaw => new Rule {Raw = newCurrentRuleRaw}); - var deletedRules = cachedRules.Except(existingCurrentRules).ToList(); - - filterListsDbContext.FilterListRules.RemoveRange(filterListsDbContext.FilterListRules - .Where(x => deletedRules.Select(y => y.Id).Contains(x.RuleId)) - .Where(x => x.FilterListId == snapshot.FilterListId)); - - //TODO: consider never removing FilterListsRules but rather marking a flag as deprecated to expose when rule was removed - - if (newCurrentRules.Any() || deletedRules.Any()) + // update UpdatedDateUtc + if (newSnapshotRulesRaw.Any() || deletedFilterListRules.Any()) { var list = filterListsDbContext.FilterLists.FindAsync(snapshot.FilterListId).Result; list.UpdatedDateUtc = DateTime.UtcNow; filterListsDbContext.FilterLists.Update(list); } + //TODO: update FilterList.ScrapedDateUtc + await filterListsDbContext.SaveChangesAsync(); } diff --git a/src/FilterLists.Services/Services/SeedService.cs b/src/FilterLists.Services/Services/SeedService.cs index 1a929e45e..ddecda9eb 100644 --- a/src/FilterLists.Services/Services/SeedService.cs +++ b/src/FilterLists.Services/Services/SeedService.cs @@ -15,7 +15,7 @@ public SeedService(FilterListsDbContext filterListsDbContext) this.filterListsDbContext = filterListsDbContext; } - public async Task> GetAll() where TEntity : class + public async Task> GetAllAsync() where TEntity : class { return await filterListsDbContext.Set().AsNoTracking().ProjectTo().ToArrayAsync(); }