diff --git a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs index f576980f6..99eccf22f 100644 --- a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs +++ b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs @@ -17,7 +17,7 @@ public ScrapeController(ScrapeService scrapeService) public async Task Index() { #if DEBUG - await scrapeService.ScrapeAsync(5); + await scrapeService.ScrapeAsync(3); return Ok(); #else return StatusCode(403); diff --git a/src/FilterLists.Services/IEnumerableExtensions.cs b/src/FilterLists.Services/IEnumerableExtensions.cs deleted file mode 100644 index d36f28a7e..000000000 --- a/src/FilterLists.Services/IEnumerableExtensions.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace FilterLists.Services -{ - public static class IEnumerableExtensions - { - public static void ForEach(this IEnumerable items, Action action) - { - foreach (var item in items) - action(item); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/Services/ScrapeService.cs b/src/FilterLists.Services/Services/ScrapeService.cs index 7a2db85d2..ef2073058 100644 --- a/src/FilterLists.Services/Services/ScrapeService.cs +++ b/src/FilterLists.Services/Services/ScrapeService.cs @@ -25,7 +25,7 @@ public async Task ScrapeAsync(int batchSize) { var lists = await GetNextFilterListsToScrape(batchSize); var snapshots = await GetSnapshots(lists); - snapshots.ForEach(async snapshot => await AddOrUpdateRules(snapshot)); + await SaveSnapshots(snapshots); } private async Task> GetNextFilterListsToScrape(int batchSize) @@ -36,8 +36,10 @@ private async Task> GetNextFilterListsToScrape private static async Task> GetSnapshots(IEnumerable lists) { - return await Task.WhenAll(lists.Select(async list => - new Snapshot {Content = await TryGetContent(list.ViewUrl), FilterListId = list.Id})); + return await Task.WhenAll(lists + .Select(async list => + new Snapshot {Content = await TryGetContent(list.ViewUrl), FilterListId = list.Id}) + .Where(x => x.Result.RawRules != null)); } private static async Task TryGetContent(string url) @@ -66,25 +68,42 @@ private static async Task GetHttpResponseMessageContent(string url) return null; } + private async Task SaveSnapshots(IEnumerable snapshots) + { + foreach (var snapshot in snapshots) + { + await AddOrUpdateRules(snapshot); + } + } + private async Task AddOrUpdateRules(Snapshot snapshot) { + dbContext.ChangeTracker.AutoDetectChangesEnabled = false; + // add new Rules var preExistingSnapshotRules = dbContext.Rules.Where(x => snapshot.RawRules.Contains(x.Raw)); var newSnapshotRawRules = snapshot.RawRules.Except(preExistingSnapshotRules.Select(x => x.Raw)); - var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRuleRaw => new Rule {Raw = newSnapshotRuleRaw}); + var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRuleRaw => new Rule {Raw = newSnapshotRuleRaw}) + .ToList(); dbContext.Rules.AddRange(newSnapshotRules); // remove deleted FilterListRules - var preExistingFilterListRules = dbContext.FilterListRules.Where(x => x.FilterListId == snapshot.FilterListId); - var deletedFilterListRules = preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId)); + var preExistingFilterListRules = + dbContext.FilterListRules.Where(x => x.FilterListId == snapshot.FilterListId); + var deletedFilterListRules = + preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId)); dbContext.FilterListRules.RemoveRange(deletedFilterListRules); - // add new FilterListRules - var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule => - new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule}).Except(preExistingFilterListRules); + // add FilterListRules for pre-existing Rules + var preExistingSnapshotFilterListRules = preExistingSnapshotRules + .Select(newSnapshotRule => + new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule}).ToList() + .Except(preExistingFilterListRules).ToList(); dbContext.FilterListRules.AddRange(preExistingSnapshotFilterListRules); + + // add new FilterListRules var newFilterListRules = newSnapshotRules.Select(newSnapshotRule => - new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule}); + new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule}).ToList(); dbContext.FilterListRules.AddRange(newFilterListRules); // update UpdatedDateUtc @@ -107,9 +126,8 @@ private class FilterListViewUrlDto private class Snapshot { - public string Content - { - set => RawRules = value.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); + public string Content { + set => RawRules = value?.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries); } public int FilterListId { get; set; }