diff --git a/src/FilterLists.Services/FilterListService/FilterListService.cs b/src/FilterLists.Services/FilterListService/FilterListService.cs index 88f8a27f0..9e2d67b17 100644 --- a/src/FilterLists.Services/FilterListService/FilterListService.cs +++ b/src/FilterLists.Services/FilterListService/FilterListService.cs @@ -4,7 +4,6 @@ using AutoMapper.QueryableExtensions; using FilterLists.Data; using FilterLists.Data.Entities; -using FilterLists.Services.FilterListService.Models; using Microsoft.EntityFrameworkCore; namespace FilterLists.Services.FilterListService diff --git a/src/FilterLists.Services/FilterListService/Models/FilterListSummaryDto.cs b/src/FilterLists.Services/FilterListService/FilterListSummaryDto.cs similarity index 79% rename from src/FilterLists.Services/FilterListService/Models/FilterListSummaryDto.cs rename to src/FilterLists.Services/FilterListService/FilterListSummaryDto.cs index 8ae452140..946378471 100644 --- a/src/FilterLists.Services/FilterListService/Models/FilterListSummaryDto.cs +++ b/src/FilterLists.Services/FilterListService/FilterListSummaryDto.cs @@ -1,4 +1,4 @@ -namespace FilterLists.Services.FilterListService.Models +namespace FilterLists.Services.FilterListService { public class FilterListSummaryDto { diff --git a/src/FilterLists.Services/ScrapeService/Models/FilterListViewUrlDto.cs b/src/FilterLists.Services/ScrapeService/FilterListViewUrlDto.cs similarity index 70% rename from src/FilterLists.Services/ScrapeService/Models/FilterListViewUrlDto.cs rename to src/FilterLists.Services/ScrapeService/FilterListViewUrlDto.cs index 9084d11e6..2fdfee176 100644 --- a/src/FilterLists.Services/ScrapeService/Models/FilterListViewUrlDto.cs +++ b/src/FilterLists.Services/ScrapeService/FilterListViewUrlDto.cs @@ -1,4 +1,4 @@ -namespace FilterLists.Services.ScrapeService.Models +namespace FilterLists.Services.ScrapeService { public class FilterListViewUrlDto { diff --git a/src/FilterLists.Services/ScrapeService/Models/Snapshot.cs b/src/FilterLists.Services/ScrapeService/Models/Snapshot.cs deleted file mode 100644 index 5754045df..000000000 --- a/src/FilterLists.Services/ScrapeService/Models/Snapshot.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; - -namespace FilterLists.Services.ScrapeService.Models -{ - public class Snapshot - { - public string Content - { - set - { - if (value == null) return; - var rawRules = value.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); - for (var i = 0; i < rawRules.Length; i++) - rawRules[i] = LintStringForMySql(rawRules[i]); - RawRules = rawRules; - } - } - - public int FilterListId { get; set; } - - public string[] RawRules { get; private set; } - - private static string LintStringForMySql(string rule) - { - rule = TrimSingleBackslashFromEnd(rule); - rule = DropIfContainsBackslashSingleQuote(rule); - return rule; - } - - private static string TrimSingleBackslashFromEnd(string rule) - { - return rule.EndsWith(@"\") && !rule.EndsWith(@"\\") ? rule.Remove(rule.Length - 1) : rule; - } - - private static string DropIfContainsBackslashSingleQuote(string rule) - { - //TODO: resolve issue and/or log rule - return rule.Contains(@"\'") ? null : rule; - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/ScrapeService/RuleLinterExtensions.cs b/src/FilterLists.Services/ScrapeService/RuleLinterExtensions.cs new file mode 100644 index 000000000..3d2ff2e92 --- /dev/null +++ b/src/FilterLists.Services/ScrapeService/RuleLinterExtensions.cs @@ -0,0 +1,23 @@ +namespace FilterLists.Services.ScrapeService +{ + public static class RuleLinterExtensions + { + public static string LintStringForMySql(this string rule) + { + rule = rule.TrimSingleBackslashFromEnd(); + rule = rule.DropIfContainsBackslashSingleQuote(); + return rule; + } + + private static string TrimSingleBackslashFromEnd(this string rule) + { + return rule.EndsWith(@"\") && !rule.EndsWith(@"\\") ? rule.Remove(rule.Length - 1) : rule; + } + + private static string DropIfContainsBackslashSingleQuote(this string rule) + { + //TODO: resolve issue and/or log rule + return rule.Contains(@"\'") ? null : rule; + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/ScrapeService/ScrapeService.cs b/src/FilterLists.Services/ScrapeService/ScrapeService.cs index 4d73ab1b8..1175c1671 100644 --- a/src/FilterLists.Services/ScrapeService/ScrapeService.cs +++ b/src/FilterLists.Services/ScrapeService/ScrapeService.cs @@ -5,9 +5,6 @@ using System.Threading.Tasks; using AutoMapper.QueryableExtensions; using FilterLists.Data; -using FilterLists.Data.Entities; -using FilterLists.Data.Entities.Junctions; -using FilterLists.Services.ScrapeService.Models; using Microsoft.EntityFrameworkCore; namespace FilterLists.Services.ScrapeService @@ -24,23 +21,22 @@ public ScrapeService(FilterListsDbContext dbContext) //TODO: call via scheduled job public async Task ScrapeAsync(int batchSize) { - var lists = await GetNextFilterListsToScrape(batchSize); + var lists = await GetNextListsToScrape(batchSize); var snapshots = await GetSnapshots(lists); await SaveSnapshots(snapshots); } - private async Task> GetNextFilterListsToScrape(int batchSize) + private async Task> GetNextListsToScrape(int batchSize) { return await dbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(batchSize) .ProjectTo().ToListAsync(); } - private static async Task> GetSnapshots(IEnumerable lists) + private async Task> GetSnapshots(IEnumerable lists) { return await Task.WhenAll(lists - .Select(async list => - new Snapshot {Content = await TryGetContent(list.ViewUrl), FilterListId = list.Id}) - .Where(x => x.Result.RawRules != null)); + .Select(async list => new Snapshot(dbContext, await TryGetContent(list.ViewUrl), list.Id)) + .Where(x => x.Result.HasRules)); } private static async Task TryGetContent(string url) @@ -69,51 +65,9 @@ private static async Task GetHttpResponseMessageContent(string url) return null; } - private async Task SaveSnapshots(IEnumerable snapshots) + private static async Task SaveSnapshots(IEnumerable snapshots) { - foreach (var snapshot in snapshots) await AddOrUpdateRules(snapshot); - } - - private async Task AddOrUpdateRules(Snapshot snapshot) - { - dbContext.ChangeTracker.AutoDetectChangesEnabled = false; - - // add new Rules - var preExistingSnapshotRules = dbContext.Rules.Where(x => snapshot.RawRules.Contains(x.Raw)); - var newSnapshotRawRules = snapshot.RawRules.Except(preExistingSnapshotRules.Select(x => x.Raw)); - var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRuleRaw => new Rule {Raw = newSnapshotRuleRaw}) - .ToList(); - dbContext.Rules.AddRange(newSnapshotRules); - - // remove deleted FilterListRules - var preExistingFilterListRules = - dbContext.FilterListRules.Where(x => x.FilterListId == snapshot.FilterListId); - var deletedFilterListRules = - preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId)); - dbContext.FilterListRules.RemoveRange(deletedFilterListRules); - - // add FilterListRules for pre-existing Rules - var preExistingSnapshotFilterListRules = preExistingSnapshotRules - .Select(newSnapshotRule => - new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule}).ToList() - .Except(preExistingFilterListRules).ToList(); - dbContext.FilterListRules.AddRange(preExistingSnapshotFilterListRules); - - // add new FilterListRules - var newFilterListRules = newSnapshotRules.Select(newSnapshotRule => - new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule}).ToList(); - dbContext.FilterListRules.AddRange(newFilterListRules); - - // update UpdatedDateUtc - var list = dbContext.FilterLists.Find(snapshot.FilterListId); - if (preExistingSnapshotFilterListRules.Any() || newFilterListRules.Any() || deletedFilterListRules.Any()) - list.UpdatedDateUtc = DateTime.UtcNow; - - // update ScrapedDateUtc - list.ScrapedDateUtc = DateTime.UtcNow; - dbContext.FilterLists.Update(list); - - await dbContext.SaveChangesAsync(); + foreach (var snapshot in snapshots) await snapshot.AddOrUpdateRules(); } } } \ No newline at end of file diff --git a/src/FilterLists.Services/ScrapeService/Snapshot.cs b/src/FilterLists.Services/ScrapeService/Snapshot.cs new file mode 100644 index 000000000..6b0f2ee74 --- /dev/null +++ b/src/FilterLists.Services/ScrapeService/Snapshot.cs @@ -0,0 +1,76 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using FilterLists.Data; +using FilterLists.Data.Entities; +using FilterLists.Data.Entities.Junctions; + +namespace FilterLists.Services.ScrapeService +{ + public class Snapshot + { + private readonly FilterListsDbContext dbContext; + + private readonly int filterListId; + + private string[] rawRules; + + public Snapshot(FilterListsDbContext dbContext, string content, int filterListId) + { + this.dbContext = dbContext; + this.filterListId = filterListId; + ParseContent(content); + } + + public bool HasRules => rawRules != null; + + private void ParseContent(string content) + { + if (content == null) return; + rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); + for (var i = 0; i < rawRules.Length; i++) + rawRules[i] = rawRules[i].LintStringForMySql(); + } + + public async Task AddOrUpdateRules() + { + dbContext.ChangeTracker.AutoDetectChangesEnabled = false; + + // add new Rules + var preExistingSnapshotRules = dbContext.Rules.Where(x => rawRules.Contains(x.Raw)); + var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw)); + var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule}).ToList(); + dbContext.Rules.AddRange(newSnapshotRules); + + // remove deleted FilterListRules + var preExistingFilterListRules = dbContext.FilterListRules.Where(x => x.FilterListId == filterListId); + var deletedFilterListRules = + preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId)); + + // add FilterListRules for pre-existing Rules + var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule => + new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList() + .Except(preExistingFilterListRules).ToList(); + + // add new FilterListRules + var newFilterListRules = newSnapshotRules.Select(newSnapshotRule => + new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList(); + + // update UpdatedDateUtc + var list = dbContext.FilterLists.Find(filterListId); + if (preExistingSnapshotFilterListRules.Any() || newFilterListRules.Any() || deletedFilterListRules.Any()) + list.UpdatedDateUtc = DateTime.UtcNow; + + // update ScrapedDateUtc + list.ScrapedDateUtc = DateTime.UtcNow; + + + dbContext.FilterListRules.RemoveRange(deletedFilterListRules); + dbContext.FilterListRules.AddRange(preExistingSnapshotFilterListRules); + dbContext.FilterListRules.AddRange(newFilterListRules); + dbContext.FilterLists.Update(list); + + await dbContext.SaveChangesAsync(); + } + } +} \ No newline at end of file