diff --git a/src/FilterLists.Services/ScrapeService/RawRuleLinterExtensions.cs b/src/FilterLists.Services/ScrapeService/RawRuleLinterExtensions.cs new file mode 100644 index 000000000..c07cb0986 --- /dev/null +++ b/src/FilterLists.Services/ScrapeService/RawRuleLinterExtensions.cs @@ -0,0 +1,34 @@ +namespace FilterLists.Services.ScrapeService +{ + public static class RawRuleLinterExtensions + { + public static string LintStringForMySql(this string rule) + { + rule = rule.TrimSingleBackslashFromEnd(); + rule = rule.DropIfContainsBackslashSingleQuote(); + rule = rule.DropIfTooLong(); + return rule; + } + + private static string TrimSingleBackslashFromEnd(this string rule) + { + if (rule != null) + return rule.EndsWith(@"\") && !rule.EndsWith(@"\\") ? rule.Remove(rule.Length - 1) : rule; + return null; + } + + private static string DropIfContainsBackslashSingleQuote(this string rule) + { + //TODO: resolve issue and/or track dropped rule + if (rule != null) + return rule.Contains(@"\'") ? null : rule; + return null; + } + + private static string DropIfTooLong(this string rule) + { + //TODO: resolve issue and/or track dropped rule + return rule?.Length > 8192 ? null : rule; + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/ScrapeService/RuleLinterExtensions.cs b/src/FilterLists.Services/ScrapeService/RuleLinterExtensions.cs deleted file mode 100644 index 3d2ff2e92..000000000 --- a/src/FilterLists.Services/ScrapeService/RuleLinterExtensions.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace FilterLists.Services.ScrapeService -{ - public static class RuleLinterExtensions - { - public static string LintStringForMySql(this string rule) - { - rule = rule.TrimSingleBackslashFromEnd(); - rule = rule.DropIfContainsBackslashSingleQuote(); - return rule; - } - - private static string TrimSingleBackslashFromEnd(this string rule) - { - return rule.EndsWith(@"\") && !rule.EndsWith(@"\\") ? rule.Remove(rule.Length - 1) : rule; - } - - private static string DropIfContainsBackslashSingleQuote(this string rule) - { - //TODO: resolve issue and/or log rule - return rule.Contains(@"\'") ? null : rule; - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/ScrapeService/ScrapeService.cs b/src/FilterLists.Services/ScrapeService/ScrapeService.cs index ac89e7ffc..54722870c 100644 --- a/src/FilterLists.Services/ScrapeService/ScrapeService.cs +++ b/src/FilterLists.Services/ScrapeService/ScrapeService.cs @@ -67,7 +67,15 @@ private static async Task GetHttpResponseMessageContent(string url) private static async Task SaveSnapshots(IEnumerable snapshots) { - foreach (var snapshot in snapshots) await snapshot.AddOrUpdateRules(); + foreach (var snapshot in snapshots) + try + { + await snapshot.AddOrUpdateRules(); + } + catch (Exception) + { + //TODO: log exception + } } } } \ No newline at end of file diff --git a/src/FilterLists.Services/ScrapeService/Snapshot.cs b/src/FilterLists.Services/ScrapeService/Snapshot.cs index 0910dc93e..d97c61891 100644 --- a/src/FilterLists.Services/ScrapeService/Snapshot.cs +++ b/src/FilterLists.Services/ScrapeService/Snapshot.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using FilterLists.Data; @@ -13,7 +14,7 @@ public class Snapshot private readonly int filterListId; - private string[] rawRules; + private List rawRules; public Snapshot(FilterListsDbContext dbContext, int filterListId, string content) { @@ -27,11 +28,13 @@ public Snapshot(FilterListsDbContext dbContext, int filterListId, string content private void ParseContent(string content) { if (content == null) return; - rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); - for (var i = 0; i < rawRules.Length; i++) + rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList(); + for (var i = 0; i < rawRules.Count; i++) rawRules[i] = rawRules[i].LintStringForMySql(); + rawRules = rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList(); } + //TODO: split large lists into batches public async Task AddOrUpdateRules() { dbContext.ChangeTracker.AutoDetectChangesEnabled = false; @@ -40,19 +43,23 @@ public async Task AddOrUpdateRules() var preExistingSnapshotRules = dbContext.Rules.Where(x => rawRules.Contains(x.Raw)); var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw)); var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule}).ToList(); + await dbContext.Rules.AddRangeAsync(newSnapshotRules); // remove deleted FilterListRules var preExistingFilterListRules = dbContext.FilterListRules.Where(x => x.FilterListId == filterListId); var deletedFilterListRules = preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId)); + dbContext.FilterListRules.RemoveRange(deletedFilterListRules); // add FilterListRules for pre-existing Rules var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule => new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList() .Except(preExistingFilterListRules).ToList(); + await dbContext.FilterListRules.AddRangeAsync(preExistingSnapshotFilterListRules); // add new FilterListRules var newFilterListRules = newSnapshotRules.Select(newSnapshotRule => new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList(); + await dbContext.FilterListRules.AddRangeAsync(newFilterListRules); // update UpdatedDateUtc var list = dbContext.FilterLists.Find(filterListId); @@ -62,10 +69,6 @@ public async Task AddOrUpdateRules() // update ScrapedDateUtc list.ScrapedDateUtc = DateTime.UtcNow; - dbContext.Rules.AddRange(newSnapshotRules); - dbContext.FilterListRules.RemoveRange(deletedFilterListRules); - dbContext.FilterListRules.AddRange(preExistingSnapshotFilterListRules); - dbContext.FilterListRules.AddRange(newFilterListRules); dbContext.FilterLists.Update(list); await dbContext.SaveChangesAsync();