diff --git a/src/FilterLists.Services/Extensions/CollectionExtensions.cs b/src/FilterLists.Services/Extensions/CollectionExtensions.cs deleted file mode 100644 index 81957558d..000000000 --- a/src/FilterLists.Services/Extensions/CollectionExtensions.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System.Collections.Generic; - -namespace FilterLists.Services.Extensions -{ - public static class CollectionExtensions - { - //https://stackoverflow.com/a/26360010/2343739 - public static void AddRange(this ICollection destination, IEnumerable source) - { - if (destination is List list) - list.AddRange(source); - else - foreach (var item in source) - destination.Add(item); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/Extensions/EnumerableExtensions.cs b/src/FilterLists.Services/Extensions/EnumerableExtensions.cs new file mode 100644 index 000000000..d6594d702 --- /dev/null +++ b/src/FilterLists.Services/Extensions/EnumerableExtensions.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; + +namespace FilterLists.Services.Extensions +{ + public static class EnumerableExtensions + { + public static void AddRange(this ICollection destination, IEnumerable source) + { + foreach (var item in source) + destination.Add(item); + } + + public static IEnumerable> GetBatches(this IEnumerable source, int batchSize) + { + var batches = new List(batchSize); + foreach (var item in source) + { + batches.Add(item); + if (batches.Count != batchSize) + continue; + yield return batches.AsReadOnly(); + batches = new List(batchSize); + } + + if (batches.Count > 0) + yield return batches.AsReadOnly(); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/FilterLists.Services.csproj b/src/FilterLists.Services/FilterLists.Services.csproj index b5df087db..6eb37a229 100644 --- a/src/FilterLists.Services/FilterLists.Services.csproj +++ b/src/FilterLists.Services/FilterLists.Services.csproj @@ -20,7 +20,6 @@ - diff --git a/src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs b/src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs index cd4a28d4f..cfe0c95e9 100644 --- a/src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs +++ b/src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs @@ -1,38 +1,35 @@ -using System; - -namespace FilterLists.Services.Snapshot +namespace FilterLists.Services.Snapshot { public static class RawRuleLinterExtensions { public static string LintRawRule(this string rule) { - rule = rule.Trim(); - rule = rule.DropIfEmpty(); - rule = rule?.DropIfTooLong(); + rule = rule?.TrimLeadingAndTrailingWhitespace(); + rule = rule?.DropIfEmpty(); rule = rule?.DropIfComment(); + rule = rule?.DropIfTooLong(); rule = rule?.DropIfContainsBackslashSingleQuote(); rule = rule?.TrimSingleBackslashFromEnd(); return rule; } - private static string DropIfEmpty(this string rule) => - rule == string.Empty ? null : rule; - - private static string DropIfTooLong(this string rule) => - rule.Length > 8192 ? null : rule; - + private static string TrimLeadingAndTrailingWhitespace(this string rule) + { + char[] charsToTrim = {' ', '\t'}; + return rule.Trim(charsToTrim); + } + + private static string DropIfEmpty(this string rule) => rule == "" ? null : rule; + private static string DropIfComment(this string rule) => - rule.StartsWith(@"!", StringComparison.Ordinal) && !rule.StartsWith(@"!#", StringComparison.Ordinal) || - rule.StartsWith(@"!##", StringComparison.Ordinal) - ? null - : rule; + rule.StartsWith(@"!") && !rule.StartsWith(@"!#") || rule.StartsWith(@"!##") ? null : rule; + + private static string DropIfTooLong(this string rule) => rule.Length > 8192 ? null : rule; private static string DropIfContainsBackslashSingleQuote(this string rule) => rule.Contains(@"\'") ? null : rule; private static string TrimSingleBackslashFromEnd(this string rule) => - rule.EndsWith(@"\", StringComparison.Ordinal) && !rule.EndsWith(@"\\", StringComparison.Ordinal) - ? rule.Remove(rule.Length - 1) - : rule; + rule.EndsWith(@"\") && !rule.EndsWith(@"\\") ? rule.Remove(rule.Length - 1) : rule; } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/SnapshotDeBatch.cs b/src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs similarity index 63% rename from src/FilterLists.Services/Snapshot/SnapshotDeBatch.cs rename to src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs index 13e35cbb0..555f5bc5c 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotDeBatch.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs @@ -8,14 +8,14 @@ namespace FilterLists.Services.Snapshot { - public class SnapshotDeBatch + public class SnapshotBatchDe { private readonly FilterListsDbContext dbContext; private readonly IEnumerable rawRules; private readonly Data.Entities.Snapshot snapshot; - private IEnumerable rules; + private IQueryable rules; - public SnapshotDeBatch(FilterListsDbContext dbContext, Data.Entities.Snapshot snapshot, + public SnapshotBatchDe(FilterListsDbContext dbContext, Data.Entities.Snapshot snapshot, IEnumerable rawRules) { this.dbContext = dbContext; @@ -23,7 +23,7 @@ public SnapshotDeBatch(FilterListsDbContext dbContext, Data.Entities.Snapshot sn this.rawRules = rawRules; } - public async Task SaveAsync() + public async Task SaveSnapshotBatchAsync() { await AddRules(); AddSnapshotRules(); @@ -32,18 +32,18 @@ public async Task SaveAsync() private async Task AddRules() { - var existingRules = GetExistingRules(); - var newRules = CreateNewRules(existingRules); + var existingRules = dbContext.Rules.Where(r => rawRules.Contains(r.Raw)); + var newRawRules = rawRules.Except(existingRules.Select(r => r.Raw)); + var newRules = newRawRules.Select(r => new Rule {Raw = r}).ToList(); rules = existingRules.Concat(newRules); await dbContext.Rules.AddRangeAsync(newRules); } - private IQueryable GetExistingRules() => dbContext.Rules.Where(r => rawRules.Contains(r.Raw)); - - private List CreateNewRules(IQueryable existingRules) => - rawRules.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList(); - - private void AddSnapshotRules() => + private void AddSnapshotRules() + { + if (snapshot.AddedSnapshotRules == null) + snapshot.AddedSnapshotRules = new List(); snapshot.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r})); + } } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/SnapshotDe.cs b/src/FilterLists.Services/Snapshot/SnapshotDe.cs index 1d94a8062..b661d4bd1 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotDe.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotDe.cs @@ -7,8 +7,8 @@ using System.Threading.Tasks; using FilterLists.Data; using FilterLists.Data.Entities.Junctions; +using FilterLists.Services.Extensions; using FilterLists.Services.Snapshot.Models; -using MoreLinq; namespace FilterLists.Services.Snapshot { @@ -28,14 +28,10 @@ public SnapshotDe(FilterListsDbContext dbContext, EmailService emailService, Fil this.dbContext = dbContext; this.emailService = emailService; this.list = list; - snapshot = new Data.Entities.Snapshot - { - FilterListId = list.Id, - AddedSnapshotRules = new List() - }; + snapshot = new Data.Entities.Snapshot {FilterListId = list.Id}; } - public async Task SaveAsync() + public async Task SaveSnapshotAsync() { using (var transaction = dbContext.Database.BeginTransaction()) { @@ -125,26 +121,26 @@ private async Task SendExceptionEmail(Exception e) private async Task SaveSnapshotInBatches(string content) { - var rawRules = ParseRawRules(content); - var snapshotBatches = CreateSnapshotBatches(rawRules); + var rawRules = GetRawRules(content); + var snapshotBatches = GetSnapshotBatches(rawRules); await SaveSnapshotBatches(snapshotBatches); } - private static IEnumerable ParseRawRules(string content) + private static IEnumerable GetRawRules(string content) { var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); for (var i = 0; i < rawRules.Length; i++) rawRules[i] = rawRules[i].LintRawRule(); - return rawRules.Where(r => r != null); + return new HashSet(rawRules.Where(r => r != null)); } - private IEnumerable CreateSnapshotBatches(IEnumerable rawRules) => - rawRules.Batch(BatchSize).Select(b => new SnapshotDeBatch(dbContext, snapshot, b)); + private IEnumerable GetSnapshotBatches(IEnumerable rawRules) => + rawRules.GetBatches(BatchSize).Select(b => new SnapshotBatchDe(dbContext, snapshot, b)); - private static async Task SaveSnapshotBatches(IEnumerable snapshotBatches) + private static async Task SaveSnapshotBatches(IEnumerable snapshotBatches) { foreach (var snapshotBatch in snapshotBatches) - await snapshotBatch.SaveAsync(); + await snapshotBatch.SaveSnapshotBatchAsync(); } private async Task DedupSnapshotRules() diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs index c53959a93..870d8ffe7 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotService.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs @@ -69,7 +69,7 @@ private IEnumerable CreateSnapshots(IEnumerable snapshots) { foreach (var snapshot in snapshots) - await snapshot.SaveAsync(); + await snapshot.SaveSnapshotAsync(); } } } \ No newline at end of file