From 46dffec60a47c4d75f5b31c217b71e67965ea7e8 Mon Sep 17 00:00:00 2001 From: Collin Barrett Date: Wed, 15 Aug 2018 06:04:44 -0500 Subject: [PATCH] misc SnapshotService cleanup ref #343 --- .../Extensions/CollectionExtensions.cs | 17 +++++++++ .../Extensions/EnumerableExtensions.cs | 29 --------------- .../FilterLists.Services.csproj | 1 + .../Snapshot/RawRuleLinterExtensions.cs | 35 ++++++++++--------- .../Snapshot/SnapshotDe.cs | 26 ++++++++------ ...{SnapshotBatchDe.cs => SnapshotDeBatch.cs} | 24 ++++++------- .../Snapshot/SnapshotService.cs | 2 +- 7 files changed, 65 insertions(+), 69 deletions(-) create mode 100644 src/FilterLists.Services/Extensions/CollectionExtensions.cs delete mode 100644 src/FilterLists.Services/Extensions/EnumerableExtensions.cs rename src/FilterLists.Services/Snapshot/{SnapshotBatchDe.cs => SnapshotDeBatch.cs} (63%) diff --git a/src/FilterLists.Services/Extensions/CollectionExtensions.cs b/src/FilterLists.Services/Extensions/CollectionExtensions.cs new file mode 100644 index 000000000..81957558d --- /dev/null +++ b/src/FilterLists.Services/Extensions/CollectionExtensions.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; + +namespace FilterLists.Services.Extensions +{ + public static class CollectionExtensions + { + //https://stackoverflow.com/a/26360010/2343739 + public static void AddRange(this ICollection destination, IEnumerable source) + { + if (destination is List list) + list.AddRange(source); + else + foreach (var item in source) + destination.Add(item); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/Extensions/EnumerableExtensions.cs b/src/FilterLists.Services/Extensions/EnumerableExtensions.cs deleted file mode 100644 index d6594d702..000000000 --- a/src/FilterLists.Services/Extensions/EnumerableExtensions.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System.Collections.Generic; - -namespace FilterLists.Services.Extensions -{ - public static class EnumerableExtensions - { - public static void AddRange(this ICollection destination, IEnumerable source) - { - foreach (var item in source) - destination.Add(item); - } - - public static IEnumerable> GetBatches(this IEnumerable source, int batchSize) - { - var batches = new List(batchSize); - foreach (var item in source) - { - batches.Add(item); - if (batches.Count != batchSize) - continue; - yield return batches.AsReadOnly(); - batches = new List(batchSize); - } - - if (batches.Count > 0) - yield return batches.AsReadOnly(); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/FilterLists.Services.csproj b/src/FilterLists.Services/FilterLists.Services.csproj index 6eb37a229..b5df087db 100644 --- a/src/FilterLists.Services/FilterLists.Services.csproj +++ b/src/FilterLists.Services/FilterLists.Services.csproj @@ -20,6 +20,7 @@ + diff --git a/src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs b/src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs index cfe0c95e9..cd4a28d4f 100644 --- a/src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs +++ b/src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs @@ -1,35 +1,38 @@ -namespace FilterLists.Services.Snapshot +using System; + +namespace FilterLists.Services.Snapshot { public static class RawRuleLinterExtensions { public static string LintRawRule(this string rule) { - rule = rule?.TrimLeadingAndTrailingWhitespace(); - rule = rule?.DropIfEmpty(); - rule = rule?.DropIfComment(); + rule = rule.Trim(); + rule = rule.DropIfEmpty(); rule = rule?.DropIfTooLong(); + rule = rule?.DropIfComment(); rule = rule?.DropIfContainsBackslashSingleQuote(); rule = rule?.TrimSingleBackslashFromEnd(); return rule; } - private static string TrimLeadingAndTrailingWhitespace(this string rule) - { - char[] charsToTrim = {' ', '\t'}; - return rule.Trim(charsToTrim); - } - - private static string DropIfEmpty(this string rule) => rule == "" ? null : rule; - + private static string DropIfEmpty(this string rule) => + rule == string.Empty ? null : rule; + + private static string DropIfTooLong(this string rule) => + rule.Length > 8192 ? null : rule; + private static string DropIfComment(this string rule) => - rule.StartsWith(@"!") && !rule.StartsWith(@"!#") || rule.StartsWith(@"!##") ? null : rule; - - private static string DropIfTooLong(this string rule) => rule.Length > 8192 ? null : rule; + rule.StartsWith(@"!", StringComparison.Ordinal) && !rule.StartsWith(@"!#", StringComparison.Ordinal) || + rule.StartsWith(@"!##", StringComparison.Ordinal) + ? null + : rule; private static string DropIfContainsBackslashSingleQuote(this string rule) => rule.Contains(@"\'") ? null : rule; private static string TrimSingleBackslashFromEnd(this string rule) => - rule.EndsWith(@"\") && !rule.EndsWith(@"\\") ? rule.Remove(rule.Length - 1) : rule; + rule.EndsWith(@"\", StringComparison.Ordinal) && !rule.EndsWith(@"\\", StringComparison.Ordinal) + ? rule.Remove(rule.Length - 1) + : rule; } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/SnapshotDe.cs b/src/FilterLists.Services/Snapshot/SnapshotDe.cs index b661d4bd1..1d94a8062 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotDe.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotDe.cs @@ -7,8 +7,8 @@ using System.Threading.Tasks; using FilterLists.Data; using FilterLists.Data.Entities.Junctions; -using FilterLists.Services.Extensions; using FilterLists.Services.Snapshot.Models; +using MoreLinq; namespace FilterLists.Services.Snapshot { @@ -28,10 +28,14 @@ public SnapshotDe(FilterListsDbContext dbContext, EmailService emailService, Fil this.dbContext = dbContext; this.emailService = emailService; this.list = list; - snapshot = new Data.Entities.Snapshot {FilterListId = list.Id}; + snapshot = new Data.Entities.Snapshot + { + FilterListId = list.Id, + AddedSnapshotRules = new List() + }; } - public async Task SaveSnapshotAsync() + public async Task SaveAsync() { using (var transaction = dbContext.Database.BeginTransaction()) { @@ -121,26 +125,26 @@ private async Task SendExceptionEmail(Exception e) private async Task SaveSnapshotInBatches(string content) { - var rawRules = GetRawRules(content); - var snapshotBatches = GetSnapshotBatches(rawRules); + var rawRules = ParseRawRules(content); + var snapshotBatches = CreateSnapshotBatches(rawRules); await SaveSnapshotBatches(snapshotBatches); } - private static IEnumerable GetRawRules(string content) + private static IEnumerable ParseRawRules(string content) { var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); for (var i = 0; i < rawRules.Length; i++) rawRules[i] = rawRules[i].LintRawRule(); - return new HashSet(rawRules.Where(r => r != null)); + return rawRules.Where(r => r != null); } - private IEnumerable GetSnapshotBatches(IEnumerable rawRules) => - rawRules.GetBatches(BatchSize).Select(b => new SnapshotBatchDe(dbContext, snapshot, b)); + private IEnumerable CreateSnapshotBatches(IEnumerable rawRules) => + rawRules.Batch(BatchSize).Select(b => new SnapshotDeBatch(dbContext, snapshot, b)); - private static async Task SaveSnapshotBatches(IEnumerable snapshotBatches) + private static async Task SaveSnapshotBatches(IEnumerable snapshotBatches) { foreach (var snapshotBatch in snapshotBatches) - await snapshotBatch.SaveSnapshotBatchAsync(); + await snapshotBatch.SaveAsync(); } private async Task DedupSnapshotRules() diff --git a/src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs b/src/FilterLists.Services/Snapshot/SnapshotDeBatch.cs similarity index 63% rename from src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs rename to src/FilterLists.Services/Snapshot/SnapshotDeBatch.cs index 555f5bc5c..13e35cbb0 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotDeBatch.cs @@ -8,14 +8,14 @@ namespace FilterLists.Services.Snapshot { - public class SnapshotBatchDe + public class SnapshotDeBatch { private readonly FilterListsDbContext dbContext; private readonly IEnumerable rawRules; private readonly Data.Entities.Snapshot snapshot; - private IQueryable rules; + private IEnumerable rules; - public SnapshotBatchDe(FilterListsDbContext dbContext, Data.Entities.Snapshot snapshot, + public SnapshotDeBatch(FilterListsDbContext dbContext, Data.Entities.Snapshot snapshot, IEnumerable rawRules) { this.dbContext = dbContext; @@ -23,7 +23,7 @@ public SnapshotBatchDe(FilterListsDbContext dbContext, Data.Entities.Snapshot sn this.rawRules = rawRules; } - public async Task SaveSnapshotBatchAsync() + public async Task SaveAsync() { await AddRules(); AddSnapshotRules(); @@ -32,18 +32,18 @@ public async Task SaveSnapshotBatchAsync() private async Task AddRules() { - var existingRules = dbContext.Rules.Where(r => rawRules.Contains(r.Raw)); - var newRawRules = rawRules.Except(existingRules.Select(r => r.Raw)); - var newRules = newRawRules.Select(r => new Rule {Raw = r}).ToList(); + var existingRules = GetExistingRules(); + var newRules = CreateNewRules(existingRules); rules = existingRules.Concat(newRules); await dbContext.Rules.AddRangeAsync(newRules); } - private void AddSnapshotRules() - { - if (snapshot.AddedSnapshotRules == null) - snapshot.AddedSnapshotRules = new List(); + private IQueryable GetExistingRules() => dbContext.Rules.Where(r => rawRules.Contains(r.Raw)); + + private List CreateNewRules(IQueryable existingRules) => + rawRules.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList(); + + private void AddSnapshotRules() => snapshot.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r})); - } } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs index 870d8ffe7..c53959a93 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotService.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs @@ -69,7 +69,7 @@ private IEnumerable CreateSnapshots(IEnumerable snapshots) { foreach (var snapshot in snapshots) - await snapshot.SaveSnapshotAsync(); + await snapshot.SaveAsync(); } } } \ No newline at end of file