diff --git a/src/FilterLists.Services/Extensions/CollectionExtensions.cs b/src/FilterLists.Services/Extensions/CollectionExtensions.cs deleted file mode 100644 index d80f7731f..000000000 --- a/src/FilterLists.Services/Extensions/CollectionExtensions.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System.Collections.Generic; - -namespace FilterLists.Services.Extensions -{ - public static class CollectionExtensions - { - public static void AddRange(this ICollection destination, IEnumerable source) - { - foreach (var item in source) - destination.Add(item); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/Extensions/EnumerableExtensions.cs b/src/FilterLists.Services/Extensions/EnumerableExtensions.cs new file mode 100644 index 000000000..b2b510006 --- /dev/null +++ b/src/FilterLists.Services/Extensions/EnumerableExtensions.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; + +namespace FilterLists.Services.Extensions +{ + public static class EnumerableExtensions + { + public static void AddRange(this ICollection destination, IEnumerable source) + { + foreach (var item in source) + destination.Add(item); + } + + public static IEnumerable> Batch(this IEnumerable source, int batchSize) + { + var batches = new List(batchSize); + foreach (var item in source) + { + batches.Add(item); + if (batches.Count != batchSize) + continue; + yield return batches.AsReadOnly(); + batches = new List(batchSize); + } + + if (batches.Count > 0) + yield return batches.AsReadOnly(); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs b/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs index fd1f01dc3..5ad81ddcf 100644 --- a/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs +++ b/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs @@ -6,6 +6,7 @@ public static class RawRuleLinterExtensions public static string LintStringForMySql(this string rule) { rule = rule.TrimLeadingAndTrailingWhitespace(); + rule = rule.DropIfEmpty(); rule = rule.TrimSingleBackslashFromEnd(); rule = rule.DropIfContainsBackslashSingleQuote(); rule = rule.DropIfTooLong(); @@ -18,6 +19,11 @@ private static string TrimLeadingAndTrailingWhitespace(this string rule) return rule.Trim(charsToTrim); } + private static string DropIfEmpty(this string rule) + { + return rule == "" ? null : rule; + } + private static string TrimSingleBackslashFromEnd(this string rule) { if (rule != null) diff --git a/src/FilterLists.Services/SnapshotService/SnapshotDe.cs b/src/FilterLists.Services/SnapshotService/SnapshotDe.cs index 19bd04d7c..9b8d40598 100644 --- a/src/FilterLists.Services/SnapshotService/SnapshotDe.cs +++ b/src/FilterLists.Services/SnapshotService/SnapshotDe.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using FilterLists.Data; using FilterLists.Data.Entities; +using FilterLists.Services.Extensions; namespace FilterLists.Services.SnapshotService { @@ -75,24 +76,18 @@ private void SaveSnapshotInBatches(string content) SaveSnapshotBatches(snapshotBatches); } - private static List GetRawRules(string content) + private static IEnumerable GetRawRules(string content) { - var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList(); - for (var i = 0; i < rawRules.Count; i++) + var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); + for (var i = 0; i < rawRules.Length; i++) rawRules[i] = rawRules[i].LintStringForMySql(); - return new List(new HashSet(rawRules.Where(x => !string.IsNullOrWhiteSpace(x)))); + return new HashSet(rawRules); } - private IEnumerable GetSnapshotBatches(List rawRules) + private IEnumerable GetSnapshotBatches(IEnumerable rawRules) { - var rawRuleBatches = GetRawRuleBatches(rawRules); - return rawRuleBatches.Select(rawRuleBatch => new SnapshotBatchDe(dbContext, snapshot, rawRuleBatch)); - } - - public static IEnumerable> GetRawRuleBatches(List rawRules) - { - for (var i = 0; i < rawRules.Count; i += BatchSize) - yield return rawRules.GetRange(i, Math.Min(BatchSize, rawRules.Count - i)); + return rawRules.Batch(BatchSize) + .Select(rawRuleBatch => new SnapshotBatchDe(dbContext, snapshot, rawRuleBatch)); } private static void SaveSnapshotBatches(IEnumerable snapshotBatches)