From 0ab08ecc421421a4535d084b01af64732b6676da Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Mon, 5 Feb 2018 11:14:46 -0600 Subject: [PATCH] move towards batched snapshots (wip) --- .../V1/Controllers/SnapshotController.cs | 1 + .../RawRuleLinterExtensions.cs | 3 +- .../SnapshotService/SnapshotBatchDe.cs | 49 +++++++++++++++ ...{SnapshotDomainEntity.cs => SnapshotDe.cs} | 60 +++++++++---------- .../SnapshotService/SnapshotService.cs | 41 ++++--------- 5 files changed, 93 insertions(+), 61 deletions(-) create mode 100644 src/FilterLists.Services/SnapshotService/SnapshotBatchDe.cs rename src/FilterLists.Services/SnapshotService/{SnapshotDomainEntity.cs => SnapshotDe.cs} (50%) diff --git a/src/FilterLists.Api/V1/Controllers/SnapshotController.cs b/src/FilterLists.Api/V1/Controllers/SnapshotController.cs index 8caa549f6..d51d9fc93 100644 --- a/src/FilterLists.Api/V1/Controllers/SnapshotController.cs +++ b/src/FilterLists.Api/V1/Controllers/SnapshotController.cs @@ -14,6 +14,7 @@ public SnapshotController(SnapshotService snapshotService) } [HttpPost] + //TODO: convert API endpoint to server-side scheduled job public async Task Index() { #if DEBUG diff --git a/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs b/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs index d11a4fdbc..c70e73521 100644 --- a/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs +++ b/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs @@ -2,6 +2,7 @@ { public static class RawRuleLinterExtensions { + //TODO: resolve issues and/or track dropped rules public static string LintStringForMySql(this string rule) { rule = rule.TrimSingleBackslashFromEnd(); @@ -19,7 +20,6 @@ private static string TrimSingleBackslashFromEnd(this string rule) private static string DropIfContainsBackslashSingleQuote(this string rule) { - //TODO: resolve issue and/or track dropped rule if (rule != null) return rule.Contains(@"\'") ? null : rule; return null; @@ -27,7 +27,6 @@ private static string DropIfContainsBackslashSingleQuote(this string rule) private static string DropIfTooLong(this string rule) { - //TODO: resolve issue and/or track dropped rule return rule?.Length > 8192 ? null : rule; } } diff --git a/src/FilterLists.Services/SnapshotService/SnapshotBatchDe.cs b/src/FilterLists.Services/SnapshotService/SnapshotBatchDe.cs new file mode 100644 index 000000000..d6e6512af --- /dev/null +++ b/src/FilterLists.Services/SnapshotService/SnapshotBatchDe.cs @@ -0,0 +1,49 @@ +using System.Collections.Generic; +using System.Linq; +using FilterLists.Data; +using FilterLists.Data.Entities; +using FilterLists.Data.Entities.Junctions; +using Microsoft.EntityFrameworkCore; + +namespace FilterLists.Services.SnapshotService +{ + public class SnapshotBatchDe + { + private readonly FilterListsDbContext dbContext; + private readonly IEnumerable rawRules; + private readonly Snapshot snapshot; + private IEnumerable newSnapshotRules; + private IEnumerable preExistingSnapshotRules; + + public SnapshotBatchDe(FilterListsDbContext dbContext, Snapshot snapshot, IEnumerable rawRules) + { + this.dbContext = dbContext; + this.snapshot = snapshot; + this.rawRules = rawRules; + } + + public void SaveSnapshotBatch() + { + AddNewRules(); + AddSnapshotRules(); + dbContext.SaveChanges(); + } + + private void AddNewRules() + { + preExistingSnapshotRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw)).AsNoTracking(); + var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw)); + newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule}) + .ToList(); + dbContext.Rules.AddRange(newSnapshotRules); + } + + private void AddSnapshotRules() + { + snapshot.SnapshotRules = preExistingSnapshotRules + .Concat(newSnapshotRules) + .Select(rule => new SnapshotRule {Rule = rule, Snapshot = snapshot}) + .ToList(); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/SnapshotService/SnapshotDomainEntity.cs b/src/FilterLists.Services/SnapshotService/SnapshotDe.cs similarity index 50% rename from src/FilterLists.Services/SnapshotService/SnapshotDomainEntity.cs rename to src/FilterLists.Services/SnapshotService/SnapshotDe.cs index 950247364..f7e5071f9 100644 --- a/src/FilterLists.Services/SnapshotService/SnapshotDomainEntity.cs +++ b/src/FilterLists.Services/SnapshotService/SnapshotDe.cs @@ -6,43 +6,35 @@ using System.Threading.Tasks; using FilterLists.Data; using FilterLists.Data.Entities; -using FilterLists.Data.Entities.Junctions; namespace FilterLists.Services.SnapshotService { - public class SnapshotDomainEntity + public class SnapshotDe { + private const int BatchSize = 100; private readonly FilterListsDbContext dbContext; private readonly FilterListViewUrlDto list; - private IEnumerable newSnapshotRules; - private IEnumerable preExistingSnapshotRules; - private List rawRules; private Snapshot snapshot; - public SnapshotDomainEntity(FilterListsDbContext dbContext, FilterListViewUrlDto list) + public SnapshotDe(FilterListsDbContext dbContext, FilterListViewUrlDto list) { this.dbContext = dbContext; this.list = list; } - public async Task SaveSnapshot() + public async Task SaveSnapshotAsync() { await AddSnapshot(); var content = await TryGetContent(); - if (content != null) - { - ParseRawRules(content); - await AddNewRules(); - AddSnapshotRules(); - } - await dbContext.SaveChangesAsync(); + if (content != null) + SaveSnapshotInBatches(content); } private async Task AddSnapshot() { - snapshot = new Snapshot {FilterList = await dbContext.FilterLists.FindAsync(list.Id)}; - dbContext.Snapshots.Add(snapshot); + snapshot = new Snapshot {FilterListId = list.Id}; + await dbContext.Snapshots.AddAsync(snapshot); } private async Task TryGetContent() @@ -71,29 +63,37 @@ private async Task GetContent() return null; } - private void ParseRawRules(string content) + private void SaveSnapshotInBatches(string content) { - rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList(); + var rawRules = GetRawRules(content); + var snapshotBatches = GetSnapshotBatches(rawRules); + SaveSnapshotBatches(snapshotBatches); + } + + private static List GetRawRules(string content) + { + var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList(); for (var i = 0; i < rawRules.Count; i++) rawRules[i] = rawRules[i].LintStringForMySql(); - rawRules = rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList(); + return rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList(); } - private async Task AddNewRules() + private IEnumerable GetSnapshotBatches(List rawRules) { - preExistingSnapshotRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw)); - var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw)); - newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule}) - .ToList(); - await dbContext.Rules.AddRangeAsync(newSnapshotRules); + var rawRuleBatches = GetRawRuleBatches(rawRules); + return rawRuleBatches.Select(rawRuleBatch => new SnapshotBatchDe(dbContext, snapshot, rawRuleBatch)); } - private void AddSnapshotRules() + public static IEnumerable> GetRawRuleBatches(List rawRules) { - snapshot.SnapshotRules = preExistingSnapshotRules - .Concat(newSnapshotRules) - .Select(rule => new SnapshotRule {Rule = rule, Snapshot = snapshot}) - .ToList(); + for (var i = 0; i < rawRules.Count; i += BatchSize) + yield return rawRules.GetRange(i, Math.Min(BatchSize, rawRules.Count - i)); + } + + private static void SaveSnapshotBatches(IEnumerable snapshotBatches) + { + foreach (var snapshotBatch in snapshotBatches) + snapshotBatch.SaveSnapshotBatch(); } } } \ No newline at end of file diff --git a/src/FilterLists.Services/SnapshotService/SnapshotService.cs b/src/FilterLists.Services/SnapshotService/SnapshotService.cs index f61f32cdf..cb330b538 100644 --- a/src/FilterLists.Services/SnapshotService/SnapshotService.cs +++ b/src/FilterLists.Services/SnapshotService/SnapshotService.cs @@ -16,53 +16,36 @@ public SnapshotService(FilterListsDbContext dbContext) this.dbContext = dbContext; } - //TODO: call via scheduled job public async Task CaptureSnapshotsAsync(int batchSize) { - var lists = await GetNextListsToCapture(batchSize); + var lists = await GetLeastRecentlyCapturedLists(batchSize); var snapshots = GetSnapshots(lists); await SaveSnapshots(snapshots); } - private async Task> GetNextListsToCapture(int batchSize) - { - var neverCapturedLists = await GetNeverCapturedLists(batchSize); - if (neverCapturedLists.Count == batchSize) - return neverCapturedLists; - var leastRecentlyCapturedLists = await GetLeastRecentlyCapturedLists(batchSize - neverCapturedLists.Count); - return neverCapturedLists.Concat(leastRecentlyCapturedLists); - } - - private async Task> GetNeverCapturedLists(int batchSize) + private async Task> GetLeastRecentlyCapturedLists(int batchSize) { return await dbContext.FilterLists - .Where(x => x.Snapshots.Count == 0) + .OrderBy(x => x.Snapshots.Any()) + .ThenBy(x => x.Snapshots + .Select(y => y.CreatedDateUtc) + .OrderByDescending(y => y) + .FirstOrDefault()) .Take(batchSize) + .AsNoTracking() .ProjectTo() .ToListAsync(); } - private async Task> GetLeastRecentlyCapturedLists(int batchSize) + private IEnumerable GetSnapshots(IEnumerable lists) { - return await dbContext.Snapshots - .GroupBy(x => x.FilterList) - .Select(x => x.OrderByDescending(y => y.CreatedDateUtc).First()) - .OrderBy(x => x.CreatedDateUtc) - .Select(x => x.FilterList) - .Take(batchSize) - .ProjectTo() - .ToListAsync(); + return lists.Select(list => new SnapshotDe(dbContext, list)); } - private IEnumerable GetSnapshots(IEnumerable lists) - { - return lists.Select(list => new SnapshotDomainEntity(dbContext, list)); - } - - private static async Task SaveSnapshots(IEnumerable snapshots) + private static async Task SaveSnapshots(IEnumerable snapshots) { foreach (var snapshot in snapshots) - await snapshot.SaveSnapshot(); + await snapshot.SaveSnapshotAsync(); } } } \ No newline at end of file