diff --git a/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs b/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs index f2b857cd3..1d4f66932 100644 --- a/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs +++ b/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs @@ -2,7 +2,7 @@ { public static class RawRuleLinterExtensions { - public static string LintStringForMySql(this string rule) + public static string LintRawRule(this string rule) { rule = rule?.TrimLeadingAndTrailingWhitespace(); rule = rule?.DropIfEmpty(); diff --git a/src/FilterLists.Services/SnapshotService/SnapshotBatchDe.cs b/src/FilterLists.Services/SnapshotService/SnapshotBatchDe.cs index 80cb4dcbb..09d0e956f 100644 --- a/src/FilterLists.Services/SnapshotService/SnapshotBatchDe.cs +++ b/src/FilterLists.Services/SnapshotService/SnapshotBatchDe.cs @@ -13,8 +13,7 @@ public class SnapshotBatchDe private readonly FilterListsDbContext dbContext; private readonly IEnumerable rawRules; private readonly Snapshot snapshot; - private IEnumerable newSnapshotRules; - private IEnumerable preExistingSnapshotRules; + private IQueryable rules; public SnapshotBatchDe(FilterListsDbContext dbContext, Snapshot snapshot, IEnumerable rawRules) { @@ -23,31 +22,26 @@ public SnapshotBatchDe(FilterListsDbContext dbContext, Snapshot snapshot, IEnume this.rawRules = rawRules; } - //TODO: support de-duplicated SnapshotRule public async Task SaveSnapshotBatchAsync() { - AddNewRules(); + AddRules(); AddSnapshotRules(); await dbContext.SaveChangesAsync(); } - private void AddNewRules() + private void AddRules() { - preExistingSnapshotRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw)); - var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw)); - newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule}) - .ToList(); - dbContext.Rules.AddRange(newSnapshotRules); + var existingRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw)); + var newRawRules = rawRules.Except(existingRules.Select(x => x.Raw)); + var newRules = newRawRules.Select(newRawRule => new Rule {Raw = newRawRule}).ToList(); + dbContext.Rules.AddRange(newRules); + rules = existingRules.Concat(newRules); } private void AddSnapshotRules() { - if (snapshot.AddedSnapshotRules == null) - snapshot.AddedSnapshotRules = new List(); - snapshot.AddedSnapshotRules.AddRange(preExistingSnapshotRules - .Concat(newSnapshotRules) - .Select(rule => new SnapshotRule {Rule = rule, AddedBySnapshot = snapshot}) - .ToList()); + var snapshotRules = rules.Select(rule => new SnapshotRule {Rule = rule, AddedBySnapshot = snapshot}); + snapshot.AddedSnapshotRules.AddRange(snapshotRules); } } } \ No newline at end of file diff --git a/src/FilterLists.Services/SnapshotService/SnapshotDe.cs b/src/FilterLists.Services/SnapshotService/SnapshotDe.cs index 07906e849..a9a445fe0 100644 --- a/src/FilterLists.Services/SnapshotService/SnapshotDe.cs +++ b/src/FilterLists.Services/SnapshotService/SnapshotDe.cs @@ -24,12 +24,21 @@ public SnapshotDe(FilterListsDbContext dbContext, FilterListViewUrlDto list) } public async Task SaveSnapshotAsync() + { + var content = await CaptureSnapshot(); + if (content != null) + { + await SaveSnapshotInBatches(content); + await DedupSnapshotRules(); + } + } + + private async Task CaptureSnapshot() { await AddSnapshot(); var content = await TryGetContent(); await dbContext.SaveChangesAsync(); - if (content != null) - await SaveSnapshotInBatches(content); + return content; } private async Task AddSnapshot() @@ -80,7 +89,7 @@ private static IEnumerable GetRawRules(string content) { var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); for (var i = 0; i < rawRules.Length; i++) - rawRules[i] = rawRules[i].LintStringForMySql(); + rawRules[i] = rawRules[i].LintRawRule(); return new HashSet(rawRules.Where(x => x != null)); } @@ -95,5 +104,25 @@ private static async Task SaveSnapshotBatches(IEnumerable snaps foreach (var snapshotBatch in snapshotBatches) await snapshotBatch.SaveSnapshotBatchAsync(); } + + //TODO: test + private async Task DedupSnapshotRules() + { + var currentSnapshotRules = + dbContext.SnapshotRules.Where(x => x.AddedBySnapshot == snapshot).Select(x => x.Rule); + + var existingSnapshotRules = dbContext.SnapshotRules.Where(x => + x.AddedBySnapshot.FilterListId == list.Id && + x.AddedBySnapshot != snapshot && + x.RemovedBySnapshot == null); + + var removedSnapshotRules = existingSnapshotRules.Where(x => !currentSnapshotRules.Contains(x.Rule)); + + removedSnapshotRules.ToList().ForEach(x => x.RemovedBySnapshot = snapshot); + + //TODO: delete any added SnapshotRules that were already in a previous snapshot and were not marked as removed + + await dbContext.SaveChangesAsync(); + } } } \ No newline at end of file diff --git a/src/FilterLists.Services/SnapshotService/SnapshotService.cs b/src/FilterLists.Services/SnapshotService/SnapshotService.cs index cb330b538..768031648 100644 --- a/src/FilterLists.Services/SnapshotService/SnapshotService.cs +++ b/src/FilterLists.Services/SnapshotService/SnapshotService.cs @@ -18,6 +18,7 @@ public SnapshotService(FilterListsDbContext dbContext) public async Task CaptureSnapshotsAsync(int batchSize) { + //TODO: rollback changes from recent snapshots if they were interrupted var lists = await GetLeastRecentlyCapturedLists(batchSize); var snapshots = GetSnapshots(lists); await SaveSnapshots(snapshots);