prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip dedup SnapshotRules

Browse source
This commit is contained in:
Collin Barrett 2018-02-10 09:52:29 -06:00
parent ba930a8211
commit b6aef2ef79
4 changed files with 44 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs
View file

@ -2,7 +2,7 @@
{
public static class RawRuleLinterExtensions
{
public static string LintStringForMySql(this string rule)
public static string LintRawRule(this string rule)
{
rule = rule?.TrimLeadingAndTrailingWhitespace();
rule = rule?.DropIfEmpty();

26
src/FilterLists.Services/SnapshotService/SnapshotBatchDe.cs
View file

@ -13,8 +13,7 @@ public class SnapshotBatchDe
private readonly FilterListsDbContext dbContext;
private readonly IEnumerable<string> rawRules;
private readonly Snapshot snapshot;
private IEnumerable<Rule> newSnapshotRules;
private IEnumerable<Rule> preExistingSnapshotRules;
private IQueryable<Rule> rules;
public SnapshotBatchDe(FilterListsDbContext dbContext, Snapshot snapshot, IEnumerable<string> rawRules)
{
@ -23,31 +22,26 @@ public SnapshotBatchDe(FilterListsDbContext dbContext, Snapshot snapshot, IEnume
this.rawRules = rawRules;
}
//TODO: support de-duplicated SnapshotRule
public async Task SaveSnapshotBatchAsync()
{
AddNewRules();
AddRules();
AddSnapshotRules();
await dbContext.SaveChangesAsync();
}
private void AddNewRules()
private void AddRules()
{
preExistingSnapshotRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw));
var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw));
newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule})
.ToList();
dbContext.Rules.AddRange(newSnapshotRules);
var existingRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw));
var newRawRules = rawRules.Except(existingRules.Select(x => x.Raw));
var newRules = newRawRules.Select(newRawRule => new Rule {Raw = newRawRule}).ToList();
dbContext.Rules.AddRange(newRules);
rules = existingRules.Concat(newRules);
}
private void AddSnapshotRules()
{
if (snapshot.AddedSnapshotRules == null)
snapshot.AddedSnapshotRules = new List<SnapshotRule>();
snapshot.AddedSnapshotRules.AddRange(preExistingSnapshotRules
.Concat(newSnapshotRules)
.Select(rule => new SnapshotRule {Rule = rule, AddedBySnapshot = snapshot})
.ToList());
var snapshotRules = rules.Select(rule => new SnapshotRule {Rule = rule, AddedBySnapshot = snapshot});
snapshot.AddedSnapshotRules.AddRange(snapshotRules);
}
}
}

35
src/FilterLists.Services/SnapshotService/SnapshotDe.cs
View file

@ -24,12 +24,21 @@ public SnapshotDe(FilterListsDbContext dbContext, FilterListViewUrlDto list)
}
public async Task SaveSnapshotAsync()
{
var content = await CaptureSnapshot();
if (content != null)
{
await SaveSnapshotInBatches(content);
await DedupSnapshotRules();
}
}
private async Task<string> CaptureSnapshot()
{
await AddSnapshot();
var content = await TryGetContent();
await dbContext.SaveChangesAsync();
if (content != null)
await SaveSnapshotInBatches(content);
return content;
}
private async Task AddSnapshot()
@ -80,7 +89,7 @@ private static IEnumerable<string> GetRawRules(string content)
{
var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < rawRules.Length; i++)
rawRules[i] = rawRules[i].LintStringForMySql();
rawRules[i] = rawRules[i].LintRawRule();
return new HashSet<string>(rawRules.Where(x => x != null));
}
@ -95,5 +104,25 @@ private static async Task SaveSnapshotBatches(IEnumerable<SnapshotBatchDe> snaps
foreach (var snapshotBatch in snapshotBatches)
await snapshotBatch.SaveSnapshotBatchAsync();
}
//TODO: test
private async Task DedupSnapshotRules()
{
var currentSnapshotRules =
dbContext.SnapshotRules.Where(x => x.AddedBySnapshot == snapshot).Select(x => x.Rule);
var existingSnapshotRules = dbContext.SnapshotRules.Where(x =>
x.AddedBySnapshot.FilterListId == list.Id &&
x.AddedBySnapshot != snapshot &&
x.RemovedBySnapshot == null);
var removedSnapshotRules = existingSnapshotRules.Where(x => !currentSnapshotRules.Contains(x.Rule));
removedSnapshotRules.ToList().ForEach(x => x.RemovedBySnapshot = snapshot);
//TODO: delete any added SnapshotRules that were already in a previous snapshot and were not marked as removed
await dbContext.SaveChangesAsync();
}
}
}

1
src/FilterLists.Services/SnapshotService/SnapshotService.cs
View file

@ -18,6 +18,7 @@ public SnapshotService(FilterListsDbContext dbContext)
public async Task CaptureSnapshotsAsync(int batchSize)
{
//TODO: rollback changes from recent snapshots if they were interrupted
var lists = await GetLeastRecentlyCapturedLists(batchSize);
var snapshots = GetSnapshots(lists);
await SaveSnapshots(snapshots);