prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

minor refactors

Browse source
This commit is contained in:
Collin M. Barrett 2018-02-10 20:01:08 -06:00
parent c75e2700a1
commit bca4baffd4
3 changed files with 36 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/FilterLists.Services/SnapshotService/SnapshotBatchDe.cs
View file

@ -32,7 +32,7 @@ public async Task SaveSnapshotBatchAsync()
private void AddRules()
{
var existingRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw));
var newRawRules = rawRules.Except(existingRules.Select(x => x.Raw));
var newRawRules = rawRules.Except(existingRules.Select(r => r.Raw));
var newRules = newRawRules.Select(newRawRule => new Rule {Raw = newRawRule}).ToList();
dbContext.Rules.AddRange(newRules);
rules = existingRules.Concat(newRules);

23
src/FilterLists.Services/SnapshotService/SnapshotDe.cs
View file

@ -93,7 +93,7 @@ private static IEnumerable<string> GetRawRules(string content)
var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < rawRules.Length; i++)
rawRules[i] = rawRules[i].LintRawRule();
return new HashSet<string>(rawRules.Where(x => x != null));
return new HashSet<string>(rawRules.Where(rr => rr != null));
}
private IEnumerable<SnapshotBatchDe> GetSnapshotBatches(IEnumerable<string> rawRules)
@ -118,24 +118,25 @@ private async Task DedupSnapshotRules()
private IQueryable<SnapshotRule> GetExistingSnapshotRules()
{
return dbContext.SnapshotRules.Where(x =>
x.AddedBySnapshot.FilterListId == list.Id &&
x.AddedBySnapshot != snapshot &&
x.RemovedBySnapshot == null);
return dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot.FilterListId == list.Id &&
sr.AddedBySnapshot != snapshot &&
sr.RemovedBySnapshot == null);
}
private void UpdateRemovedSnapshotRules(IQueryable<SnapshotRule> existingSnapshotRules)
{
var newSnapshotRules = dbContext.SnapshotRules.Where(x => x.AddedBySnapshot == snapshot);
var removedSnapshotRules = existingSnapshotRules.Where(x => !newSnapshotRules.Any(y => y.Rule == x.Rule));
removedSnapshotRules.ToList().ForEach(x => x.RemovedBySnapshot = snapshot);
var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapshot);
var removedSnapshotRules =
existingSnapshotRules.Where(sr => !newSnapshotRules.Any(nsr => nsr.Rule == sr.Rule));
removedSnapshotRules.ToList().ForEach(sr => sr.RemovedBySnapshot = snapshot);
}
private void RemoveDuplicateSnapshotRules(IQueryable<SnapshotRule> existingSnapshotRules)
{
var duplicateSnapshotRules = dbContext.SnapshotRules.Where(x =>
x.AddedBySnapshot == snapshot &&
existingSnapshotRules.Any(y => y.Rule == x.Rule));
var duplicateSnapshotRules = dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot == snapshot &&
existingSnapshotRules.Any(esr => esr.Rule == sr.Rule));
dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules);
}

42
src/FilterLists.Services/SnapshotService/SnapshotService.cs
View file

@ -19,35 +19,39 @@ public SnapshotService(FilterListsDbContext dbContext)
public async Task CaptureAsync(int batchSize)
{
RollbackIncomplete();
var lists = await GetLeastRecentlyCapturedLists(batchSize);
RollbackIncompletedSnapshot();
var lists = await GetListsToCapture(batchSize);
var snapshots = GetSnapshots(lists);
await SaveSnapshots(snapshots);
}
private void RollbackIncomplete()
private void RollbackIncompletedSnapshot()
{
var incompleteSnapshots = dbContext.Snapshots.Where(x => x.IsCompleted == false);
var incompleteSnapshots = dbContext.Snapshots.Where(ss => ss.IsCompleted == false);
dbContext.Snapshots.RemoveRange(incompleteSnapshots);
//TODO: don't assume that SnapshotDe.DedupSnapshotRules() didn't partially complete
}
private async Task<IEnumerable<FilterListViewUrlDto>> GetLeastRecentlyCapturedLists(int batchSize)
private async Task<IEnumerable<FilterListViewUrlDto>> GetListsToCapture(int batchSize)
{
return await dbContext.FilterLists.OrderBy(x => x.Snapshots.Any())
.ThenBy(x =>
x.Snapshots.Select(y => y.CreatedDateUtc)
.OrderByDescending(y => y)
.FirstOrDefault())
.Where(x => !x.Snapshots.Any() ||
x.Snapshots.Select(y => y.CreatedDateUtc)
.OrderByDescending(y => y)
.FirstOrDefault() <
DateTime.UtcNow.AddDays(-1))
.Take(batchSize)
.AsNoTracking()
.ProjectTo<FilterListViewUrlDto>()
.ToListAsync();
return await dbContext
.FilterLists
.Where(list =>
!list.Snapshots.Any() ||
list.Snapshots
.Select(ss => ss.CreatedDateUtc)
.OrderByDescending(sscd => sscd)
.FirstOrDefault() < DateTime.UtcNow.AddDays(-1))
.OrderBy(list => list.Snapshots.Any())
.ThenBy(list =>
list.Snapshots
.Select(ss => ss.CreatedDateUtc)
.OrderByDescending(sscd => sscd)
.FirstOrDefault())
.Take(batchSize)
.AsNoTracking()
.ProjectTo<FilterListViewUrlDto>()
.ToListAsync();
}
private IEnumerable<SnapshotDe> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)