prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

try using EF transactions for snapshots

Browse source 
ref #343
This commit is contained in:
Collin M. Barrett 2018-08-14 15:30:51 -05:00
parent 32ebadf992
commit 1cc8c07394
3 changed files with 60 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs
View file

@ -25,28 +25,25 @@ public SnapshotBatchDe(FilterListsDbContext dbContext, Data.Entities.Snapshot sn
public async Task SaveSnapshotBatchAsync()
{
AddRules();
await AddRules();
AddSnapshotRules();
await dbContext.SaveChangesAsync();
}
private void AddRules()
private async Task AddRules()
{
var existingRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw));
var existingRules = dbContext.Rules.Where(r => rawRules.Contains(r.Raw));
var newRawRules = rawRules.Except(existingRules.Select(r => r.Raw));
var newRules = newRawRules.Select(newRawRule => new Rule {Raw = newRawRule}).ToList();
dbContext.Rules.AddRange(newRules);
var newRules = newRawRules.Select(r => new Rule {Raw = r}).ToList();
rules = existingRules.Concat(newRules);
await dbContext.Rules.AddRangeAsync(newRules);
}
private void AddSnapshotRules()
{
var snapshotRules = new List<SnapshotRule>();
foreach (var rule in rules)
snapshotRules.Add(new SnapshotRule {Rule = rule});
if (snapshot.AddedSnapshotRules == null)
snapshot.AddedSnapshotRules = new List<SnapshotRule>();
snapshot.AddedSnapshotRules.AddRange(snapshotRules);
snapshot.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r}));
}
}
}

54
src/FilterLists.Services/Snapshot/SnapshotDe.cs
View file

@ -21,41 +21,49 @@ public class SnapshotDe
private readonly FilterListsDbContext dbContext;
private readonly EmailService emailService;
private readonly FilterListViewUrlDto list;
private Data.Entities.Snapshot snapshot;
private readonly Data.Entities.Snapshot snapshot;
public SnapshotDe(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list)
{
this.dbContext = dbContext;
this.emailService = emailService;
this.list = list;
snapshot = new Data.Entities.Snapshot {FilterListId = list.Id};
}
public async Task SaveSnapshotAsync()
{
var content = await CaptureSnapshot();
if (content != null)
using (var transaction = dbContext.Database.BeginTransaction())
{
await SaveSnapshotInBatches(content);
await DedupSnapshotRules();
}
try
{
var content = await CaptureSnapshot();
if (content != null)
{
await SaveSnapshotInBatches(content);
await DedupSnapshotRules();
await SetCompleted();
//TODO: remove after closed: https://github.com/collinbarrett/FilterLists/issues/344
await SetCompleted();
transaction.Commit();
}
}
catch (Exception e)
{
await SendExceptionEmail(e);
}
}
}
private async Task<string> CaptureSnapshot()
{
await AddSnapshot();
var content = await TryGetContent();
await dbContext.Snapshots.AddAsync(snapshot);
await dbContext.SaveChangesAsync();
return content;
}
private async Task AddSnapshot()
{
snapshot = new Data.Entities.Snapshot {FilterListId = list.Id};
await dbContext.Snapshots.AddAsync(snapshot);
}
private async Task<string> TryGetContent()
{
try
@ -70,7 +78,6 @@ private async Task<string> TryGetContent()
}
catch (Exception e)
{
//TODO: log exception (#148)
snapshot.HttpStatusCode = null;
await SendExceptionEmail(e);
return null;
@ -98,7 +105,7 @@ private async Task SendWebExceptionEmail()
{
var message = new StringBuilder();
message.AppendLine("Snapshot WebException");
message.AppendLine("FilterListId: " + snapshot.FilterList.Id);
message.AppendLine("FilterListId: " + snapshot.FilterListId);
message.AppendLine("HTTP Status Code: " + snapshot.HttpStatusCode);
await emailService.SendEmailAsync("Snapshot WebException", message.ToString());
}
@ -107,7 +114,7 @@ private async Task SendExceptionEmail(Exception e)
{
var message = new StringBuilder();
message.AppendLine("Snapshot Exception");
message.AppendLine("FilterListId: " + snapshot.FilterList.Id);
message.AppendLine("FilterListId: " + snapshot.FilterListId);
message.AppendLine("Exception: " + e.Message);
await emailService.SendEmailAsync("Snapshot Exception", message.ToString());
}
@ -124,12 +131,11 @@ private static IEnumerable<string> GetRawRules(string content)
var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < rawRules.Length; i++)
rawRules[i] = rawRules[i].LintRawRule();
return new HashSet<string>(rawRules.Where(rr => rr != null));
return new HashSet<string>(rawRules.Where(r => r != null));
}
private IEnumerable<SnapshotBatchDe> GetSnapshotBatches(IEnumerable<string> rawRules) =>
rawRules.GetBatches(BatchSize)
.Select(rawRuleBatch => new SnapshotBatchDe(dbContext, snapshot, rawRuleBatch));
rawRules.GetBatches(BatchSize).Select(b => new SnapshotBatchDe(dbContext, snapshot, b));
private static async Task SaveSnapshotBatches(IEnumerable<SnapshotBatchDe> snapshotBatches)
{
@ -147,14 +153,14 @@ private async Task DedupSnapshotRules()
private IQueryable<SnapshotRule> GetExistingSnapshotRules() =>
dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot.FilterListId == list.Id && sr.AddedBySnapshot != snapshot &&
sr.AddedBySnapshot.FilterListId == list.Id &&
sr.AddedBySnapshot != snapshot &&
sr.RemovedBySnapshot == null);
private void UpdateRemovedSnapshotRules(IQueryable<SnapshotRule> existingSnapshotRules)
{
var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapshot);
var removedSnapshotRules =
existingSnapshotRules.Where(sr => !newSnapshotRules.Any(nsr => nsr.Rule == sr.Rule));
var removedSnapshotRules = existingSnapshotRules.Where(sr => !newSnapshotRules.Any(n => n.Rule == sr.Rule));
removedSnapshotRules.ToList().ForEach(sr => sr.RemovedBySnapshot = snapshot);
}
@ -162,7 +168,7 @@ private void RemoveDuplicateSnapshotRules(IQueryable<SnapshotRule> existingSnaps
{
var duplicateSnapshotRules = dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot == snapshot &&
existingSnapshotRules.Any(esr => esr.Rule == sr.Rule));
existingSnapshotRules.Any(e => e.Rule == sr.Rule));
dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules);
}

39
src/FilterLists.Services/Snapshot/SnapshotService.cs
View file

@ -15,10 +15,13 @@ namespace FilterLists.Services.Snapshot
[UsedImplicitly]
public class SnapshotService : Service
{
//TODO: update algorithm to support non-standard list sizes and formats (#200, #201)
//TODO: https://github.com/collinbarrett/FilterLists/issues/200
//TODO: https://github.com/collinbarrett/FilterLists/issues/201
private static readonly IList<uint> IgnoreLists =
new ReadOnlyCollection<uint>(new List<uint> {48, 149, 173, 185, 186, 187, 188, 189, 352});
private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1);
public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider configurationProvider,
EmailService emailService)
: base(dbContext, configurationProvider, emailService)
@ -27,35 +30,41 @@ public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider co
public async Task CaptureAsync(int batchSize)
{
RollbackIncompletedSnapshots();
//TODO: remove after closed: https://github.com/collinbarrett/FilterLists/issues/344
await RollbackIncompletedSnapshots();
var lists = await GetListsToCapture(batchSize);
var snapshots = GetSnapshots(lists);
var snapshots = CreateSnapshots(lists);
await SaveSnapshots(snapshots);
}
private void RollbackIncompletedSnapshots()
private async Task RollbackIncompletedSnapshots()
{
var incompleteSnapshots = DbContext.Snapshots.Where(ss => ss.IsCompleted == false);
DbContext.Snapshots.RemoveRange(incompleteSnapshots);
//TODO: don't assume that SnapshotDe.DedupSnapshotRules() didn't partially complete
await DbContext.SaveChangesAsync();
}
private async Task<IEnumerable<FilterListViewUrlDto>> GetListsToCapture(int batchSize) =>
private async Task<List<FilterListViewUrlDto>> GetListsToCapture(int batchSize) =>
await DbContext
.FilterLists
.Where(list =>
(!list.Snapshots.Any() ||
list.Snapshots.Select(ss => ss.CreatedDateUtc).OrderByDescending(sscd => sscd).FirstOrDefault() <
DateTime.UtcNow.AddDays(-1)) && !IgnoreLists.Contains(list.Id))
.OrderBy(list => list.Snapshots.Any())
.ThenBy(list =>
list.Snapshots.Select(ss => ss.CreatedDateUtc).OrderByDescending(sscd => sscd).FirstOrDefault())
.Where(l => !IgnoreLists.Contains(l.Id) &&
(!l.Snapshots.Any() ||
l.Snapshots
.Select(s => s.CreatedDateUtc)
.OrderByDescending(d => d)
.First() < yesterday))
.OrderBy(l => l.Snapshots.Any())
.ThenBy(l => l.Snapshots
.Select(s => s.CreatedDateUtc)
.OrderByDescending(d => d)
.FirstOrDefault())
.Take(batchSize)
.ProjectTo<FilterListViewUrlDto>(ConfigurationProvider)
.ToListAsync();
private IEnumerable<SnapshotDe> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists) =>
lists.Select(list => new SnapshotDe(DbContext, EmailService, list));
private IEnumerable<SnapshotDe> CreateSnapshots(IEnumerable<FilterListViewUrlDto> lists) =>
lists.Select(l => new SnapshotDe(DbContext, EmailService, l));
private static async Task SaveSnapshots(IEnumerable<SnapshotDe> snapshots)
{