diff --git a/src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs b/src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs index 05a53034d..555f5bc5c 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs @@ -25,28 +25,25 @@ public SnapshotBatchDe(FilterListsDbContext dbContext, Data.Entities.Snapshot sn public async Task SaveSnapshotBatchAsync() { - AddRules(); + await AddRules(); AddSnapshotRules(); await dbContext.SaveChangesAsync(); } - private void AddRules() + private async Task AddRules() { - var existingRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw)); + var existingRules = dbContext.Rules.Where(r => rawRules.Contains(r.Raw)); var newRawRules = rawRules.Except(existingRules.Select(r => r.Raw)); - var newRules = newRawRules.Select(newRawRule => new Rule {Raw = newRawRule}).ToList(); - dbContext.Rules.AddRange(newRules); + var newRules = newRawRules.Select(r => new Rule {Raw = r}).ToList(); rules = existingRules.Concat(newRules); + await dbContext.Rules.AddRangeAsync(newRules); } private void AddSnapshotRules() { - var snapshotRules = new List(); - foreach (var rule in rules) - snapshotRules.Add(new SnapshotRule {Rule = rule}); if (snapshot.AddedSnapshotRules == null) snapshot.AddedSnapshotRules = new List(); - snapshot.AddedSnapshotRules.AddRange(snapshotRules); + snapshot.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r})); } } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/SnapshotDe.cs b/src/FilterLists.Services/Snapshot/SnapshotDe.cs index a47c70ab3..c2b799a91 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotDe.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotDe.cs @@ -21,41 +21,49 @@ public class SnapshotDe private readonly FilterListsDbContext dbContext; private readonly EmailService emailService; private readonly FilterListViewUrlDto list; - private Data.Entities.Snapshot snapshot; + private readonly Data.Entities.Snapshot snapshot; public SnapshotDe(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list) { this.dbContext = dbContext; this.emailService = emailService; this.list = list; + snapshot = new Data.Entities.Snapshot {FilterListId = list.Id}; } public async Task SaveSnapshotAsync() { - var content = await CaptureSnapshot(); - if (content != null) + using (var transaction = dbContext.Database.BeginTransaction()) { - await SaveSnapshotInBatches(content); - await DedupSnapshotRules(); - } + try + { + var content = await CaptureSnapshot(); + if (content != null) + { + await SaveSnapshotInBatches(content); + await DedupSnapshotRules(); - await SetCompleted(); + //TODO: remove after closed: https://github.com/collinbarrett/FilterLists/issues/344 + await SetCompleted(); + + transaction.Commit(); + } + } + catch (Exception e) + { + await SendExceptionEmail(e); + } + } } private async Task CaptureSnapshot() { - await AddSnapshot(); var content = await TryGetContent(); + await dbContext.Snapshots.AddAsync(snapshot); await dbContext.SaveChangesAsync(); return content; } - private async Task AddSnapshot() - { - snapshot = new Data.Entities.Snapshot {FilterListId = list.Id}; - await dbContext.Snapshots.AddAsync(snapshot); - } - private async Task TryGetContent() { try @@ -70,7 +78,6 @@ private async Task TryGetContent() } catch (Exception e) { - //TODO: log exception (#148) snapshot.HttpStatusCode = null; await SendExceptionEmail(e); return null; @@ -98,7 +105,7 @@ private async Task SendWebExceptionEmail() { var message = new StringBuilder(); message.AppendLine("Snapshot WebException"); - message.AppendLine("FilterListId: " + snapshot.FilterList.Id); + message.AppendLine("FilterListId: " + snapshot.FilterListId); message.AppendLine("HTTP Status Code: " + snapshot.HttpStatusCode); await emailService.SendEmailAsync("Snapshot WebException", message.ToString()); } @@ -107,7 +114,7 @@ private async Task SendExceptionEmail(Exception e) { var message = new StringBuilder(); message.AppendLine("Snapshot Exception"); - message.AppendLine("FilterListId: " + snapshot.FilterList.Id); + message.AppendLine("FilterListId: " + snapshot.FilterListId); message.AppendLine("Exception: " + e.Message); await emailService.SendEmailAsync("Snapshot Exception", message.ToString()); } @@ -124,12 +131,11 @@ private static IEnumerable GetRawRules(string content) var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); for (var i = 0; i < rawRules.Length; i++) rawRules[i] = rawRules[i].LintRawRule(); - return new HashSet(rawRules.Where(rr => rr != null)); + return new HashSet(rawRules.Where(r => r != null)); } private IEnumerable GetSnapshotBatches(IEnumerable rawRules) => - rawRules.GetBatches(BatchSize) - .Select(rawRuleBatch => new SnapshotBatchDe(dbContext, snapshot, rawRuleBatch)); + rawRules.GetBatches(BatchSize).Select(b => new SnapshotBatchDe(dbContext, snapshot, b)); private static async Task SaveSnapshotBatches(IEnumerable snapshotBatches) { @@ -147,14 +153,14 @@ private async Task DedupSnapshotRules() private IQueryable GetExistingSnapshotRules() => dbContext.SnapshotRules.Where(sr => - sr.AddedBySnapshot.FilterListId == list.Id && sr.AddedBySnapshot != snapshot && + sr.AddedBySnapshot.FilterListId == list.Id && + sr.AddedBySnapshot != snapshot && sr.RemovedBySnapshot == null); private void UpdateRemovedSnapshotRules(IQueryable existingSnapshotRules) { var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapshot); - var removedSnapshotRules = - existingSnapshotRules.Where(sr => !newSnapshotRules.Any(nsr => nsr.Rule == sr.Rule)); + var removedSnapshotRules = existingSnapshotRules.Where(sr => !newSnapshotRules.Any(n => n.Rule == sr.Rule)); removedSnapshotRules.ToList().ForEach(sr => sr.RemovedBySnapshot = snapshot); } @@ -162,7 +168,7 @@ private void RemoveDuplicateSnapshotRules(IQueryable existingSnaps { var duplicateSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapshot && - existingSnapshotRules.Any(esr => esr.Rule == sr.Rule)); + existingSnapshotRules.Any(e => e.Rule == sr.Rule)); dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules); } diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs index 71b59095b..870d8ffe7 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotService.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs @@ -15,10 +15,13 @@ namespace FilterLists.Services.Snapshot [UsedImplicitly] public class SnapshotService : Service { - //TODO: update algorithm to support non-standard list sizes and formats (#200, #201) + //TODO: https://github.com/collinbarrett/FilterLists/issues/200 + //TODO: https://github.com/collinbarrett/FilterLists/issues/201 private static readonly IList IgnoreLists = new ReadOnlyCollection(new List {48, 149, 173, 185, 186, 187, 188, 189, 352}); + private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1); + public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider configurationProvider, EmailService emailService) : base(dbContext, configurationProvider, emailService) @@ -27,35 +30,41 @@ public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider co public async Task CaptureAsync(int batchSize) { - RollbackIncompletedSnapshots(); + //TODO: remove after closed: https://github.com/collinbarrett/FilterLists/issues/344 + await RollbackIncompletedSnapshots(); + var lists = await GetListsToCapture(batchSize); - var snapshots = GetSnapshots(lists); + var snapshots = CreateSnapshots(lists); await SaveSnapshots(snapshots); } - private void RollbackIncompletedSnapshots() + private async Task RollbackIncompletedSnapshots() { var incompleteSnapshots = DbContext.Snapshots.Where(ss => ss.IsCompleted == false); DbContext.Snapshots.RemoveRange(incompleteSnapshots); - //TODO: don't assume that SnapshotDe.DedupSnapshotRules() didn't partially complete + await DbContext.SaveChangesAsync(); } - private async Task> GetListsToCapture(int batchSize) => + private async Task> GetListsToCapture(int batchSize) => await DbContext .FilterLists - .Where(list => - (!list.Snapshots.Any() || - list.Snapshots.Select(ss => ss.CreatedDateUtc).OrderByDescending(sscd => sscd).FirstOrDefault() < - DateTime.UtcNow.AddDays(-1)) && !IgnoreLists.Contains(list.Id)) - .OrderBy(list => list.Snapshots.Any()) - .ThenBy(list => - list.Snapshots.Select(ss => ss.CreatedDateUtc).OrderByDescending(sscd => sscd).FirstOrDefault()) + .Where(l => !IgnoreLists.Contains(l.Id) && + (!l.Snapshots.Any() || + l.Snapshots + .Select(s => s.CreatedDateUtc) + .OrderByDescending(d => d) + .First() < yesterday)) + .OrderBy(l => l.Snapshots.Any()) + .ThenBy(l => l.Snapshots + .Select(s => s.CreatedDateUtc) + .OrderByDescending(d => d) + .FirstOrDefault()) .Take(batchSize) .ProjectTo(ConfigurationProvider) .ToListAsync(); - private IEnumerable GetSnapshots(IEnumerable lists) => - lists.Select(list => new SnapshotDe(DbContext, EmailService, list)); + private IEnumerable CreateSnapshots(IEnumerable lists) => + lists.Select(l => new SnapshotDe(DbContext, EmailService, l)); private static async Task SaveSnapshots(IEnumerable snapshots) {