diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 4cca2b402..0c48ad5e6 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -25,6 +25,7 @@ public class Snapshot private readonly Data.Entities.Snapshot snapEntity; private readonly TelemetryClient telemetryClient; private readonly string userAgentString; + private HashSet lines; public Snapshot(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list, string userAgentString) @@ -41,7 +42,6 @@ public Snapshot(FilterListsDbContext dbContext, EmailService emailService, Filte telemetryClient = new TelemetryClient(); } - //TODO: add better compliance with Try/Parse pattern (https://stackoverflow.com/q/37810660/2343739) public async Task TrySaveAsync() { await AddSnapEntity(); @@ -65,50 +65,43 @@ private async Task AddSnapEntity() private async Task SaveAsync() { - using (var transaction = dbContext.Database.BeginTransaction()) + await TryGetLines(); + if (lines != null) { - var lines = await TryGetLines(); - if (lines != null) - { - await SaveInBatches(lines); - await AddRemovedSnapshotRules(); - await SetSuccessful(); - } - - transaction.Commit(); + await SaveInBatches(); + await AddRemovedSnapRules(); + await SetSuccessful(); } } - private async Task> TryGetLines() + private async Task TryGetLines() { try { - return await GetLines(); + await GetLines(); } catch (HttpRequestException hre) { await dbContext.SaveChangesAsync(); await TrackException(hre); - return null; } catch (WebException we) { snapEntity.HttpStatusCode = ((int)((HttpWebResponse)we.Response).StatusCode).ToString(); await dbContext.SaveChangesAsync(); await TrackException(we); - return null; } } - private async Task> GetLines() + private async Task GetLines() { - var lines = new HashSet(); using (var httpClient = new HttpClient()) { httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgentString); var response = await httpClient.GetAsync(list.ViewUrl, HttpCompletionOption.ResponseHeadersRead); snapEntity.HttpStatusCode = ((int)response.StatusCode).ToString(); response.EnsureSuccessStatusCode(); + lines = new HashSet(); using (var stream = await response.Content.ReadAsStreamAsync()) using (var streamReader = new StreamReader(stream)) { @@ -117,17 +110,15 @@ private async Task> GetLines() lines.AddIfNotNullOrEmpty(line.LintLine()); } } - - return lines; } - private async Task SaveInBatches(IEnumerable lines) + private async Task SaveInBatches() { - var snapshotBatches = CreateBatches(lines); + var snapshotBatches = CreateBatches(); await SaveBatches(snapshotBatches); } - private IEnumerable CreateBatches(IEnumerable lines) => + private IEnumerable CreateBatches() => lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity)); private static async Task SaveBatches(IEnumerable batches) @@ -136,26 +127,17 @@ private static async Task SaveBatches(IEnumerable batches) await batch.SaveAsync(); } - private async Task AddRemovedSnapshotRules() + private async Task AddRemovedSnapRules() { - var existingSnapshotRules = GetExistingSnapshotRules(); - AddRemovedBySnapshots(existingSnapshotRules); + dbContext.SnapshotRules + .Where(sr => + sr.AddedBySnapshot.FilterListId == list.Id && + sr.RemovedBySnapshot == null && + !lines.Contains(sr.Rule.Raw)) + .ForEach(sr => sr.RemovedBySnapshot = snapEntity); await dbContext.SaveChangesAsync(); } - private IQueryable GetExistingSnapshotRules() => - dbContext.SnapshotRules.Where(sr => - sr.AddedBySnapshot.FilterListId == list.Id && - sr.AddedBySnapshot != snapEntity && - sr.RemovedBySnapshot == null); - - private void AddRemovedBySnapshots(IQueryable existingSnapshotRules) - { - var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapEntity); - var removedSnapshotRules = existingSnapshotRules.Where(sr => !newSnapshotRules.Any(n => n.Rule == sr.Rule)); - removedSnapshotRules.ForEach(sr => sr.RemovedBySnapshot = snapEntity); - } - private async Task SetSuccessful() { snapEntity.WasSuccessful = true;