prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

minor SnapshotService refactors

Browse source
This commit is contained in:
Collin Barrett 2018-08-16 19:26:31 -05:00
parent 689e187ea9
commit 32e23f0fc0
2 changed files with 38 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
src/FilterLists.Services/Snapshot/SnapshotDe.cs
View file

@ -36,38 +36,42 @@ public SnapshotDe(FilterListsDbContext dbContext, EmailService emailService, Fil
};
}
public async Task SaveAsync()
public async Task TrySaveAsync()
{
await AddSnapshot();
using (var transaction = dbContext.Database.BeginTransaction())
await Add();
try
{
try
{
var content = await TryGetContent();
if (content != null)
{
await SaveSnapshotInBatches(content);
await DedupSnapshotRules();
await SetSuccessful();
}
transaction.Commit();
}
catch (Exception e)
{
await SendExceptionEmail(e);
}
await SaveAsync();
}
catch (Exception e)
{
await SaveHttpStatusCodeBak();
await SendExceptionEmail(e);
}
await EnsureHttpStatusCodeSaved();
}
private async Task AddSnapshot()
private async Task Add()
{
await dbContext.Snapshots.AddAsync(snapshot);
await dbContext.SaveChangesAsync();
}
private async Task SaveAsync()
{
using (var transaction = dbContext.Database.BeginTransaction())
{
var content = await TryGetContent();
if (content != null)
{
await SaveInBatches(content);
await DedupSnapshotRules();
await SetSuccessful();
}
transaction.Commit();
}
}
private async Task<string> TryGetContent()
{
try
@ -80,12 +84,6 @@ private async Task<string> TryGetContent()
await SendWebExceptionEmail();
return null;
}
catch (Exception e)
{
snapshot.HttpStatusCode = null;
await SendExceptionEmail(e);
return null;
}
}
private async Task<string> GetContent()
@ -95,8 +93,7 @@ private async Task<string> GetContent()
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentString);
using (var httpResponseMessage = await httpClient.GetAsync(list.ViewUrl))
{
snapshot.HttpStatusCode = ((int)httpResponseMessage.StatusCode).ToString();
httpStatusCodeBak = snapshot.HttpStatusCode;
snapshot.HttpStatusCode = httpStatusCodeBak = ((int)httpResponseMessage.StatusCode).ToString();
if (httpResponseMessage.IsSuccessStatusCode)
return await httpResponseMessage.Content.ReadAsStringAsync();
}
@ -106,7 +103,7 @@ private async Task<string> GetContent()
return null;
}
private async Task EnsureHttpStatusCodeSaved()
private async Task SaveHttpStatusCodeBak()
{
if (!snapshot.WasSuccessful)
{
@ -137,11 +134,11 @@ private async Task SendExceptionEmail(Exception e)
await emailService.SendEmailAsync("Snapshot Exception", message.ToString());
}
private async Task SaveSnapshotInBatches(string content)
private async Task SaveInBatches(string content)
{
var rawRules = ParseRawRules(content);
var snapshotBatches = CreateSnapshotBatches(rawRules);
await SaveSnapshotBatches(snapshotBatches);
var snapshotBatches = CreateBatches(rawRules);
await SaveBatches(snapshotBatches);
}
private static IEnumerable<string> ParseRawRules(string content)
@ -152,13 +149,13 @@ private static IEnumerable<string> ParseRawRules(string content)
return rawRules.Where(r => r != null);
}
private IEnumerable<SnapshotDeBatch> CreateSnapshotBatches(IEnumerable<string> rawRules) =>
private IEnumerable<SnapshotDeBatch> CreateBatches(IEnumerable<string> rawRules) =>
rawRules.Batch(BatchSize).Select(b => new SnapshotDeBatch(dbContext, snapshot, b));
private static async Task SaveSnapshotBatches(IEnumerable<SnapshotDeBatch> snapshotBatches)
private static async Task SaveBatches(IEnumerable<SnapshotDeBatch> snapshotBatches)
{
foreach (var snapshotBatch in snapshotBatches)
await snapshotBatch.SaveAsync();
foreach (var batch in snapshotBatches)
await batch.SaveAsync();
}
private async Task DedupSnapshotRules()
@ -179,7 +176,7 @@ private void UpdateRemovedSnapshotRules(IQueryable<SnapshotRule> existingSnapsho
{
var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapshot);
var removedSnapshotRules = existingSnapshotRules.Where(sr => !newSnapshotRules.Any(n => n.Rule == sr.Rule));
removedSnapshotRules.ToList().ForEach(sr => sr.RemovedBySnapshot = snapshot);
removedSnapshotRules.ForEach(sr => sr.RemovedBySnapshot = snapshot);
}
private void RemoveDuplicateSnapshotRules(IQueryable<SnapshotRule> existingSnapshotRules)

4
src/FilterLists.Services/Snapshot/SnapshotService.cs
View file

@ -29,7 +29,7 @@ public async Task CaptureAsync(int batchSize)
await SaveSnapshots(snapshots);
}
private async Task<List<FilterListViewUrlDto>> GetListsToCapture(int batchSize) =>
private async Task<IEnumerable<FilterListViewUrlDto>> GetListsToCapture(int batchSize) =>
await DbContext
.FilterLists
.Where(l => !l.CantSnapshot &&
@ -53,7 +53,7 @@ private IEnumerable<SnapshotDe> CreateSnapshots(IEnumerable<FilterListViewUrlDto
private static async Task SaveSnapshots(IEnumerable<SnapshotDe> snapshots)
{
foreach (var snapshot in snapshots)
await snapshot.SaveAsync();
await snapshot.TrySaveAsync();
}
}
}