prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

simplify snapshots / remove emailer for now

Browse source
This commit is contained in:
Collin Barrett 2018-08-17 18:46:18 -05:00
parent c50fb639ab
commit ea3dff8434
4 changed files with 30 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
View file

@ -33,7 +33,6 @@ public static void AddFilterListsAgentServices(this IServiceCollection services,
o.UseMySql(config.GetConnectionString("FilterListsConnection"),
m => m.MigrationsAssembly("FilterLists.Api")));
services.TryAddScoped<SnapshotService>();
services.TryAddScoped<EmailService>();
services.AddAutoMapper();
}
}

76
src/FilterLists.Services/Snapshot/SnapshotDe.cs → src/FilterLists.Services/Snapshot/Snapshot.cs
View file

@ -3,7 +3,6 @@
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Data.Entities.Junctions;
@ -12,24 +11,18 @@
namespace FilterLists.Services.Snapshot
{
public class SnapshotDe
public class Snapshot
{
private const string UserAgentString =
@"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36";
private const int BatchSize = 50;
private readonly FilterListsDbContext dbContext;
private readonly EmailService emailService;
private readonly FilterListViewUrlDto list;
private readonly Data.Entities.Snapshot snapshot;
private string httpStatusCodeBak;
private readonly Data.Entities.Snapshot snapEntity;
public SnapshotDe(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list)
public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list)
{
this.dbContext = dbContext;
this.emailService = emailService;
this.list = list;
snapshot = new Data.Entities.Snapshot
snapEntity = new Data.Entities.Snapshot
{
FilterListId = list.Id,
AddedSnapshotRules = new List<SnapshotRule>()
@ -43,16 +36,15 @@ public async Task TrySaveAsync()
{
await SaveAsync();
}
catch (Exception e)
catch (Exception)
{
await SaveHttpStatusCodeBak();
await SendExceptionEmail(e);
//TODO: log
}
}
private async Task Add()
{
dbContext.Snapshots.Add(snapshot);
dbContext.Snapshots.Add(snapEntity);
await dbContext.SaveChangesAsync();
}
@ -80,8 +72,7 @@ private async Task<string> TryGetContent()
}
catch (WebException we)
{
snapshot.HttpStatusCode = ((int)((HttpWebResponse)we.Response).StatusCode).ToString();
await SendWebExceptionEmail();
snapEntity.HttpStatusCode = ((int)((HttpWebResponse)we.Response).StatusCode).ToString();
return null;
}
}
@ -90,50 +81,17 @@ private async Task<string> GetContent()
{
using (var httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentString);
using (var httpResponseMessage = await httpClient.GetAsync(list.ViewUrl))
{
snapshot.HttpStatusCode = httpStatusCodeBak = ((int)httpResponseMessage.StatusCode).ToString();
snapEntity.HttpStatusCode = ((int)httpResponseMessage.StatusCode).ToString();
if (httpResponseMessage.IsSuccessStatusCode)
return await httpResponseMessage.Content.ReadAsStringAsync();
}
}
await SendWebExceptionEmail();
return null;
}
private async Task SaveHttpStatusCodeBak()
{
if (!snapshot.WasSuccessful)
{
snapshot.HttpStatusCode = httpStatusCodeBak;
await dbContext.SaveChangesAsync();
}
}
private async Task SendWebExceptionEmail()
{
var message = new StringBuilder();
message.AppendLine("Snapshot WebException");
message.AppendLine("FilterListId: " + snapshot.FilterListId);
message.AppendLine("HTTP Status Code: " + snapshot.HttpStatusCode);
await emailService.SendEmailAsync("Snapshot WebException", message.ToString());
}
private async Task SendExceptionEmail(Exception e)
{
var message = new StringBuilder();
message.AppendLine("Snapshot Exception");
message.AppendLine("FilterListId: " + snapshot.FilterListId);
message.AppendLine("Exception:");
message.AppendLine(e.Message);
message.AppendLine(e.StackTrace);
message.AppendLine(e.InnerException?.Message);
message.AppendLine(e.InnerException?.StackTrace);
await emailService.SendEmailAsync("Snapshot Exception", message.ToString());
}
private async Task SaveInBatches(string content)
{
var rawRules = ParseRawRules(content);
@ -149,10 +107,10 @@ private static IEnumerable<string> ParseRawRules(string content)
return rawRules.Where(r => r != null);
}
private IEnumerable<SnapshotDeBatch> CreateBatches(IEnumerable<string> rawRules) =>
rawRules.Batch(BatchSize).Select(b => new SnapshotDeBatch(dbContext, snapshot, b));
private IEnumerable<SnapshotBatch> CreateBatches(IEnumerable<string> rawRules) =>
rawRules.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity));
private static async Task SaveBatches(IEnumerable<SnapshotDeBatch> snapshotBatches)
private static async Task SaveBatches(IEnumerable<SnapshotBatch> snapshotBatches)
{
foreach (var batch in snapshotBatches)
await batch.SaveAsync();
@ -169,27 +127,27 @@ private async Task DedupSnapshotRules()
private IQueryable<SnapshotRule> GetExistingSnapshotRules() =>
dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot.FilterListId == list.Id &&
sr.AddedBySnapshot != snapshot &&
sr.AddedBySnapshot != snapEntity &&
sr.RemovedBySnapshot == null);
private void UpdateRemovedSnapshotRules(IQueryable<SnapshotRule> existingSnapshotRules)
{
var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapshot);
var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapEntity);
var removedSnapshotRules = existingSnapshotRules.Where(sr => !newSnapshotRules.Any(n => n.Rule == sr.Rule));
removedSnapshotRules.ForEach(sr => sr.RemovedBySnapshot = snapshot);
removedSnapshotRules.ForEach(sr => sr.RemovedBySnapshot = snapEntity);
}
private void RemoveDuplicateSnapshotRules(IQueryable<SnapshotRule> existingSnapshotRules)
{
var duplicateSnapshotRules = dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot == snapshot &&
sr.AddedBySnapshot == snapEntity &&
existingSnapshotRules.Any(e => e.Rule == sr.Rule));
dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules);
}
private async Task SetSuccessful()
{
snapshot.WasSuccessful = true;
snapEntity.WasSuccessful = true;
await dbContext.SaveChangesAsync();
}
}

12
src/FilterLists.Services/Snapshot/SnapshotDeBatch.cs → src/FilterLists.Services/Snapshot/SnapshotBatch.cs
View file

@ -8,18 +8,18 @@
namespace FilterLists.Services.Snapshot
{
public class SnapshotDeBatch
public class SnapshotBatch
{
private readonly FilterListsDbContext dbContext;
private readonly IEnumerable<string> rawRules;
private readonly Data.Entities.Snapshot snapshot;
private readonly Data.Entities.Snapshot snapEntity;
private IQueryable<Rule> rules;
public SnapshotDeBatch(FilterListsDbContext dbContext, Data.Entities.Snapshot snapshot,
IEnumerable<string> rawRules)
public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable<string> rawRules,
Data.Entities.Snapshot snapEntity)
{
this.dbContext = dbContext;
this.snapshot = snapshot;
this.snapEntity = snapEntity;
this.rawRules = rawRules;
}
@ -44,6 +44,6 @@ private List<Rule> CreateNewRules(IQueryable<Rule> existingRules) =>
rawRules.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList();
private void AddSnapshotRules() =>
snapshot.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r}));
snapEntity.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r}));
}
}

14
src/FilterLists.Services/Snapshot/SnapshotService.cs
View file

@ -12,12 +12,12 @@ namespace FilterLists.Services.Snapshot
{
public class SnapshotService : Service
{
private readonly EmailService emailService;
private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1);
public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider configurationProvider,
EmailService emailService)
: base(dbContext, configurationProvider) => this.emailService = emailService;
public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig)
: base(dbContext, mapConfig)
{
}
public async Task CaptureAsync(int batchSize)
{
@ -44,10 +44,10 @@ await DbContext
.ProjectTo<FilterListViewUrlDto>(MapConfig)
.ToListAsync();
private IEnumerable<SnapshotDe> CreateSnapshots(IEnumerable<FilterListViewUrlDto> lists) =>
lists.Select(l => new SnapshotDe(DbContext, emailService, l));
private IEnumerable<Snapshot> CreateSnapshots(IEnumerable<FilterListViewUrlDto> lists) =>
lists.Select(l => new Snapshot(DbContext, l));
private static async Task SaveSnapshots(IEnumerable<SnapshotDe> snapshots)
private static async Task SaveSnapshots(IEnumerable<Snapshot> snapshots)
{
foreach (var snapshot in snapshots)
await snapshot.TrySaveAsync();