diff --git a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs index 984421861..ee93937d8 100644 --- a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs +++ b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs @@ -33,7 +33,6 @@ public static void AddFilterListsAgentServices(this IServiceCollection services, o.UseMySql(config.GetConnectionString("FilterListsConnection"), m => m.MigrationsAssembly("FilterLists.Api"))); services.TryAddScoped(); - services.TryAddScoped(); services.AddAutoMapper(); } } diff --git a/src/FilterLists.Services/Snapshot/SnapshotDe.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs similarity index 60% rename from src/FilterLists.Services/Snapshot/SnapshotDe.cs rename to src/FilterLists.Services/Snapshot/Snapshot.cs index 740561e5a..0ae3e1969 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotDe.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -3,7 +3,6 @@ using System.Linq; using System.Net; using System.Net.Http; -using System.Text; using System.Threading.Tasks; using FilterLists.Data; using FilterLists.Data.Entities.Junctions; @@ -12,24 +11,18 @@ namespace FilterLists.Services.Snapshot { - public class SnapshotDe + public class Snapshot { - private const string UserAgentString = - @"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"; - private const int BatchSize = 50; private readonly FilterListsDbContext dbContext; - private readonly EmailService emailService; private readonly FilterListViewUrlDto list; - private readonly Data.Entities.Snapshot snapshot; - private string httpStatusCodeBak; + private readonly Data.Entities.Snapshot snapEntity; - public SnapshotDe(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list) + public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list) { this.dbContext = dbContext; - this.emailService = emailService; this.list = list; - snapshot = new Data.Entities.Snapshot + snapEntity = new Data.Entities.Snapshot { FilterListId = list.Id, AddedSnapshotRules = new List() @@ -43,16 +36,15 @@ public async Task TrySaveAsync() { await SaveAsync(); } - catch (Exception e) + catch (Exception) { - await SaveHttpStatusCodeBak(); - await SendExceptionEmail(e); + //TODO: log } } private async Task Add() { - dbContext.Snapshots.Add(snapshot); + dbContext.Snapshots.Add(snapEntity); await dbContext.SaveChangesAsync(); } @@ -80,8 +72,7 @@ private async Task TryGetContent() } catch (WebException we) { - snapshot.HttpStatusCode = ((int)((HttpWebResponse)we.Response).StatusCode).ToString(); - await SendWebExceptionEmail(); + snapEntity.HttpStatusCode = ((int)((HttpWebResponse)we.Response).StatusCode).ToString(); return null; } } @@ -90,50 +81,17 @@ private async Task GetContent() { using (var httpClient = new HttpClient()) { - httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentString); using (var httpResponseMessage = await httpClient.GetAsync(list.ViewUrl)) { - snapshot.HttpStatusCode = httpStatusCodeBak = ((int)httpResponseMessage.StatusCode).ToString(); + snapEntity.HttpStatusCode = ((int)httpResponseMessage.StatusCode).ToString(); if (httpResponseMessage.IsSuccessStatusCode) return await httpResponseMessage.Content.ReadAsStringAsync(); } } - await SendWebExceptionEmail(); return null; } - private async Task SaveHttpStatusCodeBak() - { - if (!snapshot.WasSuccessful) - { - snapshot.HttpStatusCode = httpStatusCodeBak; - await dbContext.SaveChangesAsync(); - } - } - - private async Task SendWebExceptionEmail() - { - var message = new StringBuilder(); - message.AppendLine("Snapshot WebException"); - message.AppendLine("FilterListId: " + snapshot.FilterListId); - message.AppendLine("HTTP Status Code: " + snapshot.HttpStatusCode); - await emailService.SendEmailAsync("Snapshot WebException", message.ToString()); - } - - private async Task SendExceptionEmail(Exception e) - { - var message = new StringBuilder(); - message.AppendLine("Snapshot Exception"); - message.AppendLine("FilterListId: " + snapshot.FilterListId); - message.AppendLine("Exception:"); - message.AppendLine(e.Message); - message.AppendLine(e.StackTrace); - message.AppendLine(e.InnerException?.Message); - message.AppendLine(e.InnerException?.StackTrace); - await emailService.SendEmailAsync("Snapshot Exception", message.ToString()); - } - private async Task SaveInBatches(string content) { var rawRules = ParseRawRules(content); @@ -149,10 +107,10 @@ private static IEnumerable ParseRawRules(string content) return rawRules.Where(r => r != null); } - private IEnumerable CreateBatches(IEnumerable rawRules) => - rawRules.Batch(BatchSize).Select(b => new SnapshotDeBatch(dbContext, snapshot, b)); + private IEnumerable CreateBatches(IEnumerable rawRules) => + rawRules.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity)); - private static async Task SaveBatches(IEnumerable snapshotBatches) + private static async Task SaveBatches(IEnumerable snapshotBatches) { foreach (var batch in snapshotBatches) await batch.SaveAsync(); @@ -169,27 +127,27 @@ private async Task DedupSnapshotRules() private IQueryable GetExistingSnapshotRules() => dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot.FilterListId == list.Id && - sr.AddedBySnapshot != snapshot && + sr.AddedBySnapshot != snapEntity && sr.RemovedBySnapshot == null); private void UpdateRemovedSnapshotRules(IQueryable existingSnapshotRules) { - var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapshot); + var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapEntity); var removedSnapshotRules = existingSnapshotRules.Where(sr => !newSnapshotRules.Any(n => n.Rule == sr.Rule)); - removedSnapshotRules.ForEach(sr => sr.RemovedBySnapshot = snapshot); + removedSnapshotRules.ForEach(sr => sr.RemovedBySnapshot = snapEntity); } private void RemoveDuplicateSnapshotRules(IQueryable existingSnapshotRules) { var duplicateSnapshotRules = dbContext.SnapshotRules.Where(sr => - sr.AddedBySnapshot == snapshot && + sr.AddedBySnapshot == snapEntity && existingSnapshotRules.Any(e => e.Rule == sr.Rule)); dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules); } private async Task SetSuccessful() { - snapshot.WasSuccessful = true; + snapEntity.WasSuccessful = true; await dbContext.SaveChangesAsync(); } } diff --git a/src/FilterLists.Services/Snapshot/SnapshotDeBatch.cs b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs similarity index 77% rename from src/FilterLists.Services/Snapshot/SnapshotDeBatch.cs rename to src/FilterLists.Services/Snapshot/SnapshotBatch.cs index bbdc11962..708b81052 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotDeBatch.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs @@ -8,18 +8,18 @@ namespace FilterLists.Services.Snapshot { - public class SnapshotDeBatch + public class SnapshotBatch { private readonly FilterListsDbContext dbContext; private readonly IEnumerable rawRules; - private readonly Data.Entities.Snapshot snapshot; + private readonly Data.Entities.Snapshot snapEntity; private IQueryable rules; - public SnapshotDeBatch(FilterListsDbContext dbContext, Data.Entities.Snapshot snapshot, - IEnumerable rawRules) + public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable rawRules, + Data.Entities.Snapshot snapEntity) { this.dbContext = dbContext; - this.snapshot = snapshot; + this.snapEntity = snapEntity; this.rawRules = rawRules; } @@ -44,6 +44,6 @@ private List CreateNewRules(IQueryable existingRules) => rawRules.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList(); private void AddSnapshotRules() => - snapshot.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r})); + snapEntity.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r})); } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs index d85dc50cc..35e062a5a 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotService.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs @@ -12,12 +12,12 @@ namespace FilterLists.Services.Snapshot { public class SnapshotService : Service { - private readonly EmailService emailService; private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1); - public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider configurationProvider, - EmailService emailService) - : base(dbContext, configurationProvider) => this.emailService = emailService; + public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig) + : base(dbContext, mapConfig) + { + } public async Task CaptureAsync(int batchSize) { @@ -44,10 +44,10 @@ await DbContext .ProjectTo(MapConfig) .ToListAsync(); - private IEnumerable CreateSnapshots(IEnumerable lists) => - lists.Select(l => new SnapshotDe(DbContext, emailService, l)); + private IEnumerable CreateSnapshots(IEnumerable lists) => + lists.Select(l => new Snapshot(DbContext, l)); - private static async Task SaveSnapshots(IEnumerable snapshots) + private static async Task SaveSnapshots(IEnumerable snapshots) { foreach (var snapshot in snapshots) await snapshot.TrySaveAsync();