diff --git a/src/FilterLists.Services/Extensions/CollectionExtensions.cs b/src/FilterLists.Services/Extensions/CollectionExtensions.cs index 81957558d..68c6042a1 100644 --- a/src/FilterLists.Services/Extensions/CollectionExtensions.cs +++ b/src/FilterLists.Services/Extensions/CollectionExtensions.cs @@ -13,5 +13,11 @@ public static void AddRange(this ICollection destination, IEnumerable s foreach (var item in source) destination.Add(item); } + + public static void AddIfNotNullOrEmpty(this ICollection set, string item) + { + if (!string.IsNullOrEmpty(item)) + set.Add(item); + } } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/LineLinterExtensions.cs b/src/FilterLists.Services/Snapshot/LineLinterExtensions.cs new file mode 100644 index 000000000..14675b0b5 --- /dev/null +++ b/src/FilterLists.Services/Snapshot/LineLinterExtensions.cs @@ -0,0 +1,15 @@ +namespace FilterLists.Services.Snapshot +{ + public static class LineLinterExtensions + { + public static string LintLine(this string line) + { + line = line.Trim(); + line = line.DropIfTooLong(); + return line; + } + + private static string DropIfTooLong(this string line) => + line.Length > 8192 ? null : line; + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs b/src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs deleted file mode 100644 index adac350f8..000000000 --- a/src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; - -namespace FilterLists.Services.Snapshot -{ - public static class RawRuleLinterExtensions - { - public static string LintRawRule(this string rule) - { - rule = rule.Trim(); - rule = rule.DropIfTooLong(); - rule = rule?.DropIfComment(); - rule = rule?.DropIfContainsBackslashSingleQuote(); - rule = rule?.TrimSingleBackslashFromEnd(); - rule = rule?.DropIfEmpty(); - return rule; - } - - private static string DropIfTooLong(this string rule) => - rule.Length > 8192 ? null : rule; - - private static string DropIfComment(this string rule) => - rule.StartsWith(@"!", StringComparison.Ordinal) && !rule.StartsWith(@"!#", StringComparison.Ordinal) || - rule.StartsWith(@"!##", StringComparison.Ordinal) - ? null - : rule; - - private static string DropIfContainsBackslashSingleQuote(this string rule) => - rule.Contains(@"\'") ? null : rule; - - private static string TrimSingleBackslashFromEnd(this string rule) => - rule.EndsWith(@"\", StringComparison.Ordinal) && !rule.EndsWith(@"\\", StringComparison.Ordinal) - ? rule.Remove(rule.Length - 1) - : rule; - - private static string DropIfEmpty(this string rule) => - rule == string.Empty ? null : rule; - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 066d6e7e4..e2857cf11 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -1,11 +1,13 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Threading.Tasks; using FilterLists.Data; using FilterLists.Data.Entities.Junctions; +using FilterLists.Services.Extensions; using FilterLists.Services.Snapshot.Models; using MoreLinq; @@ -29,20 +31,22 @@ public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list) }; } + //TODO: add better compliance with Try/Parse pattern (https://stackoverflow.com/q/37810660/2343739) public async Task TrySaveAsync() { - await Add(); + await AddSnapEntity(); try { await SaveAsync(); } catch (Exception) { + //allow other snapshots to continue //TODO: log } } - private async Task Add() + private async Task AddSnapEntity() { dbContext.Snapshots.Add(snapEntity); await dbContext.SaveChangesAsync(); @@ -52,10 +56,10 @@ private async Task SaveAsync() { using (var transaction = dbContext.Database.BeginTransaction()) { - var content = await TryGetContent(); - if (content != null) + var lines = await TryGetLines(); + if (lines != null) { - await SaveInBatches(content); + await SaveInBatches(lines); await DedupSnapshotRules(); await SetSuccessful(); } @@ -64,11 +68,15 @@ private async Task SaveAsync() } } - private async Task TryGetContent() + private async Task> TryGetLines() { try { - return await GetContent(); + return await GetLines(); + } + catch (HttpRequestException) + { + return null; } catch (WebException we) { @@ -77,49 +85,45 @@ private async Task TryGetContent() } } - private async Task GetContent() + private async Task> GetLines() { + var lines = new HashSet(); using (var httpClient = new HttpClient()) { - using (var httpResponseMessage = await httpClient.GetAsync(list.ViewUrl)) + var response = await httpClient.GetAsync(list.ViewUrl, HttpCompletionOption.ResponseHeadersRead); + snapEntity.HttpStatusCode = ((int)response.StatusCode).ToString(); + response.EnsureSuccessStatusCode(); + using (var stream = await response.Content.ReadAsStreamAsync()) + using (var streamReader = new StreamReader(stream)) { - snapEntity.HttpStatusCode = ((int)httpResponseMessage.StatusCode).ToString(); - if (httpResponseMessage.IsSuccessStatusCode) - return await httpResponseMessage.Content.ReadAsStringAsync(); + string line; + while ((line = await streamReader.ReadLineAsync()) != null) + lines.AddIfNotNullOrEmpty(line.LintLine()); } } - return null; + return lines; } - private async Task SaveInBatches(string content) + private async Task SaveInBatches(IEnumerable lines) { - var rawRules = ParseRawRules(content); - var snapshotBatches = CreateBatches(rawRules); + var snapshotBatches = CreateBatches(lines); await SaveBatches(snapshotBatches); } - private static IEnumerable ParseRawRules(string content) - { - var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); - for (var i = 0; i < rawRules.Length; i++) - rawRules[i] = rawRules[i].LintRawRule(); - return new HashSet(rawRules.Where(r => r != null)); - } + private IEnumerable CreateBatches(IEnumerable lines) => + lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity)); - private IEnumerable CreateBatches(IEnumerable rawRules) => - rawRules.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity)); - - private static async Task SaveBatches(IEnumerable snapshotBatches) + private static async Task SaveBatches(IEnumerable batches) { - foreach (var batch in snapshotBatches) + foreach (var batch in batches) await batch.SaveAsync(); } private async Task DedupSnapshotRules() { var existingSnapshotRules = GetExistingSnapshotRules(); - UpdateRemovedSnapshotRules(existingSnapshotRules); + AddRemovedBySnapshots(existingSnapshotRules); RemoveDuplicateSnapshotRules(existingSnapshotRules); await dbContext.SaveChangesAsync(); } @@ -130,7 +134,7 @@ private IQueryable GetExistingSnapshotRules() => sr.AddedBySnapshot != snapEntity && sr.RemovedBySnapshot == null); - private void UpdateRemovedSnapshotRules(IQueryable existingSnapshotRules) + private void AddRemovedBySnapshots(IQueryable existingSnapshotRules) { var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapEntity); var removedSnapshotRules = existingSnapshotRules.Where(sr => !newSnapshotRules.Any(n => n.Rule == sr.Rule)); @@ -141,7 +145,8 @@ private void RemoveDuplicateSnapshotRules(IQueryable existingSnaps { var duplicateSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapEntity && - existingSnapshotRules.Any(e => e.Rule == sr.Rule)); + existingSnapshotRules.Any(e => + e.Rule == sr.Rule && e.RemovedBySnapshot == null)); dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules); } diff --git a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs index 2a3aa10b4..0b2f4cfb8 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs @@ -11,14 +11,14 @@ namespace FilterLists.Services.Snapshot public class SnapshotBatch { private readonly FilterListsDbContext dbContext; - private readonly IEnumerable rawRules; + private readonly IEnumerable lines; private readonly Data.Entities.Snapshot snapEntity; - public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable rawRules, + public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable lines, Data.Entities.Snapshot snapEntity) { this.dbContext = dbContext; - this.rawRules = rawRules; + this.lines = lines; this.snapEntity = snapEntity; } @@ -32,10 +32,10 @@ public async Task SaveAsync() await dbContext.SaveChangesAsync(); } - private IQueryable GetExistingRules() => dbContext.Rules.Where(r => rawRules.Contains(r.Raw)); + private IQueryable GetExistingRules() => dbContext.Rules.Where(r => lines.Contains(r.Raw)); private List CreateNewRules(IQueryable existingRules) => - rawRules.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList(); + lines.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList(); private void AddSnapshotRules(IQueryable rules) => snapEntity.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r}));