prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update end-of-line algo

Browse source 
closes #352
This commit is contained in:
Collin M. Barrett 2018-08-18 15:13:47 -05:00
parent 740da95e1b
commit b962a562ec
5 changed files with 62 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/FilterLists.Services/Extensions/CollectionExtensions.cs
View file

@ -13,5 +13,11 @@ public static void AddRange<T>(this ICollection<T> destination, IEnumerable<T> s
foreach (var item in source)
destination.Add(item);
}
public static void AddIfNotNullOrEmpty(this ICollection<string> set, string item)
{
if (!string.IsNullOrEmpty(item))
set.Add(item);
}
}
}

15
src/FilterLists.Services/Snapshot/LineLinterExtensions.cs Normal file
View file

@ -0,0 +1,15 @@
namespace FilterLists.Services.Snapshot
{
public static class LineLinterExtensions
{
public static string LintLine(this string line)
{
line = line.Trim();
line = line.DropIfTooLong();
return line;
}
private static string DropIfTooLong(this string line) =>
line.Length > 8192 ? null : line;
}
}

38
src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs
View file

@ -1,38 +0,0 @@
using System;
namespace FilterLists.Services.Snapshot
{
public static class RawRuleLinterExtensions
{
public static string LintRawRule(this string rule)
{
rule = rule.Trim();
rule = rule.DropIfTooLong();
rule = rule?.DropIfComment();
rule = rule?.DropIfContainsBackslashSingleQuote();
rule = rule?.TrimSingleBackslashFromEnd();
rule = rule?.DropIfEmpty();
return rule;
}
private static string DropIfTooLong(this string rule) =>
rule.Length > 8192 ? null : rule;
private static string DropIfComment(this string rule) =>
rule.StartsWith(@"!", StringComparison.Ordinal) && !rule.StartsWith(@"!#", StringComparison.Ordinal) ||
rule.StartsWith(@"!##", StringComparison.Ordinal)
? null
: rule;
private static string DropIfContainsBackslashSingleQuote(this string rule) =>
rule.Contains(@"\'") ? null : rule;
private static string TrimSingleBackslashFromEnd(this string rule) =>
rule.EndsWith(@"\", StringComparison.Ordinal) && !rule.EndsWith(@"\\", StringComparison.Ordinal)
? rule.Remove(rule.Length - 1)
: rule;
private static string DropIfEmpty(this string rule) =>
rule == string.Empty ? null : rule;
}
}

67
src/FilterLists.Services/Snapshot/Snapshot.cs
View file

@ -1,11 +1,13 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Data.Entities.Junctions;
using FilterLists.Services.Extensions;
using FilterLists.Services.Snapshot.Models;
using MoreLinq;
@ -29,20 +31,22 @@ public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list)
};
}
//TODO: add better compliance with Try/Parse pattern (https://stackoverflow.com/q/37810660/2343739)
public async Task TrySaveAsync()
{
await Add();
await AddSnapEntity();
try
{
await SaveAsync();
}
catch (Exception)
{
//allow other snapshots to continue
//TODO: log
}
}
private async Task Add()
private async Task AddSnapEntity()
{
dbContext.Snapshots.Add(snapEntity);
await dbContext.SaveChangesAsync();
@ -52,10 +56,10 @@ private async Task SaveAsync()
{
using (var transaction = dbContext.Database.BeginTransaction())
{
var content = await TryGetContent();
if (content != null)
var lines = await TryGetLines();
if (lines != null)
{
await SaveInBatches(content);
await SaveInBatches(lines);
await DedupSnapshotRules();
await SetSuccessful();
}
@ -64,11 +68,15 @@ private async Task SaveAsync()
}
}
private async Task<string> TryGetContent()
private async Task<IEnumerable<string>> TryGetLines()
{
try
{
return await GetContent();
return await GetLines();
}
catch (HttpRequestException)
{
return null;
}
catch (WebException we)
{
@ -77,49 +85,45 @@ private async Task<string> TryGetContent()
}
}
private async Task<string> GetContent()
private async Task<IEnumerable<string>> GetLines()
{
var lines = new HashSet<string>();
using (var httpClient = new HttpClient())
{
using (var httpResponseMessage = await httpClient.GetAsync(list.ViewUrl))
var response = await httpClient.GetAsync(list.ViewUrl, HttpCompletionOption.ResponseHeadersRead);
snapEntity.HttpStatusCode = ((int)response.StatusCode).ToString();
response.EnsureSuccessStatusCode();
using (var stream = await response.Content.ReadAsStreamAsync())
using (var streamReader = new StreamReader(stream))
{
snapEntity.HttpStatusCode = ((int)httpResponseMessage.StatusCode).ToString();
if (httpResponseMessage.IsSuccessStatusCode)
return await httpResponseMessage.Content.ReadAsStringAsync();
string line;
while ((line = await streamReader.ReadLineAsync()) != null)
lines.AddIfNotNullOrEmpty(line.LintLine());
}
}
return null;
return lines;
}
private async Task SaveInBatches(string content)
private async Task SaveInBatches(IEnumerable<string> lines)
{
var rawRules = ParseRawRules(content);
var snapshotBatches = CreateBatches(rawRules);
var snapshotBatches = CreateBatches(lines);
await SaveBatches(snapshotBatches);
}
private static IEnumerable<string> ParseRawRules(string content)
{
var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < rawRules.Length; i++)
rawRules[i] = rawRules[i].LintRawRule();
return new HashSet<string>(rawRules.Where(r => r != null));
}
private IEnumerable<SnapshotBatch> CreateBatches(IEnumerable<string> lines) =>
lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity));
private IEnumerable<SnapshotBatch> CreateBatches(IEnumerable<string> rawRules) =>
rawRules.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity));
private static async Task SaveBatches(IEnumerable<SnapshotBatch> snapshotBatches)
private static async Task SaveBatches(IEnumerable<SnapshotBatch> batches)
{
foreach (var batch in snapshotBatches)
foreach (var batch in batches)
await batch.SaveAsync();
}
private async Task DedupSnapshotRules()
{
var existingSnapshotRules = GetExistingSnapshotRules();
UpdateRemovedSnapshotRules(existingSnapshotRules);
AddRemovedBySnapshots(existingSnapshotRules);
RemoveDuplicateSnapshotRules(existingSnapshotRules);
await dbContext.SaveChangesAsync();
}
@ -130,7 +134,7 @@ private IQueryable<SnapshotRule> GetExistingSnapshotRules() =>
sr.AddedBySnapshot != snapEntity &&
sr.RemovedBySnapshot == null);
private void UpdateRemovedSnapshotRules(IQueryable<SnapshotRule> existingSnapshotRules)
private void AddRemovedBySnapshots(IQueryable<SnapshotRule> existingSnapshotRules)
{
var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapEntity);
var removedSnapshotRules = existingSnapshotRules.Where(sr => !newSnapshotRules.Any(n => n.Rule == sr.Rule));
@ -141,7 +145,8 @@ private void RemoveDuplicateSnapshotRules(IQueryable<SnapshotRule> existingSnaps
{
var duplicateSnapshotRules = dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot == snapEntity &&
existingSnapshotRules.Any(e => e.Rule == sr.Rule));
existingSnapshotRules.Any(e =>
e.Rule == sr.Rule && e.RemovedBySnapshot == null));
dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules);
}

10
src/FilterLists.Services/Snapshot/SnapshotBatch.cs
View file

@ -11,14 +11,14 @@ namespace FilterLists.Services.Snapshot
public class SnapshotBatch
{
private readonly FilterListsDbContext dbContext;
private readonly IEnumerable<string> rawRules;
private readonly IEnumerable<string> lines;
private readonly Data.Entities.Snapshot snapEntity;
public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable<string> rawRules,
public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable<string> lines,
Data.Entities.Snapshot snapEntity)
{
this.dbContext = dbContext;
this.rawRules = rawRules;
this.lines = lines;
this.snapEntity = snapEntity;
}
@ -32,10 +32,10 @@ public async Task SaveAsync()
await dbContext.SaveChangesAsync();
}
private IQueryable<Rule> GetExistingRules() => dbContext.Rules.Where(r => rawRules.Contains(r.Raw));
private IQueryable<Rule> GetExistingRules() => dbContext.Rules.Where(r => lines.Contains(r.Raw));
private List<Rule> CreateNewRules(IQueryable<Rule> existingRules) =>
rawRules.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList();
lines.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList();
private void AddSnapshotRules(IQueryable<Rule> rules) =>
snapEntity.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r}));