prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

misc SnapshotService cleanup

Browse source 
ref #343
This commit is contained in:
Collin Barrett 2018-08-15 06:04:44 -05:00
parent 4e7d282de1
commit 46dffec60a
7 changed files with 65 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/FilterLists.Services/Extensions/CollectionExtensions.cs Normal file
View file

@ -0,0 +1,17 @@
using System.Collections.Generic;
namespace FilterLists.Services.Extensions
{
public static class CollectionExtensions
{
//https://stackoverflow.com/a/26360010/2343739
public static void AddRange<T>(this ICollection<T> destination, IEnumerable<T> source)
{
if (destination is List<T> list)
list.AddRange(source);
else
foreach (var item in source)
destination.Add(item);
}
}
}

29
src/FilterLists.Services/Extensions/EnumerableExtensions.cs
View file

@ -1,29 +0,0 @@
using System.Collections.Generic;
namespace FilterLists.Services.Extensions
{
public static class EnumerableExtensions
{
public static void AddRange<T>(this ICollection<T> destination, IEnumerable<T> source)
{
foreach (var item in source)
destination.Add(item);
}
public static IEnumerable<IEnumerable<T>> GetBatches<T>(this IEnumerable<T> source, int batchSize)
{
var batches = new List<T>(batchSize);
foreach (var item in source)
{
batches.Add(item);
if (batches.Count != batchSize)
continue;
yield return batches.AsReadOnly();
batches = new List<T>(batchSize);
}
if (batches.Count > 0)
yield return batches.AsReadOnly();
}
}
}

1
src/FilterLists.Services/FilterLists.Services.csproj
View file

@ -20,6 +20,7 @@
<PackageReference Include="JetBrains.Annotations" Version="2018.2.1" />
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.4.0" />
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha" />
<PackageReference Include="morelinq" Version="3.0.0" />
<PackageReference Include="SendGrid" Version="9.9.0" />
</ItemGroup>

35
src/FilterLists.Services/Snapshot/RawRuleLinterExtensions.cs
View file

@ -1,35 +1,38 @@
namespace FilterLists.Services.Snapshot
using System;
namespace FilterLists.Services.Snapshot
{
public static class RawRuleLinterExtensions
{
public static string LintRawRule(this string rule)
{
rule = rule?.TrimLeadingAndTrailingWhitespace();
rule = rule?.DropIfEmpty();
rule = rule?.DropIfComment();
rule = rule.Trim();
rule = rule.DropIfEmpty();
rule = rule?.DropIfTooLong();
rule = rule?.DropIfComment();
rule = rule?.DropIfContainsBackslashSingleQuote();
rule = rule?.TrimSingleBackslashFromEnd();
return rule;
}
private static string TrimLeadingAndTrailingWhitespace(this string rule)
{
char[] charsToTrim = {' ', '\t'};
return rule.Trim(charsToTrim);
}
private static string DropIfEmpty(this string rule) => rule == "" ? null : rule;
private static string DropIfEmpty(this string rule) =>
rule == string.Empty ? null : rule;
private static string DropIfTooLong(this string rule) =>
rule.Length > 8192 ? null : rule;
private static string DropIfComment(this string rule) =>
rule.StartsWith(@"!") && !rule.StartsWith(@"!#") || rule.StartsWith(@"!##") ? null : rule;
private static string DropIfTooLong(this string rule) => rule.Length > 8192 ? null : rule;
rule.StartsWith(@"!", StringComparison.Ordinal) && !rule.StartsWith(@"!#", StringComparison.Ordinal) ||
rule.StartsWith(@"!##", StringComparison.Ordinal)
? null
: rule;
private static string DropIfContainsBackslashSingleQuote(this string rule) =>
rule.Contains(@"\'") ? null : rule;
private static string TrimSingleBackslashFromEnd(this string rule) =>
rule.EndsWith(@"\") && !rule.EndsWith(@"\\") ? rule.Remove(rule.Length - 1) : rule;
rule.EndsWith(@"\", StringComparison.Ordinal) && !rule.EndsWith(@"\\", StringComparison.Ordinal)
? rule.Remove(rule.Length - 1)
: rule;
}
}

26
src/FilterLists.Services/Snapshot/SnapshotDe.cs
View file

@ -7,8 +7,8 @@
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Data.Entities.Junctions;
using FilterLists.Services.Extensions;
using FilterLists.Services.Snapshot.Models;
using MoreLinq;
namespace FilterLists.Services.Snapshot
{
@ -28,10 +28,14 @@ public SnapshotDe(FilterListsDbContext dbContext, EmailService emailService, Fil
this.dbContext = dbContext;
this.emailService = emailService;
this.list = list;
snapshot = new Data.Entities.Snapshot {FilterListId = list.Id};
snapshot = new Data.Entities.Snapshot
{
FilterListId = list.Id,
AddedSnapshotRules = new List<SnapshotRule>()
};
}
public async Task SaveSnapshotAsync()
public async Task SaveAsync()
{
using (var transaction = dbContext.Database.BeginTransaction())
{
@ -121,26 +125,26 @@ private async Task SendExceptionEmail(Exception e)
private async Task SaveSnapshotInBatches(string content)
{
var rawRules = GetRawRules(content);
var snapshotBatches = GetSnapshotBatches(rawRules);
var rawRules = ParseRawRules(content);
var snapshotBatches = CreateSnapshotBatches(rawRules);
await SaveSnapshotBatches(snapshotBatches);
}
private static IEnumerable<string> GetRawRules(string content)
private static IEnumerable<string> ParseRawRules(string content)
{
var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < rawRules.Length; i++)
rawRules[i] = rawRules[i].LintRawRule();
return new HashSet<string>(rawRules.Where(r => r != null));
return rawRules.Where(r => r != null);
}
private IEnumerable<SnapshotBatchDe> GetSnapshotBatches(IEnumerable<string> rawRules) =>
rawRules.GetBatches(BatchSize).Select(b => new SnapshotBatchDe(dbContext, snapshot, b));
private IEnumerable<SnapshotDeBatch> CreateSnapshotBatches(IEnumerable<string> rawRules) =>
rawRules.Batch(BatchSize).Select(b => new SnapshotDeBatch(dbContext, snapshot, b));
private static async Task SaveSnapshotBatches(IEnumerable<SnapshotBatchDe> snapshotBatches)
private static async Task SaveSnapshotBatches(IEnumerable<SnapshotDeBatch> snapshotBatches)
{
foreach (var snapshotBatch in snapshotBatches)
await snapshotBatch.SaveSnapshotBatchAsync();
await snapshotBatch.SaveAsync();
}
private async Task DedupSnapshotRules()

24
src/FilterLists.Services/Snapshot/SnapshotBatchDe.cs → src/FilterLists.Services/Snapshot/SnapshotDeBatch.cs
View file

@ -8,14 +8,14 @@
namespace FilterLists.Services.Snapshot
{
public class SnapshotBatchDe
public class SnapshotDeBatch
{
private readonly FilterListsDbContext dbContext;
private readonly IEnumerable<string> rawRules;
private readonly Data.Entities.Snapshot snapshot;
private IQueryable<Rule> rules;
private IEnumerable<Rule> rules;
public SnapshotBatchDe(FilterListsDbContext dbContext, Data.Entities.Snapshot snapshot,
public SnapshotDeBatch(FilterListsDbContext dbContext, Data.Entities.Snapshot snapshot,
IEnumerable<string> rawRules)
{
this.dbContext = dbContext;
@ -23,7 +23,7 @@ public SnapshotBatchDe(FilterListsDbContext dbContext, Data.Entities.Snapshot sn
this.rawRules = rawRules;
}
public async Task SaveSnapshotBatchAsync()
public async Task SaveAsync()
{
await AddRules();
AddSnapshotRules();
@ -32,18 +32,18 @@ public async Task SaveSnapshotBatchAsync()
private async Task AddRules()
{
var existingRules = dbContext.Rules.Where(r => rawRules.Contains(r.Raw));
var newRawRules = rawRules.Except(existingRules.Select(r => r.Raw));
var newRules = newRawRules.Select(r => new Rule {Raw = r}).ToList();
var existingRules = GetExistingRules();
var newRules = CreateNewRules(existingRules);
rules = existingRules.Concat(newRules);
await dbContext.Rules.AddRangeAsync(newRules);
}
private void AddSnapshotRules()
{
if (snapshot.AddedSnapshotRules == null)
snapshot.AddedSnapshotRules = new List<SnapshotRule>();
private IQueryable<Rule> GetExistingRules() => dbContext.Rules.Where(r => rawRules.Contains(r.Raw));
private List<Rule> CreateNewRules(IQueryable<Rule> existingRules) =>
rawRules.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList();
private void AddSnapshotRules() =>
snapshot.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r}));
}
}
}

2
src/FilterLists.Services/Snapshot/SnapshotService.cs
View file

@ -69,7 +69,7 @@ private IEnumerable<SnapshotDe> CreateSnapshots(IEnumerable<FilterListViewUrlDto
private static async Task SaveSnapshots(IEnumerable<SnapshotDe> snapshots)
{
foreach (var snapshot in snapshots)
await snapshot.SaveSnapshotAsync();
await snapshot.SaveAsync();
}
}
}