prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

move towards batched snapshots (wip)

Browse source
This commit is contained in:
Collin M. Barrett 2018-02-05 11:14:46 -06:00
parent 21bbff279f
commit 0ab08ecc42
5 changed files with 93 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/FilterLists.Api/V1/Controllers/SnapshotController.cs
View file

@ -14,6 +14,7 @@ public SnapshotController(SnapshotService snapshotService)
}
[HttpPost]
//TODO: convert API endpoint to server-side scheduled job
public async Task<IActionResult> Index()
{
#if DEBUG

3
src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs
View file

@ -2,6 +2,7 @@
{
public static class RawRuleLinterExtensions
{
//TODO: resolve issues and/or track dropped rules
public static string LintStringForMySql(this string rule)
{
rule = rule.TrimSingleBackslashFromEnd();
@ -19,7 +20,6 @@ private static string TrimSingleBackslashFromEnd(this string rule)
private static string DropIfContainsBackslashSingleQuote(this string rule)
{
//TODO: resolve issue and/or track dropped rule
if (rule != null)
return rule.Contains(@"\'") ? null : rule;
return null;
@ -27,7 +27,6 @@ private static string DropIfContainsBackslashSingleQuote(this string rule)
private static string DropIfTooLong(this string rule)
{
//TODO: resolve issue and/or track dropped rule
return rule?.Length > 8192 ? null : rule;
}
}

49
src/FilterLists.Services/SnapshotService/SnapshotBatchDe.cs Normal file
View file

@ -0,0 +1,49 @@
using System.Collections.Generic;
using System.Linq;
using FilterLists.Data;
using FilterLists.Data.Entities;
using FilterLists.Data.Entities.Junctions;
using Microsoft.EntityFrameworkCore;
namespace FilterLists.Services.SnapshotService
{
public class SnapshotBatchDe
{
private readonly FilterListsDbContext dbContext;
private readonly IEnumerable<string> rawRules;
private readonly Snapshot snapshot;
private IEnumerable<Rule> newSnapshotRules;
private IEnumerable<Rule> preExistingSnapshotRules;
public SnapshotBatchDe(FilterListsDbContext dbContext, Snapshot snapshot, IEnumerable<string> rawRules)
{
this.dbContext = dbContext;
this.snapshot = snapshot;
this.rawRules = rawRules;
}
public void SaveSnapshotBatch()
{
AddNewRules();
AddSnapshotRules();
dbContext.SaveChanges();
}
private void AddNewRules()
{
preExistingSnapshotRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw)).AsNoTracking();
var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw));
newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule})
.ToList();
dbContext.Rules.AddRange(newSnapshotRules);
}
private void AddSnapshotRules()
{
snapshot.SnapshotRules = preExistingSnapshotRules
.Concat(newSnapshotRules)
.Select(rule => new SnapshotRule {Rule = rule, Snapshot = snapshot})
.ToList();
}
}
}

60
src/FilterLists.Services/SnapshotService/SnapshotDomainEntity.cs → src/FilterLists.Services/SnapshotService/SnapshotDe.cs
View file

@ -6,43 +6,35 @@
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Data.Entities;
using FilterLists.Data.Entities.Junctions;
namespace FilterLists.Services.SnapshotService
{
public class SnapshotDomainEntity
public class SnapshotDe
{
private const int BatchSize = 100;
private readonly FilterListsDbContext dbContext;
private readonly FilterListViewUrlDto list;
private IEnumerable<Rule> newSnapshotRules;
private IEnumerable<Rule> preExistingSnapshotRules;
private List<string> rawRules;
private Snapshot snapshot;
public SnapshotDomainEntity(FilterListsDbContext dbContext, FilterListViewUrlDto list)
public SnapshotDe(FilterListsDbContext dbContext, FilterListViewUrlDto list)
{
this.dbContext = dbContext;
this.list = list;
}
public async Task SaveSnapshot()
public async Task SaveSnapshotAsync()
{
await AddSnapshot();
var content = await TryGetContent();
if (content != null)
{
ParseRawRules(content);
await AddNewRules();
AddSnapshotRules();
}
await dbContext.SaveChangesAsync();
if (content != null)
SaveSnapshotInBatches(content);
}
private async Task AddSnapshot()
{
snapshot = new Snapshot {FilterList = await dbContext.FilterLists.FindAsync(list.Id)};
dbContext.Snapshots.Add(snapshot);
snapshot = new Snapshot {FilterListId = list.Id};
await dbContext.Snapshots.AddAsync(snapshot);
}
private async Task<string> TryGetContent()
@ -71,29 +63,37 @@ private async Task<string> GetContent()
return null;
}
private void ParseRawRules(string content)
private void SaveSnapshotInBatches(string content)
{
rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList();
var rawRules = GetRawRules(content);
var snapshotBatches = GetSnapshotBatches(rawRules);
SaveSnapshotBatches(snapshotBatches);
}
private static List<string> GetRawRules(string content)
{
var rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList();
for (var i = 0; i < rawRules.Count; i++)
rawRules[i] = rawRules[i].LintStringForMySql();
rawRules = rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
return rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
}
private async Task AddNewRules()
private IEnumerable<SnapshotBatchDe> GetSnapshotBatches(List<string> rawRules)
{
preExistingSnapshotRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw));
var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw));
newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule})
.ToList();
await dbContext.Rules.AddRangeAsync(newSnapshotRules);
var rawRuleBatches = GetRawRuleBatches(rawRules);
return rawRuleBatches.Select(rawRuleBatch => new SnapshotBatchDe(dbContext, snapshot, rawRuleBatch));
}
private void AddSnapshotRules()
public static IEnumerable<IEnumerable<string>> GetRawRuleBatches(List<string> rawRules)
{
snapshot.SnapshotRules = preExistingSnapshotRules
.Concat(newSnapshotRules)
.Select(rule => new SnapshotRule {Rule = rule, Snapshot = snapshot})
.ToList();
for (var i = 0; i < rawRules.Count; i += BatchSize)
yield return rawRules.GetRange(i, Math.Min(BatchSize, rawRules.Count - i));
}
private static void SaveSnapshotBatches(IEnumerable<SnapshotBatchDe> snapshotBatches)
{
foreach (var snapshotBatch in snapshotBatches)
snapshotBatch.SaveSnapshotBatch();
}
}
}

41
src/FilterLists.Services/SnapshotService/SnapshotService.cs
View file

@ -16,53 +16,36 @@ public SnapshotService(FilterListsDbContext dbContext)
this.dbContext = dbContext;
}
//TODO: call via scheduled job
public async Task CaptureSnapshotsAsync(int batchSize)
{
var lists = await GetNextListsToCapture(batchSize);
var lists = await GetLeastRecentlyCapturedLists(batchSize);
var snapshots = GetSnapshots(lists);
await SaveSnapshots(snapshots);
}
private async Task<IEnumerable<FilterListViewUrlDto>> GetNextListsToCapture(int batchSize)
{
var neverCapturedLists = await GetNeverCapturedLists(batchSize);
if (neverCapturedLists.Count == batchSize)
return neverCapturedLists;
var leastRecentlyCapturedLists = await GetLeastRecentlyCapturedLists(batchSize - neverCapturedLists.Count);
return neverCapturedLists.Concat(leastRecentlyCapturedLists);
}
private async Task<List<FilterListViewUrlDto>> GetNeverCapturedLists(int batchSize)
private async Task<IEnumerable<FilterListViewUrlDto>> GetLeastRecentlyCapturedLists(int batchSize)
{
return await dbContext.FilterLists
.Where(x => x.Snapshots.Count == 0)
.OrderBy(x => x.Snapshots.Any())
.ThenBy(x => x.Snapshots
.Select(y => y.CreatedDateUtc)
.OrderByDescending(y => y)
.FirstOrDefault())
.Take(batchSize)
.AsNoTracking()
.ProjectTo<FilterListViewUrlDto>()
.ToListAsync();
}
private async Task<List<FilterListViewUrlDto>> GetLeastRecentlyCapturedLists(int batchSize)
private IEnumerable<SnapshotDe> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)
{
return await dbContext.Snapshots
.GroupBy(x => x.FilterList)
.Select(x => x.OrderByDescending(y => y.CreatedDateUtc).First())
.OrderBy(x => x.CreatedDateUtc)
.Select(x => x.FilterList)
.Take(batchSize)
.ProjectTo<FilterListViewUrlDto>()
.ToListAsync();
return lists.Select(list => new SnapshotDe(dbContext, list));
}
private IEnumerable<SnapshotDomainEntity> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)
{
return lists.Select(list => new SnapshotDomainEntity(dbContext, list));
}
private static async Task SaveSnapshots(IEnumerable<SnapshotDomainEntity> snapshots)
private static async Task SaveSnapshots(IEnumerable<SnapshotDe> snapshots)
{
foreach (var snapshot in snapshots)
await snapshot.SaveSnapshot();
await snapshot.SaveSnapshotAsync();
}
}
}