prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor ScrapeService architecture

Browse source
This commit is contained in:
Collin M. Barrett 2018-02-01 10:05:51 -06:00
parent 07c88be4e4
commit c456ac704c
7 changed files with 108 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/FilterLists.Services/FilterListService/FilterListService.cs
View file

@ -4,7 +4,6 @@
using AutoMapper.QueryableExtensions;
using FilterLists.Data;
using FilterLists.Data.Entities;
using FilterLists.Services.FilterListService.Models;
using Microsoft.EntityFrameworkCore;
namespace FilterLists.Services.FilterListService

2
src/FilterLists.Services/FilterListService/Models/FilterListSummaryDto.cs → src/FilterLists.Services/FilterListService/FilterListSummaryDto.cs
View file

@ -1,4 +1,4 @@
namespace FilterLists.Services.FilterListService.Models
namespace FilterLists.Services.FilterListService
{
public class FilterListSummaryDto
{

2
src/FilterLists.Services/ScrapeService/Models/FilterListViewUrlDto.cs → src/FilterLists.Services/ScrapeService/FilterListViewUrlDto.cs
View file

@ -1,4 +1,4 @@
namespace FilterLists.Services.ScrapeService.Models
namespace FilterLists.Services.ScrapeService
{
public class FilterListViewUrlDto
{

41
src/FilterLists.Services/ScrapeService/Models/Snapshot.cs
View file

@ -1,41 +0,0 @@
using System;
namespace FilterLists.Services.ScrapeService.Models
{
public class Snapshot
{
public string Content
{
set
{
if (value == null) return;
var rawRules = value.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < rawRules.Length; i++)
rawRules[i] = LintStringForMySql(rawRules[i]);
RawRules = rawRules;
}
}
public int FilterListId { get; set; }
public string[] RawRules { get; private set; }
private static string LintStringForMySql(string rule)
{
rule = TrimSingleBackslashFromEnd(rule);
rule = DropIfContainsBackslashSingleQuote(rule);
return rule;
}
private static string TrimSingleBackslashFromEnd(string rule)
{
return rule.EndsWith(@"\") && !rule.EndsWith(@"\\") ? rule.Remove(rule.Length - 1) : rule;
}
private static string DropIfContainsBackslashSingleQuote(string rule)
{
//TODO: resolve issue and/or log rule
return rule.Contains(@"\'") ? null : rule;
}
}
}

23
src/FilterLists.Services/ScrapeService/RuleLinterExtensions.cs Normal file
View file

@ -0,0 +1,23 @@
namespace FilterLists.Services.ScrapeService
{
public static class RuleLinterExtensions
{
public static string LintStringForMySql(this string rule)
{
rule = rule.TrimSingleBackslashFromEnd();
rule = rule.DropIfContainsBackslashSingleQuote();
return rule;
}
private static string TrimSingleBackslashFromEnd(this string rule)
{
return rule.EndsWith(@"\") && !rule.EndsWith(@"\\") ? rule.Remove(rule.Length - 1) : rule;
}
private static string DropIfContainsBackslashSingleQuote(this string rule)
{
//TODO: resolve issue and/or log rule
return rule.Contains(@"\'") ? null : rule;
}
}
}

60
src/FilterLists.Services/ScrapeService/ScrapeService.cs
View file

@ -5,9 +5,6 @@
using System.Threading.Tasks;
using AutoMapper.QueryableExtensions;
using FilterLists.Data;
using FilterLists.Data.Entities;
using FilterLists.Data.Entities.Junctions;
using FilterLists.Services.ScrapeService.Models;
using Microsoft.EntityFrameworkCore;
namespace FilterLists.Services.ScrapeService
@ -24,23 +21,22 @@ public ScrapeService(FilterListsDbContext dbContext)
//TODO: call via scheduled job
public async Task ScrapeAsync(int batchSize)
{
var lists = await GetNextFilterListsToScrape(batchSize);
var lists = await GetNextListsToScrape(batchSize);
var snapshots = await GetSnapshots(lists);
await SaveSnapshots(snapshots);
}
private async Task<IEnumerable<FilterListViewUrlDto>> GetNextFilterListsToScrape(int batchSize)
private async Task<List<FilterListViewUrlDto>> GetNextListsToScrape(int batchSize)
{
return await dbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(batchSize)
.ProjectTo<FilterListViewUrlDto>().ToListAsync();
}
private static async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)
private async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)
{
return await Task.WhenAll(lists
.Select(async list =>
new Snapshot {Content = await TryGetContent(list.ViewUrl), FilterListId = list.Id})
.Where(x => x.Result.RawRules != null));
.Select(async list => new Snapshot(dbContext, await TryGetContent(list.ViewUrl), list.Id))
.Where(x => x.Result.HasRules));
}
private static async Task<string> TryGetContent(string url)
@ -69,51 +65,9 @@ private static async Task<string> GetHttpResponseMessageContent(string url)
return null;
}
private async Task SaveSnapshots(IEnumerable<Snapshot> snapshots)
private static async Task SaveSnapshots(IEnumerable<Snapshot> snapshots)
{
foreach (var snapshot in snapshots) await AddOrUpdateRules(snapshot);
}
private async Task AddOrUpdateRules(Snapshot snapshot)
{
dbContext.ChangeTracker.AutoDetectChangesEnabled = false;
// add new Rules
var preExistingSnapshotRules = dbContext.Rules.Where(x => snapshot.RawRules.Contains(x.Raw));
var newSnapshotRawRules = snapshot.RawRules.Except(preExistingSnapshotRules.Select(x => x.Raw));
var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRuleRaw => new Rule {Raw = newSnapshotRuleRaw})
.ToList();
dbContext.Rules.AddRange(newSnapshotRules);
// remove deleted FilterListRules
var preExistingFilterListRules =
dbContext.FilterListRules.Where(x => x.FilterListId == snapshot.FilterListId);
var deletedFilterListRules =
preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId));
dbContext.FilterListRules.RemoveRange(deletedFilterListRules);
// add FilterListRules for pre-existing Rules
var preExistingSnapshotFilterListRules = preExistingSnapshotRules
.Select(newSnapshotRule =>
new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule}).ToList()
.Except(preExistingFilterListRules).ToList();
dbContext.FilterListRules.AddRange(preExistingSnapshotFilterListRules);
// add new FilterListRules
var newFilterListRules = newSnapshotRules.Select(newSnapshotRule =>
new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule}).ToList();
dbContext.FilterListRules.AddRange(newFilterListRules);
// update UpdatedDateUtc
var list = dbContext.FilterLists.Find(snapshot.FilterListId);
if (preExistingSnapshotFilterListRules.Any() || newFilterListRules.Any() || deletedFilterListRules.Any())
list.UpdatedDateUtc = DateTime.UtcNow;
// update ScrapedDateUtc
list.ScrapedDateUtc = DateTime.UtcNow;
dbContext.FilterLists.Update(list);
await dbContext.SaveChangesAsync();
foreach (var snapshot in snapshots) await snapshot.AddOrUpdateRules();
}
}
}

76
src/FilterLists.Services/ScrapeService/Snapshot.cs Normal file
View file

@ -0,0 +1,76 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Data.Entities;
using FilterLists.Data.Entities.Junctions;
namespace FilterLists.Services.ScrapeService
{
public class Snapshot
{
private readonly FilterListsDbContext dbContext;
private readonly int filterListId;
private string[] rawRules;
public Snapshot(FilterListsDbContext dbContext, string content, int filterListId)
{
this.dbContext = dbContext;
this.filterListId = filterListId;
ParseContent(content);
}
public bool HasRules => rawRules != null;
private void ParseContent(string content)
{
if (content == null) return;
rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < rawRules.Length; i++)
rawRules[i] = rawRules[i].LintStringForMySql();
}
public async Task AddOrUpdateRules()
{
dbContext.ChangeTracker.AutoDetectChangesEnabled = false;
// add new Rules
var preExistingSnapshotRules = dbContext.Rules.Where(x => rawRules.Contains(x.Raw));
var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw));
var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule}).ToList();
dbContext.Rules.AddRange(newSnapshotRules);
// remove deleted FilterListRules
var preExistingFilterListRules = dbContext.FilterListRules.Where(x => x.FilterListId == filterListId);
var deletedFilterListRules =
preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId));
// add FilterListRules for pre-existing Rules
var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule =>
new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList()
.Except(preExistingFilterListRules).ToList();
// add new FilterListRules
var newFilterListRules = newSnapshotRules.Select(newSnapshotRule =>
new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList();
// update UpdatedDateUtc
var list = dbContext.FilterLists.Find(filterListId);
if (preExistingSnapshotFilterListRules.Any() || newFilterListRules.Any() || deletedFilterListRules.Any())
list.UpdatedDateUtc = DateTime.UtcNow;
// update ScrapedDateUtc
list.ScrapedDateUtc = DateTime.UtcNow;
dbContext.FilterListRules.RemoveRange(deletedFilterListRules);
dbContext.FilterListRules.AddRange(preExistingSnapshotFilterListRules);
dbContext.FilterListRules.AddRange(newFilterListRules);
dbContext.FilterLists.Update(list);
await dbContext.SaveChangesAsync();
}
}
}