prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

ScrapeService tweaks

Browse source
This commit is contained in:
Collin M. Barrett 2018-02-02 15:55:50 -06:00
parent 2b1ffa1018
commit edf8f1b527
4 changed files with 53 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
src/FilterLists.Services/ScrapeService/RawRuleLinterExtensions.cs Normal file
View file

@ -0,0 +1,34 @@
namespace FilterLists.Services.ScrapeService
{
public static class RawRuleLinterExtensions
{
public static string LintStringForMySql(this string rule)
{
rule = rule.TrimSingleBackslashFromEnd();
rule = rule.DropIfContainsBackslashSingleQuote();
rule = rule.DropIfTooLong();
return rule;
}
private static string TrimSingleBackslashFromEnd(this string rule)
{
if (rule != null)
return rule.EndsWith(@"\") && !rule.EndsWith(@"\\") ? rule.Remove(rule.Length - 1) : rule;
return null;
}
private static string DropIfContainsBackslashSingleQuote(this string rule)
{
//TODO: resolve issue and/or track dropped rule
if (rule != null)
return rule.Contains(@"\'") ? null : rule;
return null;
}
private static string DropIfTooLong(this string rule)
{
//TODO: resolve issue and/or track dropped rule
return rule?.Length > 8192 ? null : rule;
}
}
}

23
src/FilterLists.Services/ScrapeService/RuleLinterExtensions.cs
View file

@ -1,23 +0,0 @@
namespace FilterLists.Services.ScrapeService
{
public static class RuleLinterExtensions
{
public static string LintStringForMySql(this string rule)
{
rule = rule.TrimSingleBackslashFromEnd();
rule = rule.DropIfContainsBackslashSingleQuote();
return rule;
}
private static string TrimSingleBackslashFromEnd(this string rule)
{
return rule.EndsWith(@"\") && !rule.EndsWith(@"\\") ? rule.Remove(rule.Length - 1) : rule;
}
private static string DropIfContainsBackslashSingleQuote(this string rule)
{
//TODO: resolve issue and/or log rule
return rule.Contains(@"\'") ? null : rule;
}
}
}

10
src/FilterLists.Services/ScrapeService/ScrapeService.cs
View file

@ -67,7 +67,15 @@ private static async Task<string> GetHttpResponseMessageContent(string url)
private static async Task SaveSnapshots(IEnumerable<Snapshot> snapshots)
{
foreach (var snapshot in snapshots) await snapshot.AddOrUpdateRules();
foreach (var snapshot in snapshots)
try
{
await snapshot.AddOrUpdateRules();
}
catch (Exception)
{
//TODO: log exception
}
}
}
}

17
src/FilterLists.Services/ScrapeService/Snapshot.cs
View file

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FilterLists.Data;
@ -13,7 +14,7 @@ public class Snapshot
private readonly int filterListId;
private string[] rawRules;
private List<string> rawRules;
public Snapshot(FilterListsDbContext dbContext, int filterListId, string content)
{
@ -27,11 +28,13 @@ public Snapshot(FilterListsDbContext dbContext, int filterListId, string content
private void ParseContent(string content)
{
if (content == null) return;
rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < rawRules.Length; i++)
rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList();
for (var i = 0; i < rawRules.Count; i++)
rawRules[i] = rawRules[i].LintStringForMySql();
rawRules = rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
}
//TODO: split large lists into batches
public async Task AddOrUpdateRules()
{
dbContext.ChangeTracker.AutoDetectChangesEnabled = false;
@ -40,19 +43,23 @@ public async Task AddOrUpdateRules()
var preExistingSnapshotRules = dbContext.Rules.Where(x => rawRules.Contains(x.Raw));
var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw));
var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule}).ToList();
await dbContext.Rules.AddRangeAsync(newSnapshotRules);
// remove deleted FilterListRules
var preExistingFilterListRules = dbContext.FilterListRules.Where(x => x.FilterListId == filterListId);
var deletedFilterListRules = preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId));
dbContext.FilterListRules.RemoveRange(deletedFilterListRules);
// add FilterListRules for pre-existing Rules
var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule =>
new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList()
.Except(preExistingFilterListRules).ToList();
await dbContext.FilterListRules.AddRangeAsync(preExistingSnapshotFilterListRules);
// add new FilterListRules
var newFilterListRules = newSnapshotRules.Select(newSnapshotRule =>
new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList();
await dbContext.FilterListRules.AddRangeAsync(newFilterListRules);
// update UpdatedDateUtc
var list = dbContext.FilterLists.Find(filterListId);
@ -62,10 +69,6 @@ public async Task AddOrUpdateRules()
// update ScrapedDateUtc
list.ScrapedDateUtc = DateTime.UtcNow;
dbContext.Rules.AddRange(newSnapshotRules);
dbContext.FilterListRules.RemoveRange(deletedFilterListRules);
dbContext.FilterListRules.AddRange(preExistingSnapshotFilterListRules);
dbContext.FilterListRules.AddRange(newFilterListRules);
dbContext.FilterLists.Update(list);
await dbContext.SaveChangesAsync();