prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

more work on ScrapeService

Browse source
This commit is contained in:
Collin M. Barrett 2018-01-30 10:34:30 -06:00
parent b1ce29ebb0
commit 8acee09dc7
3 changed files with 32 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/FilterLists.Api/V1/Controllers/ScrapeController.cs
View file

@ -17,7 +17,7 @@ public ScrapeController(ScrapeService scrapeService)
public async Task<IActionResult> Index()
{
#if DEBUG
await scrapeService.ScrapeAsync(5);
await scrapeService.ScrapeAsync(3);
return Ok();
#else
return StatusCode(403);

14
src/FilterLists.Services/IEnumerableExtensions.cs
View file

@ -1,14 +0,0 @@
using System;
using System.Collections.Generic;
namespace FilterLists.Services
{
public static class IEnumerableExtensions
{
public static void ForEach<T>(this IEnumerable<T> items, Action<T> action)
{
foreach (var item in items)
action(item);
}
}
}

44
src/FilterLists.Services/Services/ScrapeService.cs
View file

@ -25,7 +25,7 @@ public async Task ScrapeAsync(int batchSize)
{
var lists = await GetNextFilterListsToScrape(batchSize);
var snapshots = await GetSnapshots(lists);
snapshots.ForEach(async snapshot => await AddOrUpdateRules(snapshot));
await SaveSnapshots(snapshots);
}
private async Task<IEnumerable<FilterListViewUrlDto>> GetNextFilterListsToScrape(int batchSize)
@ -36,8 +36,10 @@ private async Task<IEnumerable<FilterListViewUrlDto>> GetNextFilterListsToScrape
private static async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)
{
return await Task.WhenAll(lists.Select(async list =>
new Snapshot {Content = await TryGetContent(list.ViewUrl), FilterListId = list.Id}));
return await Task.WhenAll(lists
.Select(async list =>
new Snapshot {Content = await TryGetContent(list.ViewUrl), FilterListId = list.Id})
.Where(x => x.Result.RawRules != null));
}
private static async Task<string> TryGetContent(string url)
@ -66,25 +68,42 @@ private static async Task<string> GetHttpResponseMessageContent(string url)
return null;
}
private async Task SaveSnapshots(IEnumerable<Snapshot> snapshots)
{
foreach (var snapshot in snapshots)
{
await AddOrUpdateRules(snapshot);
}
}
private async Task AddOrUpdateRules(Snapshot snapshot)
{
dbContext.ChangeTracker.AutoDetectChangesEnabled = false;
// add new Rules
var preExistingSnapshotRules = dbContext.Rules.Where(x => snapshot.RawRules.Contains(x.Raw));
var newSnapshotRawRules = snapshot.RawRules.Except(preExistingSnapshotRules.Select(x => x.Raw));
var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRuleRaw => new Rule {Raw = newSnapshotRuleRaw});
var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRuleRaw => new Rule {Raw = newSnapshotRuleRaw})
.ToList();
dbContext.Rules.AddRange(newSnapshotRules);
// remove deleted FilterListRules
var preExistingFilterListRules = dbContext.FilterListRules.Where(x => x.FilterListId == snapshot.FilterListId);
var deletedFilterListRules = preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId));
var preExistingFilterListRules =
dbContext.FilterListRules.Where(x => x.FilterListId == snapshot.FilterListId);
var deletedFilterListRules =
preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId));
dbContext.FilterListRules.RemoveRange(deletedFilterListRules);
// add new FilterListRules
var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule =>
new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule}).Except(preExistingFilterListRules);
// add FilterListRules for pre-existing Rules
var preExistingSnapshotFilterListRules = preExistingSnapshotRules
.Select(newSnapshotRule =>
new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule}).ToList()
.Except(preExistingFilterListRules).ToList();
dbContext.FilterListRules.AddRange(preExistingSnapshotFilterListRules);
// add new FilterListRules
var newFilterListRules = newSnapshotRules.Select(newSnapshotRule =>
new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule});
new FilterListRule {FilterListId = snapshot.FilterListId, Rule = newSnapshotRule}).ToList();
dbContext.FilterListRules.AddRange(newFilterListRules);
// update UpdatedDateUtc
@ -107,9 +126,8 @@ private class FilterListViewUrlDto
private class Snapshot
{
public string Content
{
set => RawRules = value.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
public string Content {
set => RawRules = value?.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries);
}
public int FilterListId { get; set; }