prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

more Scraper work

Browse source
This commit is contained in:
Collin Barrett 2018-01-27 20:14:46 -06:00
parent b2fb15a20b
commit 2715e36722
1 changed files with 15 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
src/FilterLists.Services/Services/ScrapeService.cs
View file

@ -5,8 +5,6 @@
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Data.Entities;
using FilterLists.Data.Entities.Junctions;
using Microsoft.EntityFrameworkCore;
namespace FilterLists.Services.Services
{
@ -42,12 +40,10 @@ private static async Task<string> GetContent(string url)
try
{
using (var httpClient = new HttpClient())
using (var httpResponseMessage = await httpClient.GetAsync(url))
{
using (var httpResponseMessage = await httpClient.GetAsync(url))
{
if (httpResponseMessage.IsSuccessStatusCode)
return await httpResponseMessage.Content.ReadAsStringAsync();
}
if (httpResponseMessage.IsSuccessStatusCode)
return await httpResponseMessage.Content.ReadAsStringAsync();
}
}
catch (Exception)
@ -66,46 +62,36 @@ private async Task AddOrUpdateRules(IEnumerable<Snapshot> snapshots)
await AddOrUpdateRules(snapshot);
}
//TODO: finish and validate
private async Task AddOrUpdateRules(Snapshot snapshot)
{
var cachedRules = filterListsDbContext.FilterListRules
.Where(x => x.FilterListId == snapshot.FilterListId)
.Select(x => x.Rule);
IEnumerable<string> currentRulesRaw =
snapshot.Content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.None);
var currentRulesRaw =
snapshot.Content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
var existingCurrentRules = filterListsDbContext.Rules.Where(x => currentRulesRaw.Contains(x.Raw));
var newCurrentRulesRaw = currentRulesRaw.Except(existingCurrentRules.Select(x => x.Raw));
var newCurrentRules = newCurrentRulesRaw.Select(newCurrentRuleRaw => new Rule {Raw = newCurrentRuleRaw});
var deletedRules = cachedRules.Except(existingCurrentRules).ToList();
filterListsDbContext.FilterListRules.RemoveRange(filterListsDbContext.FilterListRules
.Where(x => deletedRules.Select(y => y.Id).Contains(x.RuleId))
.Where(x => x.FilterListId == snapshot.FilterListId));
//TODO: update FilterList.UpdatedDateUtc
foreach (var line in currentRulesRaw)
if (newCurrentRules.Any() || deletedRules.Any())
{
if (string.IsNullOrWhiteSpace(line)) continue;
var rule = new Rule {Raw = line};
if (!await filterListsDbContext.Rules.AnyAsync(x => x.Raw == rule.Raw))
filterListsDbContext.Rules.Add(rule);
var filterListRule = new FilterListRule {FilterListId = snapshot.FilterListId, Rule = rule};
if (!await filterListsDbContext.FilterListRules.AnyAsync(x =>
x.RuleId == rule.Id && x.FilterListId == snapshot.FilterListId))
filterListsDbContext.FilterListRules.Add(filterListRule);
filterListsDbContext.FilterListRules.Add(filterListRule);
var list = filterListsDbContext.FilterLists.FindAsync(snapshot.FilterListId).Result;
list.UpdatedDateUtc = DateTime.UtcNow;
filterListsDbContext.FilterLists.Update(list);
}
try
{
await filterListsDbContext.SaveChangesAsync();
}
catch (Exception)
{
//TODO: log exception
}
await filterListsDbContext.SaveChangesAsync();
}
private class FilterListDto