prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

work on ScrapeService (currently in broken state)

Browse source
This commit is contained in:
Collin Barrett 2018-01-29 06:43:20 -06:00
parent 92c004a60a
commit d14c4f9518
1 changed files with 57 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
src/FilterLists.Services/Services/ScrapeService.cs
View file

@ -6,6 +6,7 @@
using AutoMapper.QueryableExtensions;
using FilterLists.Data;
using FilterLists.Data.Entities;
using FilterLists.Data.Entities.Junctions;
using Microsoft.EntityFrameworkCore;
namespace FilterLists.Services.Services
@ -22,9 +23,10 @@ public ScrapeService(FilterListsDbContext filterListsDbContext)
//TODO: call via scheduled job
public async Task ScrapeAsync(int batchSize)
{
var lists = await GetNextFilterListDtosToScrape(batchSize);
var snapshots = await GetSnapshots(lists);
AddOrUpdateRules(snapshots);
var filterListDtos = await GetNextFilterListDtosToScrape(batchSize);
var snapshots = await GetSnapshots(filterListDtos);
//var temp = ParseRules(snapshots);
//AddOrUpdateRules(temp);
}
private async Task<IEnumerable<FilterListDto>> GetNextFilterListDtosToScrape(int batchSize)
@ -33,22 +35,17 @@ private async Task<IEnumerable<FilterListDto>> GetNextFilterListDtosToScrape(int
.ProjectTo<FilterListDto>().ToListAsync();
}
private static async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListDto> lists)
private static async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListDto> filterListDtos)
{
return await Task.WhenAll(lists.Select(async list =>
new Snapshot {Content = await GetContent(list.ViewUrl), FilterListId = list.Id}));
return await Task.WhenAll(filterListDtos.Select(async filterListDto =>
new Snapshot {Content = await TryGetContent(filterListDto.ViewUrl), FilterListId = filterListDto.Id}));
}
private static async Task<string> GetContent(string url)
private static async Task<string> TryGetContent(string url)
{
try
{
using (var httpClient = new HttpClient())
using (var httpResponseMessage = await httpClient.GetAsync(url))
{
if (httpResponseMessage.IsSuccessStatusCode)
return await httpResponseMessage.Content.ReadAsStringAsync();
}
await GetHttpResponseMessageContent(url);
}
catch (Exception)
{
@ -56,49 +53,72 @@ private static async Task<string> GetContent(string url)
return null;
}
return null;
}
private static async Task<string> GetHttpResponseMessageContent(string url)
{
using (var httpClient = new HttpClient())
using (var httpResponseMessage = await httpClient.GetAsync(url))
{
if (httpResponseMessage.IsSuccessStatusCode)
return await httpResponseMessage.Content.ReadAsStringAsync();
}
//TODO: log httpResponseMessage.StatusCode
return null;
}
private void AddOrUpdateRules(IEnumerable<Snapshot> snapshots)
private void ParseRules(IEnumerable<Snapshot> snapshots)
{
foreach (var snapshot in snapshots)
AddOrUpdateRules(snapshot);
ParseRules(snapshot);
}
//TODO: finish and validate
private void AddOrUpdateRules(Snapshot snapshot)
private void ParseRules(Snapshot snapshot)
{
// add new Rules
var snapshotRulesRaw =
snapshot.Content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries);
var preExistingSnapshotRules = filterListsDbContext.Rules.Where(x => snapshotRulesRaw.Contains(x.Raw));
var newSnapshotRulesRaw = snapshotRulesRaw.Except(preExistingSnapshotRules.Select(x => x.Raw));
var snapshotRulesRawDto =
new SnapshotRulesRaw {FilterListId = snapshot.FilterListId, RulesRaw = snapshotRulesRaw};
//return snapshotRulesRawDto;
}
private void AddOrUpdateRules(SnapshotRulesRaw snapshotRulesRaw)
{
// add new Rules
var preExistingSnapshotRules =
filterListsDbContext.Rules.Where(x => snapshotRulesRaw.RulesRaw.Contains(x.Raw));
var newSnapshotRulesRaw = snapshotRulesRaw.RulesRaw.Except(preExistingSnapshotRules.Select(x => x.Raw));
var newSnapshotRules =
newSnapshotRulesRaw.Select(newSnapshotRuleRaw => new Rule {Raw = newSnapshotRuleRaw});
filterListsDbContext.Rules.AddRange(newSnapshotRules);
// remove deleted FilterListRules
var preExistingFilterListRules =
filterListsDbContext.FilterListRules.Where(x => x.FilterListId == snapshot.FilterListId);
filterListsDbContext.FilterListRules.Where(x => x.FilterListId == snapshotRulesRaw.FilterListId);
var deletedFilterListRules =
preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId));
filterListsDbContext.FilterListRules.RemoveRange(deletedFilterListRules);
// add new FilterListRules
// add new (or previously missing junction) FilterListRules
var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule =>
new FilterListRule {FilterListId = snapshotRulesRaw.FilterListId, Rule = newSnapshotRule});
filterListsDbContext.FilterListRules.AddRange(preExistingSnapshotFilterListRules);
var newFilterListRules = newSnapshotRules.Select(newSnapshotRule =>
new FilterListRule {FilterListId = snapshotRulesRaw.FilterListId, Rule = newSnapshotRule});
filterListsDbContext.FilterListRules.AddRange(newFilterListRules);
// update UpdatedDateUtc
if (newSnapshotRulesRaw.Any() || deletedFilterListRules.Any())
{
var list = filterListsDbContext.FilterLists.FindAsync(snapshot.FilterListId).Result;
list.UpdatedDateUtc = DateTime.UtcNow;
filterListsDbContext.FilterLists.Update(list);
}
var list = filterListsDbContext.FilterLists.Find(snapshotRulesRaw.FilterListId);
//if (preExistingSnapshotFilterListRules.Any() || newFilterListRules.Any() || deletedFilterListRules.Any()))
list.UpdatedDateUtc = DateTime.UtcNow;
//TODO: update FilterList.ScrapedDateUtc
// update ScrapedDateUtc
list.ScrapedDateUtc = DateTime.UtcNow;
filterListsDbContext.FilterLists.Update(list);
filterListsDbContext.SaveChangesAsync();
filterListsDbContext.SaveChanges();
}
private class FilterListDto
@ -112,5 +132,11 @@ private class Snapshot
public string Content { get; set; }
public int FilterListId { get; set; }
}
private class SnapshotRulesRaw
{
public int FilterListId { get; set; }
public IEnumerable<string> RulesRaw { get; set; }
}
}
}