diff --git a/src/FilterLists.Services/Services/ScrapeService.cs b/src/FilterLists.Services/Services/ScrapeService.cs index 6f839a2cb..96fb0ecc9 100644 --- a/src/FilterLists.Services/Services/ScrapeService.cs +++ b/src/FilterLists.Services/Services/ScrapeService.cs @@ -6,6 +6,7 @@ using AutoMapper.QueryableExtensions; using FilterLists.Data; using FilterLists.Data.Entities; +using FilterLists.Data.Entities.Junctions; using Microsoft.EntityFrameworkCore; namespace FilterLists.Services.Services @@ -22,9 +23,10 @@ public ScrapeService(FilterListsDbContext filterListsDbContext) //TODO: call via scheduled job public async Task ScrapeAsync(int batchSize) { - var lists = await GetNextFilterListDtosToScrape(batchSize); - var snapshots = await GetSnapshots(lists); - AddOrUpdateRules(snapshots); + var filterListDtos = await GetNextFilterListDtosToScrape(batchSize); + var snapshots = await GetSnapshots(filterListDtos); + //var temp = ParseRules(snapshots); + //AddOrUpdateRules(temp); } private async Task> GetNextFilterListDtosToScrape(int batchSize) @@ -33,22 +35,17 @@ private async Task> GetNextFilterListDtosToScrape(int .ProjectTo().ToListAsync(); } - private static async Task> GetSnapshots(IEnumerable lists) + private static async Task> GetSnapshots(IEnumerable filterListDtos) { - return await Task.WhenAll(lists.Select(async list => - new Snapshot {Content = await GetContent(list.ViewUrl), FilterListId = list.Id})); + return await Task.WhenAll(filterListDtos.Select(async filterListDto => + new Snapshot {Content = await TryGetContent(filterListDto.ViewUrl), FilterListId = filterListDto.Id})); } - private static async Task GetContent(string url) + private static async Task TryGetContent(string url) { try { - using (var httpClient = new HttpClient()) - using (var httpResponseMessage = await httpClient.GetAsync(url)) - { - if (httpResponseMessage.IsSuccessStatusCode) - return await httpResponseMessage.Content.ReadAsStringAsync(); - } + await GetHttpResponseMessageContent(url); } catch (Exception) { @@ -56,49 +53,72 @@ private static async Task GetContent(string url) return null; } + return null; + } + + private static async Task GetHttpResponseMessageContent(string url) + { + using (var httpClient = new HttpClient()) + using (var httpResponseMessage = await httpClient.GetAsync(url)) + { + if (httpResponseMessage.IsSuccessStatusCode) + return await httpResponseMessage.Content.ReadAsStringAsync(); + } + //TODO: log httpResponseMessage.StatusCode return null; } - private void AddOrUpdateRules(IEnumerable snapshots) + private void ParseRules(IEnumerable snapshots) { foreach (var snapshot in snapshots) - AddOrUpdateRules(snapshot); + ParseRules(snapshot); } - //TODO: finish and validate - private void AddOrUpdateRules(Snapshot snapshot) + private void ParseRules(Snapshot snapshot) { - // add new Rules var snapshotRulesRaw = snapshot.Content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); - var preExistingSnapshotRules = filterListsDbContext.Rules.Where(x => snapshotRulesRaw.Contains(x.Raw)); - var newSnapshotRulesRaw = snapshotRulesRaw.Except(preExistingSnapshotRules.Select(x => x.Raw)); + var snapshotRulesRawDto = + new SnapshotRulesRaw {FilterListId = snapshot.FilterListId, RulesRaw = snapshotRulesRaw}; + //return snapshotRulesRawDto; + } + + private void AddOrUpdateRules(SnapshotRulesRaw snapshotRulesRaw) + { + // add new Rules + var preExistingSnapshotRules = + filterListsDbContext.Rules.Where(x => snapshotRulesRaw.RulesRaw.Contains(x.Raw)); + var newSnapshotRulesRaw = snapshotRulesRaw.RulesRaw.Except(preExistingSnapshotRules.Select(x => x.Raw)); var newSnapshotRules = newSnapshotRulesRaw.Select(newSnapshotRuleRaw => new Rule {Raw = newSnapshotRuleRaw}); filterListsDbContext.Rules.AddRange(newSnapshotRules); // remove deleted FilterListRules var preExistingFilterListRules = - filterListsDbContext.FilterListRules.Where(x => x.FilterListId == snapshot.FilterListId); + filterListsDbContext.FilterListRules.Where(x => x.FilterListId == snapshotRulesRaw.FilterListId); var deletedFilterListRules = preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId)); filterListsDbContext.FilterListRules.RemoveRange(deletedFilterListRules); - // add new FilterListRules - + // add new (or previously missing junction) FilterListRules + var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule => + new FilterListRule {FilterListId = snapshotRulesRaw.FilterListId, Rule = newSnapshotRule}); + filterListsDbContext.FilterListRules.AddRange(preExistingSnapshotFilterListRules); + var newFilterListRules = newSnapshotRules.Select(newSnapshotRule => + new FilterListRule {FilterListId = snapshotRulesRaw.FilterListId, Rule = newSnapshotRule}); + filterListsDbContext.FilterListRules.AddRange(newFilterListRules); // update UpdatedDateUtc - if (newSnapshotRulesRaw.Any() || deletedFilterListRules.Any()) - { - var list = filterListsDbContext.FilterLists.FindAsync(snapshot.FilterListId).Result; - list.UpdatedDateUtc = DateTime.UtcNow; - filterListsDbContext.FilterLists.Update(list); - } + var list = filterListsDbContext.FilterLists.Find(snapshotRulesRaw.FilterListId); + //if (preExistingSnapshotFilterListRules.Any() || newFilterListRules.Any() || deletedFilterListRules.Any())) + list.UpdatedDateUtc = DateTime.UtcNow; - //TODO: update FilterList.ScrapedDateUtc + // update ScrapedDateUtc + list.ScrapedDateUtc = DateTime.UtcNow; + filterListsDbContext.FilterLists.Update(list); - filterListsDbContext.SaveChangesAsync(); + filterListsDbContext.SaveChanges(); } private class FilterListDto @@ -112,5 +132,11 @@ private class Snapshot public string Content { get; set; } public int FilterListId { get; set; } } + + private class SnapshotRulesRaw + { + public int FilterListId { get; set; } + public IEnumerable RulesRaw { get; set; } + } } } \ No newline at end of file