diff --git a/src/FilterLists.Services/Services/ScrapeService.cs b/src/FilterLists.Services/Services/ScrapeService.cs index 96fb0ecc9..6f839a2cb 100644 --- a/src/FilterLists.Services/Services/ScrapeService.cs +++ b/src/FilterLists.Services/Services/ScrapeService.cs @@ -6,7 +6,6 @@ using AutoMapper.QueryableExtensions; using FilterLists.Data; using FilterLists.Data.Entities; -using FilterLists.Data.Entities.Junctions; using Microsoft.EntityFrameworkCore; namespace FilterLists.Services.Services @@ -23,10 +22,9 @@ public ScrapeService(FilterListsDbContext filterListsDbContext) //TODO: call via scheduled job public async Task ScrapeAsync(int batchSize) { - var filterListDtos = await GetNextFilterListDtosToScrape(batchSize); - var snapshots = await GetSnapshots(filterListDtos); - //var temp = ParseRules(snapshots); - //AddOrUpdateRules(temp); + var lists = await GetNextFilterListDtosToScrape(batchSize); + var snapshots = await GetSnapshots(lists); + AddOrUpdateRules(snapshots); } private async Task> GetNextFilterListDtosToScrape(int batchSize) @@ -35,17 +33,22 @@ private async Task> GetNextFilterListDtosToScrape(int .ProjectTo().ToListAsync(); } - private static async Task> GetSnapshots(IEnumerable filterListDtos) + private static async Task> GetSnapshots(IEnumerable lists) { - return await Task.WhenAll(filterListDtos.Select(async filterListDto => - new Snapshot {Content = await TryGetContent(filterListDto.ViewUrl), FilterListId = filterListDto.Id})); + return await Task.WhenAll(lists.Select(async list => + new Snapshot {Content = await GetContent(list.ViewUrl), FilterListId = list.Id})); } - private static async Task TryGetContent(string url) + private static async Task GetContent(string url) { try { - await GetHttpResponseMessageContent(url); + using (var httpClient = new HttpClient()) + using (var httpResponseMessage = await httpClient.GetAsync(url)) + { + if (httpResponseMessage.IsSuccessStatusCode) + return await httpResponseMessage.Content.ReadAsStringAsync(); + } } catch (Exception) { @@ -53,72 +56,49 @@ private static async Task TryGetContent(string url) return null; } - return null; - } - - private static async Task GetHttpResponseMessageContent(string url) - { - using (var httpClient = new HttpClient()) - using (var httpResponseMessage = await httpClient.GetAsync(url)) - { - if (httpResponseMessage.IsSuccessStatusCode) - return await httpResponseMessage.Content.ReadAsStringAsync(); - } - //TODO: log httpResponseMessage.StatusCode return null; } - private void ParseRules(IEnumerable snapshots) + private void AddOrUpdateRules(IEnumerable snapshots) { foreach (var snapshot in snapshots) - ParseRules(snapshot); + AddOrUpdateRules(snapshot); } - private void ParseRules(Snapshot snapshot) - { - var snapshotRulesRaw = - snapshot.Content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); - var snapshotRulesRawDto = - new SnapshotRulesRaw {FilterListId = snapshot.FilterListId, RulesRaw = snapshotRulesRaw}; - //return snapshotRulesRawDto; - } - - private void AddOrUpdateRules(SnapshotRulesRaw snapshotRulesRaw) + //TODO: finish and validate + private void AddOrUpdateRules(Snapshot snapshot) { // add new Rules - var preExistingSnapshotRules = - filterListsDbContext.Rules.Where(x => snapshotRulesRaw.RulesRaw.Contains(x.Raw)); - var newSnapshotRulesRaw = snapshotRulesRaw.RulesRaw.Except(preExistingSnapshotRules.Select(x => x.Raw)); + var snapshotRulesRaw = + snapshot.Content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries); + var preExistingSnapshotRules = filterListsDbContext.Rules.Where(x => snapshotRulesRaw.Contains(x.Raw)); + var newSnapshotRulesRaw = snapshotRulesRaw.Except(preExistingSnapshotRules.Select(x => x.Raw)); var newSnapshotRules = newSnapshotRulesRaw.Select(newSnapshotRuleRaw => new Rule {Raw = newSnapshotRuleRaw}); filterListsDbContext.Rules.AddRange(newSnapshotRules); // remove deleted FilterListRules var preExistingFilterListRules = - filterListsDbContext.FilterListRules.Where(x => x.FilterListId == snapshotRulesRaw.FilterListId); + filterListsDbContext.FilterListRules.Where(x => x.FilterListId == snapshot.FilterListId); var deletedFilterListRules = preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId)); filterListsDbContext.FilterListRules.RemoveRange(deletedFilterListRules); - // add new (or previously missing junction) FilterListRules - var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule => - new FilterListRule {FilterListId = snapshotRulesRaw.FilterListId, Rule = newSnapshotRule}); - filterListsDbContext.FilterListRules.AddRange(preExistingSnapshotFilterListRules); - var newFilterListRules = newSnapshotRules.Select(newSnapshotRule => - new FilterListRule {FilterListId = snapshotRulesRaw.FilterListId, Rule = newSnapshotRule}); - filterListsDbContext.FilterListRules.AddRange(newFilterListRules); + // add new FilterListRules + // update UpdatedDateUtc - var list = filterListsDbContext.FilterLists.Find(snapshotRulesRaw.FilterListId); - //if (preExistingSnapshotFilterListRules.Any() || newFilterListRules.Any() || deletedFilterListRules.Any())) - list.UpdatedDateUtc = DateTime.UtcNow; + if (newSnapshotRulesRaw.Any() || deletedFilterListRules.Any()) + { + var list = filterListsDbContext.FilterLists.FindAsync(snapshot.FilterListId).Result; + list.UpdatedDateUtc = DateTime.UtcNow; + filterListsDbContext.FilterLists.Update(list); + } - // update ScrapedDateUtc - list.ScrapedDateUtc = DateTime.UtcNow; - filterListsDbContext.FilterLists.Update(list); + //TODO: update FilterList.ScrapedDateUtc - filterListsDbContext.SaveChanges(); + filterListsDbContext.SaveChangesAsync(); } private class FilterListDto @@ -132,11 +112,5 @@ private class Snapshot public string Content { get; set; } public int FilterListId { get; set; } } - - private class SnapshotRulesRaw - { - public int FilterListId { get; set; } - public IEnumerable RulesRaw { get; set; } - } } } \ No newline at end of file