diff --git a/src/FilterLists.Services/Services/ScrapeService.cs b/src/FilterLists.Services/Services/ScrapeService.cs index aa996c682..d994e667d 100644 --- a/src/FilterLists.Services/Services/ScrapeService.cs +++ b/src/FilterLists.Services/Services/ScrapeService.cs @@ -27,34 +27,39 @@ public async Task ScrapeAsync(int batchSize) AddOrUpdateRules(snapshots); } - private async Task> GetNextFilterListDtosToScrape(int batchSize) + private async Task> GetNextFilterListDtosToScrape(int batchSize) { return await filterListsDbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(batchSize) - .ProjectTo().ToListAsync(); + .ProjectTo().ToListAsync(); } - private static async Task> GetSnapshots(IEnumerable lists) + private static async Task> GetSnapshots(IEnumerable lists) { return await Task.WhenAll(lists.Select(async list => - new Snapshot {Content = await GetContent(list.ViewUrl), FilterListId = list.Id})); + new Snapshot {Content = await TryGetContent(list.ViewUrl), FilterListId = list.Id})); } - private static async Task GetContent(string url) + private static async Task TryGetContent(string url) { try { - using (var httpClient = new HttpClient()) - using (var httpResponseMessage = await httpClient.GetAsync(url)) - { - if (httpResponseMessage.IsSuccessStatusCode) - return await httpResponseMessage.Content.ReadAsStringAsync(); - } + return await GetSuccessfulHttpResponseMessageContent(url); } catch (Exception) { //TODO: log exception return null; } + } + + private static async Task GetSuccessfulHttpResponseMessageContent(string url) + { + using (var httpClient = new HttpClient()) + using (var httpResponseMessage = await httpClient.GetAsync(url)) + { + if (httpResponseMessage.IsSuccessStatusCode) + return await httpResponseMessage.Content.ReadAsStringAsync(); + } //TODO: log httpResponseMessage.StatusCode return null; @@ -100,7 +105,7 @@ private void AddOrUpdateRules(Snapshot snapshot) filterListsDbContext.SaveChangesAsync(); } - private class FilterListDto + private class FilterListViewUrlDto { public int Id { get; set; } public string ViewUrl { get; set; }