prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

separate exception handling from logic for GetContent()

Browse source
This commit is contained in:
Collin M. Barrett 2018-01-29 15:30:53 -06:00
parent c1772a8680
commit 6995755248
1 changed files with 17 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
src/FilterLists.Services/Services/ScrapeService.cs
View file

@ -27,34 +27,39 @@ public async Task ScrapeAsync(int batchSize)
AddOrUpdateRules(snapshots);
}
private async Task<IEnumerable<FilterListDto>> GetNextFilterListDtosToScrape(int batchSize)
private async Task<IEnumerable<FilterListViewUrlDto>> GetNextFilterListDtosToScrape(int batchSize)
{
return await filterListsDbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(batchSize)
.ProjectTo<FilterListDto>().ToListAsync();
.ProjectTo<FilterListViewUrlDto>().ToListAsync();
}
private static async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListDto> lists)
private static async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)
{
return await Task.WhenAll(lists.Select(async list =>
new Snapshot {Content = await GetContent(list.ViewUrl), FilterListId = list.Id}));
new Snapshot {Content = await TryGetContent(list.ViewUrl), FilterListId = list.Id}));
}
private static async Task<string> GetContent(string url)
private static async Task<string> TryGetContent(string url)
{
try
{
using (var httpClient = new HttpClient())
using (var httpResponseMessage = await httpClient.GetAsync(url))
{
if (httpResponseMessage.IsSuccessStatusCode)
return await httpResponseMessage.Content.ReadAsStringAsync();
}
return await GetSuccessfulHttpResponseMessageContent(url);
}
catch (Exception)
{
//TODO: log exception
return null;
}
}
private static async Task<string> GetSuccessfulHttpResponseMessageContent(string url)
{
using (var httpClient = new HttpClient())
using (var httpResponseMessage = await httpClient.GetAsync(url))
{
if (httpResponseMessage.IsSuccessStatusCode)
return await httpResponseMessage.Content.ReadAsStringAsync();
}
//TODO: log httpResponseMessage.StatusCode
return null;
@ -100,7 +105,7 @@ private void AddOrUpdateRules(Snapshot snapshot)
filterListsDbContext.SaveChangesAsync();
}
private class FilterListDto
private class FilterListViewUrlDto
{
public int Id { get; set; }
public string ViewUrl { get; set; }