prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

got async scraper semi-working

Browse source
This commit is contained in:
Collin M. Barrett 2018-01-26 18:34:53 -06:00
parent b21af4fb7d
commit 6caef4c4d6
2 changed files with 16 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/FilterLists.Api/V1/Controllers/ScrapeController.cs
View file

@ -1,4 +1,5 @@
using FilterLists.Services.Services;
using System.Threading.Tasks;
using FilterLists.Services.Services;
using Microsoft.AspNetCore.Mvc;
namespace FilterLists.Api.V1.Controllers
@ -13,10 +14,10 @@ public ScrapeController(ScrapeService scrapeService)
}
[HttpPost]
public IActionResult Index()
public async Task<IActionResult> Index()
{
//TODO: convert endpoint into scheduled job, don't expose on prod
//scrapeService.Scrape();
//await scrapeService.Scrape();
return Ok();
}
}

21
src/FilterLists.Services/Services/ScrapeService.cs
View file

@ -1,5 +1,4 @@
using System;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using FilterLists.Data;
@ -16,28 +15,32 @@ public ScrapeService(FilterListsDbContext filterListsDbContext)
this.filterListsDbContext = filterListsDbContext;
}
public void Scrape()
public async Task Scrape()
{
var lists = filterListsDbContext.FilterLists.Take(5);
foreach (var list in lists) AddSnapshot(list);
filterListsDbContext.SaveChangesAsync();
var lists = filterListsDbContext.FilterLists;
foreach (var list in lists) await AddSnapshot(list);
await filterListsDbContext.SaveChangesAsync();
}
private void AddSnapshot(FilterList list)
private async Task AddSnapshot(FilterList list)
{
var snapshot = GetContent(list.ViewUrl);
var snapshot = await GetContent(list.ViewUrl);
try
{
if (snapshot.Result == null) return;
if (snapshot == null) return;
}
catch (AggregateException)
{
return;
}
catch (HttpRequestException)
{
return;
}
filterListsDbContext.Snapshots.Add(new Snapshot
{
Content = snapshot.Result,
Content = snapshot,
FilterListId = list.Id
});
}