From 6caef4c4d61ce02a34a09b221d3d8fa01a201830 Mon Sep 17 00:00:00 2001
From: "Collin M. Barrett" <collinbarrett@users.noreply.github.com>
Date: Fri, 26 Jan 2018 18:34:53 -0600
Subject: [PATCH] got async scraper semi-working

---
 .../V1/Controllers/ScrapeController.cs        |  7 ++++---
 .../Services/ScrapeService.cs                 | 21 +++++++++++--------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs
index aafee5f5c..e6b0451d0 100644
--- a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs
+++ b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs
@@ -1,4 +1,5 @@
-using FilterLists.Services.Services;
+using System.Threading.Tasks;
+using FilterLists.Services.Services;
 using Microsoft.AspNetCore.Mvc;
 
 namespace FilterLists.Api.V1.Controllers
@@ -13,10 +14,10 @@ public ScrapeController(ScrapeService scrapeService)
         }
 
         [HttpPost]
-        public IActionResult Index()
+        public async Task<IActionResult> Index()
         {
             //TODO: convert endpoint into scheduled job, don't expose on prod
-            //scrapeService.Scrape();
+            //await scrapeService.Scrape();
             return Ok();
         }
     }
diff --git a/src/FilterLists.Services/Services/ScrapeService.cs b/src/FilterLists.Services/Services/ScrapeService.cs
index fda5d75a5..1085e641a 100644
--- a/src/FilterLists.Services/Services/ScrapeService.cs
+++ b/src/FilterLists.Services/Services/ScrapeService.cs
@@ -1,5 +1,4 @@
 using System;
-using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
 using FilterLists.Data;
@@ -16,28 +15,32 @@ public ScrapeService(FilterListsDbContext filterListsDbContext)
             this.filterListsDbContext = filterListsDbContext;
         }
 
-        public void Scrape()
+        public async Task Scrape()
         {
-            var lists = filterListsDbContext.FilterLists.Take(5);
-            foreach (var list in lists) AddSnapshot(list);
-            filterListsDbContext.SaveChangesAsync();
+            var lists = filterListsDbContext.FilterLists;
+            foreach (var list in lists) await AddSnapshot(list);
+            await filterListsDbContext.SaveChangesAsync();
         }
 
-        private void AddSnapshot(FilterList list)
+        private async Task AddSnapshot(FilterList list)
         {
-            var snapshot = GetContent(list.ViewUrl);
+            var snapshot = await GetContent(list.ViewUrl);
             try
             {
-                if (snapshot.Result == null) return;
+                if (snapshot == null) return;
             }
             catch (AggregateException)
             {
                 return;
             }
+            catch (HttpRequestException)
+            {
+                return;
+            }
 
             filterListsDbContext.Snapshots.Add(new Snapshot
             {
-                Content = snapshot.Result,
+                Content = snapshot,
                 FilterListId = list.Id
             });
         }