From 7f980b76b4e7d2211063ce08d89b2ee787a2ba3a Mon Sep 17 00:00:00 2001
From: "Collin M. Barrett" <collinbarrett@users.noreply.github.com>
Date: Fri, 26 Jan 2018 12:59:31 -0600
Subject: [PATCH] begin work on ScrapeService

---
 .../Properties/launchSettings.json            |  2 +-
 .../V1/Controllers/ScrapeController.cs        | 22 +++++++++
 .../SnapshotTypeConfiguration.cs              |  2 +-
 .../Extensions/ConfigureServicesCollection.cs |  1 +
 .../Services/ScrapeService.cs                 | 45 +++++++++++++++++++
 5 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 src/FilterLists.Api/V1/Controllers/ScrapeController.cs
 create mode 100644 src/FilterLists.Services/Services/ScrapeService.cs

diff --git a/src/FilterLists.Api/Properties/launchSettings.json b/src/FilterLists.Api/Properties/launchSettings.json
index bceadb5a5..4433e921a 100644
--- a/src/FilterLists.Api/Properties/launchSettings.json
+++ b/src/FilterLists.Api/Properties/launchSettings.json
@@ -2,7 +2,7 @@
   "profiles": {
     "FilterLists.Api": {
       "commandName": "Project",
-      "launchBrowser": true,
+      "launchBrowser": false,
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
       },
diff --git a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs
new file mode 100644
index 000000000..befa40a18
--- /dev/null
+++ b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs
@@ -0,0 +1,22 @@
+using FilterLists.Services.Services;
+using Microsoft.AspNetCore.Mvc;
+
+namespace FilterLists.Api.V1.Controllers
+{
+    public class ScrapeController : BaseController
+    {
+        private readonly ScrapeService scrapeService;
+
+        public ScrapeController(ScrapeService scrapeService)
+        {
+            this.scrapeService = scrapeService;
+        }
+
+        [HttpPost]
+        public void Index()
+        {
+            //TODO: Convert endpoint into scheduled job, don't expose on prod
+            //scrapeService.Scrape();
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
index 159368a53..65e50fde2 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
@@ -10,7 +10,7 @@ public override void Configure(EntityTypeBuilder<Snapshot> entityTypeBuilder)
         {
             entityTypeBuilder.ToTable("snapshots");
             entityTypeBuilder.Property(x => x.Content)
-                .HasColumnType("TEXT")
+                .HasColumnType("MEDIUMTEXT")
                 .IsRequired();
             entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
             base.Configure(entityTypeBuilder);
diff --git a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
index c4869309f..7a2bba75b 100644
--- a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
+++ b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
@@ -18,6 +18,7 @@ public static void AddFilterListsServices(this IServiceCollection services, ICon
                     b => b.MigrationsAssembly("FilterLists.Api")));
             services.TryAddScoped<SeedService>();
             services.TryAddScoped<FilterListService>();
+            services.TryAddScoped<ScrapeService>();
             services.AddAutoMapper();
         }
     }
diff --git a/src/FilterLists.Services/Services/ScrapeService.cs b/src/FilterLists.Services/Services/ScrapeService.cs
new file mode 100644
index 000000000..e569d4523
--- /dev/null
+++ b/src/FilterLists.Services/Services/ScrapeService.cs
@@ -0,0 +1,45 @@
+using System;
+using System.Linq;
+using System.Net.Http;
+using System.Threading.Tasks;
+using FilterLists.Data;
+using FilterLists.Data.Entities;
+
+namespace FilterLists.Services.Services
+{
+    public class ScrapeService
+    {
+        private readonly FilterListsDbContext filterListsDbContext;
+
+        public ScrapeService(FilterListsDbContext filterListsDbContext)
+        {
+            this.filterListsDbContext = filterListsDbContext;
+        }
+
+        public void Scrape()
+        {
+            var list = filterListsDbContext.FilterLists.First();
+            var snapshot = GetContent(list.ViewUrl);
+            filterListsDbContext.Snapshots.Add(new Snapshot
+            {
+                Content = snapshot.Result,
+                FilterListId = list.Id
+            });
+            filterListsDbContext.SaveChangesAsync();
+        }
+
+        private static async Task<string> GetContent(string url)
+        {
+            using (var httpClient = new HttpClient())
+            {
+                using (var httpResponseMessage = await httpClient.GetAsync(url))
+                {
+                    if (httpResponseMessage.IsSuccessStatusCode)
+                        return await httpResponseMessage.Content.ReadAsStringAsync();
+                }
+            }
+
+            return null;
+        }
+    }
+}
\ No newline at end of file