From d2532067ca2aec1dc3010793a257b56eff978c7d Mon Sep 17 00:00:00 2001
From: "Collin M. Barrett" <collinbarrett@users.noreply.github.com>
Date: Fri, 26 Jan 2018 14:13:51 -0600
Subject: [PATCH] further work on scraper

---
 .../20180126190529_addSnapshot.Designer.cs    | 507 ++++++++++++++++++
 .../Migrations/20180126190529_addSnapshot.cs  |  47 ++
 .../FilterListsDbContextModelSnapshot.cs      | 138 +++--
 .../V1/Controllers/ScrapeController.cs        |   5 +-
 .../Services/ScrapeService.cs                 |  18 +-
 5 files changed, 659 insertions(+), 56 deletions(-)
 create mode 100644 src/FilterLists.Api/Migrations/20180126190529_addSnapshot.Designer.cs
 create mode 100644 src/FilterLists.Api/Migrations/20180126190529_addSnapshot.cs

diff --git a/src/FilterLists.Api/Migrations/20180126190529_addSnapshot.Designer.cs b/src/FilterLists.Api/Migrations/20180126190529_addSnapshot.Designer.cs
new file mode 100644
index 000000000..5f2915a07
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180126190529_addSnapshot.Designer.cs
@@ -0,0 +1,507 @@
+// <auto-generated />
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage;
+using Microsoft.EntityFrameworkCore.Storage.Internal;
+using System;
+
+namespace FilterLists.Api.Migrations
+{
+    [DbContext(typeof(FilterListsDbContext))]
+    [Migration("20180126190529_addSnapshot")]
+    partial class addSnapshot
+    {
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
+                .HasAnnotation("ProductVersion", "2.0.1-rtm-125");
+
+            modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Description")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("DescriptionSourceUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<DateTime?>("DiscontinuedDate");
+
+                    b.Property<string>("DonateUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("EmailAddress")
+                        .HasColumnType("NVARCHAR(126)");
+
+                    b.Property<string>("ForumUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("HomeUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("IssuesUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<int?>("LicenseId");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("NVARCHAR(126)");
+
+                    b.Property<string>("SubmissionUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<int?>("SyntaxId");
+
+                    b.Property<string>("ViewUrl")
+                        .HasColumnType("TEXT");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("LicenseId");
+
+                    b.HasIndex("SyntaxId");
+
+                    b.ToTable("filterlists");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+                {
+                    b.Property<int>("FilterListId");
+
+                    b.Property<int>("LanguageId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("FilterListId", "LanguageId");
+
+                    b.HasIndex("LanguageId");
+
+                    b.ToTable("filterlists_languages");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+                {
+                    b.Property<int>("FilterListId");
+
+                    b.Property<int>("MaintainerId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("FilterListId", "MaintainerId");
+
+                    b.HasIndex("MaintainerId");
+
+                    b.ToTable("filterlists_maintainers");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
+                {
+                    b.Property<int>("FilterListId");
+
+                    b.Property<int>("RuleId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("FilterListId", "RuleId");
+
+                    b.HasIndex("RuleId");
+
+                    b.ToTable("filterlists_rules");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+                {
+                    b.Property<int>("ForkFilterListId");
+
+                    b.Property<int>("UpstreamFilterListId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+                    b.HasIndex("UpstreamFilterListId");
+
+                    b.ToTable("forks");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+                {
+                    b.Property<int>("MergeFilterListId");
+
+                    b.Property<int>("UpstreamFilterListId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+                    b.HasIndex("UpstreamFilterListId");
+
+                    b.ToTable("merges");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+                {
+                    b.Property<int>("SoftwareId");
+
+                    b.Property<int>("SyntaxId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("SoftwareId", "SyntaxId");
+
+                    b.HasIndex("SyntaxId");
+
+                    b.ToTable("software_syntaxes");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Iso6391")
+                        .HasColumnType("NVARCHAR(2)");
+
+                    b.Property<string>("Iso6392")
+                        .HasColumnType("NVARCHAR(3)");
+
+                    b.Property<string>("Iso6392B")
+                        .HasColumnType("NVARCHAR(3)");
+
+                    b.Property<string>("Iso6392T")
+                        .HasColumnType("NVARCHAR(3)");
+
+                    b.Property<string>("Iso6393")
+                        .HasColumnType("NVARCHAR(3)");
+
+                    b.Property<string>("LocalName")
+                        .HasColumnType("NVARCHAR(126)");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .HasColumnType("NVARCHAR(126)");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("languages");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("DescriptionUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("NVARCHAR(126)");
+
+                    b.Property<bool>("PermissiveAdaptation");
+
+                    b.Property<bool>("PermissiveCommercial");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("licenses");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("EmailAddress")
+                        .HasColumnType("NVARCHAR(126)");
+
+                    b.Property<string>("HomeUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("NVARCHAR(126)");
+
+                    b.Property<string>("TwitterHandle")
+                        .HasColumnType("NVARCHAR(126)");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("maintainers");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Raw")
+                        .HasColumnType("NVARCHAR(2083)");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("rules");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<string>("Content")
+                        .IsRequired()
+                        .HasColumnType("MEDIUMTEXT");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<int>("FilterListId");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("FilterListId");
+
+                    b.ToTable("snapshots");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("DownloadUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("HomeUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("NVARCHAR(126)");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("software");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("DefinitionUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("NVARCHAR(126)");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("syntaxes");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.License")
+                        .WithMany("FilterLists")
+                        .HasForeignKey("LicenseId");
+
+                    b.HasOne("FilterLists.Data.Entities.Syntax")
+                        .WithMany("FilterLists")
+                        .HasForeignKey("SyntaxId");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                        .WithMany("FilterListLanguages")
+                        .HasForeignKey("FilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.Language", "Language")
+                        .WithMany("FilterListLanguages")
+                        .HasForeignKey("LanguageId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                        .WithMany("FilterListMaintainers")
+                        .HasForeignKey("FilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+                        .WithMany("FilterListMaintainers")
+                        .HasForeignKey("MaintainerId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                        .WithMany("FilterListRules")
+                        .HasForeignKey("FilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+                        .WithMany("FilterListRules")
+                        .HasForeignKey("RuleId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+                        .WithMany()
+                        .HasForeignKey("ForkFilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+                        .WithMany()
+                        .HasForeignKey("UpstreamFilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+                        .WithMany()
+                        .HasForeignKey("MergeFilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+                        .WithMany()
+                        .HasForeignKey("UpstreamFilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.Software", "Software")
+                        .WithMany("SoftwareSyntaxes")
+                        .HasForeignKey("SoftwareId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+                        .WithMany("SoftwareSyntaxes")
+                        .HasForeignKey("SyntaxId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList")
+                        .WithMany("Snapshots")
+                        .HasForeignKey("FilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}
diff --git a/src/FilterLists.Api/Migrations/20180126190529_addSnapshot.cs b/src/FilterLists.Api/Migrations/20180126190529_addSnapshot.cs
new file mode 100644
index 000000000..88b1daa51
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180126190529_addSnapshot.cs
@@ -0,0 +1,47 @@
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+using System;
+using System.Collections.Generic;
+
+namespace FilterLists.Api.Migrations
+{
+    public partial class addSnapshot : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "snapshots",
+                columns: table => new
+                {
+                    Id = table.Column<int>(nullable: false)
+                        .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
+                    Content = table.Column<string>(type: "MEDIUMTEXT", nullable: false),
+                    CreatedDateUtc = table.Column<DateTime>(type: "TIMESTAMP", nullable: false)
+                        .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
+                    FilterListId = table.Column<int>(nullable: false),
+                    ModifiedDateUtc = table.Column<DateTime>(type: "TIMESTAMP", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_snapshots", x => x.Id);
+                    table.ForeignKey(
+                        name: "FK_snapshots_filterlists_FilterListId",
+                        column: x => x.FilterListId,
+                        principalTable: "filterlists",
+                        principalColumn: "Id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateIndex(
+                name: "IX_snapshots_FilterListId",
+                table: "snapshots",
+                column: "FilterListId");
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "snapshots");
+        }
+    }
+}
diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
index bfb493ca7..a903bd3f5 100644
--- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
+++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
@@ -79,7 +79,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.ToTable("filterlists");
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.FilterListLanguage", b =>
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
                 {
                     b.Property<int>("FilterListId");
 
@@ -100,7 +100,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.ToTable("filterlists_languages");
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.FilterListMaintainer", b =>
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
                 {
                     b.Property<int>("FilterListId");
 
@@ -121,7 +121,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.ToTable("filterlists_maintainers");
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.FilterListRule", b =>
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
                 {
                     b.Property<int>("FilterListId");
 
@@ -142,7 +142,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.ToTable("filterlists_rules");
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.Fork", b =>
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
                 {
                     b.Property<int>("ForkFilterListId");
 
@@ -163,6 +163,48 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.ToTable("forks");
                 });
 
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+                {
+                    b.Property<int>("MergeFilterListId");
+
+                    b.Property<int>("UpstreamFilterListId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+                    b.HasIndex("UpstreamFilterListId");
+
+                    b.ToTable("merges");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+                {
+                    b.Property<int>("SoftwareId");
+
+                    b.Property<int>("SyntaxId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("SoftwareId", "SyntaxId");
+
+                    b.HasIndex("SyntaxId");
+
+                    b.ToTable("software_syntaxes");
+                });
+
             modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
                 {
                     b.Property<int>("Id")
@@ -262,27 +304,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.ToTable("maintainers");
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.Merge", b =>
-                {
-                    b.Property<int>("MergeFilterListId");
-
-                    b.Property<int>("UpstreamFilterListId");
-
-                    b.Property<DateTime>("CreatedDateUtc")
-                        .ValueGeneratedOnAdd()
-                        .HasColumnType("TIMESTAMP");
-
-                    b.Property<DateTime>("ModifiedDateUtc")
-                        .ValueGeneratedOnAddOrUpdate()
-                        .HasColumnType("TIMESTAMP");
-
-                    b.HasKey("MergeFilterListId", "UpstreamFilterListId");
-
-                    b.HasIndex("UpstreamFilterListId");
-
-                    b.ToTable("merges");
-                });
-
             modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
                 {
                     b.Property<int>("Id")
@@ -304,6 +325,32 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.ToTable("rules");
                 });
 
+            modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<string>("Content")
+                        .IsRequired()
+                        .HasColumnType("MEDIUMTEXT");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<int>("FilterListId");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("FilterListId");
+
+                    b.ToTable("snapshots");
+                });
+
             modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
                 {
                     b.Property<int>("Id")
@@ -332,27 +379,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.ToTable("software");
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.SoftwareSyntax", b =>
-                {
-                    b.Property<int>("SoftwareId");
-
-                    b.Property<int>("SyntaxId");
-
-                    b.Property<DateTime>("CreatedDateUtc")
-                        .ValueGeneratedOnAdd()
-                        .HasColumnType("TIMESTAMP");
-
-                    b.Property<DateTime>("ModifiedDateUtc")
-                        .ValueGeneratedOnAddOrUpdate()
-                        .HasColumnType("TIMESTAMP");
-
-                    b.HasKey("SoftwareId", "SyntaxId");
-
-                    b.HasIndex("SyntaxId");
-
-                    b.ToTable("software_syntaxes");
-                });
-
             modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
                 {
                     b.Property<int>("Id")
@@ -389,7 +415,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                         .HasForeignKey("SyntaxId");
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.FilterListLanguage", b =>
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
                 {
                     b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
                         .WithMany("FilterListLanguages")
@@ -402,7 +428,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                         .OnDelete(DeleteBehavior.Cascade);
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.FilterListMaintainer", b =>
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
                 {
                     b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
                         .WithMany("FilterListMaintainers")
@@ -415,7 +441,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                         .OnDelete(DeleteBehavior.Cascade);
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.FilterListRule", b =>
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
                 {
                     b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
                         .WithMany("FilterListRules")
@@ -428,7 +454,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                         .OnDelete(DeleteBehavior.Cascade);
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.Fork", b =>
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
                 {
                     b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
                         .WithMany()
@@ -441,7 +467,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                         .OnDelete(DeleteBehavior.Cascade);
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.Merge", b =>
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
                 {
                     b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
                         .WithMany()
@@ -454,7 +480,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                         .OnDelete(DeleteBehavior.Cascade);
                 });
 
-            modelBuilder.Entity("FilterLists.Data.Entities.SoftwareSyntax", b =>
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
                 {
                     b.HasOne("FilterLists.Data.Entities.Software", "Software")
                         .WithMany("SoftwareSyntaxes")
@@ -466,6 +492,14 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                         .HasForeignKey("SyntaxId")
                         .OnDelete(DeleteBehavior.Cascade);
                 });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList")
+                        .WithMany("Snapshots")
+                        .HasForeignKey("FilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
 #pragma warning restore 612, 618
         }
     }
diff --git a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs
index befa40a18..aafee5f5c 100644
--- a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs
+++ b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs
@@ -13,10 +13,11 @@ public ScrapeController(ScrapeService scrapeService)
         }
 
         [HttpPost]
-        public void Index()
+        public IActionResult Index()
         {
-            //TODO: Convert endpoint into scheduled job, don't expose on prod
+            //TODO: convert endpoint into scheduled job, don't expose on prod
             //scrapeService.Scrape();
+            return Ok();
         }
     }
 }
\ No newline at end of file
diff --git a/src/FilterLists.Services/Services/ScrapeService.cs b/src/FilterLists.Services/Services/ScrapeService.cs
index e569d4523..fda5d75a5 100644
--- a/src/FilterLists.Services/Services/ScrapeService.cs
+++ b/src/FilterLists.Services/Services/ScrapeService.cs
@@ -18,14 +18,28 @@ public ScrapeService(FilterListsDbContext filterListsDbContext)
 
         public void Scrape()
         {
-            var list = filterListsDbContext.FilterLists.First();
+            var lists = filterListsDbContext.FilterLists.Take(5);
+            foreach (var list in lists) AddSnapshot(list);
+            filterListsDbContext.SaveChangesAsync();
+        }
+
+        private void AddSnapshot(FilterList list)
+        {
             var snapshot = GetContent(list.ViewUrl);
+            try
+            {
+                if (snapshot.Result == null) return;
+            }
+            catch (AggregateException)
+            {
+                return;
+            }
+
             filterListsDbContext.Snapshots.Add(new Snapshot
             {
                 Content = snapshot.Result,
                 FilterListId = list.Id
             });
-            filterListsDbContext.SaveChangesAsync();
         }
 
         private static async Task<string> GetContent(string url)