initial add scrapes, add navs to forks/merges

src/FilterLists.Api/Migrations/20180202235529_AddScrapes.Designer.cs

@@ -0,0 +1,503 @@
+// <auto-generated />
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage;
+using System;
+
+namespace FilterLists.Api.Migrations
+{
+    [DbContext(typeof(FilterListsDbContext))]
+    [Migration("20180202235529_AddScrapes")]
+    partial class AddScrapes
+    {
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
+                .HasAnnotation("ProductVersion", "2.0.1-rtm-125");
+
+            modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("SMALLINT UNSIGNED");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Description")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("DescriptionSourceUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<DateTime?>("DiscontinuedDate");
+
+                    b.Property<string>("DonateUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("EmailAddress")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("VARCHAR(126)")
+                        .HasDefaultValueSql("NULL");
+
+                    b.Property<string>("ForumUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("HomeUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("IssuesUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<int?>("LicenseId");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("VARCHAR(126)");
+
+                    b.Property<string>("PolicyUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("SubmissionUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<int?>("SyntaxId");
+
+                    b.Property<string>("ViewUrl")
+                        .IsRequired()
+                        .HasColumnType("TEXT");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("LicenseId");
+
+                    b.HasIndex("SyntaxId");
+
+                    b.ToTable("filterlists");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+                {
+                    b.Property<int>("FilterListId");
+
+                    b.Property<int>("LanguageId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("FilterListId", "LanguageId");
+
+                    b.HasIndex("LanguageId");
+
+                    b.ToTable("filterlists_languages");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+                {
+                    b.Property<int>("FilterListId");
+
+                    b.Property<int>("MaintainerId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("FilterListId", "MaintainerId");
+
+                    b.HasIndex("MaintainerId");
+
+                    b.ToTable("filterlists_maintainers");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
+                {
+                    b.Property<int>("FilterListId");
+
+                    b.Property<int>("RuleId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("FilterListId", "RuleId");
+
+                    b.HasIndex("RuleId");
+
+                    b.ToTable("filterlists_rules");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+                {
+                    b.Property<int>("ForkFilterListId");
+
+                    b.Property<int>("UpstreamFilterListId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+                    b.HasIndex("UpstreamFilterListId");
+
+                    b.ToTable("forks");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+                {
+                    b.Property<int>("MergeFilterListId");
+
+                    b.Property<int>("UpstreamFilterListId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+                    b.HasIndex("UpstreamFilterListId");
+
+                    b.ToTable("merges");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+                {
+                    b.Property<int>("SoftwareId");
+
+                    b.Property<int>("SyntaxId");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.HasKey("SoftwareId", "SyntaxId");
+
+                    b.HasIndex("SyntaxId");
+
+                    b.ToTable("software_syntaxes");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("SMALLINT UNSIGNED");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Iso6391")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("VARCHAR(2)")
+                        .HasDefaultValueSql("NULL");
+
+                    b.Property<string>("Iso6392")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("VARCHAR(3)")
+                        .HasDefaultValueSql("NULL");
+
+                    b.Property<string>("Iso6392B")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("VARCHAR(3)")
+                        .HasDefaultValueSql("NULL");
+
+                    b.Property<string>("Iso6392T")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("VARCHAR(3)")
+                        .HasDefaultValueSql("NULL");
+
+                    b.Property<string>("Iso6393")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("VARCHAR(3)")
+                        .HasDefaultValueSql("NULL");
+
+                    b.Property<string>("LocalName")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("VARCHAR(126)")
+                        .HasDefaultValueSql("NULL");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("VARCHAR(126)")
+                        .HasDefaultValueSql("NULL");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("languages");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("SMALLINT UNSIGNED");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("DescriptionUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("VARCHAR(126)");
+
+                    b.Property<bool>("PermissiveAdaptation");
+
+                    b.Property<bool>("PermissiveCommercial");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("licenses");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("SMALLINT UNSIGNED");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("EmailAddress")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("VARCHAR(126)")
+                        .HasDefaultValueSql("NULL");
+
+                    b.Property<string>("HomeUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("VARCHAR(126)");
+
+                    b.Property<string>("TwitterHandle")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("VARCHAR(126)")
+                        .HasDefaultValueSql("NULL");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("maintainers");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("BIGINT UNSIGNED");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Raw")
+                        .IsRequired()
+                        .HasColumnType("VARCHAR(8192)");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("rules");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("SMALLINT UNSIGNED");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<int>("FilterListId");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("FilterListId");
+
+                    b.ToTable("scrapes");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("SMALLINT UNSIGNED");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("DownloadUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("HomeUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("VARCHAR(126)");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("software");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("SMALLINT UNSIGNED");
+
+                    b.Property<DateTime>("CreatedDateUtc")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("DefinitionUrl")
+                        .HasColumnType("TEXT");
+
+                    b.Property<DateTime>("ModifiedDateUtc")
+                        .ValueGeneratedOnAddOrUpdate()
+                        .HasColumnType("TIMESTAMP");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("VARCHAR(126)");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("syntaxes");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.License")
+                        .WithMany("FilterLists")
+                        .HasForeignKey("LicenseId");
+
+                    b.HasOne("FilterLists.Data.Entities.Syntax")
+                        .WithMany("FilterLists")
+                        .HasForeignKey("SyntaxId");
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                        .WithMany("FilterListLanguages")
+                        .HasForeignKey("FilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.Language", "Language")
+                        .WithMany("FilterListLanguages")
+                        .HasForeignKey("LanguageId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                        .WithMany("FilterListMaintainers")
+                        .HasForeignKey("FilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+                        .WithMany("FilterListMaintainers")
+                        .HasForeignKey("MaintainerId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                        .WithMany("FilterListRules")
+                        .HasForeignKey("FilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+                        .WithMany("FilterListRules")
+                        .HasForeignKey("RuleId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+                        .WithMany("ForkFilterLists")
+                        .HasForeignKey("ForkFilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+                        .WithMany("UpstreamForkFilterLists")
+                        .HasForeignKey("UpstreamFilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+                        .WithMany("MergeFilterLists")
+                        .HasForeignKey("MergeFilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+                        .WithMany("UpstreamMergeFilterLists")
+                        .HasForeignKey("UpstreamFilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.Software", "Software")
+                        .WithMany("SoftwareSyntaxes")
+                        .HasForeignKey("SoftwareId")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+                        .WithMany("SoftwareSyntaxes")
+                        .HasForeignKey("SyntaxId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
+                {
+                    b.HasOne("FilterLists.Data.Entities.FilterList")
+                        .WithMany("Scrapes")
+                        .HasForeignKey("FilterListId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

src/FilterLists.Api/Migrations/20180202235529_AddScrapes.cs

@@ -0,0 +1,90 @@
+using System;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+    public partial class AddScrapes : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                "ModifiedDateUtc",
+                "rules");
+
+            migrationBuilder.DropColumn(
+                "ScrapedDateUtc",
+                "filterlists");
+
+            migrationBuilder.DropColumn(
+                "UpdatedDateUtc",
+                "filterlists");
+
+            migrationBuilder.AlterColumn<string>(
+                "ViewUrl",
+                "filterlists",
+                "TEXT",
+                nullable: false,
+                oldClrType: typeof(string),
+                oldType: "TEXT NOT NULL",
+                oldNullable: true);
+
+            migrationBuilder.CreateTable(
+                "scrapes",
+                table => new
+                {
+                    Id = table.Column<int>("SMALLINT UNSIGNED", nullable: false)
+                        .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
+                    CreatedDateUtc = table.Column<DateTime>("TIMESTAMP", nullable: false)
+                        .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
+                    FilterListId = table.Column<int>(nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_scrapes", x => x.Id);
+                    table.ForeignKey(
+                        "FK_scrapes_filterlists_FilterListId",
+                        x => x.FilterListId,
+                        "filterlists",
+                        "Id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateIndex(
+                "IX_scrapes_FilterListId",
+                "scrapes",
+                "FilterListId");
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                "scrapes");
+
+            migrationBuilder.AddColumn<DateTime>(
+                "ModifiedDateUtc",
+                "rules",
+                "TIMESTAMP",
+                nullable: false,
+                defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
+            migrationBuilder.AlterColumn<string>(
+                "ViewUrl",
+                "filterlists",
+                "TEXT NOT NULL",
+                nullable: true,
+                oldClrType: typeof(string),
+                oldType: "TEXT");
+
+            migrationBuilder.AddColumn<DateTime>(
+                "ScrapedDateUtc",
+                "filterlists",
+                nullable: true);
+
+            migrationBuilder.AddColumn<DateTime>(
+                "UpdatedDateUtc",
+                "filterlists",
+                nullable: true);
+        }
+    }
+}

src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs

@@ -66,17 +66,14 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                 b.Property<string>("PolicyUrl")
                     .HasColumnType("TEXT");
 
-                b.Property<DateTime?>("ScrapedDateUtc");
-
                 b.Property<string>("SubmissionUrl")
                     .HasColumnType("TEXT");
 
                 b.Property<int?>("SyntaxId");
 
-                b.Property<DateTime?>("UpdatedDateUtc");
-
                 b.Property<string>("ViewUrl")
-                    .HasColumnType("TEXT NOT NULL");
+                    .IsRequired()
+                    .HasColumnType("TEXT");
 
                 b.HasKey("Id");
 
@@ -319,10 +316,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     .ValueGeneratedOnAdd()
                     .HasColumnType("TIMESTAMP");
 
-                b.Property<DateTime>("ModifiedDateUtc")
-                    .ValueGeneratedOnAddOrUpdate()
-                    .HasColumnType("TIMESTAMP");
-
                 b.Property<string>("Raw")
                     .IsRequired()
                     .HasColumnType("VARCHAR(8192)");
@@ -332,6 +325,25 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                 b.ToTable("rules");
             });
 
+            modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
+            {
+                b.Property<int>("Id")
+                    .ValueGeneratedOnAdd()
+                    .HasColumnType("SMALLINT UNSIGNED");
+
+                b.Property<DateTime>("CreatedDateUtc")
+                    .ValueGeneratedOnAdd()
+                    .HasColumnType("TIMESTAMP");
+
+                b.Property<int>("FilterListId");
+
+                b.HasKey("Id");
+
+                b.HasIndex("FilterListId");
+
+                b.ToTable("scrapes");
+            });
+
             modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
             {
                 b.Property<int>("Id")
@@ -440,12 +452,12 @@ protected override void BuildModel(ModelBuilder modelBuilder)
             modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
             {
                 b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
-                    .WithMany()
+                    .WithMany("ForkFilterLists")
                     .HasForeignKey("ForkFilterListId")
                     .OnDelete(DeleteBehavior.Cascade);
 
                 b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
-                    .WithMany()
+                    .WithMany("UpstreamForkFilterLists")
                     .HasForeignKey("UpstreamFilterListId")
                     .OnDelete(DeleteBehavior.Cascade);
             });
@@ -453,12 +465,12 @@ protected override void BuildModel(ModelBuilder modelBuilder)
             modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
             {
                 b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
-                    .WithMany()
+                    .WithMany("MergeFilterLists")
                     .HasForeignKey("MergeFilterListId")
                     .OnDelete(DeleteBehavior.Cascade);
 
                 b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
-                    .WithMany()
+                    .WithMany("UpstreamMergeFilterLists")
                     .HasForeignKey("UpstreamFilterListId")
                     .OnDelete(DeleteBehavior.Cascade);
             });
@@ -475,6 +487,14 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     .HasForeignKey("SyntaxId")
                     .OnDelete(DeleteBehavior.Cascade);
             });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList")
+                    .WithMany("Scrapes")
+                    .HasForeignKey("FilterListId")
+                    .OnDelete(DeleteBehavior.Cascade);
+            });
 #pragma warning restore 612, 618
         }
     }

src/FilterLists.Data/Entities/FilterList.cs

@@ -11,20 +11,22 @@ public class FilterList : BaseEntity
         public DateTime? DiscontinuedDate { get; set; }
         public string DonateUrl { get; set; }
         public string EmailAddress { get; set; }
+        public ICollection<Fork> ForkFilterLists { get; set; }
+        public ICollection<Fork> UpstreamForkFilterLists { get; set; }
         public string ForumUrl { get; set; }
         public string HomeUrl { get; set; }
         public string IssuesUrl { get; set; }
+        public ICollection<FilterListLanguage> FilterListLanguages { get; set; }
         public int? LicenseId { get; set; }
+        public ICollection<FilterListMaintainer> FilterListMaintainers { get; set; }
+        public ICollection<Merge> MergeFilterLists { get; set; }
+        public ICollection<Merge> UpstreamMergeFilterLists { get; set; }
         public string Name { get; set; }
         public string PolicyUrl { get; set; }
-        public DateTime? ScrapedDateUtc { get; set; }
+        public ICollection<FilterListRule> FilterListRules { get; set; }
+        public ICollection<Scrape> Scrapes { get; set; }
         public string SubmissionUrl { get; set; }
         public int? SyntaxId { get; set; }
-        public DateTime? UpdatedDateUtc { get; set; }
         public string ViewUrl { get; set; }
-
-        public ICollection<FilterListLanguage> FilterListLanguages { get; set; }
-        public ICollection<FilterListMaintainer> FilterListMaintainers { get; set; }
-        public ICollection<FilterListRule> FilterListRules { get; set; }
     }
 }

src/FilterLists.Data/Entities/Language.cs

@@ -5,6 +5,7 @@ namespace FilterLists.Data.Entities
 {
     public class Language : BaseEntity
     {
+        public ICollection<FilterListLanguage> FilterListLanguages { get; set; }
         public string Iso6391 { get; set; }
         public string Iso6392 { get; set; }
         public string Iso6392B { get; set; }
@@ -12,7 +13,5 @@ public class Language : BaseEntity
         public string Iso6393 { get; set; }
         public string LocalName { get; set; }
         public string Name { get; set; }
-
-        public ICollection<FilterListLanguage> FilterListLanguages { get; set; }
     }
 }

src/FilterLists.Data/Entities/Maintainer.cs

@@ -6,10 +6,9 @@ namespace FilterLists.Data.Entities
     public class Maintainer : BaseEntity
     {
         public string EmailAddress { get; set; }
+        public ICollection<FilterListMaintainer> FilterListMaintainers { get; set; }
         public string HomeUrl { get; set; }
         public string Name { get; set; }
         public string TwitterHandle { get; set; }
-
-        public ICollection<FilterListMaintainer> FilterListMaintainers { get; set; }
     }
 }

src/FilterLists.Data/Entities/Rule.cs

@@ -5,8 +5,7 @@ namespace FilterLists.Data.Entities
 {
     public class Rule : BaseEntity
     {
-        public string Raw { get; set; }
-
         public ICollection<FilterListRule> FilterListRules { get; set; }
+        public string Raw { get; set; }
     }
 }

src/FilterLists.Data/Entities/Scrape.cs

@@ -0,0 +1,7 @@
+namespace FilterLists.Data.Entities
+{
+    public class Scrape : BaseEntity
+    {
+        public int FilterListId { get; set; }
+    }
+}

src/FilterLists.Data/Entities/Software.cs

@@ -8,7 +8,6 @@ public class Software : BaseEntity
         public string DownloadUrl { get; set; }
         public string HomeUrl { get; set; }
         public string Name { get; set; }
-
         public ICollection<SoftwareSyntax> SoftwareSyntaxes { get; set; }
     }
 }

src/FilterLists.Data/Entities/Syntax.cs

@@ -8,7 +8,6 @@ public class Syntax : BaseEntity
         public string DefinitionUrl { get; set; }
         public ICollection<FilterList> FilterLists { get; set; }
         public string Name { get; set; }
-
         public ICollection<SoftwareSyntax> SoftwareSyntaxes { get; set; }
     }
 }

src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs

@@ -35,7 +35,8 @@ public override void Configure(EntityTypeBuilder<FilterList> entityTypeBuilder)
             entityTypeBuilder.Property(x => x.SubmissionUrl)
                 .HasColumnType("TEXT");
             entityTypeBuilder.Property(x => x.ViewUrl)
-                .HasColumnType("TEXT NOT NULL");
+                .HasColumnType("TEXT")
+                .IsRequired();
         }
     }
 }

src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs

@@ -15,6 +15,8 @@ public override void Configure(EntityTypeBuilder<Rule> entityTypeBuilder)
             entityTypeBuilder.Property(x => x.Id)
                 .HasColumnType("BIGINT UNSIGNED");
 
+            entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
+
             entityTypeBuilder.Property(x => x.Raw)
                 .HasColumnType("VARCHAR(8192)")
                 .IsRequired();

src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs

@@ -0,0 +1,18 @@
+using FilterLists.Data.Entities;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace FilterLists.Data.EntityTypeConfigurations
+{
+    public class ScrapeTypeConfiguration : BaseEntityTypeConfiguration<Scrape>
+    {
+        public override void Configure(EntityTypeBuilder<Scrape> entityTypeBuilder)
+        {
+            base.Configure(entityTypeBuilder);
+
+            entityTypeBuilder.ToTable("scrapes");
+
+            entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
+        }
+    }
+}

src/FilterLists.Data/FilterListsDbContext.cs

@@ -27,6 +27,7 @@ private static void ApplyConfigurationsEntities(ModelBuilder modelBuilder)
             modelBuilder.ApplyConfiguration(new LicenseTypeConfiguration());
             modelBuilder.ApplyConfiguration(new MaintainerTypeConfiguration());
             modelBuilder.ApplyConfiguration(new RuleTypeConfiguration());
+            modelBuilder.ApplyConfiguration(new ScrapeTypeConfiguration());
             modelBuilder.ApplyConfiguration(new SoftwareTypeConfiguration());
             modelBuilder.ApplyConfiguration(new SyntaxTypeConfiguration());
         }
@@ -37,10 +38,36 @@ private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder)
             modelBuilder.ApplyConfiguration(new FilterListMaintainerTypeConfiguration());
             modelBuilder.ApplyConfiguration(new FilterListRuleTypeConfiguration());
             modelBuilder.ApplyConfiguration(new ForkTypeConfiguration());
+            ConfigureForkSelfReferencing(modelBuilder);
             modelBuilder.ApplyConfiguration(new MergeTypeConfiguration());
+            ConfigureMergeSelfReferencing(modelBuilder);
             modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration());
         }
 
+        private static void ConfigureForkSelfReferencing(ModelBuilder modelBuilder)
+        {
+            modelBuilder.Entity<Fork>()
+                .HasOne(x => x.ForkFilterList)
+                .WithMany(x => x.ForkFilterLists)
+                .HasForeignKey(x => x.ForkFilterListId);
+            modelBuilder.Entity<Fork>()
+                .HasOne(x => x.UpstreamFilterList)
+                .WithMany(x => x.UpstreamForkFilterLists)
+                .HasForeignKey(x => x.UpstreamFilterListId);
+        }
+
+        private static void ConfigureMergeSelfReferencing(ModelBuilder modelBuilder)
+        {
+            modelBuilder.Entity<Merge>()
+                .HasOne(x => x.MergeFilterList)
+                .WithMany(x => x.MergeFilterLists)
+                .HasForeignKey(x => x.MergeFilterListId);
+            modelBuilder.Entity<Merge>()
+                .HasOne(x => x.UpstreamFilterList)
+                .WithMany(x => x.UpstreamMergeFilterLists)
+                .HasForeignKey(x => x.UpstreamFilterListId);
+        }
+
         #region Entities
 
         public DbSet<FilterList> FilterLists { get; set; }
@@ -48,6 +75,7 @@ private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder)
         public DbSet<License> Licenses { get; set; }
         public DbSet<Maintainer> Maintainers { get; set; }
         public DbSet<Rule> Rules { get; set; }
+        public DbSet<Scrape> Scrapes { get; set; }
         public DbSet<Software> Software { get; set; }
         public DbSet<Syntax> Syntaxes { get; set; }
 

src/FilterLists.Services/ScrapeService/ScrapeService.cs

@@ -28,8 +28,8 @@ public async Task ScrapeAsync(int batchSize)
 
         private async Task<List<FilterListViewUrlDto>> GetNextListsToScrape(int batchSize)
         {
-            return await dbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(batchSize)
-                .ProjectTo<FilterListViewUrlDto>().ToListAsync();
+            //TODO: order by least recently scraped
+            return await dbContext.FilterLists.Take(batchSize).ProjectTo<FilterListViewUrlDto>().ToListAsync();
         }
 
         private async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)

src/FilterLists.Services/ScrapeService/Snapshot.cs

@@ -61,15 +61,7 @@ public async Task AddOrUpdateRules()
                 new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList();
             await dbContext.FilterListRules.AddRangeAsync(newFilterListRules);
 
-            // update UpdatedDateUtc
-            var list = dbContext.FilterLists.Find(filterListId);
-            if (preExistingSnapshotFilterListRules.Any() || newFilterListRules.Any() || deletedFilterListRules.Any())
-                list.UpdatedDateUtc = DateTime.UtcNow;
-
-            // update ScrapedDateUtc
-            list.ScrapedDateUtc = DateTime.UtcNow;
-
-            dbContext.FilterLists.Update(list);
+            // update scrapes
 
             await dbContext.SaveChangesAsync();
         }