add batch size optimizer

ref #344

src/FilterLists.Api/Migrations/20180824223021_AddBatchSize.Designer.cs

@@ -0,0 +1,650 @@
+// <auto-generated />
+
+using System;
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+    [DbContext(typeof(FilterListsDbContext))]
+    [Migration("20180824223021_AddBatchSize")]
+    partial class AddBatchSize
+    {
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "2.1.2-rtm-30932")
+                .HasAnnotation("Relational:MaxIdentifierLength", 64);
+
+            modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+            {
+                b.Property<ushort>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<bool>("CantSnapshot");
+
+                b.Property<string>("ChatUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("Description")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("DescriptionSourceUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("DiscontinuedDate")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("DATE")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("DonateUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("EmailAddress")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("ForumUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("HomeUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("IssuesUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<byte>("LicenseId")
+                 .ValueGeneratedOnAdd()
+                 .HasDefaultValue((byte)5);
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.Property<string>("PolicyUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("PublishedDate")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("DATE")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("SubmissionUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<byte?>("SyntaxId");
+
+                b.Property<string>("ViewUrl")
+                 .HasColumnType("TEXT");
+
+                b.HasKey("Id");
+
+                b.HasIndex("LicenseId");
+
+                b.HasIndex("SyntaxId");
+
+                b.ToTable("filterlists");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+            {
+                b.Property<ushort>("FilterListId");
+
+                b.Property<ushort>("LanguageId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("FilterListId", "LanguageId");
+
+                b.HasIndex("LanguageId");
+
+                b.ToTable("filterlists_languages");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+            {
+                b.Property<ushort>("FilterListId");
+
+                b.Property<ushort>("MaintainerId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("FilterListId", "MaintainerId");
+
+                b.HasIndex("MaintainerId");
+
+                b.ToTable("filterlists_maintainers");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+            {
+                b.Property<ushort>("FilterListId");
+
+                b.Property<byte>("TagId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("FilterListId", "TagId");
+
+                b.HasIndex("TagId");
+
+                b.ToTable("filterlists_tags");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+            {
+                b.Property<ushort>("ForkFilterListId");
+
+                b.Property<ushort>("UpstreamFilterListId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+                b.HasIndex("UpstreamFilterListId");
+
+                b.ToTable("forks");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+            {
+                b.Property<ushort>("MergeFilterListId");
+
+                b.Property<ushort>("UpstreamFilterListId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+                b.HasIndex("UpstreamFilterListId");
+
+                b.ToTable("merges");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+            {
+                b.Property<uint>("SnapshotId");
+
+                b.Property<uint>("RuleId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("SnapshotId", "RuleId");
+
+                b.HasIndex("RuleId");
+
+                b.ToTable("snapshots_rules");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+            {
+                b.Property<byte>("SyntaxId");
+
+                b.Property<byte>("SoftwareId");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.HasKey("SyntaxId", "SoftwareId");
+
+                b.HasIndex("SoftwareId");
+
+                b.ToTable("software_syntaxes");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+            {
+                b.Property<ushort>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("Iso6391")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(2)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6392")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6392B")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6392T")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("Iso6393")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(3)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("LocalName")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.HasKey("Id");
+
+                b.ToTable("languages");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+            {
+                b.Property<byte>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TINYINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("DescriptionUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.Property<bool>("PermissiveAdaptation");
+
+                b.Property<bool>("PermissiveCommercial");
+
+                b.HasKey("Id");
+
+                b.ToTable("licenses");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+            {
+                b.Property<ushort>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("EmailAddress")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<string>("HomeUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.Property<string>("TwitterHandle")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("VARCHAR(126)")
+                 .HasDefaultValueSql("NULL");
+
+                b.HasKey("Id");
+
+                b.ToTable("maintainers");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+            {
+                b.Property<uint>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("INT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("Raw")
+                 .IsRequired()
+                 .HasColumnType("LONGTEXT");
+
+                b.HasKey("Id");
+
+                b.ToTable("rules");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+            {
+                b.Property<uint>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("MEDIUMINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<short?>("BatchSize")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<ushort>("FilterListId");
+
+                b.Property<short?>("HttpStatusCode")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT")
+                 .HasDefaultValueSql("NULL");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<bool>("WasSuccessful");
+
+                b.Property<DateTime?>("WaybackTimestamp")
+                 .HasColumnType("TIMESTAMP");
+
+                b.HasKey("Id");
+
+                b.HasIndex("FilterListId");
+
+                b.ToTable("snapshots");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+            {
+                b.Property<byte>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TINYINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("DownloadUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<string>("HomeUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.HasKey("Id");
+
+                b.ToTable("software");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+            {
+                b.Property<byte>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TINYINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("DefinitionUrl")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.HasKey("Id");
+
+                b.ToTable("syntaxes");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Tag", b =>
+            {
+                b.Property<byte>("Id")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TINYINT UNSIGNED")
+                 .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+                b.Property<DateTime?>("CreatedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp()");
+
+                b.Property<string>("Description")
+                 .HasColumnType("TEXT");
+
+                b.Property<DateTime?>("ModifiedDateUtc")
+                 .IsRequired()
+                 .ValueGeneratedOnAddOrUpdate()
+                 .HasColumnType("TIMESTAMP")
+                 .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+                b.Property<string>("Name")
+                 .IsRequired()
+                 .HasColumnType("VARCHAR(126)");
+
+                b.HasKey("Id");
+
+                b.ToTable("tags");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.License", "License")
+                 .WithMany("FilterLists")
+                 .HasForeignKey("LicenseId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+                 .WithMany("FilterLists")
+                 .HasForeignKey("SyntaxId");
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                 .WithMany("FilterListLanguages")
+                 .HasForeignKey("FilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Language", "Language")
+                 .WithMany("FilterListLanguages")
+                 .HasForeignKey("LanguageId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                 .WithMany("FilterListMaintainers")
+                 .HasForeignKey("FilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+                 .WithMany("FilterListMaintainers")
+                 .HasForeignKey("MaintainerId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                 .WithMany("FilterListTags")
+                 .HasForeignKey("FilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Tag", "Tag")
+                 .WithMany("FilterListTags")
+                 .HasForeignKey("TagId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+                 .WithMany("ForkFilterLists")
+                 .HasForeignKey("ForkFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+                 .WithMany("UpstreamForkFilterLists")
+                 .HasForeignKey("UpstreamFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+                 .WithMany("MergeFilterLists")
+                 .HasForeignKey("MergeFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+                 .WithMany("UpstreamMergeFilterLists")
+                 .HasForeignKey("UpstreamFilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+                 .WithMany("SnapshotRules")
+                 .HasForeignKey("RuleId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
+                 .WithMany("SnapshotRules")
+                 .HasForeignKey("SnapshotId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.Software", "Software")
+                 .WithMany("SoftwareSyntaxes")
+                 .HasForeignKey("SoftwareId")
+                 .OnDelete(DeleteBehavior.Cascade);
+
+                b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+                 .WithMany("SoftwareSyntaxes")
+                 .HasForeignKey("SyntaxId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+
+            modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+            {
+                b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+                 .WithMany("Snapshots")
+                 .HasForeignKey("FilterListId")
+                 .OnDelete(DeleteBehavior.Cascade);
+            });
+#pragma warning restore 612, 618
+        }
+    }
+}

src/FilterLists.Api/Migrations/20180824223021_AddBatchSize.cs

@@ -0,0 +1,46 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+    public partial class AddBatchSize : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AlterColumn<short>(
+                "HttpStatusCode",
+                "snapshots",
+                "SMALLINT",
+                nullable: true,
+                defaultValueSql: "NULL",
+                oldClrType: typeof(ushort),
+                oldType: "SMALLINT UNSIGNED",
+                oldNullable: true,
+                oldDefaultValueSql: "NULL");
+
+            migrationBuilder.AddColumn<short>(
+                "BatchSize",
+                "snapshots",
+                "SMALLINT",
+                nullable: true,
+                defaultValueSql: "NULL");
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                "BatchSize",
+                "snapshots");
+
+            migrationBuilder.AlterColumn<ushort>(
+                "HttpStatusCode",
+                "snapshots",
+                "SMALLINT UNSIGNED",
+                nullable: true,
+                defaultValueSql: "NULL",
+                oldClrType: typeof(short),
+                oldType: "SMALLINT",
+                oldNullable: true,
+                oldDefaultValueSql: "NULL");
+        }
+    }
+}

src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs

@@ -400,6 +400,11 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                  .HasColumnType("MEDIUMINT UNSIGNED")
                  .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
 
+                b.Property<short?>("BatchSize")
+                 .ValueGeneratedOnAdd()
+                 .HasColumnType("SMALLINT")
+                 .HasDefaultValueSql("NULL");
+
                 b.Property<DateTime?>("CreatedDateUtc")
                  .IsRequired()
                  .ValueGeneratedOnAdd()
@@ -408,9 +413,9 @@ protected override void BuildModel(ModelBuilder modelBuilder)
 
                 b.Property<ushort>("FilterListId");
 
-                b.Property<ushort?>("HttpStatusCode")
+                b.Property<short?>("HttpStatusCode")
                  .ValueGeneratedOnAdd()
-                 .HasColumnType("SMALLINT UNSIGNED")
+                 .HasColumnType("SMALLINT")
                  .HasDefaultValueSql("NULL");
 
                 b.Property<DateTime?>("ModifiedDateUtc")

src/FilterLists.Data/Entities/Snapshot.cs

@@ -6,9 +6,10 @@ namespace FilterLists.Data.Entities
 {
     public class Snapshot : BaseEntity
     {
+        public int? BatchSize { get; set; }
         public uint FilterListId { get; set; }
         public FilterList FilterList { get; set; }
-        public uint? HttpStatusCode { get; set; }
+        public int? HttpStatusCode { get; set; }
         public bool WasSuccessful { get; set; }
         public ICollection<SnapshotRule> SnapshotRules { get; set; }
         public DateTime? WaybackTimestamp { get; set; }

src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs

@@ -12,8 +12,11 @@ public override void Configure(EntityTypeBuilder<Snapshot> entityTypeBuilder)
             entityTypeBuilder.ToTable("snapshots");
             entityTypeBuilder.Property(x => x.Id)
                              .HasColumnType("MEDIUMINT UNSIGNED");
+            entityTypeBuilder.Property(x => x.BatchSize)
+                             .HasColumnType("SMALLINT")
+                             .HasDefaultValueSql("NULL");
             entityTypeBuilder.Property(x => x.HttpStatusCode)
-                             .HasColumnType("SMALLINT UNSIGNED")
+                             .HasColumnType("SMALLINT")
                              .HasDefaultValueSql("NULL");
             entityTypeBuilder.Property(x => x.WaybackTimestamp)
                              .HasColumnType("TIMESTAMP");

src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs

@@ -33,6 +33,7 @@ public static void AddFilterListsAgentServices(this IServiceCollection services,
                         o.UseMySql(config.GetConnectionString("FilterListsConnection"),
                             m => m.MigrationsAssembly("FilterLists.Api")));
             services.TryAddScoped<SnapshotService>();
+            services.TryAddScoped<BatchSizeService>();
             services.AddAutoMapper();
         }
     }

src/FilterLists.Services/Snapshot/Batch.cs

@@ -7,14 +7,13 @@
 
 namespace FilterLists.Services.Snapshot
 {
-    public class SnapshotBatch
+    public class Batch
     {
         private readonly FilterListsDbContext dbContext;
         private readonly IEnumerable<string> lines;
         private readonly Data.Entities.Snapshot snapEntity;
 
-        public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable<string> lines,
-            Data.Entities.Snapshot snapEntity)
+        public Batch(FilterListsDbContext dbContext, IEnumerable<string> lines, Data.Entities.Snapshot snapEntity)
         {
             this.dbContext = dbContext;
             this.lines = lines;

src/FilterLists.Services/Snapshot/BatchSizeService.cs

@@ -0,0 +1,52 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+
+namespace FilterLists.Services.Snapshot
+{
+    public class BatchSizeService : Service
+    {
+        private const int DefaultBatchSize = 500;
+        private const float PercentMultiplier = 0.05F;
+
+        public BatchSizeService(FilterListsDbContext dbContext) : base(dbContext)
+        {
+        }
+
+        public async Task<int> GetBatchSize()
+        {
+            var recentSnapPerfs = await GetRecentSnapPerfs();
+            return recentSnapPerfs.Count == 2
+                ? (recentSnapPerfs[0].RulesPerMs >= recentSnapPerfs[1].RulesPerMs
+                    ? (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 + PercentMultiplier))
+                    : (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 - PercentMultiplier)))
+                : DefaultBatchSize;
+        }
+
+        private async Task<List<SnapshotPerformance>> GetRecentSnapPerfs() =>
+            await DbContext.Snapshots
+                           .Where(s => s.WasSuccessful && s.BatchSize.HasValue)
+                           .OrderByDescending(s => s.CreatedDateUtc)
+                           .Take(2)
+                           .Select(s => new SnapshotPerformance
+                           {
+                               BatchSize = s.BatchSize.Value,
+                               CreatedDateUtc = s.CreatedDateUtc.Value,
+                               ModifiedDateUtc = s.ModifiedDateUtc.Value,
+                               RulesCount = s.SnapshotRules.Count
+                           })
+                           .ToListAsync();
+
+        private class SnapshotPerformance
+        {
+            public int BatchSize { get; set; }
+            public int RulesCount { private get; set; }
+            public DateTime CreatedDateUtc { private get; set; }
+            public DateTime ModifiedDateUtc { private get; set; }
+            public int RulesPerMs => (ModifiedDateUtc - CreatedDateUtc).Milliseconds / RulesCount;
+        }
+    }
+}

src/FilterLists.Services/Snapshot/Snapshot.cs

@@ -17,7 +17,7 @@ namespace FilterLists.Services.Snapshot
 {
     public class Snapshot
     {
-        private const int BatchSize = 500;
+        private readonly BatchSizeService batchSizeService;
         private readonly FilterListsDbContext dbContext;
         public readonly FilterListViewUrlDto List;
         protected readonly Data.Entities.Snapshot SnapEntity;
@@ -31,8 +31,10 @@ public Snapshot()
         }
 
         [UsedImplicitly]
-        public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, string uaString)
+        public Snapshot(BatchSizeService batchSizeService, FilterListsDbContext dbContext, FilterListViewUrlDto list,
+            string uaString)
         {
+            this.batchSizeService = batchSizeService;
             this.dbContext = dbContext;
             List = list;
             ListUrl = list.ViewUrl;
@@ -89,7 +91,7 @@ private async Task TryGetLines()
             }
             catch (WebException we)
             {
-                SnapEntity.HttpStatusCode = (uint)((HttpWebResponse)we.Response).StatusCode;
+                SnapEntity.HttpStatusCode = (int)((HttpWebResponse)we.Response).StatusCode;
                 await dbContext.SaveChangesAsync();
                 TrackException(we);
             }
@@ -101,7 +103,7 @@ private async Task GetLines()
             {
                 httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(uaString);
                 var response = await httpClient.GetAsync(ListUrl, HttpCompletionOption.ResponseHeadersRead);
-                SnapEntity.HttpStatusCode = (uint)response.StatusCode;
+                SnapEntity.HttpStatusCode = (int)response.StatusCode;
                 response.EnsureSuccessStatusCode();
                 lines = new HashSet<string>();
                 using (var stream = await response.Content.ReadAsStreamAsync())
@@ -116,14 +118,17 @@ private async Task GetLines()
 
         private async Task SaveInBatches()
         {
-            var snapBatches = CreateBatches();
+            var snapBatches = await CreateBatches();
             await SaveBatches(snapBatches);
         }
 
-        private IEnumerable<SnapshotBatch> CreateBatches() =>
-            lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, SnapEntity));
+        private async Task<IEnumerable<Batch>> CreateBatches()
+        {
+            SnapEntity.BatchSize = await batchSizeService.GetBatchSize();
+            return lines.Batch(SnapEntity.BatchSize.Value).Select(b => new Batch(dbContext, b, SnapEntity));
+        }
 
-        private static async Task SaveBatches(IEnumerable<SnapshotBatch> batches)
+        private static async Task SaveBatches(IEnumerable<Batch> batches)
         {
             foreach (var batch in batches)
                 await batch.SaveAsync();

src/FilterLists.Services/Snapshot/SnapshotService.cs

@@ -13,13 +13,13 @@ namespace FilterLists.Services.Snapshot
 {
     public class SnapshotService : Service
     {
+        private readonly BatchSizeService batchSizeService;
         private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1);
         private string uaString;
 
-        public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig)
-            : base(dbContext, mapConfig)
-        {
-        }
+        public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig,
+            BatchSizeService batchSizeService)
+            : base(dbContext, mapConfig) => this.batchSizeService = batchSizeService;
 
         public async Task CaptureAsync(int batchSize)
         {
@@ -67,7 +67,8 @@ private async Task<List<TSnap>> CreateAndSaveSnaps<TSnap>(IEnumerable<FilterList
 
         private IEnumerable<TSnap> CreateSnaps<TSnap>(IEnumerable<FilterListViewUrlDto> lists)
             where TSnap : Snapshot, new() =>
-            lists.Select(l => Activator.CreateInstance(typeof(TSnap), DbContext, l, uaString) as TSnap);
+            lists.Select(l =>
+                Activator.CreateInstance(typeof(TSnap), batchSizeService, DbContext, l, uaString) as TSnap);
 
         private static async Task SaveSnaps(IEnumerable<Snapshot> snaps)
         {

src/FilterLists.Services/Snapshot/SnapshotWayback.cs

@@ -13,8 +13,9 @@ public SnapshotWayback()
         }
 
         [UsedImplicitly]
-        public SnapshotWayback(FilterListsDbContext dbContext, FilterListViewUrlDto list, string uaString)
-            : base(dbContext, list, uaString)
+        public SnapshotWayback(BatchSizeService batchSizeService, FilterListsDbContext dbContext,
+            FilterListViewUrlDto list, string uaString)
+            : base(batchSizeService, dbContext, list, uaString)
         {
         }