diff --git a/src/FilterLists.Api/Migrations/20180824223021_AddBatchSize.Designer.cs b/src/FilterLists.Api/Migrations/20180824223021_AddBatchSize.Designer.cs
new file mode 100644
index 000000000..78ae0cf2d
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180824223021_AddBatchSize.Designer.cs
@@ -0,0 +1,650 @@
+//
+
+using System;
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+ [DbContext(typeof(FilterListsDbContext))]
+ [Migration("20180824223021_AddBatchSize")]
+ partial class AddBatchSize
+ {
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasAnnotation("ProductVersion", "2.1.2-rtm-30932")
+ .HasAnnotation("Relational:MaxIdentifierLength", 64);
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CantSnapshot");
+
+ b.Property("ChatUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Description")
+ .HasColumnType("TEXT");
+
+ b.Property("DescriptionSourceUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("DiscontinuedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("DATE")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("DonateUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ForumUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("IssuesUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("LicenseId")
+ .ValueGeneratedOnAdd()
+ .HasDefaultValue((byte)5);
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PolicyUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("PublishedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("DATE")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("SubmissionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("SyntaxId");
+
+ b.Property("ViewUrl")
+ .HasColumnType("TEXT");
+
+ b.HasKey("Id");
+
+ b.HasIndex("LicenseId");
+
+ b.HasIndex("SyntaxId");
+
+ b.ToTable("filterlists");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("LanguageId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "LanguageId");
+
+ b.HasIndex("LanguageId");
+
+ b.ToTable("filterlists_languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("MaintainerId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "MaintainerId");
+
+ b.HasIndex("MaintainerId");
+
+ b.ToTable("filterlists_maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("TagId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "TagId");
+
+ b.HasIndex("TagId");
+
+ b.ToTable("filterlists_tags");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.Property("ForkFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("forks");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.Property("MergeFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("merges");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.Property("SnapshotId");
+
+ b.Property("RuleId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("SnapshotId", "RuleId");
+
+ b.HasIndex("RuleId");
+
+ b.ToTable("snapshots_rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.Property("SyntaxId");
+
+ b.Property("SoftwareId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("SyntaxId", "SoftwareId");
+
+ b.HasIndex("SoftwareId");
+
+ b.ToTable("software_syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Iso6391")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(2)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392B")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392T")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6393")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("LocalName")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DescriptionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PermissiveAdaptation");
+
+ b.Property("PermissiveCommercial");
+
+ b.HasKey("Id");
+
+ b.ToTable("licenses");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("TwitterHandle")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("INT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Raw")
+ .IsRequired()
+ .HasColumnType("LONGTEXT");
+
+ b.HasKey("Id");
+
+ b.ToTable("rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("MEDIUMINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("BatchSize")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("FilterListId");
+
+ b.Property("HttpStatusCode")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("WasSuccessful");
+
+ b.Property("WaybackTimestamp")
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("Id");
+
+ b.HasIndex("FilterListId");
+
+ b.ToTable("snapshots");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DownloadUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("software");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DefinitionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Tag", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Description")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("tags");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.License", "License")
+ .WithMany("FilterLists")
+ .HasForeignKey("LicenseId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("FilterLists")
+ .HasForeignKey("SyntaxId");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Language", "Language")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("LanguageId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("MaintainerId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListTags")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Tag", "Tag")
+ .WithMany("FilterListTags")
+ .HasForeignKey("TagId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+ .WithMany("ForkFilterLists")
+ .HasForeignKey("ForkFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamForkFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+ .WithMany("MergeFilterLists")
+ .HasForeignKey("MergeFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamMergeFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("RuleId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("SnapshotId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Software", "Software")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SoftwareId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SyntaxId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("Snapshots")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+#pragma warning restore 612, 618
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Api/Migrations/20180824223021_AddBatchSize.cs b/src/FilterLists.Api/Migrations/20180824223021_AddBatchSize.cs
new file mode 100644
index 000000000..eed496568
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180824223021_AddBatchSize.cs
@@ -0,0 +1,46 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+ public partial class AddBatchSize : Migration
+ {
+ protected override void Up(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.AlterColumn(
+ "HttpStatusCode",
+ "snapshots",
+ "SMALLINT",
+ nullable: true,
+ defaultValueSql: "NULL",
+ oldClrType: typeof(ushort),
+ oldType: "SMALLINT UNSIGNED",
+ oldNullable: true,
+ oldDefaultValueSql: "NULL");
+
+ migrationBuilder.AddColumn(
+ "BatchSize",
+ "snapshots",
+ "SMALLINT",
+ nullable: true,
+ defaultValueSql: "NULL");
+ }
+
+ protected override void Down(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropColumn(
+ "BatchSize",
+ "snapshots");
+
+ migrationBuilder.AlterColumn(
+ "HttpStatusCode",
+ "snapshots",
+ "SMALLINT UNSIGNED",
+ nullable: true,
+ defaultValueSql: "NULL",
+ oldClrType: typeof(short),
+ oldType: "SMALLINT",
+ oldNullable: true,
+ oldDefaultValueSql: "NULL");
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
index 33e8ef9fa..11ddd4fa3 100644
--- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
+++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
@@ -400,6 +400,11 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.HasColumnType("MEDIUMINT UNSIGNED")
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+ b.Property("BatchSize")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT")
+ .HasDefaultValueSql("NULL");
+
b.Property("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
@@ -408,9 +413,9 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property("FilterListId");
- b.Property("HttpStatusCode")
+ b.Property("HttpStatusCode")
.ValueGeneratedOnAdd()
- .HasColumnType("SMALLINT UNSIGNED")
+ .HasColumnType("SMALLINT")
.HasDefaultValueSql("NULL");
b.Property("ModifiedDateUtc")
diff --git a/src/FilterLists.Data/Entities/Snapshot.cs b/src/FilterLists.Data/Entities/Snapshot.cs
index 785223479..0d8882423 100644
--- a/src/FilterLists.Data/Entities/Snapshot.cs
+++ b/src/FilterLists.Data/Entities/Snapshot.cs
@@ -6,9 +6,10 @@ namespace FilterLists.Data.Entities
{
public class Snapshot : BaseEntity
{
+ public int? BatchSize { get; set; }
public uint FilterListId { get; set; }
public FilterList FilterList { get; set; }
- public uint? HttpStatusCode { get; set; }
+ public int? HttpStatusCode { get; set; }
public bool WasSuccessful { get; set; }
public ICollection SnapshotRules { get; set; }
public DateTime? WaybackTimestamp { get; set; }
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
index a9e33a8ef..a58994fe5 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
@@ -12,8 +12,11 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder)
entityTypeBuilder.ToTable("snapshots");
entityTypeBuilder.Property(x => x.Id)
.HasColumnType("MEDIUMINT UNSIGNED");
+ entityTypeBuilder.Property(x => x.BatchSize)
+ .HasColumnType("SMALLINT")
+ .HasDefaultValueSql("NULL");
entityTypeBuilder.Property(x => x.HttpStatusCode)
- .HasColumnType("SMALLINT UNSIGNED")
+ .HasColumnType("SMALLINT")
.HasDefaultValueSql("NULL");
entityTypeBuilder.Property(x => x.WaybackTimestamp)
.HasColumnType("TIMESTAMP");
diff --git a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
index ee93937d8..23d292a49 100644
--- a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
+++ b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
@@ -33,6 +33,7 @@ public static void AddFilterListsAgentServices(this IServiceCollection services,
o.UseMySql(config.GetConnectionString("FilterListsConnection"),
m => m.MigrationsAssembly("FilterLists.Api")));
services.TryAddScoped();
+ services.TryAddScoped();
services.AddAutoMapper();
}
}
diff --git a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs b/src/FilterLists.Services/Snapshot/Batch.cs
similarity index 89%
rename from src/FilterLists.Services/Snapshot/SnapshotBatch.cs
rename to src/FilterLists.Services/Snapshot/Batch.cs
index dd3cb225f..c53467780 100644
--- a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs
+++ b/src/FilterLists.Services/Snapshot/Batch.cs
@@ -7,14 +7,13 @@
namespace FilterLists.Services.Snapshot
{
- public class SnapshotBatch
+ public class Batch
{
private readonly FilterListsDbContext dbContext;
private readonly IEnumerable lines;
private readonly Data.Entities.Snapshot snapEntity;
- public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable lines,
- Data.Entities.Snapshot snapEntity)
+ public Batch(FilterListsDbContext dbContext, IEnumerable lines, Data.Entities.Snapshot snapEntity)
{
this.dbContext = dbContext;
this.lines = lines;
diff --git a/src/FilterLists.Services/Snapshot/BatchSizeService.cs b/src/FilterLists.Services/Snapshot/BatchSizeService.cs
new file mode 100644
index 000000000..985e44a39
--- /dev/null
+++ b/src/FilterLists.Services/Snapshot/BatchSizeService.cs
@@ -0,0 +1,52 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+
+namespace FilterLists.Services.Snapshot
+{
+ public class BatchSizeService : Service
+ {
+ private const int DefaultBatchSize = 500;
+ private const float PercentMultiplier = 0.05F;
+
+ public BatchSizeService(FilterListsDbContext dbContext) : base(dbContext)
+ {
+ }
+
+ public async Task GetBatchSize()
+ {
+ var recentSnapPerfs = await GetRecentSnapPerfs();
+ return recentSnapPerfs.Count == 2
+ ? (recentSnapPerfs[0].RulesPerMs >= recentSnapPerfs[1].RulesPerMs
+ ? (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 + PercentMultiplier))
+ : (int)Math.Round(recentSnapPerfs[0].BatchSize * (1 - PercentMultiplier)))
+ : DefaultBatchSize;
+ }
+
+ private async Task> GetRecentSnapPerfs() =>
+ await DbContext.Snapshots
+ .Where(s => s.WasSuccessful && s.BatchSize.HasValue)
+ .OrderByDescending(s => s.CreatedDateUtc)
+ .Take(2)
+ .Select(s => new SnapshotPerformance
+ {
+ BatchSize = s.BatchSize.Value,
+ CreatedDateUtc = s.CreatedDateUtc.Value,
+ ModifiedDateUtc = s.ModifiedDateUtc.Value,
+ RulesCount = s.SnapshotRules.Count
+ })
+ .ToListAsync();
+
+ private class SnapshotPerformance
+ {
+ public int BatchSize { get; set; }
+ public int RulesCount { private get; set; }
+ public DateTime CreatedDateUtc { private get; set; }
+ public DateTime ModifiedDateUtc { private get; set; }
+ public int RulesPerMs => (ModifiedDateUtc - CreatedDateUtc).Milliseconds / RulesCount;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs
index be0f94c3d..a9b1c1722 100644
--- a/src/FilterLists.Services/Snapshot/Snapshot.cs
+++ b/src/FilterLists.Services/Snapshot/Snapshot.cs
@@ -17,7 +17,7 @@ namespace FilterLists.Services.Snapshot
{
public class Snapshot
{
- private const int BatchSize = 500;
+ private readonly BatchSizeService batchSizeService;
private readonly FilterListsDbContext dbContext;
public readonly FilterListViewUrlDto List;
protected readonly Data.Entities.Snapshot SnapEntity;
@@ -31,8 +31,10 @@ public Snapshot()
}
[UsedImplicitly]
- public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, string uaString)
+ public Snapshot(BatchSizeService batchSizeService, FilterListsDbContext dbContext, FilterListViewUrlDto list,
+ string uaString)
{
+ this.batchSizeService = batchSizeService;
this.dbContext = dbContext;
List = list;
ListUrl = list.ViewUrl;
@@ -89,7 +91,7 @@ private async Task TryGetLines()
}
catch (WebException we)
{
- SnapEntity.HttpStatusCode = (uint)((HttpWebResponse)we.Response).StatusCode;
+ SnapEntity.HttpStatusCode = (int)((HttpWebResponse)we.Response).StatusCode;
await dbContext.SaveChangesAsync();
TrackException(we);
}
@@ -101,7 +103,7 @@ private async Task GetLines()
{
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(uaString);
var response = await httpClient.GetAsync(ListUrl, HttpCompletionOption.ResponseHeadersRead);
- SnapEntity.HttpStatusCode = (uint)response.StatusCode;
+ SnapEntity.HttpStatusCode = (int)response.StatusCode;
response.EnsureSuccessStatusCode();
lines = new HashSet();
using (var stream = await response.Content.ReadAsStreamAsync())
@@ -116,14 +118,17 @@ private async Task GetLines()
private async Task SaveInBatches()
{
- var snapBatches = CreateBatches();
+ var snapBatches = await CreateBatches();
await SaveBatches(snapBatches);
}
- private IEnumerable CreateBatches() =>
- lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, SnapEntity));
+ private async Task> CreateBatches()
+ {
+ SnapEntity.BatchSize = await batchSizeService.GetBatchSize();
+ return lines.Batch(SnapEntity.BatchSize.Value).Select(b => new Batch(dbContext, b, SnapEntity));
+ }
- private static async Task SaveBatches(IEnumerable batches)
+ private static async Task SaveBatches(IEnumerable batches)
{
foreach (var batch in batches)
await batch.SaveAsync();
diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs
index 132a61b29..7dae4d93b 100644
--- a/src/FilterLists.Services/Snapshot/SnapshotService.cs
+++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs
@@ -13,13 +13,13 @@ namespace FilterLists.Services.Snapshot
{
public class SnapshotService : Service
{
+ private readonly BatchSizeService batchSizeService;
private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1);
private string uaString;
- public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig)
- : base(dbContext, mapConfig)
- {
- }
+ public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig,
+ BatchSizeService batchSizeService)
+ : base(dbContext, mapConfig) => this.batchSizeService = batchSizeService;
public async Task CaptureAsync(int batchSize)
{
@@ -67,7 +67,8 @@ private async Task> CreateAndSaveSnaps(IEnumerable CreateSnaps(IEnumerable lists)
where TSnap : Snapshot, new() =>
- lists.Select(l => Activator.CreateInstance(typeof(TSnap), DbContext, l, uaString) as TSnap);
+ lists.Select(l =>
+ Activator.CreateInstance(typeof(TSnap), batchSizeService, DbContext, l, uaString) as TSnap);
private static async Task SaveSnaps(IEnumerable snaps)
{
diff --git a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs
index f9914aeb9..aee3af79d 100644
--- a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs
+++ b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs
@@ -13,8 +13,9 @@ public SnapshotWayback()
}
[UsedImplicitly]
- public SnapshotWayback(FilterListsDbContext dbContext, FilterListViewUrlDto list, string uaString)
- : base(dbContext, list, uaString)
+ public SnapshotWayback(BatchSizeService batchSizeService, FilterListsDbContext dbContext,
+ FilterListViewUrlDto list, string uaString)
+ : base(batchSizeService, dbContext, list, uaString)
{
}