diff --git a/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.Designer.cs b/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.Designer.cs
new file mode 100644
index 000000000..798a93ce5
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.Designer.cs
@@ -0,0 +1,505 @@
+//
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage;
+using System;
+
+namespace FilterLists.Api.Migrations
+{
+ [DbContext(typeof(FilterListsDbContext))]
+ [Migration("20180203203947_AddSnapshot")]
+ partial class AddSnapshot
+ {
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
+ .HasAnnotation("ProductVersion", "2.0.1-rtm-125");
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Description")
+ .HasColumnType("TEXT");
+
+ b.Property("DescriptionSourceUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("DiscontinuedDate");
+
+ b.Property("DonateUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ForumUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("IssuesUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("LicenseId");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PolicyUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("SubmissionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("SyntaxId");
+
+ b.Property("ViewUrl")
+ .HasColumnType("TEXT");
+
+ b.HasKey("Id");
+
+ b.HasIndex("LicenseId");
+
+ b.HasIndex("SyntaxId");
+
+ b.ToTable("filterlists");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("LanguageId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("FilterListId", "LanguageId");
+
+ b.HasIndex("LanguageId");
+
+ b.ToTable("filterlists_languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("MaintainerId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("FilterListId", "MaintainerId");
+
+ b.HasIndex("MaintainerId");
+
+ b.ToTable("filterlists_maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.Property("ForkFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("forks");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.Property("MergeFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("merges");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.Property("SnapshotId");
+
+ b.Property("RuleId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("SnapshotId", "RuleId");
+
+ b.HasIndex("RuleId");
+
+ b.ToTable("snapshots_rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.Property("SoftwareId");
+
+ b.Property("SyntaxId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("SoftwareId", "SyntaxId");
+
+ b.HasIndex("SyntaxId");
+
+ b.ToTable("software_syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Iso6391")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(2)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392B")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392T")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6393")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("LocalName")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("DescriptionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PermissiveAdaptation");
+
+ b.Property("PermissiveCommercial");
+
+ b.HasKey("Id");
+
+ b.ToTable("licenses");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("TwitterHandle")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("BIGINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Raw")
+ .IsRequired()
+ .HasColumnType("VARCHAR(8192)");
+
+ b.HasKey("Id");
+
+ b.ToTable("rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("MEDIUMINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("FilterListId");
+
+ b.Property("HttpStatusCode")
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.HasKey("Id");
+
+ b.HasIndex("FilterListId");
+
+ b.ToTable("snapshots");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("DownloadUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("software");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("DefinitionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.License", "License")
+ .WithMany("FilterLists")
+ .HasForeignKey("LicenseId");
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("FilterLists")
+ .HasForeignKey("SyntaxId");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Language", "Language")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("LanguageId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("MaintainerId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+ .WithMany("ForkFilterLists")
+ .HasForeignKey("ForkFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamForkFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+ .WithMany("MergeFilterLists")
+ .HasForeignKey("MergeFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamMergeFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("RuleId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("SnapshotId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Software", "Software")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SoftwareId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SyntaxId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("Snapshots")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+#pragma warning restore 612, 618
+ }
+ }
+}
diff --git a/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.cs b/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.cs
new file mode 100644
index 000000000..8fa55bdee
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.cs
@@ -0,0 +1,143 @@
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+using System;
+using System.Collections.Generic;
+
+namespace FilterLists.Api.Migrations
+{
+ public partial class AddSnapshot : Migration
+ {
+ protected override void Up(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropTable(
+ name: "filterlists_rules");
+
+ migrationBuilder.DropTable(
+ name: "scrapes");
+
+ migrationBuilder.CreateTable(
+ name: "snapshots",
+ columns: table => new
+ {
+ Id = table.Column(type: "MEDIUMINT UNSIGNED", nullable: false)
+ .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
+ CreatedDateUtc = table.Column(type: "TIMESTAMP", nullable: false)
+ .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
+ FilterListId = table.Column(nullable: false),
+ HttpStatusCode = table.Column(type: "SMALLINT UNSIGNED", nullable: true)
+ },
+ constraints: table =>
+ {
+ table.PrimaryKey("PK_snapshots", x => x.Id);
+ table.ForeignKey(
+ name: "FK_snapshots_filterlists_FilterListId",
+ column: x => x.FilterListId,
+ principalTable: "filterlists",
+ principalColumn: "Id",
+ onDelete: ReferentialAction.Cascade);
+ });
+
+ migrationBuilder.CreateTable(
+ name: "snapshots_rules",
+ columns: table => new
+ {
+ SnapshotId = table.Column(nullable: false),
+ RuleId = table.Column(nullable: false),
+ CreatedDateUtc = table.Column(type: "TIMESTAMP", nullable: false)
+ .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
+ },
+ constraints: table =>
+ {
+ table.PrimaryKey("PK_snapshots_rules", x => new { x.SnapshotId, x.RuleId });
+ table.ForeignKey(
+ name: "FK_snapshots_rules_rules_RuleId",
+ column: x => x.RuleId,
+ principalTable: "rules",
+ principalColumn: "Id",
+ onDelete: ReferentialAction.Cascade);
+ table.ForeignKey(
+ name: "FK_snapshots_rules_snapshots_SnapshotId",
+ column: x => x.SnapshotId,
+ principalTable: "snapshots",
+ principalColumn: "Id",
+ onDelete: ReferentialAction.Cascade);
+ });
+
+ migrationBuilder.CreateIndex(
+ name: "IX_snapshots_FilterListId",
+ table: "snapshots",
+ column: "FilterListId");
+
+ migrationBuilder.CreateIndex(
+ name: "IX_snapshots_rules_RuleId",
+ table: "snapshots_rules",
+ column: "RuleId");
+ }
+
+ protected override void Down(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropTable(
+ name: "snapshots_rules");
+
+ migrationBuilder.DropTable(
+ name: "snapshots");
+
+ migrationBuilder.CreateTable(
+ name: "filterlists_rules",
+ columns: table => new
+ {
+ FilterListId = table.Column(nullable: false),
+ RuleId = table.Column(nullable: false),
+ CreatedDateUtc = table.Column(type: "TIMESTAMP", nullable: false)
+ .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
+ },
+ constraints: table =>
+ {
+ table.PrimaryKey("PK_filterlists_rules", x => new { x.FilterListId, x.RuleId });
+ table.ForeignKey(
+ name: "FK_filterlists_rules_filterlists_FilterListId",
+ column: x => x.FilterListId,
+ principalTable: "filterlists",
+ principalColumn: "Id",
+ onDelete: ReferentialAction.Cascade);
+ table.ForeignKey(
+ name: "FK_filterlists_rules_rules_RuleId",
+ column: x => x.RuleId,
+ principalTable: "rules",
+ principalColumn: "Id",
+ onDelete: ReferentialAction.Cascade);
+ });
+
+ migrationBuilder.CreateTable(
+ name: "scrapes",
+ columns: table => new
+ {
+ Id = table.Column(type: "SMALLINT UNSIGNED", nullable: false)
+ .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
+ CreatedDateUtc = table.Column(type: "TIMESTAMP", nullable: false)
+ .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
+ FilterListId = table.Column(nullable: false)
+ },
+ constraints: table =>
+ {
+ table.PrimaryKey("PK_scrapes", x => x.Id);
+ table.ForeignKey(
+ name: "FK_scrapes_filterlists_FilterListId",
+ column: x => x.FilterListId,
+ principalTable: "filterlists",
+ principalColumn: "Id",
+ onDelete: ReferentialAction.Cascade);
+ });
+
+ migrationBuilder.CreateIndex(
+ name: "IX_filterlists_rules_RuleId",
+ table: "filterlists_rules",
+ column: "RuleId");
+
+ migrationBuilder.CreateIndex(
+ name: "IX_scrapes_FilterListId",
+ table: "scrapes",
+ column: "FilterListId");
+ }
+ }
+}
diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
index fb5792fa6..ff6153e25 100644
--- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
+++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
@@ -118,23 +118,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.ToTable("filterlists_maintainers");
});
- modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
- {
- b.Property("FilterListId");
-
- b.Property("RuleId");
-
- b.Property("CreatedDateUtc")
- .ValueGeneratedOnAdd()
- .HasColumnType("TIMESTAMP");
-
- b.HasKey("FilterListId", "RuleId");
-
- b.HasIndex("RuleId");
-
- b.ToTable("filterlists_rules");
- });
-
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
{
b.Property("ForkFilterListId");
@@ -169,6 +152,23 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.ToTable("merges");
});
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.Property("SnapshotId");
+
+ b.Property("RuleId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("SnapshotId", "RuleId");
+
+ b.HasIndex("RuleId");
+
+ b.ToTable("snapshots_rules");
+ });
+
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
{
b.Property("SoftwareId");
@@ -325,11 +325,11 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.ToTable("rules");
});
- modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
{
b.Property("Id")
.ValueGeneratedOnAdd()
- .HasColumnType("SMALLINT UNSIGNED");
+ .HasColumnType("MEDIUMINT UNSIGNED");
b.Property("CreatedDateUtc")
.ValueGeneratedOnAdd()
@@ -337,11 +337,14 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property("FilterListId");
+ b.Property("HttpStatusCode")
+ .HasColumnType("SMALLINT UNSIGNED");
+
b.HasKey("Id");
b.HasIndex("FilterListId");
- b.ToTable("scrapes");
+ b.ToTable("snapshots");
});
modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
@@ -401,11 +404,11 @@ protected override void BuildModel(ModelBuilder modelBuilder)
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
{
- b.HasOne("FilterLists.Data.Entities.License")
+ b.HasOne("FilterLists.Data.Entities.License", "License")
.WithMany("FilterLists")
.HasForeignKey("LicenseId");
- b.HasOne("FilterLists.Data.Entities.Syntax")
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
.WithMany("FilterLists")
.HasForeignKey("SyntaxId");
});
@@ -436,19 +439,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.OnDelete(DeleteBehavior.Cascade);
});
- modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
- {
- b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
- .WithMany("FilterListRules")
- .HasForeignKey("FilterListId")
- .OnDelete(DeleteBehavior.Cascade);
-
- b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
- .WithMany("FilterListRules")
- .HasForeignKey("RuleId")
- .OnDelete(DeleteBehavior.Cascade);
- });
-
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
@@ -475,6 +465,19 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.OnDelete(DeleteBehavior.Cascade);
});
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("RuleId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("SnapshotId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
{
b.HasOne("FilterLists.Data.Entities.Software", "Software")
@@ -488,10 +491,10 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.OnDelete(DeleteBehavior.Cascade);
});
- modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
{
- b.HasOne("FilterLists.Data.Entities.FilterList")
- .WithMany("Scrapes")
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("Snapshots")
.HasForeignKey("FilterListId")
.OnDelete(DeleteBehavior.Cascade);
});
diff --git a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs
deleted file mode 100644
index baecb63ee..000000000
--- a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs
+++ /dev/null
@@ -1,27 +0,0 @@
-using System.Threading.Tasks;
-using FilterLists.Services.ScrapeService;
-using Microsoft.AspNetCore.Mvc;
-
-namespace FilterLists.Api.V1.Controllers
-{
- public class ScrapeController : BaseController
- {
- private readonly ScrapeService scrapeService;
-
- public ScrapeController(ScrapeService scrapeService)
- {
- this.scrapeService = scrapeService;
- }
-
- [HttpPost]
- public async Task Index()
- {
-#if DEBUG
- await scrapeService.ScrapeAsync(3);
- return Ok();
-#else
- return StatusCode(403);
-#endif
- }
- }
-}
\ No newline at end of file
diff --git a/src/FilterLists.Api/V1/Controllers/SnapshotController.cs b/src/FilterLists.Api/V1/Controllers/SnapshotController.cs
new file mode 100644
index 000000000..8caa549f6
--- /dev/null
+++ b/src/FilterLists.Api/V1/Controllers/SnapshotController.cs
@@ -0,0 +1,27 @@
+using System.Threading.Tasks;
+using FilterLists.Services.SnapshotService;
+using Microsoft.AspNetCore.Mvc;
+
+namespace FilterLists.Api.V1.Controllers
+{
+ public class SnapshotController : BaseController
+ {
+ private readonly SnapshotService snapshotService;
+
+ public SnapshotController(SnapshotService snapshotService)
+ {
+ this.snapshotService = snapshotService;
+ }
+
+ [HttpPost]
+ public async Task Index()
+ {
+#if DEBUG
+ await snapshotService.CaptureSnapshotsAsync(1);
+ return Ok();
+#else
+ return StatusCode(403);
+#endif
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Data/Entities/FilterList.cs b/src/FilterLists.Data/Entities/FilterList.cs
index 2725d5ef4..132ee75e7 100644
--- a/src/FilterLists.Data/Entities/FilterList.cs
+++ b/src/FilterLists.Data/Entities/FilterList.cs
@@ -24,8 +24,7 @@ public class FilterList : BaseEntity
public ICollection UpstreamMergeFilterLists { get; set; }
public string Name { get; set; }
public string PolicyUrl { get; set; }
- public ICollection FilterListRules { get; set; }
- public ICollection Scrapes { get; set; }
+ public ICollection Snapshots { get; set; }
public string SubmissionUrl { get; set; }
public int? SyntaxId { get; set; }
public Syntax Syntax { get; set; }
diff --git a/src/FilterLists.Data/Entities/Junctions/FilterListRule.cs b/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs
similarity index 50%
rename from src/FilterLists.Data/Entities/Junctions/FilterListRule.cs
rename to src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs
index e8d709aeb..5f5b9f1fc 100644
--- a/src/FilterLists.Data/Entities/Junctions/FilterListRule.cs
+++ b/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs
@@ -1,9 +1,9 @@
namespace FilterLists.Data.Entities.Junctions
{
- public class FilterListRule : BaseJunction
+ public class SnapshotRule : BaseJunction
{
- public int FilterListId { get; set; }
- public FilterList FilterList { get; set; }
+ public int SnapshotId { get; set; }
+ public Snapshot Snapshot { get; set; }
public int RuleId { get; set; }
public Rule Rule { get; set; }
diff --git a/src/FilterLists.Data/Entities/Rule.cs b/src/FilterLists.Data/Entities/Rule.cs
index 41b15b313..a96c24885 100644
--- a/src/FilterLists.Data/Entities/Rule.cs
+++ b/src/FilterLists.Data/Entities/Rule.cs
@@ -5,7 +5,7 @@ namespace FilterLists.Data.Entities
{
public class Rule : BaseEntity
{
- public ICollection FilterListRules { get; set; }
public string Raw { get; set; }
+ public ICollection SnapshotRules { get; set; }
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/Entities/Scrape.cs b/src/FilterLists.Data/Entities/Scrape.cs
deleted file mode 100644
index 695048dd6..000000000
--- a/src/FilterLists.Data/Entities/Scrape.cs
+++ /dev/null
@@ -1,8 +0,0 @@
-namespace FilterLists.Data.Entities
-{
- public class Scrape : BaseEntity
- {
- public int FilterListId { get; set; }
- public FilterList FilterList { get; set; }
- }
-}
\ No newline at end of file
diff --git a/src/FilterLists.Data/Entities/Snapshot.cs b/src/FilterLists.Data/Entities/Snapshot.cs
new file mode 100644
index 000000000..8ed26fb6d
--- /dev/null
+++ b/src/FilterLists.Data/Entities/Snapshot.cs
@@ -0,0 +1,13 @@
+using System.Collections.Generic;
+using FilterLists.Data.Entities.Junctions;
+
+namespace FilterLists.Data.Entities
+{
+ public class Snapshot : BaseEntity
+ {
+ public int FilterListId { get; set; }
+ public FilterList FilterList { get; set; }
+ public int? HttpStatusCode { get; set; }
+ public ICollection SnapshotRules { get; set; }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityTypeConfiguration.cs
index 255c3c0e0..c6ba92df2 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityTypeConfiguration.cs
@@ -9,13 +9,13 @@ public class BaseEntityTypeConfiguration : IEntityTypeConfiguration entityTypeBuilder)
{
entityTypeBuilder.Property(x => x.Id)
- .HasColumnType("SMALLINT UNSIGNED");
+ .HasColumnType("SMALLINT UNSIGNED");
entityTypeBuilder.Property(x => x.CreatedDateUtc)
- .HasColumnType("TIMESTAMP")
- .ValueGeneratedOnAdd();
+ .HasColumnType("TIMESTAMP")
+ .ValueGeneratedOnAdd();
entityTypeBuilder.Property(x => x.ModifiedDateUtc)
- .HasColumnType("TIMESTAMP")
- .ValueGeneratedOnAddOrUpdate();
+ .HasColumnType("TIMESTAMP")
+ .ValueGeneratedOnAddOrUpdate();
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs
index 349d8850f..7b0a87465 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs
@@ -9,33 +9,31 @@ public class FilterListTypeConfiguration : BaseEntityTypeConfiguration entityTypeBuilder)
{
base.Configure(entityTypeBuilder);
-
entityTypeBuilder.ToTable("filterlists");
-
entityTypeBuilder.Property(x => x.Description)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.DescriptionSourceUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.DonateUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.EmailAddress)
- .HasColumnType("VARCHAR(126)")
- .HasDefaultValueSql("NULL");
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
entityTypeBuilder.Property(x => x.ForumUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.HomeUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.IssuesUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.Name)
- .HasColumnType("VARCHAR(126)")
- .IsRequired();
+ .HasColumnType("VARCHAR(126)")
+ .IsRequired();
entityTypeBuilder.Property(x => x.PolicyUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.SubmissionUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.ViewUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/BaseJunctionTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/BaseJunctionTypeConfiguration.cs
index 56e871224..959847b01 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/BaseJunctionTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/BaseJunctionTypeConfiguration.cs
@@ -10,8 +10,8 @@ public class BaseJunctionTypeConfiguration : IEntityTypeConfiguration
public virtual void Configure(EntityTypeBuilder entityTypeBuilder)
{
entityTypeBuilder.Property(x => x.CreatedDateUtc)
- .HasColumnType("TIMESTAMP")
- .ValueGeneratedOnAdd();
+ .HasColumnType("TIMESTAMP")
+ .ValueGeneratedOnAdd();
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/FilterListRuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/FilterListRuleTypeConfiguration.cs
deleted file mode 100644
index debfa368b..000000000
--- a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/FilterListRuleTypeConfiguration.cs
+++ /dev/null
@@ -1,16 +0,0 @@
-using FilterLists.Data.Entities.Junctions;
-using Microsoft.EntityFrameworkCore;
-using Microsoft.EntityFrameworkCore.Metadata.Builders;
-
-namespace FilterLists.Data.EntityTypeConfigurations.Junctions
-{
- public class FilterListRuleTypeConfiguration : BaseJunctionTypeConfiguration
- {
- public override void Configure(EntityTypeBuilder entityTypeBuilder)
- {
- base.Configure(entityTypeBuilder);
- entityTypeBuilder.ToTable("filterlists_rules");
- entityTypeBuilder.HasKey(x => new {x.FilterListId, x.RuleId});
- }
- }
-}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/ForkTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/ForkTypeConfiguration.cs
index baf37eb72..c1b6e9abd 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/ForkTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/ForkTypeConfiguration.cs
@@ -11,6 +11,12 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder)
base.Configure(entityTypeBuilder);
entityTypeBuilder.ToTable("forks");
entityTypeBuilder.HasKey(x => new {x.ForkFilterListId, x.UpstreamFilterListId});
+ entityTypeBuilder.HasOne(x => x.ForkFilterList)
+ .WithMany(x => x.ForkFilterLists)
+ .HasForeignKey(x => x.ForkFilterListId);
+ entityTypeBuilder.HasOne(x => x.UpstreamFilterList)
+ .WithMany(x => x.UpstreamForkFilterLists)
+ .HasForeignKey(x => x.UpstreamFilterListId);
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/MergeTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/MergeTypeConfiguration.cs
index a5b199801..c527526ce 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/MergeTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/MergeTypeConfiguration.cs
@@ -11,6 +11,12 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder)
base.Configure(entityTypeBuilder);
entityTypeBuilder.ToTable("merges");
entityTypeBuilder.HasKey(x => new {x.MergeFilterListId, x.UpstreamFilterListId});
+ entityTypeBuilder.HasOne(x => x.MergeFilterList)
+ .WithMany(x => x.MergeFilterLists)
+ .HasForeignKey(x => x.MergeFilterListId);
+ entityTypeBuilder.HasOne(x => x.UpstreamFilterList)
+ .WithMany(x => x.UpstreamMergeFilterLists)
+ .HasForeignKey(x => x.UpstreamFilterListId);
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs
new file mode 100644
index 000000000..f21c612fb
--- /dev/null
+++ b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs
@@ -0,0 +1,16 @@
+using FilterLists.Data.Entities.Junctions;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace FilterLists.Data.EntityTypeConfigurations.Junctions
+{
+ public class SnapshotRuleTypeConfiguration : BaseJunctionTypeConfiguration
+ {
+ public override void Configure(EntityTypeBuilder entityTypeBuilder)
+ {
+ base.Configure(entityTypeBuilder);
+ entityTypeBuilder.ToTable("snapshots_rules");
+ entityTypeBuilder.HasKey(x => new {x.SnapshotId, x.RuleId});
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs
index 5eaf2ba75..7afad32ce 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs
@@ -9,30 +9,28 @@ public class LanguageTypeConfiguration : BaseEntityTypeConfiguration
public override void Configure(EntityTypeBuilder entityTypeBuilder)
{
base.Configure(entityTypeBuilder);
-
entityTypeBuilder.ToTable("languages");
-
entityTypeBuilder.Property(x => x.Iso6391)
- .HasColumnType("VARCHAR(2)")
- .HasDefaultValueSql("NULL");
+ .HasColumnType("VARCHAR(2)")
+ .HasDefaultValueSql("NULL");
entityTypeBuilder.Property(x => x.Iso6392)
- .HasColumnType("VARCHAR(3)")
- .HasDefaultValueSql("NULL");
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
entityTypeBuilder.Property(x => x.Iso6392B)
- .HasColumnType("VARCHAR(3)")
- .HasDefaultValueSql("NULL");
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
entityTypeBuilder.Property(x => x.Iso6392T)
- .HasColumnType("VARCHAR(3)")
- .HasDefaultValueSql("NULL");
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
entityTypeBuilder.Property(x => x.Iso6393)
- .HasColumnType("VARCHAR(3)")
- .HasDefaultValueSql("NULL");
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
entityTypeBuilder.Property(x => x.LocalName)
- .HasColumnType("VARCHAR(126)")
- .HasDefaultValueSql("NULL");
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
entityTypeBuilder.Property(x => x.Name)
- .HasColumnType("VARCHAR(126)")
- .HasDefaultValueSql("NULL");
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs
index ad74507e7..89d543cf3 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs
@@ -9,14 +9,12 @@ public class LicenseTypeConfiguration : BaseEntityTypeConfiguration
public override void Configure(EntityTypeBuilder entityTypeBuilder)
{
base.Configure(entityTypeBuilder);
-
entityTypeBuilder.ToTable("licenses");
-
entityTypeBuilder.Property(x => x.DescriptionUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.Name)
- .HasColumnType("VARCHAR(126)")
- .IsRequired();
+ .HasColumnType("VARCHAR(126)")
+ .IsRequired();
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs
index 7a37f3e87..4f37dc804 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs
@@ -9,20 +9,18 @@ public class MaintainerTypeConfiguration : BaseEntityTypeConfiguration entityTypeBuilder)
{
base.Configure(entityTypeBuilder);
-
entityTypeBuilder.ToTable("maintainers");
-
entityTypeBuilder.Property(x => x.EmailAddress)
- .HasColumnType("VARCHAR(126)")
- .HasDefaultValueSql("NULL");
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
entityTypeBuilder.Property(x => x.HomeUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.Name)
- .HasColumnType("VARCHAR(126)")
- .IsRequired();
+ .HasColumnType("VARCHAR(126)")
+ .IsRequired();
entityTypeBuilder.Property(x => x.TwitterHandle)
- .HasColumnType("VARCHAR(126)")
- .HasDefaultValueSql("NULL"); ;
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs
index f921ac24e..0e2f29864 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs
@@ -9,17 +9,13 @@ public class RuleTypeConfiguration : BaseEntityTypeConfiguration
public override void Configure(EntityTypeBuilder entityTypeBuilder)
{
base.Configure(entityTypeBuilder);
-
entityTypeBuilder.ToTable("rules");
-
entityTypeBuilder.Property(x => x.Id)
- .HasColumnType("BIGINT UNSIGNED");
-
+ .HasColumnType("BIGINT UNSIGNED");
entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
-
entityTypeBuilder.Property(x => x.Raw)
- .HasColumnType("VARCHAR(8192)")
- .IsRequired();
+ .HasColumnType("VARCHAR(8192)")
+ .IsRequired();
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs
deleted file mode 100644
index b2d851ebf..000000000
--- a/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs
+++ /dev/null
@@ -1,18 +0,0 @@
-using FilterLists.Data.Entities;
-using Microsoft.EntityFrameworkCore;
-using Microsoft.EntityFrameworkCore.Metadata.Builders;
-
-namespace FilterLists.Data.EntityTypeConfigurations
-{
- public class ScrapeTypeConfiguration : BaseEntityTypeConfiguration
- {
- public override void Configure(EntityTypeBuilder entityTypeBuilder)
- {
- base.Configure(entityTypeBuilder);
-
- entityTypeBuilder.ToTable("scrapes");
-
- entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
- }
- }
-}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
new file mode 100644
index 000000000..67dae7553
--- /dev/null
+++ b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
@@ -0,0 +1,20 @@
+using FilterLists.Data.Entities;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace FilterLists.Data.EntityTypeConfigurations
+{
+ public class SnapshotTypeConfiguration : BaseEntityTypeConfiguration
+ {
+ public override void Configure(EntityTypeBuilder entityTypeBuilder)
+ {
+ base.Configure(entityTypeBuilder);
+ entityTypeBuilder.ToTable("snapshots");
+ entityTypeBuilder.Property(x => x.Id)
+ .HasColumnType("MEDIUMINT UNSIGNED");
+ entityTypeBuilder.Property(x => x.HttpStatusCode)
+ .HasColumnType("SMALLINT UNSIGNED");
+ entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs
index 07311fece..0c02e8fce 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs
@@ -9,16 +9,14 @@ public class SoftwareTypeConfiguration : BaseEntityTypeConfiguration
public override void Configure(EntityTypeBuilder entityTypeBuilder)
{
base.Configure(entityTypeBuilder);
-
entityTypeBuilder.ToTable("software");
-
entityTypeBuilder.Property(x => x.DownloadUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.HomeUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.Name)
- .HasColumnType("VARCHAR(126)")
- .IsRequired();
+ .HasColumnType("VARCHAR(126)")
+ .IsRequired();
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs
index a5207fbb4..b1c84c3f0 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs
@@ -9,14 +9,12 @@ public class SyntaxTypeConfiguration : BaseEntityTypeConfiguration
public override void Configure(EntityTypeBuilder entityTypeBuilder)
{
base.Configure(entityTypeBuilder);
-
entityTypeBuilder.ToTable("syntaxes");
-
entityTypeBuilder.Property(x => x.DefinitionUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.Name)
- .HasColumnType("VARCHAR(126)")
- .IsRequired();
+ .HasColumnType("VARCHAR(126)")
+ .IsRequired();
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/FilterListsDbContext.cs b/src/FilterLists.Data/FilterListsDbContext.cs
index 64fc89969..58db3ca61 100644
--- a/src/FilterLists.Data/FilterListsDbContext.cs
+++ b/src/FilterLists.Data/FilterListsDbContext.cs
@@ -8,18 +8,19 @@ namespace FilterLists.Data
{
public class FilterListsDbContext : DbContext
{
- public FilterListsDbContext(DbContextOptions options)
- : base(options)
+ public FilterListsDbContext(DbContextOptions options) : base(options)
{
}
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
+ base.OnModelCreating(modelBuilder);
ApplyConfigurationsEntities(modelBuilder);
ApplyConfigurationsJunctions(modelBuilder);
- base.OnModelCreating(modelBuilder);
}
+ #region Entities
+
private static void ApplyConfigurationsEntities(ModelBuilder modelBuilder)
{
modelBuilder.ApplyConfiguration(new FilterListTypeConfiguration());
@@ -27,55 +28,17 @@ private static void ApplyConfigurationsEntities(ModelBuilder modelBuilder)
modelBuilder.ApplyConfiguration(new LicenseTypeConfiguration());
modelBuilder.ApplyConfiguration(new MaintainerTypeConfiguration());
modelBuilder.ApplyConfiguration(new RuleTypeConfiguration());
- modelBuilder.ApplyConfiguration(new ScrapeTypeConfiguration());
+ modelBuilder.ApplyConfiguration(new SnapshotTypeConfiguration());
modelBuilder.ApplyConfiguration(new SoftwareTypeConfiguration());
modelBuilder.ApplyConfiguration(new SyntaxTypeConfiguration());
}
- private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder)
- {
- modelBuilder.ApplyConfiguration(new FilterListLanguageTypeConfiguration());
- modelBuilder.ApplyConfiguration(new FilterListMaintainerTypeConfiguration());
- modelBuilder.ApplyConfiguration(new FilterListRuleTypeConfiguration());
- modelBuilder.ApplyConfiguration(new ForkTypeConfiguration());
- ConfigureForkSelfReferencing(modelBuilder);
- modelBuilder.ApplyConfiguration(new MergeTypeConfiguration());
- ConfigureMergeSelfReferencing(modelBuilder);
- modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration());
- }
-
- private static void ConfigureForkSelfReferencing(ModelBuilder modelBuilder)
- {
- modelBuilder.Entity()
- .HasOne(x => x.ForkFilterList)
- .WithMany(x => x.ForkFilterLists)
- .HasForeignKey(x => x.ForkFilterListId);
- modelBuilder.Entity()
- .HasOne(x => x.UpstreamFilterList)
- .WithMany(x => x.UpstreamForkFilterLists)
- .HasForeignKey(x => x.UpstreamFilterListId);
- }
-
- private static void ConfigureMergeSelfReferencing(ModelBuilder modelBuilder)
- {
- modelBuilder.Entity()
- .HasOne(x => x.MergeFilterList)
- .WithMany(x => x.MergeFilterLists)
- .HasForeignKey(x => x.MergeFilterListId);
- modelBuilder.Entity()
- .HasOne(x => x.UpstreamFilterList)
- .WithMany(x => x.UpstreamMergeFilterLists)
- .HasForeignKey(x => x.UpstreamFilterListId);
- }
-
- #region Entities
-
public DbSet FilterLists { get; set; }
public DbSet Languages { get; set; }
public DbSet Licenses { get; set; }
public DbSet Maintainers { get; set; }
public DbSet Rules { get; set; }
- public DbSet Scrapes { get; set; }
+ public DbSet Snapshots { get; set; }
public DbSet Software { get; set; }
public DbSet Syntaxes { get; set; }
@@ -83,11 +46,21 @@ private static void ConfigureMergeSelfReferencing(ModelBuilder modelBuilder)
#region Junctions
+ private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder)
+ {
+ modelBuilder.ApplyConfiguration(new FilterListLanguageTypeConfiguration());
+ modelBuilder.ApplyConfiguration(new FilterListMaintainerTypeConfiguration());
+ modelBuilder.ApplyConfiguration(new ForkTypeConfiguration());
+ modelBuilder.ApplyConfiguration(new MergeTypeConfiguration());
+ modelBuilder.ApplyConfiguration(new SnapshotRuleTypeConfiguration());
+ modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration());
+ }
+
public DbSet FilterListLanguages { get; set; }
public DbSet FilterListMaintainers { get; set; }
- public DbSet FilterListRules { get; set; }
public DbSet Forks { get; set; }
public DbSet Merges { get; set; }
+ public DbSet SnapshotRules { get; set; }
public DbSet SoftwareSyntaxes { get; set; }
#endregion
diff --git a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs
index 4aa933749..01ab21ea4 100644
--- a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs
+++ b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs
@@ -75,7 +75,6 @@ private static List GetSeedRows(string dataPath)
private static string CreateRowValues(IEnumerable properties, TEntityType row)
{
- //TODO: use EF nav entity rather than foreign key so foreign key doesn't need to be explicitly in model (ie, License rather than LicenseId)
return (from property in properties
let value = row.GetType().GetProperty(property.Name).GetValue(row)
select FormatDataForMySql(property, value)).Aggregate("",
diff --git a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
index ab2dfb04c..456ed110c 100644
--- a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
+++ b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
@@ -17,7 +17,7 @@ public static void AddFilterListsServices(this IServiceCollection services, ICon
b => b.MigrationsAssembly("FilterLists.Api")));
services.TryAddScoped();
services.TryAddScoped();
- services.TryAddScoped();
+ services.TryAddScoped();
services.AddAutoMapper();
}
}
diff --git a/src/FilterLists.Services/ScrapeService/ScrapeService.cs b/src/FilterLists.Services/ScrapeService/ScrapeService.cs
deleted file mode 100644
index 8a5f77365..000000000
--- a/src/FilterLists.Services/ScrapeService/ScrapeService.cs
+++ /dev/null
@@ -1,105 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net.Http;
-using System.Threading.Tasks;
-using AutoMapper.QueryableExtensions;
-using FilterLists.Data;
-using Microsoft.EntityFrameworkCore;
-
-namespace FilterLists.Services.ScrapeService
-{
- public class ScrapeService
- {
- private readonly FilterListsDbContext dbContext;
-
- public ScrapeService(FilterListsDbContext dbContext)
- {
- this.dbContext = dbContext;
- }
-
- //TODO: call via scheduled job
- public async Task ScrapeAsync(int batchSize)
- {
- var lists = await GetNextListsToScrape(batchSize);
- var snapshots = await GetSnapshots(lists);
- await SaveSnapshots(snapshots);
- }
-
- private async Task> GetNextListsToScrape(int batchSize)
- {
- var neverScraped = await GetNeverScrapedLists(batchSize);
- if (neverScraped.Count == batchSize)
- return neverScraped;
- var leastRecentlyScraped = await GetLeastRecentlyScrapedLists(batchSize - neverScraped.Count);
- return neverScraped.Concat(leastRecentlyScraped).ToList();
- }
-
- private async Task> GetNeverScrapedLists(int batchSize)
- {
- return await dbContext.FilterLists
- .Where(x => x.Scrapes.Count == 0)
- .Take(batchSize)
- .ProjectTo()
- .ToListAsync();
- }
-
- private async Task> GetLeastRecentlyScrapedLists(int batchSize)
- {
- return await dbContext.Scrapes
- .GroupBy(x => x.FilterList)
- .Select(x => x.OrderByDescending(y => y.CreatedDateUtc).First())
- .OrderBy(x => x.CreatedDateUtc)
- .Select(x => x.FilterList)
- .Take(batchSize)
- .ProjectTo()
- .ToListAsync();
- }
-
- private async Task> GetSnapshots(IEnumerable lists)
- {
- return await Task.WhenAll(lists
- .Select(async list => new Snapshot(dbContext, list.Id, await TryGetContent(list.ViewUrl)))
- .Where(x => x.Result.HasRules));
- }
-
- private static async Task TryGetContent(string url)
- {
- try
- {
- return await GetHttpResponseMessageContent(url);
- }
- catch (Exception)
- {
- //TODO: log exception
- return null;
- }
- }
-
- private static async Task GetHttpResponseMessageContent(string url)
- {
- using (var httpClient = new HttpClient())
- using (var httpResponseMessage = await httpClient.GetAsync(url))
- {
- if (httpResponseMessage.IsSuccessStatusCode)
- return await httpResponseMessage.Content.ReadAsStringAsync();
- }
-
- //TODO: log httpResponseMessage.StatusCode
- return null;
- }
-
- private static async Task SaveSnapshots(IEnumerable snapshots)
- {
- foreach (var snapshot in snapshots)
- try
- {
- await snapshot.AddOrUpdateRules();
- }
- catch (Exception)
- {
- //TODO: log exception
- }
- }
- }
-}
\ No newline at end of file
diff --git a/src/FilterLists.Services/ScrapeService/Snapshot.cs b/src/FilterLists.Services/ScrapeService/Snapshot.cs
deleted file mode 100644
index 3da339b91..000000000
--- a/src/FilterLists.Services/ScrapeService/Snapshot.cs
+++ /dev/null
@@ -1,69 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using FilterLists.Data;
-using FilterLists.Data.Entities;
-using FilterLists.Data.Entities.Junctions;
-
-namespace FilterLists.Services.ScrapeService
-{
- public class Snapshot
- {
- private readonly FilterListsDbContext dbContext;
-
- private readonly int filterListId;
-
- private List rawRules;
-
- public Snapshot(FilterListsDbContext dbContext, int filterListId, string content)
- {
- this.dbContext = dbContext;
- this.filterListId = filterListId;
- ParseContent(content);
- }
-
- public bool HasRules => rawRules != null;
-
- private void ParseContent(string content)
- {
- if (content == null) return;
- rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList();
- for (var i = 0; i < rawRules.Count; i++)
- rawRules[i] = rawRules[i].LintStringForMySql();
- rawRules = rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
- }
-
- //TODO: split large lists into batches
- public async Task AddOrUpdateRules()
- {
- dbContext.ChangeTracker.AutoDetectChangesEnabled = false;
-
- // add new Rules
- var preExistingSnapshotRules = dbContext.Rules.Where(x => rawRules.Contains(x.Raw));
- var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw));
- var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule}).ToList();
- await dbContext.Rules.AddRangeAsync(newSnapshotRules);
-
- // remove deleted FilterListRules
- var preExistingFilterListRules = dbContext.FilterListRules.Where(x => x.FilterListId == filterListId);
- var deletedFilterListRules = preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId));
- dbContext.FilterListRules.RemoveRange(deletedFilterListRules);
-
- // add FilterListRules for pre-existing Rules
- var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule =>
- new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList()
- .Except(preExistingFilterListRules).ToList();
- await dbContext.FilterListRules.AddRangeAsync(preExistingSnapshotFilterListRules);
-
- // add new FilterListRules
- var newFilterListRules = newSnapshotRules.Select(newSnapshotRule =>
- new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList();
- await dbContext.FilterListRules.AddRangeAsync(newFilterListRules);
-
- // update scrapes
-
- await dbContext.SaveChangesAsync();
- }
- }
-}
\ No newline at end of file
diff --git a/src/FilterLists.Services/ScrapeService/FilterListViewUrlDto.cs b/src/FilterLists.Services/SnapshotService/FilterListViewUrlDto.cs
similarity index 72%
rename from src/FilterLists.Services/ScrapeService/FilterListViewUrlDto.cs
rename to src/FilterLists.Services/SnapshotService/FilterListViewUrlDto.cs
index 2fdfee176..b62621007 100644
--- a/src/FilterLists.Services/ScrapeService/FilterListViewUrlDto.cs
+++ b/src/FilterLists.Services/SnapshotService/FilterListViewUrlDto.cs
@@ -1,4 +1,4 @@
-namespace FilterLists.Services.ScrapeService
+namespace FilterLists.Services.SnapshotService
{
public class FilterListViewUrlDto
{
diff --git a/src/FilterLists.Services/ScrapeService/RawRuleLinterExtensions.cs b/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs
similarity index 95%
rename from src/FilterLists.Services/ScrapeService/RawRuleLinterExtensions.cs
rename to src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs
index c07cb0986..d11a4fdbc 100644
--- a/src/FilterLists.Services/ScrapeService/RawRuleLinterExtensions.cs
+++ b/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs
@@ -1,4 +1,4 @@
-namespace FilterLists.Services.ScrapeService
+namespace FilterLists.Services.SnapshotService
{
public static class RawRuleLinterExtensions
{
diff --git a/src/FilterLists.Services/SnapshotService/SnapshotDomainEntity.cs b/src/FilterLists.Services/SnapshotService/SnapshotDomainEntity.cs
new file mode 100644
index 000000000..950247364
--- /dev/null
+++ b/src/FilterLists.Services/SnapshotService/SnapshotDomainEntity.cs
@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using FilterLists.Data;
+using FilterLists.Data.Entities;
+using FilterLists.Data.Entities.Junctions;
+
+namespace FilterLists.Services.SnapshotService
+{
+ public class SnapshotDomainEntity
+ {
+ private readonly FilterListsDbContext dbContext;
+ private readonly FilterListViewUrlDto list;
+ private IEnumerable newSnapshotRules;
+ private IEnumerable preExistingSnapshotRules;
+ private List rawRules;
+ private Snapshot snapshot;
+
+ public SnapshotDomainEntity(FilterListsDbContext dbContext, FilterListViewUrlDto list)
+ {
+ this.dbContext = dbContext;
+ this.list = list;
+ }
+
+ public async Task SaveSnapshot()
+ {
+ await AddSnapshot();
+ var content = await TryGetContent();
+ if (content != null)
+ {
+ ParseRawRules(content);
+ await AddNewRules();
+ AddSnapshotRules();
+ }
+
+ await dbContext.SaveChangesAsync();
+ }
+
+ private async Task AddSnapshot()
+ {
+ snapshot = new Snapshot {FilterList = await dbContext.FilterLists.FindAsync(list.Id)};
+ dbContext.Snapshots.Add(snapshot);
+ }
+
+ private async Task TryGetContent()
+ {
+ try
+ {
+ return await GetContent();
+ }
+ catch (WebException we)
+ {
+ snapshot.HttpStatusCode = (int)((HttpWebResponse)we.Response).StatusCode;
+ return null;
+ }
+ }
+
+ private async Task GetContent()
+ {
+ using (var httpClient = new HttpClient())
+ using (var httpResponseMessage = await httpClient.GetAsync(list.ViewUrl))
+ {
+ snapshot.HttpStatusCode = (int)httpResponseMessage.StatusCode;
+ if (httpResponseMessage.IsSuccessStatusCode)
+ return await httpResponseMessage.Content.ReadAsStringAsync();
+ }
+
+ return null;
+ }
+
+ private void ParseRawRules(string content)
+ {
+ rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList();
+ for (var i = 0; i < rawRules.Count; i++)
+ rawRules[i] = rawRules[i].LintStringForMySql();
+ rawRules = rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
+ }
+
+ private async Task AddNewRules()
+ {
+ preExistingSnapshotRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw));
+ var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw));
+ newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule})
+ .ToList();
+ await dbContext.Rules.AddRangeAsync(newSnapshotRules);
+ }
+
+ private void AddSnapshotRules()
+ {
+ snapshot.SnapshotRules = preExistingSnapshotRules
+ .Concat(newSnapshotRules)
+ .Select(rule => new SnapshotRule {Rule = rule, Snapshot = snapshot})
+ .ToList();
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Services/SnapshotService/SnapshotService.cs b/src/FilterLists.Services/SnapshotService/SnapshotService.cs
new file mode 100644
index 000000000..f61f32cdf
--- /dev/null
+++ b/src/FilterLists.Services/SnapshotService/SnapshotService.cs
@@ -0,0 +1,68 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using AutoMapper.QueryableExtensions;
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+
+namespace FilterLists.Services.SnapshotService
+{
+ public class SnapshotService
+ {
+ private readonly FilterListsDbContext dbContext;
+
+ public SnapshotService(FilterListsDbContext dbContext)
+ {
+ this.dbContext = dbContext;
+ }
+
+ //TODO: call via scheduled job
+ public async Task CaptureSnapshotsAsync(int batchSize)
+ {
+ var lists = await GetNextListsToCapture(batchSize);
+ var snapshots = GetSnapshots(lists);
+ await SaveSnapshots(snapshots);
+ }
+
+ private async Task> GetNextListsToCapture(int batchSize)
+ {
+ var neverCapturedLists = await GetNeverCapturedLists(batchSize);
+ if (neverCapturedLists.Count == batchSize)
+ return neverCapturedLists;
+ var leastRecentlyCapturedLists = await GetLeastRecentlyCapturedLists(batchSize - neverCapturedLists.Count);
+ return neverCapturedLists.Concat(leastRecentlyCapturedLists);
+ }
+
+ private async Task> GetNeverCapturedLists(int batchSize)
+ {
+ return await dbContext.FilterLists
+ .Where(x => x.Snapshots.Count == 0)
+ .Take(batchSize)
+ .ProjectTo()
+ .ToListAsync();
+ }
+
+ private async Task> GetLeastRecentlyCapturedLists(int batchSize)
+ {
+ return await dbContext.Snapshots
+ .GroupBy(x => x.FilterList)
+ .Select(x => x.OrderByDescending(y => y.CreatedDateUtc).First())
+ .OrderBy(x => x.CreatedDateUtc)
+ .Select(x => x.FilterList)
+ .Take(batchSize)
+ .ProjectTo()
+ .ToListAsync();
+ }
+
+ private IEnumerable GetSnapshots(IEnumerable lists)
+ {
+ return lists.Select(list => new SnapshotDomainEntity(dbContext, list));
+ }
+
+ private static async Task SaveSnapshots(IEnumerable snapshots)
+ {
+ foreach (var snapshot in snapshots)
+ await snapshot.SaveSnapshot();
+ }
+ }
+}
\ No newline at end of file