From 93da2989087991ccb2df7e286ac64a0d8e5bd458 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sat, 3 Feb 2018 15:49:32 -0600 Subject: [PATCH] major SnapshotService refactor --- .../20180203203947_AddSnapshot.Designer.cs | 505 ++++++++++++++++++ .../Migrations/20180203203947_AddSnapshot.cs | 143 +++++ .../FilterListsDbContextModelSnapshot.cs | 79 +-- .../V1/Controllers/ScrapeController.cs | 27 - .../V1/Controllers/SnapshotController.cs | 27 + src/FilterLists.Data/Entities/FilterList.cs | 3 +- .../{FilterListRule.cs => SnapshotRule.cs} | 6 +- src/FilterLists.Data/Entities/Rule.cs | 2 +- src/FilterLists.Data/Entities/Scrape.cs | 8 - src/FilterLists.Data/Entities/Snapshot.cs | 13 + .../BaseEntityTypeConfiguration.cs | 10 +- .../FilterListTypeConfiguration.cs | 28 +- .../BaseJunctionTypeConfiguration.cs | 4 +- .../FilterListRuleTypeConfiguration.cs | 16 - .../Junctions/ForkTypeConfiguration.cs | 6 + .../Junctions/MergeTypeConfiguration.cs | 6 + .../SnapshotRuleTypeConfiguration.cs | 16 + .../LanguageTypeConfiguration.cs | 30 +- .../LicenseTypeConfiguration.cs | 8 +- .../MaintainerTypeConfiguration.cs | 16 +- .../RuleTypeConfiguration.cs | 10 +- .../ScrapeTypeConfiguration.cs | 18 - .../SnapshotTypeConfiguration.cs | 20 + .../SoftwareTypeConfiguration.cs | 10 +- .../SyntaxTypeConfiguration.cs | 8 +- src/FilterLists.Data/FilterListsDbContext.cs | 61 +-- .../Extensions/SeedFilterListsDbContext.cs | 1 - .../Extensions/ConfigureServicesCollection.cs | 2 +- .../ScrapeService/ScrapeService.cs | 105 ---- .../ScrapeService/Snapshot.cs | 69 --- .../FilterListViewUrlDto.cs | 2 +- .../RawRuleLinterExtensions.cs | 2 +- .../SnapshotService/SnapshotDomainEntity.cs | 99 ++++ .../SnapshotService/SnapshotService.cs | 68 +++ 34 files changed, 1023 insertions(+), 405 deletions(-) create mode 100644 src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.Designer.cs create mode 100644 src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.cs delete mode 100644 src/FilterLists.Api/V1/Controllers/ScrapeController.cs create mode 100644 src/FilterLists.Api/V1/Controllers/SnapshotController.cs rename src/FilterLists.Data/Entities/Junctions/{FilterListRule.cs => SnapshotRule.cs} (50%) delete mode 100644 src/FilterLists.Data/Entities/Scrape.cs create mode 100644 src/FilterLists.Data/Entities/Snapshot.cs delete mode 100644 src/FilterLists.Data/EntityTypeConfigurations/Junctions/FilterListRuleTypeConfiguration.cs create mode 100644 src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs delete mode 100644 src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs create mode 100644 src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs delete mode 100644 src/FilterLists.Services/ScrapeService/ScrapeService.cs delete mode 100644 src/FilterLists.Services/ScrapeService/Snapshot.cs rename src/FilterLists.Services/{ScrapeService => SnapshotService}/FilterListViewUrlDto.cs (72%) rename src/FilterLists.Services/{ScrapeService => SnapshotService}/RawRuleLinterExtensions.cs (95%) create mode 100644 src/FilterLists.Services/SnapshotService/SnapshotDomainEntity.cs create mode 100644 src/FilterLists.Services/SnapshotService/SnapshotService.cs diff --git a/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.Designer.cs b/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.Designer.cs new file mode 100644 index 000000000..798a93ce5 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.Designer.cs @@ -0,0 +1,505 @@ +// +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage; +using System; + +namespace FilterLists.Api.Migrations +{ + [DbContext(typeof(FilterListsDbContext))] + [Migration("20180203203947_AddSnapshot")] + partial class AddSnapshot + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn) + .HasAnnotation("ProductVersion", "2.0.1-rtm-125"); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("DescriptionSourceUrl") + .HasColumnType("TEXT"); + + b.Property("DiscontinuedDate"); + + b.Property("DonateUrl") + .HasColumnType("TEXT"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ForumUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("IssuesUrl") + .HasColumnType("TEXT"); + + b.Property("LicenseId"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PolicyUrl") + .HasColumnType("TEXT"); + + b.Property("SubmissionUrl") + .HasColumnType("TEXT"); + + b.Property("SyntaxId"); + + b.Property("ViewUrl") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("LicenseId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("filterlists"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.Property("FilterListId"); + + b.Property("LanguageId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "LanguageId"); + + b.HasIndex("LanguageId"); + + b.ToTable("filterlists_languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.Property("FilterListId"); + + b.Property("MaintainerId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "MaintainerId"); + + b.HasIndex("MaintainerId"); + + b.ToTable("filterlists_maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.Property("ForkFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("ForkFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("forks"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.Property("MergeFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("MergeFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("merges"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.Property("SnapshotId"); + + b.Property("RuleId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("SnapshotId", "RuleId"); + + b.HasIndex("RuleId"); + + b.ToTable("snapshots_rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.Property("SoftwareId"); + + b.Property("SyntaxId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("SoftwareId", "SyntaxId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("software_syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Language", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Iso6391") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(2)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392B") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392T") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6393") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("LocalName") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.License", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DescriptionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PermissiveAdaptation"); + + b.Property("PermissiveCommercial"); + + b.HasKey("Id"); + + b.ToTable("licenses"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("TwitterHandle") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Rule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("BIGINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Raw") + .IsRequired() + .HasColumnType("VARCHAR(8192)"); + + b.HasKey("Id"); + + b.ToTable("rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("MEDIUMINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("FilterListId"); + + b.Property("HttpStatusCode") + .HasColumnType("SMALLINT UNSIGNED"); + + b.HasKey("Id"); + + b.HasIndex("FilterListId"); + + b.ToTable("snapshots"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DownloadUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("software"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DefinitionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.HasOne("FilterLists.Data.Entities.License", "License") + .WithMany("FilterLists") + .HasForeignKey("LicenseId"); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("FilterLists") + .HasForeignKey("SyntaxId"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListLanguages") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Language", "Language") + .WithMany("FilterListLanguages") + .HasForeignKey("LanguageId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListMaintainers") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer") + .WithMany("FilterListMaintainers") + .HasForeignKey("MaintainerId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") + .WithMany("ForkFilterLists") + .HasForeignKey("ForkFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamForkFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList") + .WithMany("MergeFilterLists") + .HasForeignKey("MergeFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamMergeFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.HasOne("FilterLists.Data.Entities.Rule", "Rule") + .WithMany("SnapshotRules") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot") + .WithMany("SnapshotRules") + .HasForeignKey("SnapshotId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.HasOne("FilterLists.Data.Entities.Software", "Software") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SoftwareId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SyntaxId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("Snapshots") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.cs b/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.cs new file mode 100644 index 000000000..8fa55bdee --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.cs @@ -0,0 +1,143 @@ +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using System; +using System.Collections.Generic; + +namespace FilterLists.Api.Migrations +{ + public partial class AddSnapshot : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "filterlists_rules"); + + migrationBuilder.DropTable( + name: "scrapes"); + + migrationBuilder.CreateTable( + name: "snapshots", + columns: table => new + { + Id = table.Column(type: "MEDIUMINT UNSIGNED", nullable: false) + .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn), + CreatedDateUtc = table.Column(type: "TIMESTAMP", nullable: false) + .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn), + FilterListId = table.Column(nullable: false), + HttpStatusCode = table.Column(type: "SMALLINT UNSIGNED", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_snapshots", x => x.Id); + table.ForeignKey( + name: "FK_snapshots_filterlists_FilterListId", + column: x => x.FilterListId, + principalTable: "filterlists", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "snapshots_rules", + columns: table => new + { + SnapshotId = table.Column(nullable: false), + RuleId = table.Column(nullable: false), + CreatedDateUtc = table.Column(type: "TIMESTAMP", nullable: false) + .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn) + }, + constraints: table => + { + table.PrimaryKey("PK_snapshots_rules", x => new { x.SnapshotId, x.RuleId }); + table.ForeignKey( + name: "FK_snapshots_rules_rules_RuleId", + column: x => x.RuleId, + principalTable: "rules", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_snapshots_rules_snapshots_SnapshotId", + column: x => x.SnapshotId, + principalTable: "snapshots", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_snapshots_FilterListId", + table: "snapshots", + column: "FilterListId"); + + migrationBuilder.CreateIndex( + name: "IX_snapshots_rules_RuleId", + table: "snapshots_rules", + column: "RuleId"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "snapshots_rules"); + + migrationBuilder.DropTable( + name: "snapshots"); + + migrationBuilder.CreateTable( + name: "filterlists_rules", + columns: table => new + { + FilterListId = table.Column(nullable: false), + RuleId = table.Column(nullable: false), + CreatedDateUtc = table.Column(type: "TIMESTAMP", nullable: false) + .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn) + }, + constraints: table => + { + table.PrimaryKey("PK_filterlists_rules", x => new { x.FilterListId, x.RuleId }); + table.ForeignKey( + name: "FK_filterlists_rules_filterlists_FilterListId", + column: x => x.FilterListId, + principalTable: "filterlists", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_filterlists_rules_rules_RuleId", + column: x => x.RuleId, + principalTable: "rules", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "scrapes", + columns: table => new + { + Id = table.Column(type: "SMALLINT UNSIGNED", nullable: false) + .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn), + CreatedDateUtc = table.Column(type: "TIMESTAMP", nullable: false) + .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn), + FilterListId = table.Column(nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_scrapes", x => x.Id); + table.ForeignKey( + name: "FK_scrapes_filterlists_FilterListId", + column: x => x.FilterListId, + principalTable: "filterlists", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_filterlists_rules_RuleId", + table: "filterlists_rules", + column: "RuleId"); + + migrationBuilder.CreateIndex( + name: "IX_scrapes_FilterListId", + table: "scrapes", + column: "FilterListId"); + } + } +} diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs index fb5792fa6..ff6153e25 100644 --- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs +++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs @@ -118,23 +118,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("filterlists_maintainers"); }); - modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b => - { - b.Property("FilterListId"); - - b.Property("RuleId"); - - b.Property("CreatedDateUtc") - .ValueGeneratedOnAdd() - .HasColumnType("TIMESTAMP"); - - b.HasKey("FilterListId", "RuleId"); - - b.HasIndex("RuleId"); - - b.ToTable("filterlists_rules"); - }); - modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => { b.Property("ForkFilterListId"); @@ -169,6 +152,23 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("merges"); }); + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.Property("SnapshotId"); + + b.Property("RuleId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("SnapshotId", "RuleId"); + + b.HasIndex("RuleId"); + + b.ToTable("snapshots_rules"); + }); + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => { b.Property("SoftwareId"); @@ -325,11 +325,11 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("rules"); }); - modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b => + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => { b.Property("Id") .ValueGeneratedOnAdd() - .HasColumnType("SMALLINT UNSIGNED"); + .HasColumnType("MEDIUMINT UNSIGNED"); b.Property("CreatedDateUtc") .ValueGeneratedOnAdd() @@ -337,11 +337,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("FilterListId"); + b.Property("HttpStatusCode") + .HasColumnType("SMALLINT UNSIGNED"); + b.HasKey("Id"); b.HasIndex("FilterListId"); - b.ToTable("scrapes"); + b.ToTable("snapshots"); }); modelBuilder.Entity("FilterLists.Data.Entities.Software", b => @@ -401,11 +404,11 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => { - b.HasOne("FilterLists.Data.Entities.License") + b.HasOne("FilterLists.Data.Entities.License", "License") .WithMany("FilterLists") .HasForeignKey("LicenseId"); - b.HasOne("FilterLists.Data.Entities.Syntax") + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") .WithMany("FilterLists") .HasForeignKey("SyntaxId"); }); @@ -436,19 +439,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .OnDelete(DeleteBehavior.Cascade); }); - modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b => - { - b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") - .WithMany("FilterListRules") - .HasForeignKey("FilterListId") - .OnDelete(DeleteBehavior.Cascade); - - b.HasOne("FilterLists.Data.Entities.Rule", "Rule") - .WithMany("FilterListRules") - .HasForeignKey("RuleId") - .OnDelete(DeleteBehavior.Cascade); - }); - modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => { b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") @@ -475,6 +465,19 @@ protected override void BuildModel(ModelBuilder modelBuilder) .OnDelete(DeleteBehavior.Cascade); }); + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.HasOne("FilterLists.Data.Entities.Rule", "Rule") + .WithMany("SnapshotRules") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot") + .WithMany("SnapshotRules") + .HasForeignKey("SnapshotId") + .OnDelete(DeleteBehavior.Cascade); + }); + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => { b.HasOne("FilterLists.Data.Entities.Software", "Software") @@ -488,10 +491,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .OnDelete(DeleteBehavior.Cascade); }); - modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b => + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => { - b.HasOne("FilterLists.Data.Entities.FilterList") - .WithMany("Scrapes") + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("Snapshots") .HasForeignKey("FilterListId") .OnDelete(DeleteBehavior.Cascade); }); diff --git a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs b/src/FilterLists.Api/V1/Controllers/ScrapeController.cs deleted file mode 100644 index baecb63ee..000000000 --- a/src/FilterLists.Api/V1/Controllers/ScrapeController.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System.Threading.Tasks; -using FilterLists.Services.ScrapeService; -using Microsoft.AspNetCore.Mvc; - -namespace FilterLists.Api.V1.Controllers -{ - public class ScrapeController : BaseController - { - private readonly ScrapeService scrapeService; - - public ScrapeController(ScrapeService scrapeService) - { - this.scrapeService = scrapeService; - } - - [HttpPost] - public async Task Index() - { -#if DEBUG - await scrapeService.ScrapeAsync(3); - return Ok(); -#else - return StatusCode(403); -#endif - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Api/V1/Controllers/SnapshotController.cs b/src/FilterLists.Api/V1/Controllers/SnapshotController.cs new file mode 100644 index 000000000..8caa549f6 --- /dev/null +++ b/src/FilterLists.Api/V1/Controllers/SnapshotController.cs @@ -0,0 +1,27 @@ +using System.Threading.Tasks; +using FilterLists.Services.SnapshotService; +using Microsoft.AspNetCore.Mvc; + +namespace FilterLists.Api.V1.Controllers +{ + public class SnapshotController : BaseController + { + private readonly SnapshotService snapshotService; + + public SnapshotController(SnapshotService snapshotService) + { + this.snapshotService = snapshotService; + } + + [HttpPost] + public async Task Index() + { +#if DEBUG + await snapshotService.CaptureSnapshotsAsync(1); + return Ok(); +#else + return StatusCode(403); +#endif + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Data/Entities/FilterList.cs b/src/FilterLists.Data/Entities/FilterList.cs index 2725d5ef4..132ee75e7 100644 --- a/src/FilterLists.Data/Entities/FilterList.cs +++ b/src/FilterLists.Data/Entities/FilterList.cs @@ -24,8 +24,7 @@ public class FilterList : BaseEntity public ICollection UpstreamMergeFilterLists { get; set; } public string Name { get; set; } public string PolicyUrl { get; set; } - public ICollection FilterListRules { get; set; } - public ICollection Scrapes { get; set; } + public ICollection Snapshots { get; set; } public string SubmissionUrl { get; set; } public int? SyntaxId { get; set; } public Syntax Syntax { get; set; } diff --git a/src/FilterLists.Data/Entities/Junctions/FilterListRule.cs b/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs similarity index 50% rename from src/FilterLists.Data/Entities/Junctions/FilterListRule.cs rename to src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs index e8d709aeb..5f5b9f1fc 100644 --- a/src/FilterLists.Data/Entities/Junctions/FilterListRule.cs +++ b/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs @@ -1,9 +1,9 @@ namespace FilterLists.Data.Entities.Junctions { - public class FilterListRule : BaseJunction + public class SnapshotRule : BaseJunction { - public int FilterListId { get; set; } - public FilterList FilterList { get; set; } + public int SnapshotId { get; set; } + public Snapshot Snapshot { get; set; } public int RuleId { get; set; } public Rule Rule { get; set; } diff --git a/src/FilterLists.Data/Entities/Rule.cs b/src/FilterLists.Data/Entities/Rule.cs index 41b15b313..a96c24885 100644 --- a/src/FilterLists.Data/Entities/Rule.cs +++ b/src/FilterLists.Data/Entities/Rule.cs @@ -5,7 +5,7 @@ namespace FilterLists.Data.Entities { public class Rule : BaseEntity { - public ICollection FilterListRules { get; set; } public string Raw { get; set; } + public ICollection SnapshotRules { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/Entities/Scrape.cs b/src/FilterLists.Data/Entities/Scrape.cs deleted file mode 100644 index 695048dd6..000000000 --- a/src/FilterLists.Data/Entities/Scrape.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace FilterLists.Data.Entities -{ - public class Scrape : BaseEntity - { - public int FilterListId { get; set; } - public FilterList FilterList { get; set; } - } -} \ No newline at end of file diff --git a/src/FilterLists.Data/Entities/Snapshot.cs b/src/FilterLists.Data/Entities/Snapshot.cs new file mode 100644 index 000000000..8ed26fb6d --- /dev/null +++ b/src/FilterLists.Data/Entities/Snapshot.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; +using FilterLists.Data.Entities.Junctions; + +namespace FilterLists.Data.Entities +{ + public class Snapshot : BaseEntity + { + public int FilterListId { get; set; } + public FilterList FilterList { get; set; } + public int? HttpStatusCode { get; set; } + public ICollection SnapshotRules { get; set; } + } +} \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityTypeConfiguration.cs index 255c3c0e0..c6ba92df2 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityTypeConfiguration.cs @@ -9,13 +9,13 @@ public class BaseEntityTypeConfiguration : IEntityTypeConfiguration entityTypeBuilder) { entityTypeBuilder.Property(x => x.Id) - .HasColumnType("SMALLINT UNSIGNED"); + .HasColumnType("SMALLINT UNSIGNED"); entityTypeBuilder.Property(x => x.CreatedDateUtc) - .HasColumnType("TIMESTAMP") - .ValueGeneratedOnAdd(); + .HasColumnType("TIMESTAMP") + .ValueGeneratedOnAdd(); entityTypeBuilder.Property(x => x.ModifiedDateUtc) - .HasColumnType("TIMESTAMP") - .ValueGeneratedOnAddOrUpdate(); + .HasColumnType("TIMESTAMP") + .ValueGeneratedOnAddOrUpdate(); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs index 349d8850f..7b0a87465 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs @@ -9,33 +9,31 @@ public class FilterListTypeConfiguration : BaseEntityTypeConfiguration entityTypeBuilder) { base.Configure(entityTypeBuilder); - entityTypeBuilder.ToTable("filterlists"); - entityTypeBuilder.Property(x => x.Description) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.DescriptionSourceUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.DonateUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.EmailAddress) - .HasColumnType("VARCHAR(126)") - .HasDefaultValueSql("NULL"); + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); entityTypeBuilder.Property(x => x.ForumUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.HomeUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.IssuesUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.Name) - .HasColumnType("VARCHAR(126)") - .IsRequired(); + .HasColumnType("VARCHAR(126)") + .IsRequired(); entityTypeBuilder.Property(x => x.PolicyUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.SubmissionUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.ViewUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/BaseJunctionTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/BaseJunctionTypeConfiguration.cs index 56e871224..959847b01 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/BaseJunctionTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/BaseJunctionTypeConfiguration.cs @@ -10,8 +10,8 @@ public class BaseJunctionTypeConfiguration : IEntityTypeConfiguration public virtual void Configure(EntityTypeBuilder entityTypeBuilder) { entityTypeBuilder.Property(x => x.CreatedDateUtc) - .HasColumnType("TIMESTAMP") - .ValueGeneratedOnAdd(); + .HasColumnType("TIMESTAMP") + .ValueGeneratedOnAdd(); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/FilterListRuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/FilterListRuleTypeConfiguration.cs deleted file mode 100644 index debfa368b..000000000 --- a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/FilterListRuleTypeConfiguration.cs +++ /dev/null @@ -1,16 +0,0 @@ -using FilterLists.Data.Entities.Junctions; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Metadata.Builders; - -namespace FilterLists.Data.EntityTypeConfigurations.Junctions -{ - public class FilterListRuleTypeConfiguration : BaseJunctionTypeConfiguration - { - public override void Configure(EntityTypeBuilder entityTypeBuilder) - { - base.Configure(entityTypeBuilder); - entityTypeBuilder.ToTable("filterlists_rules"); - entityTypeBuilder.HasKey(x => new {x.FilterListId, x.RuleId}); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/ForkTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/ForkTypeConfiguration.cs index baf37eb72..c1b6e9abd 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/ForkTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/ForkTypeConfiguration.cs @@ -11,6 +11,12 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) base.Configure(entityTypeBuilder); entityTypeBuilder.ToTable("forks"); entityTypeBuilder.HasKey(x => new {x.ForkFilterListId, x.UpstreamFilterListId}); + entityTypeBuilder.HasOne(x => x.ForkFilterList) + .WithMany(x => x.ForkFilterLists) + .HasForeignKey(x => x.ForkFilterListId); + entityTypeBuilder.HasOne(x => x.UpstreamFilterList) + .WithMany(x => x.UpstreamForkFilterLists) + .HasForeignKey(x => x.UpstreamFilterListId); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/MergeTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/MergeTypeConfiguration.cs index a5b199801..c527526ce 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/MergeTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/MergeTypeConfiguration.cs @@ -11,6 +11,12 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) base.Configure(entityTypeBuilder); entityTypeBuilder.ToTable("merges"); entityTypeBuilder.HasKey(x => new {x.MergeFilterListId, x.UpstreamFilterListId}); + entityTypeBuilder.HasOne(x => x.MergeFilterList) + .WithMany(x => x.MergeFilterLists) + .HasForeignKey(x => x.MergeFilterListId); + entityTypeBuilder.HasOne(x => x.UpstreamFilterList) + .WithMany(x => x.UpstreamMergeFilterLists) + .HasForeignKey(x => x.UpstreamFilterListId); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs new file mode 100644 index 000000000..f21c612fb --- /dev/null +++ b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs @@ -0,0 +1,16 @@ +using FilterLists.Data.Entities.Junctions; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace FilterLists.Data.EntityTypeConfigurations.Junctions +{ + public class SnapshotRuleTypeConfiguration : BaseJunctionTypeConfiguration + { + public override void Configure(EntityTypeBuilder entityTypeBuilder) + { + base.Configure(entityTypeBuilder); + entityTypeBuilder.ToTable("snapshots_rules"); + entityTypeBuilder.HasKey(x => new {x.SnapshotId, x.RuleId}); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs index 5eaf2ba75..7afad32ce 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs @@ -9,30 +9,28 @@ public class LanguageTypeConfiguration : BaseEntityTypeConfiguration public override void Configure(EntityTypeBuilder entityTypeBuilder) { base.Configure(entityTypeBuilder); - entityTypeBuilder.ToTable("languages"); - entityTypeBuilder.Property(x => x.Iso6391) - .HasColumnType("VARCHAR(2)") - .HasDefaultValueSql("NULL"); + .HasColumnType("VARCHAR(2)") + .HasDefaultValueSql("NULL"); entityTypeBuilder.Property(x => x.Iso6392) - .HasColumnType("VARCHAR(3)") - .HasDefaultValueSql("NULL"); + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); entityTypeBuilder.Property(x => x.Iso6392B) - .HasColumnType("VARCHAR(3)") - .HasDefaultValueSql("NULL"); + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); entityTypeBuilder.Property(x => x.Iso6392T) - .HasColumnType("VARCHAR(3)") - .HasDefaultValueSql("NULL"); + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); entityTypeBuilder.Property(x => x.Iso6393) - .HasColumnType("VARCHAR(3)") - .HasDefaultValueSql("NULL"); + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); entityTypeBuilder.Property(x => x.LocalName) - .HasColumnType("VARCHAR(126)") - .HasDefaultValueSql("NULL"); + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); entityTypeBuilder.Property(x => x.Name) - .HasColumnType("VARCHAR(126)") - .HasDefaultValueSql("NULL"); + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs index ad74507e7..89d543cf3 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs @@ -9,14 +9,12 @@ public class LicenseTypeConfiguration : BaseEntityTypeConfiguration public override void Configure(EntityTypeBuilder entityTypeBuilder) { base.Configure(entityTypeBuilder); - entityTypeBuilder.ToTable("licenses"); - entityTypeBuilder.Property(x => x.DescriptionUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.Name) - .HasColumnType("VARCHAR(126)") - .IsRequired(); + .HasColumnType("VARCHAR(126)") + .IsRequired(); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs index 7a37f3e87..4f37dc804 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs @@ -9,20 +9,18 @@ public class MaintainerTypeConfiguration : BaseEntityTypeConfiguration entityTypeBuilder) { base.Configure(entityTypeBuilder); - entityTypeBuilder.ToTable("maintainers"); - entityTypeBuilder.Property(x => x.EmailAddress) - .HasColumnType("VARCHAR(126)") - .HasDefaultValueSql("NULL"); + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); entityTypeBuilder.Property(x => x.HomeUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.Name) - .HasColumnType("VARCHAR(126)") - .IsRequired(); + .HasColumnType("VARCHAR(126)") + .IsRequired(); entityTypeBuilder.Property(x => x.TwitterHandle) - .HasColumnType("VARCHAR(126)") - .HasDefaultValueSql("NULL"); ; + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs index f921ac24e..0e2f29864 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs @@ -9,17 +9,13 @@ public class RuleTypeConfiguration : BaseEntityTypeConfiguration public override void Configure(EntityTypeBuilder entityTypeBuilder) { base.Configure(entityTypeBuilder); - entityTypeBuilder.ToTable("rules"); - entityTypeBuilder.Property(x => x.Id) - .HasColumnType("BIGINT UNSIGNED"); - + .HasColumnType("BIGINT UNSIGNED"); entityTypeBuilder.Ignore(x => x.ModifiedDateUtc); - entityTypeBuilder.Property(x => x.Raw) - .HasColumnType("VARCHAR(8192)") - .IsRequired(); + .HasColumnType("VARCHAR(8192)") + .IsRequired(); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs deleted file mode 100644 index b2d851ebf..000000000 --- a/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs +++ /dev/null @@ -1,18 +0,0 @@ -using FilterLists.Data.Entities; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Metadata.Builders; - -namespace FilterLists.Data.EntityTypeConfigurations -{ - public class ScrapeTypeConfiguration : BaseEntityTypeConfiguration - { - public override void Configure(EntityTypeBuilder entityTypeBuilder) - { - base.Configure(entityTypeBuilder); - - entityTypeBuilder.ToTable("scrapes"); - - entityTypeBuilder.Ignore(x => x.ModifiedDateUtc); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs new file mode 100644 index 000000000..67dae7553 --- /dev/null +++ b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs @@ -0,0 +1,20 @@ +using FilterLists.Data.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace FilterLists.Data.EntityTypeConfigurations +{ + public class SnapshotTypeConfiguration : BaseEntityTypeConfiguration + { + public override void Configure(EntityTypeBuilder entityTypeBuilder) + { + base.Configure(entityTypeBuilder); + entityTypeBuilder.ToTable("snapshots"); + entityTypeBuilder.Property(x => x.Id) + .HasColumnType("MEDIUMINT UNSIGNED"); + entityTypeBuilder.Property(x => x.HttpStatusCode) + .HasColumnType("SMALLINT UNSIGNED"); + entityTypeBuilder.Ignore(x => x.ModifiedDateUtc); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs index 07311fece..0c02e8fce 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs @@ -9,16 +9,14 @@ public class SoftwareTypeConfiguration : BaseEntityTypeConfiguration public override void Configure(EntityTypeBuilder entityTypeBuilder) { base.Configure(entityTypeBuilder); - entityTypeBuilder.ToTable("software"); - entityTypeBuilder.Property(x => x.DownloadUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.HomeUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.Name) - .HasColumnType("VARCHAR(126)") - .IsRequired(); + .HasColumnType("VARCHAR(126)") + .IsRequired(); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs index a5207fbb4..b1c84c3f0 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs @@ -9,14 +9,12 @@ public class SyntaxTypeConfiguration : BaseEntityTypeConfiguration public override void Configure(EntityTypeBuilder entityTypeBuilder) { base.Configure(entityTypeBuilder); - entityTypeBuilder.ToTable("syntaxes"); - entityTypeBuilder.Property(x => x.DefinitionUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.Name) - .HasColumnType("VARCHAR(126)") - .IsRequired(); + .HasColumnType("VARCHAR(126)") + .IsRequired(); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/FilterListsDbContext.cs b/src/FilterLists.Data/FilterListsDbContext.cs index 64fc89969..58db3ca61 100644 --- a/src/FilterLists.Data/FilterListsDbContext.cs +++ b/src/FilterLists.Data/FilterListsDbContext.cs @@ -8,18 +8,19 @@ namespace FilterLists.Data { public class FilterListsDbContext : DbContext { - public FilterListsDbContext(DbContextOptions options) - : base(options) + public FilterListsDbContext(DbContextOptions options) : base(options) { } protected override void OnModelCreating(ModelBuilder modelBuilder) { + base.OnModelCreating(modelBuilder); ApplyConfigurationsEntities(modelBuilder); ApplyConfigurationsJunctions(modelBuilder); - base.OnModelCreating(modelBuilder); } + #region Entities + private static void ApplyConfigurationsEntities(ModelBuilder modelBuilder) { modelBuilder.ApplyConfiguration(new FilterListTypeConfiguration()); @@ -27,55 +28,17 @@ private static void ApplyConfigurationsEntities(ModelBuilder modelBuilder) modelBuilder.ApplyConfiguration(new LicenseTypeConfiguration()); modelBuilder.ApplyConfiguration(new MaintainerTypeConfiguration()); modelBuilder.ApplyConfiguration(new RuleTypeConfiguration()); - modelBuilder.ApplyConfiguration(new ScrapeTypeConfiguration()); + modelBuilder.ApplyConfiguration(new SnapshotTypeConfiguration()); modelBuilder.ApplyConfiguration(new SoftwareTypeConfiguration()); modelBuilder.ApplyConfiguration(new SyntaxTypeConfiguration()); } - private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder) - { - modelBuilder.ApplyConfiguration(new FilterListLanguageTypeConfiguration()); - modelBuilder.ApplyConfiguration(new FilterListMaintainerTypeConfiguration()); - modelBuilder.ApplyConfiguration(new FilterListRuleTypeConfiguration()); - modelBuilder.ApplyConfiguration(new ForkTypeConfiguration()); - ConfigureForkSelfReferencing(modelBuilder); - modelBuilder.ApplyConfiguration(new MergeTypeConfiguration()); - ConfigureMergeSelfReferencing(modelBuilder); - modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration()); - } - - private static void ConfigureForkSelfReferencing(ModelBuilder modelBuilder) - { - modelBuilder.Entity() - .HasOne(x => x.ForkFilterList) - .WithMany(x => x.ForkFilterLists) - .HasForeignKey(x => x.ForkFilterListId); - modelBuilder.Entity() - .HasOne(x => x.UpstreamFilterList) - .WithMany(x => x.UpstreamForkFilterLists) - .HasForeignKey(x => x.UpstreamFilterListId); - } - - private static void ConfigureMergeSelfReferencing(ModelBuilder modelBuilder) - { - modelBuilder.Entity() - .HasOne(x => x.MergeFilterList) - .WithMany(x => x.MergeFilterLists) - .HasForeignKey(x => x.MergeFilterListId); - modelBuilder.Entity() - .HasOne(x => x.UpstreamFilterList) - .WithMany(x => x.UpstreamMergeFilterLists) - .HasForeignKey(x => x.UpstreamFilterListId); - } - - #region Entities - public DbSet FilterLists { get; set; } public DbSet Languages { get; set; } public DbSet Licenses { get; set; } public DbSet Maintainers { get; set; } public DbSet Rules { get; set; } - public DbSet Scrapes { get; set; } + public DbSet Snapshots { get; set; } public DbSet Software { get; set; } public DbSet Syntaxes { get; set; } @@ -83,11 +46,21 @@ private static void ConfigureMergeSelfReferencing(ModelBuilder modelBuilder) #region Junctions + private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder) + { + modelBuilder.ApplyConfiguration(new FilterListLanguageTypeConfiguration()); + modelBuilder.ApplyConfiguration(new FilterListMaintainerTypeConfiguration()); + modelBuilder.ApplyConfiguration(new ForkTypeConfiguration()); + modelBuilder.ApplyConfiguration(new MergeTypeConfiguration()); + modelBuilder.ApplyConfiguration(new SnapshotRuleTypeConfiguration()); + modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration()); + } + public DbSet FilterListLanguages { get; set; } public DbSet FilterListMaintainers { get; set; } - public DbSet FilterListRules { get; set; } public DbSet Forks { get; set; } public DbSet Merges { get; set; } + public DbSet SnapshotRules { get; set; } public DbSet SoftwareSyntaxes { get; set; } #endregion diff --git a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs index 4aa933749..01ab21ea4 100644 --- a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs +++ b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs @@ -75,7 +75,6 @@ private static List GetSeedRows(string dataPath) private static string CreateRowValues(IEnumerable properties, TEntityType row) { - //TODO: use EF nav entity rather than foreign key so foreign key doesn't need to be explicitly in model (ie, License rather than LicenseId) return (from property in properties let value = row.GetType().GetProperty(property.Name).GetValue(row) select FormatDataForMySql(property, value)).Aggregate("", diff --git a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs index ab2dfb04c..456ed110c 100644 --- a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs +++ b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs @@ -17,7 +17,7 @@ public static void AddFilterListsServices(this IServiceCollection services, ICon b => b.MigrationsAssembly("FilterLists.Api"))); services.TryAddScoped(); services.TryAddScoped(); - services.TryAddScoped(); + services.TryAddScoped(); services.AddAutoMapper(); } } diff --git a/src/FilterLists.Services/ScrapeService/ScrapeService.cs b/src/FilterLists.Services/ScrapeService/ScrapeService.cs deleted file mode 100644 index 8a5f77365..000000000 --- a/src/FilterLists.Services/ScrapeService/ScrapeService.cs +++ /dev/null @@ -1,105 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Threading.Tasks; -using AutoMapper.QueryableExtensions; -using FilterLists.Data; -using Microsoft.EntityFrameworkCore; - -namespace FilterLists.Services.ScrapeService -{ - public class ScrapeService - { - private readonly FilterListsDbContext dbContext; - - public ScrapeService(FilterListsDbContext dbContext) - { - this.dbContext = dbContext; - } - - //TODO: call via scheduled job - public async Task ScrapeAsync(int batchSize) - { - var lists = await GetNextListsToScrape(batchSize); - var snapshots = await GetSnapshots(lists); - await SaveSnapshots(snapshots); - } - - private async Task> GetNextListsToScrape(int batchSize) - { - var neverScraped = await GetNeverScrapedLists(batchSize); - if (neverScraped.Count == batchSize) - return neverScraped; - var leastRecentlyScraped = await GetLeastRecentlyScrapedLists(batchSize - neverScraped.Count); - return neverScraped.Concat(leastRecentlyScraped).ToList(); - } - - private async Task> GetNeverScrapedLists(int batchSize) - { - return await dbContext.FilterLists - .Where(x => x.Scrapes.Count == 0) - .Take(batchSize) - .ProjectTo() - .ToListAsync(); - } - - private async Task> GetLeastRecentlyScrapedLists(int batchSize) - { - return await dbContext.Scrapes - .GroupBy(x => x.FilterList) - .Select(x => x.OrderByDescending(y => y.CreatedDateUtc).First()) - .OrderBy(x => x.CreatedDateUtc) - .Select(x => x.FilterList) - .Take(batchSize) - .ProjectTo() - .ToListAsync(); - } - - private async Task> GetSnapshots(IEnumerable lists) - { - return await Task.WhenAll(lists - .Select(async list => new Snapshot(dbContext, list.Id, await TryGetContent(list.ViewUrl))) - .Where(x => x.Result.HasRules)); - } - - private static async Task TryGetContent(string url) - { - try - { - return await GetHttpResponseMessageContent(url); - } - catch (Exception) - { - //TODO: log exception - return null; - } - } - - private static async Task GetHttpResponseMessageContent(string url) - { - using (var httpClient = new HttpClient()) - using (var httpResponseMessage = await httpClient.GetAsync(url)) - { - if (httpResponseMessage.IsSuccessStatusCode) - return await httpResponseMessage.Content.ReadAsStringAsync(); - } - - //TODO: log httpResponseMessage.StatusCode - return null; - } - - private static async Task SaveSnapshots(IEnumerable snapshots) - { - foreach (var snapshot in snapshots) - try - { - await snapshot.AddOrUpdateRules(); - } - catch (Exception) - { - //TODO: log exception - } - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/ScrapeService/Snapshot.cs b/src/FilterLists.Services/ScrapeService/Snapshot.cs deleted file mode 100644 index 3da339b91..000000000 --- a/src/FilterLists.Services/ScrapeService/Snapshot.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using FilterLists.Data; -using FilterLists.Data.Entities; -using FilterLists.Data.Entities.Junctions; - -namespace FilterLists.Services.ScrapeService -{ - public class Snapshot - { - private readonly FilterListsDbContext dbContext; - - private readonly int filterListId; - - private List rawRules; - - public Snapshot(FilterListsDbContext dbContext, int filterListId, string content) - { - this.dbContext = dbContext; - this.filterListId = filterListId; - ParseContent(content); - } - - public bool HasRules => rawRules != null; - - private void ParseContent(string content) - { - if (content == null) return; - rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList(); - for (var i = 0; i < rawRules.Count; i++) - rawRules[i] = rawRules[i].LintStringForMySql(); - rawRules = rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList(); - } - - //TODO: split large lists into batches - public async Task AddOrUpdateRules() - { - dbContext.ChangeTracker.AutoDetectChangesEnabled = false; - - // add new Rules - var preExistingSnapshotRules = dbContext.Rules.Where(x => rawRules.Contains(x.Raw)); - var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw)); - var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule}).ToList(); - await dbContext.Rules.AddRangeAsync(newSnapshotRules); - - // remove deleted FilterListRules - var preExistingFilterListRules = dbContext.FilterListRules.Where(x => x.FilterListId == filterListId); - var deletedFilterListRules = preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId)); - dbContext.FilterListRules.RemoveRange(deletedFilterListRules); - - // add FilterListRules for pre-existing Rules - var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule => - new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList() - .Except(preExistingFilterListRules).ToList(); - await dbContext.FilterListRules.AddRangeAsync(preExistingSnapshotFilterListRules); - - // add new FilterListRules - var newFilterListRules = newSnapshotRules.Select(newSnapshotRule => - new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList(); - await dbContext.FilterListRules.AddRangeAsync(newFilterListRules); - - // update scrapes - - await dbContext.SaveChangesAsync(); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/ScrapeService/FilterListViewUrlDto.cs b/src/FilterLists.Services/SnapshotService/FilterListViewUrlDto.cs similarity index 72% rename from src/FilterLists.Services/ScrapeService/FilterListViewUrlDto.cs rename to src/FilterLists.Services/SnapshotService/FilterListViewUrlDto.cs index 2fdfee176..b62621007 100644 --- a/src/FilterLists.Services/ScrapeService/FilterListViewUrlDto.cs +++ b/src/FilterLists.Services/SnapshotService/FilterListViewUrlDto.cs @@ -1,4 +1,4 @@ -namespace FilterLists.Services.ScrapeService +namespace FilterLists.Services.SnapshotService { public class FilterListViewUrlDto { diff --git a/src/FilterLists.Services/ScrapeService/RawRuleLinterExtensions.cs b/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs similarity index 95% rename from src/FilterLists.Services/ScrapeService/RawRuleLinterExtensions.cs rename to src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs index c07cb0986..d11a4fdbc 100644 --- a/src/FilterLists.Services/ScrapeService/RawRuleLinterExtensions.cs +++ b/src/FilterLists.Services/SnapshotService/RawRuleLinterExtensions.cs @@ -1,4 +1,4 @@ -namespace FilterLists.Services.ScrapeService +namespace FilterLists.Services.SnapshotService { public static class RawRuleLinterExtensions { diff --git a/src/FilterLists.Services/SnapshotService/SnapshotDomainEntity.cs b/src/FilterLists.Services/SnapshotService/SnapshotDomainEntity.cs new file mode 100644 index 000000000..950247364 --- /dev/null +++ b/src/FilterLists.Services/SnapshotService/SnapshotDomainEntity.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using FilterLists.Data; +using FilterLists.Data.Entities; +using FilterLists.Data.Entities.Junctions; + +namespace FilterLists.Services.SnapshotService +{ + public class SnapshotDomainEntity + { + private readonly FilterListsDbContext dbContext; + private readonly FilterListViewUrlDto list; + private IEnumerable newSnapshotRules; + private IEnumerable preExistingSnapshotRules; + private List rawRules; + private Snapshot snapshot; + + public SnapshotDomainEntity(FilterListsDbContext dbContext, FilterListViewUrlDto list) + { + this.dbContext = dbContext; + this.list = list; + } + + public async Task SaveSnapshot() + { + await AddSnapshot(); + var content = await TryGetContent(); + if (content != null) + { + ParseRawRules(content); + await AddNewRules(); + AddSnapshotRules(); + } + + await dbContext.SaveChangesAsync(); + } + + private async Task AddSnapshot() + { + snapshot = new Snapshot {FilterList = await dbContext.FilterLists.FindAsync(list.Id)}; + dbContext.Snapshots.Add(snapshot); + } + + private async Task TryGetContent() + { + try + { + return await GetContent(); + } + catch (WebException we) + { + snapshot.HttpStatusCode = (int)((HttpWebResponse)we.Response).StatusCode; + return null; + } + } + + private async Task GetContent() + { + using (var httpClient = new HttpClient()) + using (var httpResponseMessage = await httpClient.GetAsync(list.ViewUrl)) + { + snapshot.HttpStatusCode = (int)httpResponseMessage.StatusCode; + if (httpResponseMessage.IsSuccessStatusCode) + return await httpResponseMessage.Content.ReadAsStringAsync(); + } + + return null; + } + + private void ParseRawRules(string content) + { + rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList(); + for (var i = 0; i < rawRules.Count; i++) + rawRules[i] = rawRules[i].LintStringForMySql(); + rawRules = rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList(); + } + + private async Task AddNewRules() + { + preExistingSnapshotRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw)); + var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw)); + newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule}) + .ToList(); + await dbContext.Rules.AddRangeAsync(newSnapshotRules); + } + + private void AddSnapshotRules() + { + snapshot.SnapshotRules = preExistingSnapshotRules + .Concat(newSnapshotRules) + .Select(rule => new SnapshotRule {Rule = rule, Snapshot = snapshot}) + .ToList(); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/SnapshotService/SnapshotService.cs b/src/FilterLists.Services/SnapshotService/SnapshotService.cs new file mode 100644 index 000000000..f61f32cdf --- /dev/null +++ b/src/FilterLists.Services/SnapshotService/SnapshotService.cs @@ -0,0 +1,68 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AutoMapper.QueryableExtensions; +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; + +namespace FilterLists.Services.SnapshotService +{ + public class SnapshotService + { + private readonly FilterListsDbContext dbContext; + + public SnapshotService(FilterListsDbContext dbContext) + { + this.dbContext = dbContext; + } + + //TODO: call via scheduled job + public async Task CaptureSnapshotsAsync(int batchSize) + { + var lists = await GetNextListsToCapture(batchSize); + var snapshots = GetSnapshots(lists); + await SaveSnapshots(snapshots); + } + + private async Task> GetNextListsToCapture(int batchSize) + { + var neverCapturedLists = await GetNeverCapturedLists(batchSize); + if (neverCapturedLists.Count == batchSize) + return neverCapturedLists; + var leastRecentlyCapturedLists = await GetLeastRecentlyCapturedLists(batchSize - neverCapturedLists.Count); + return neverCapturedLists.Concat(leastRecentlyCapturedLists); + } + + private async Task> GetNeverCapturedLists(int batchSize) + { + return await dbContext.FilterLists + .Where(x => x.Snapshots.Count == 0) + .Take(batchSize) + .ProjectTo() + .ToListAsync(); + } + + private async Task> GetLeastRecentlyCapturedLists(int batchSize) + { + return await dbContext.Snapshots + .GroupBy(x => x.FilterList) + .Select(x => x.OrderByDescending(y => y.CreatedDateUtc).First()) + .OrderBy(x => x.CreatedDateUtc) + .Select(x => x.FilterList) + .Take(batchSize) + .ProjectTo() + .ToListAsync(); + } + + private IEnumerable GetSnapshots(IEnumerable lists) + { + return lists.Select(list => new SnapshotDomainEntity(dbContext, list)); + } + + private static async Task SaveSnapshots(IEnumerable snapshots) + { + foreach (var snapshot in snapshots) + await snapshot.SaveSnapshot(); + } + } +} \ No newline at end of file