mirror of
https://github.com/collinbarrett/FilterLists.git
synced 2026-03-11 09:04:27 +00:00
major SnapshotService refactor
This commit is contained in:
Collin M. Barrett
parent
858d0618f7
commit
93da298908
34 changed files with 1023 additions and 405 deletions
505
src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.Designer.cs
generated
Normal file
505
src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.Designer.cs
generated
Normal file
|
|
@ -0,0 +1,505 @@
|
|||
// <auto-generated />
|
||||
using FilterLists.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Metadata;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Storage;
|
||||
using System;
|
||||
|
||||
namespace FilterLists.Api.Migrations
|
||||
{
|
||||
[DbContext(typeof(FilterListsDbContext))]
|
||||
[Migration("20180203203947_AddSnapshot")]
|
||||
partial class AddSnapshot
|
||||
{
|
||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
|
||||
.HasAnnotation("ProductVersion", "2.0.1-rtm-125");
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Description")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("DescriptionSourceUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime?>("DiscontinuedDate");
|
||||
|
||||
b.Property<string>("DonateUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("EmailAddress")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
|
||||
b.Property<string>("ForumUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("HomeUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("IssuesUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<int?>("LicenseId");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("VARCHAR(126)");
|
||||
|
||||
b.Property<string>("PolicyUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("SubmissionUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<int?>("SyntaxId");
|
||||
|
||||
b.Property<string>("ViewUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("LicenseId");
|
||||
|
||||
b.HasIndex("SyntaxId");
|
||||
|
||||
b.ToTable("filterlists");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<int>("LanguageId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("FilterListId", "LanguageId");
|
||||
|
||||
b.HasIndex("LanguageId");
|
||||
|
||||
b.ToTable("filterlists_languages");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<int>("MaintainerId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("FilterListId", "MaintainerId");
|
||||
|
||||
b.HasIndex("MaintainerId");
|
||||
|
||||
b.ToTable("filterlists_maintainers");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
|
||||
{
|
||||
b.Property<int>("ForkFilterListId");
|
||||
|
||||
b.Property<int>("UpstreamFilterListId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("ForkFilterListId", "UpstreamFilterListId");
|
||||
|
||||
b.HasIndex("UpstreamFilterListId");
|
||||
|
||||
b.ToTable("forks");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
|
||||
{
|
||||
b.Property<int>("MergeFilterListId");
|
||||
|
||||
b.Property<int>("UpstreamFilterListId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("MergeFilterListId", "UpstreamFilterListId");
|
||||
|
||||
b.HasIndex("UpstreamFilterListId");
|
||||
|
||||
b.ToTable("merges");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
|
||||
{
|
||||
b.Property<int>("SnapshotId");
|
||||
|
||||
b.Property<int>("RuleId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("SnapshotId", "RuleId");
|
||||
|
||||
b.HasIndex("RuleId");
|
||||
|
||||
b.ToTable("snapshots_rules");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
|
||||
{
|
||||
b.Property<int>("SoftwareId");
|
||||
|
||||
b.Property<int>("SyntaxId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("SoftwareId", "SyntaxId");
|
||||
|
||||
b.HasIndex("SyntaxId");
|
||||
|
||||
b.ToTable("software_syntaxes");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Iso6391")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("VARCHAR(2)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
|
||||
b.Property<string>("Iso6392")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
|
||||
b.Property<string>("Iso6392B")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
|
||||
b.Property<string>("Iso6392T")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
|
||||
b.Property<string>("Iso6393")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
|
||||
b.Property<string>("LocalName")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("languages");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("DescriptionUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("VARCHAR(126)");
|
||||
|
||||
b.Property<bool>("PermissiveAdaptation");
|
||||
|
||||
b.Property<bool>("PermissiveCommercial");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("licenses");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("EmailAddress")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
|
||||
b.Property<string>("HomeUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("VARCHAR(126)");
|
||||
|
||||
b.Property<string>("TwitterHandle")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("maintainers");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("BIGINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Raw")
|
||||
.IsRequired()
|
||||
.HasColumnType("VARCHAR(8192)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("rules");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("MEDIUMINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<int?>("HttpStatusCode")
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("FilterListId");
|
||||
|
||||
b.ToTable("snapshots");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("DownloadUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("HomeUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("VARCHAR(126)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("software");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("DefinitionUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("VARCHAR(126)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("syntaxes");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.License", "License")
|
||||
.WithMany("FilterLists")
|
||||
.HasForeignKey("LicenseId");
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
|
||||
.WithMany("FilterLists")
|
||||
.HasForeignKey("SyntaxId");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListLanguages")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Language", "Language")
|
||||
.WithMany("FilterListLanguages")
|
||||
.HasForeignKey("LanguageId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListMaintainers")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
|
||||
.WithMany("FilterListMaintainers")
|
||||
.HasForeignKey("MaintainerId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
|
||||
.WithMany("ForkFilterLists")
|
||||
.HasForeignKey("ForkFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
|
||||
.WithMany("UpstreamForkFilterLists")
|
||||
.HasForeignKey("UpstreamFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
|
||||
.WithMany("MergeFilterLists")
|
||||
.HasForeignKey("MergeFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
|
||||
.WithMany("UpstreamMergeFilterLists")
|
||||
.HasForeignKey("UpstreamFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
|
||||
.WithMany("SnapshotRules")
|
||||
.HasForeignKey("RuleId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
|
||||
.WithMany("SnapshotRules")
|
||||
.HasForeignKey("SnapshotId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.Software", "Software")
|
||||
.WithMany("SoftwareSyntaxes")
|
||||
.HasForeignKey("SoftwareId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
|
||||
.WithMany("SoftwareSyntaxes")
|
||||
.HasForeignKey("SyntaxId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("Snapshots")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
#pragma warning restore 612, 618
|
||||
}
|
||||
}
|
||||
}
|
||||
143
src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.cs
Normal file
143
src/FilterLists.Api/Migrations/20180203203947_AddSnapshot.cs
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
using Microsoft.EntityFrameworkCore.Metadata;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace FilterLists.Api.Migrations
|
||||
{
|
||||
public partial class AddSnapshot : Migration
|
||||
{
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropTable(
|
||||
name: "filterlists_rules");
|
||||
|
||||
migrationBuilder.DropTable(
|
||||
name: "scrapes");
|
||||
|
||||
migrationBuilder.CreateTable(
|
||||
name: "snapshots",
|
||||
columns: table => new
|
||||
{
|
||||
Id = table.Column<int>(type: "MEDIUMINT UNSIGNED", nullable: false)
|
||||
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
|
||||
CreatedDateUtc = table.Column<DateTime>(type: "TIMESTAMP", nullable: false)
|
||||
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
|
||||
FilterListId = table.Column<int>(nullable: false),
|
||||
HttpStatusCode = table.Column<int>(type: "SMALLINT UNSIGNED", nullable: true)
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_snapshots", x => x.Id);
|
||||
table.ForeignKey(
|
||||
name: "FK_snapshots_filterlists_FilterListId",
|
||||
column: x => x.FilterListId,
|
||||
principalTable: "filterlists",
|
||||
principalColumn: "Id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateTable(
|
||||
name: "snapshots_rules",
|
||||
columns: table => new
|
||||
{
|
||||
SnapshotId = table.Column<int>(nullable: false),
|
||||
RuleId = table.Column<int>(nullable: false),
|
||||
CreatedDateUtc = table.Column<DateTime>(type: "TIMESTAMP", nullable: false)
|
||||
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_snapshots_rules", x => new { x.SnapshotId, x.RuleId });
|
||||
table.ForeignKey(
|
||||
name: "FK_snapshots_rules_rules_RuleId",
|
||||
column: x => x.RuleId,
|
||||
principalTable: "rules",
|
||||
principalColumn: "Id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
table.ForeignKey(
|
||||
name: "FK_snapshots_rules_snapshots_SnapshotId",
|
||||
column: x => x.SnapshotId,
|
||||
principalTable: "snapshots",
|
||||
principalColumn: "Id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_snapshots_FilterListId",
|
||||
table: "snapshots",
|
||||
column: "FilterListId");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_snapshots_rules_RuleId",
|
||||
table: "snapshots_rules",
|
||||
column: "RuleId");
|
||||
}
|
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropTable(
|
||||
name: "snapshots_rules");
|
||||
|
||||
migrationBuilder.DropTable(
|
||||
name: "snapshots");
|
||||
|
||||
migrationBuilder.CreateTable(
|
||||
name: "filterlists_rules",
|
||||
columns: table => new
|
||||
{
|
||||
FilterListId = table.Column<int>(nullable: false),
|
||||
RuleId = table.Column<int>(nullable: false),
|
||||
CreatedDateUtc = table.Column<DateTime>(type: "TIMESTAMP", nullable: false)
|
||||
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_filterlists_rules", x => new { x.FilterListId, x.RuleId });
|
||||
table.ForeignKey(
|
||||
name: "FK_filterlists_rules_filterlists_FilterListId",
|
||||
column: x => x.FilterListId,
|
||||
principalTable: "filterlists",
|
||||
principalColumn: "Id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
table.ForeignKey(
|
||||
name: "FK_filterlists_rules_rules_RuleId",
|
||||
column: x => x.RuleId,
|
||||
principalTable: "rules",
|
||||
principalColumn: "Id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateTable(
|
||||
name: "scrapes",
|
||||
columns: table => new
|
||||
{
|
||||
Id = table.Column<int>(type: "SMALLINT UNSIGNED", nullable: false)
|
||||
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
|
||||
CreatedDateUtc = table.Column<DateTime>(type: "TIMESTAMP", nullable: false)
|
||||
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
|
||||
FilterListId = table.Column<int>(nullable: false)
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_scrapes", x => x.Id);
|
||||
table.ForeignKey(
|
||||
name: "FK_scrapes_filterlists_FilterListId",
|
||||
column: x => x.FilterListId,
|
||||
principalTable: "filterlists",
|
||||
principalColumn: "Id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_filterlists_rules_RuleId",
|
||||
table: "filterlists_rules",
|
||||
column: "RuleId");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_scrapes_FilterListId",
|
||||
table: "scrapes",
|
||||
column: "FilterListId");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -118,23 +118,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.ToTable("filterlists_maintainers");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<int>("RuleId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("FilterListId", "RuleId");
|
||||
|
||||
b.HasIndex("RuleId");
|
||||
|
||||
b.ToTable("filterlists_rules");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
|
||||
{
|
||||
b.Property<int>("ForkFilterListId");
|
||||
|
|
@ -169,6 +152,23 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.ToTable("merges");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
|
||||
{
|
||||
b.Property<int>("SnapshotId");
|
||||
|
||||
b.Property<int>("RuleId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("SnapshotId", "RuleId");
|
||||
|
||||
b.HasIndex("RuleId");
|
||||
|
||||
b.ToTable("snapshots_rules");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
|
||||
{
|
||||
b.Property<int>("SoftwareId");
|
||||
|
|
@ -325,11 +325,11 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.ToTable("rules");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
.HasColumnType("MEDIUMINT UNSIGNED");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
|
|
@ -337,11 +337,14 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<int?>("HttpStatusCode")
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("FilterListId");
|
||||
|
||||
b.ToTable("scrapes");
|
||||
b.ToTable("snapshots");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
|
||||
|
|
@ -401,11 +404,11 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.License")
|
||||
b.HasOne("FilterLists.Data.Entities.License", "License")
|
||||
.WithMany("FilterLists")
|
||||
.HasForeignKey("LicenseId");
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Syntax")
|
||||
b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
|
||||
.WithMany("FilterLists")
|
||||
.HasForeignKey("SyntaxId");
|
||||
});
|
||||
|
|
@ -436,19 +439,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListRules")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
|
||||
.WithMany("FilterListRules")
|
||||
.HasForeignKey("RuleId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
|
||||
|
|
@ -475,6 +465,19 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
|
||||
.WithMany("SnapshotRules")
|
||||
.HasForeignKey("RuleId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
|
||||
.WithMany("SnapshotRules")
|
||||
.HasForeignKey("SnapshotId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.Software", "Software")
|
||||
|
|
@ -488,10 +491,10 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList")
|
||||
.WithMany("Scrapes")
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("Snapshots")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
using System.Threading.Tasks;
|
||||
using FilterLists.Services.ScrapeService;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace FilterLists.Api.V1.Controllers
|
||||
{
|
||||
public class ScrapeController : BaseController
|
||||
{
|
||||
private readonly ScrapeService scrapeService;
|
||||
|
||||
public ScrapeController(ScrapeService scrapeService)
|
||||
{
|
||||
this.scrapeService = scrapeService;
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> Index()
|
||||
{
|
||||
#if DEBUG
|
||||
await scrapeService.ScrapeAsync(3);
|
||||
return Ok();
|
||||
#else
|
||||
return StatusCode(403);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
27
src/FilterLists.Api/V1/Controllers/SnapshotController.cs
Normal file
27
src/FilterLists.Api/V1/Controllers/SnapshotController.cs
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
using System.Threading.Tasks;
|
||||
using FilterLists.Services.SnapshotService;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace FilterLists.Api.V1.Controllers
|
||||
{
|
||||
public class SnapshotController : BaseController
|
||||
{
|
||||
private readonly SnapshotService snapshotService;
|
||||
|
||||
public SnapshotController(SnapshotService snapshotService)
|
||||
{
|
||||
this.snapshotService = snapshotService;
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> Index()
|
||||
{
|
||||
#if DEBUG
|
||||
await snapshotService.CaptureSnapshotsAsync(1);
|
||||
return Ok();
|
||||
#else
|
||||
return StatusCode(403);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -24,8 +24,7 @@ public class FilterList : BaseEntity
|
|||
public ICollection<Merge> UpstreamMergeFilterLists { get; set; }
|
||||
public string Name { get; set; }
|
||||
public string PolicyUrl { get; set; }
|
||||
public ICollection<FilterListRule> FilterListRules { get; set; }
|
||||
public ICollection<Scrape> Scrapes { get; set; }
|
||||
public ICollection<Snapshot> Snapshots { get; set; }
|
||||
public string SubmissionUrl { get; set; }
|
||||
public int? SyntaxId { get; set; }
|
||||
public Syntax Syntax { get; set; }
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
namespace FilterLists.Data.Entities.Junctions
|
||||
{
|
||||
public class FilterListRule : BaseJunction
|
||||
public class SnapshotRule : BaseJunction
|
||||
{
|
||||
public int FilterListId { get; set; }
|
||||
public FilterList FilterList { get; set; }
|
||||
public int SnapshotId { get; set; }
|
||||
public Snapshot Snapshot { get; set; }
|
||||
|
||||
public int RuleId { get; set; }
|
||||
public Rule Rule { get; set; }
|
||||
|
|
@ -5,7 +5,7 @@ namespace FilterLists.Data.Entities
|
|||
{
|
||||
public class Rule : BaseEntity
|
||||
{
|
||||
public ICollection<FilterListRule> FilterListRules { get; set; }
|
||||
public string Raw { get; set; }
|
||||
public ICollection<SnapshotRule> SnapshotRules { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
namespace FilterLists.Data.Entities
|
||||
{
|
||||
public class Scrape : BaseEntity
|
||||
{
|
||||
public int FilterListId { get; set; }
|
||||
public FilterList FilterList { get; set; }
|
||||
}
|
||||
}
|
||||
13
src/FilterLists.Data/Entities/Snapshot.cs
Normal file
13
src/FilterLists.Data/Entities/Snapshot.cs
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
using System.Collections.Generic;
|
||||
using FilterLists.Data.Entities.Junctions;
|
||||
|
||||
namespace FilterLists.Data.Entities
|
||||
{
|
||||
public class Snapshot : BaseEntity
|
||||
{
|
||||
public int FilterListId { get; set; }
|
||||
public FilterList FilterList { get; set; }
|
||||
public int? HttpStatusCode { get; set; }
|
||||
public ICollection<SnapshotRule> SnapshotRules { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -9,13 +9,13 @@ public class BaseEntityTypeConfiguration<TEntity> : IEntityTypeConfiguration<TEn
|
|||
public virtual void Configure(EntityTypeBuilder<TEntity> entityTypeBuilder)
|
||||
{
|
||||
entityTypeBuilder.Property(x => x.Id)
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
entityTypeBuilder.Property(x => x.CreatedDateUtc)
|
||||
.HasColumnType("TIMESTAMP")
|
||||
.ValueGeneratedOnAdd();
|
||||
.HasColumnType("TIMESTAMP")
|
||||
.ValueGeneratedOnAdd();
|
||||
entityTypeBuilder.Property(x => x.ModifiedDateUtc)
|
||||
.HasColumnType("TIMESTAMP")
|
||||
.ValueGeneratedOnAddOrUpdate();
|
||||
.HasColumnType("TIMESTAMP")
|
||||
.ValueGeneratedOnAddOrUpdate();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,33 +9,31 @@ public class FilterListTypeConfiguration : BaseEntityTypeConfiguration<FilterLis
|
|||
public override void Configure(EntityTypeBuilder<FilterList> entityTypeBuilder)
|
||||
{
|
||||
base.Configure(entityTypeBuilder);
|
||||
|
||||
entityTypeBuilder.ToTable("filterlists");
|
||||
|
||||
entityTypeBuilder.Property(x => x.Description)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.DescriptionSourceUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.DonateUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.EmailAddress)
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
entityTypeBuilder.Property(x => x.ForumUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.HomeUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.IssuesUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.Name)
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.IsRequired();
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.IsRequired();
|
||||
entityTypeBuilder.Property(x => x.PolicyUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.SubmissionUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.ViewUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -10,8 +10,8 @@ public class BaseJunctionTypeConfiguration<TJunction> : IEntityTypeConfiguration
|
|||
public virtual void Configure(EntityTypeBuilder<TJunction> entityTypeBuilder)
|
||||
{
|
||||
entityTypeBuilder.Property(x => x.CreatedDateUtc)
|
||||
.HasColumnType("TIMESTAMP")
|
||||
.ValueGeneratedOnAdd();
|
||||
.HasColumnType("TIMESTAMP")
|
||||
.ValueGeneratedOnAdd();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
using FilterLists.Data.Entities.Junctions;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
|
||||
namespace FilterLists.Data.EntityTypeConfigurations.Junctions
|
||||
{
|
||||
public class FilterListRuleTypeConfiguration : BaseJunctionTypeConfiguration<FilterListRule>
|
||||
{
|
||||
public override void Configure(EntityTypeBuilder<FilterListRule> entityTypeBuilder)
|
||||
{
|
||||
base.Configure(entityTypeBuilder);
|
||||
entityTypeBuilder.ToTable("filterlists_rules");
|
||||
entityTypeBuilder.HasKey(x => new {x.FilterListId, x.RuleId});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -11,6 +11,12 @@ public override void Configure(EntityTypeBuilder<Fork> entityTypeBuilder)
|
|||
base.Configure(entityTypeBuilder);
|
||||
entityTypeBuilder.ToTable("forks");
|
||||
entityTypeBuilder.HasKey(x => new {x.ForkFilterListId, x.UpstreamFilterListId});
|
||||
entityTypeBuilder.HasOne(x => x.ForkFilterList)
|
||||
.WithMany(x => x.ForkFilterLists)
|
||||
.HasForeignKey(x => x.ForkFilterListId);
|
||||
entityTypeBuilder.HasOne(x => x.UpstreamFilterList)
|
||||
.WithMany(x => x.UpstreamForkFilterLists)
|
||||
.HasForeignKey(x => x.UpstreamFilterListId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -11,6 +11,12 @@ public override void Configure(EntityTypeBuilder<Merge> entityTypeBuilder)
|
|||
base.Configure(entityTypeBuilder);
|
||||
entityTypeBuilder.ToTable("merges");
|
||||
entityTypeBuilder.HasKey(x => new {x.MergeFilterListId, x.UpstreamFilterListId});
|
||||
entityTypeBuilder.HasOne(x => x.MergeFilterList)
|
||||
.WithMany(x => x.MergeFilterLists)
|
||||
.HasForeignKey(x => x.MergeFilterListId);
|
||||
entityTypeBuilder.HasOne(x => x.UpstreamFilterList)
|
||||
.WithMany(x => x.UpstreamMergeFilterLists)
|
||||
.HasForeignKey(x => x.UpstreamFilterListId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
using FilterLists.Data.Entities.Junctions;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
|
||||
namespace FilterLists.Data.EntityTypeConfigurations.Junctions
|
||||
{
|
||||
public class SnapshotRuleTypeConfiguration : BaseJunctionTypeConfiguration<SnapshotRule>
|
||||
{
|
||||
public override void Configure(EntityTypeBuilder<SnapshotRule> entityTypeBuilder)
|
||||
{
|
||||
base.Configure(entityTypeBuilder);
|
||||
entityTypeBuilder.ToTable("snapshots_rules");
|
||||
entityTypeBuilder.HasKey(x => new {x.SnapshotId, x.RuleId});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,30 +9,28 @@ public class LanguageTypeConfiguration : BaseEntityTypeConfiguration<Language>
|
|||
public override void Configure(EntityTypeBuilder<Language> entityTypeBuilder)
|
||||
{
|
||||
base.Configure(entityTypeBuilder);
|
||||
|
||||
entityTypeBuilder.ToTable("languages");
|
||||
|
||||
entityTypeBuilder.Property(x => x.Iso6391)
|
||||
.HasColumnType("VARCHAR(2)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
.HasColumnType("VARCHAR(2)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
entityTypeBuilder.Property(x => x.Iso6392)
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
entityTypeBuilder.Property(x => x.Iso6392B)
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
entityTypeBuilder.Property(x => x.Iso6392T)
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
entityTypeBuilder.Property(x => x.Iso6393)
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
.HasColumnType("VARCHAR(3)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
entityTypeBuilder.Property(x => x.LocalName)
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
entityTypeBuilder.Property(x => x.Name)
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,14 +9,12 @@ public class LicenseTypeConfiguration : BaseEntityTypeConfiguration<License>
|
|||
public override void Configure(EntityTypeBuilder<License> entityTypeBuilder)
|
||||
{
|
||||
base.Configure(entityTypeBuilder);
|
||||
|
||||
entityTypeBuilder.ToTable("licenses");
|
||||
|
||||
entityTypeBuilder.Property(x => x.DescriptionUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.Name)
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.IsRequired();
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.IsRequired();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,20 +9,18 @@ public class MaintainerTypeConfiguration : BaseEntityTypeConfiguration<Maintaine
|
|||
public override void Configure(EntityTypeBuilder<Maintainer> entityTypeBuilder)
|
||||
{
|
||||
base.Configure(entityTypeBuilder);
|
||||
|
||||
entityTypeBuilder.ToTable("maintainers");
|
||||
|
||||
entityTypeBuilder.Property(x => x.EmailAddress)
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
entityTypeBuilder.Property(x => x.HomeUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.Name)
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.IsRequired();
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.IsRequired();
|
||||
entityTypeBuilder.Property(x => x.TwitterHandle)
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL"); ;
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.HasDefaultValueSql("NULL");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,17 +9,13 @@ public class RuleTypeConfiguration : BaseEntityTypeConfiguration<Rule>
|
|||
public override void Configure(EntityTypeBuilder<Rule> entityTypeBuilder)
|
||||
{
|
||||
base.Configure(entityTypeBuilder);
|
||||
|
||||
entityTypeBuilder.ToTable("rules");
|
||||
|
||||
entityTypeBuilder.Property(x => x.Id)
|
||||
.HasColumnType("BIGINT UNSIGNED");
|
||||
|
||||
.HasColumnType("BIGINT UNSIGNED");
|
||||
entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
|
||||
|
||||
entityTypeBuilder.Property(x => x.Raw)
|
||||
.HasColumnType("VARCHAR(8192)")
|
||||
.IsRequired();
|
||||
.HasColumnType("VARCHAR(8192)")
|
||||
.IsRequired();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
using FilterLists.Data.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
|
||||
namespace FilterLists.Data.EntityTypeConfigurations
|
||||
{
|
||||
public class ScrapeTypeConfiguration : BaseEntityTypeConfiguration<Scrape>
|
||||
{
|
||||
public override void Configure(EntityTypeBuilder<Scrape> entityTypeBuilder)
|
||||
{
|
||||
base.Configure(entityTypeBuilder);
|
||||
|
||||
entityTypeBuilder.ToTable("scrapes");
|
||||
|
||||
entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
using FilterLists.Data.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
|
||||
namespace FilterLists.Data.EntityTypeConfigurations
|
||||
{
|
||||
public class SnapshotTypeConfiguration : BaseEntityTypeConfiguration<Snapshot>
|
||||
{
|
||||
public override void Configure(EntityTypeBuilder<Snapshot> entityTypeBuilder)
|
||||
{
|
||||
base.Configure(entityTypeBuilder);
|
||||
entityTypeBuilder.ToTable("snapshots");
|
||||
entityTypeBuilder.Property(x => x.Id)
|
||||
.HasColumnType("MEDIUMINT UNSIGNED");
|
||||
entityTypeBuilder.Property(x => x.HttpStatusCode)
|
||||
.HasColumnType("SMALLINT UNSIGNED");
|
||||
entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,16 +9,14 @@ public class SoftwareTypeConfiguration : BaseEntityTypeConfiguration<Software>
|
|||
public override void Configure(EntityTypeBuilder<Software> entityTypeBuilder)
|
||||
{
|
||||
base.Configure(entityTypeBuilder);
|
||||
|
||||
entityTypeBuilder.ToTable("software");
|
||||
|
||||
entityTypeBuilder.Property(x => x.DownloadUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.HomeUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.Name)
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.IsRequired();
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.IsRequired();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,14 +9,12 @@ public class SyntaxTypeConfiguration : BaseEntityTypeConfiguration<Syntax>
|
|||
public override void Configure(EntityTypeBuilder<Syntax> entityTypeBuilder)
|
||||
{
|
||||
base.Configure(entityTypeBuilder);
|
||||
|
||||
entityTypeBuilder.ToTable("syntaxes");
|
||||
|
||||
entityTypeBuilder.Property(x => x.DefinitionUrl)
|
||||
.HasColumnType("TEXT");
|
||||
.HasColumnType("TEXT");
|
||||
entityTypeBuilder.Property(x => x.Name)
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.IsRequired();
|
||||
.HasColumnType("VARCHAR(126)")
|
||||
.IsRequired();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -8,18 +8,19 @@ namespace FilterLists.Data
|
|||
{
|
||||
public class FilterListsDbContext : DbContext
|
||||
{
|
||||
public FilterListsDbContext(DbContextOptions options)
|
||||
: base(options)
|
||||
public FilterListsDbContext(DbContextOptions options) : base(options)
|
||||
{
|
||||
}
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
{
|
||||
base.OnModelCreating(modelBuilder);
|
||||
ApplyConfigurationsEntities(modelBuilder);
|
||||
ApplyConfigurationsJunctions(modelBuilder);
|
||||
base.OnModelCreating(modelBuilder);
|
||||
}
|
||||
|
||||
#region Entities
|
||||
|
||||
private static void ApplyConfigurationsEntities(ModelBuilder modelBuilder)
|
||||
{
|
||||
modelBuilder.ApplyConfiguration(new FilterListTypeConfiguration());
|
||||
|
|
@ -27,55 +28,17 @@ private static void ApplyConfigurationsEntities(ModelBuilder modelBuilder)
|
|||
modelBuilder.ApplyConfiguration(new LicenseTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new MaintainerTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new RuleTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new ScrapeTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SnapshotTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SoftwareTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SyntaxTypeConfiguration());
|
||||
}
|
||||
|
||||
private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder)
|
||||
{
|
||||
modelBuilder.ApplyConfiguration(new FilterListLanguageTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new FilterListMaintainerTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new FilterListRuleTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new ForkTypeConfiguration());
|
||||
ConfigureForkSelfReferencing(modelBuilder);
|
||||
modelBuilder.ApplyConfiguration(new MergeTypeConfiguration());
|
||||
ConfigureMergeSelfReferencing(modelBuilder);
|
||||
modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration());
|
||||
}
|
||||
|
||||
private static void ConfigureForkSelfReferencing(ModelBuilder modelBuilder)
|
||||
{
|
||||
modelBuilder.Entity<Fork>()
|
||||
.HasOne(x => x.ForkFilterList)
|
||||
.WithMany(x => x.ForkFilterLists)
|
||||
.HasForeignKey(x => x.ForkFilterListId);
|
||||
modelBuilder.Entity<Fork>()
|
||||
.HasOne(x => x.UpstreamFilterList)
|
||||
.WithMany(x => x.UpstreamForkFilterLists)
|
||||
.HasForeignKey(x => x.UpstreamFilterListId);
|
||||
}
|
||||
|
||||
private static void ConfigureMergeSelfReferencing(ModelBuilder modelBuilder)
|
||||
{
|
||||
modelBuilder.Entity<Merge>()
|
||||
.HasOne(x => x.MergeFilterList)
|
||||
.WithMany(x => x.MergeFilterLists)
|
||||
.HasForeignKey(x => x.MergeFilterListId);
|
||||
modelBuilder.Entity<Merge>()
|
||||
.HasOne(x => x.UpstreamFilterList)
|
||||
.WithMany(x => x.UpstreamMergeFilterLists)
|
||||
.HasForeignKey(x => x.UpstreamFilterListId);
|
||||
}
|
||||
|
||||
#region Entities
|
||||
|
||||
public DbSet<FilterList> FilterLists { get; set; }
|
||||
public DbSet<Language> Languages { get; set; }
|
||||
public DbSet<License> Licenses { get; set; }
|
||||
public DbSet<Maintainer> Maintainers { get; set; }
|
||||
public DbSet<Rule> Rules { get; set; }
|
||||
public DbSet<Scrape> Scrapes { get; set; }
|
||||
public DbSet<Snapshot> Snapshots { get; set; }
|
||||
public DbSet<Software> Software { get; set; }
|
||||
public DbSet<Syntax> Syntaxes { get; set; }
|
||||
|
||||
|
|
@ -83,11 +46,21 @@ private static void ConfigureMergeSelfReferencing(ModelBuilder modelBuilder)
|
|||
|
||||
#region Junctions
|
||||
|
||||
private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder)
|
||||
{
|
||||
modelBuilder.ApplyConfiguration(new FilterListLanguageTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new FilterListMaintainerTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new ForkTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new MergeTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SnapshotRuleTypeConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration());
|
||||
}
|
||||
|
||||
public DbSet<FilterListLanguage> FilterListLanguages { get; set; }
|
||||
public DbSet<FilterListMaintainer> FilterListMaintainers { get; set; }
|
||||
public DbSet<FilterListRule> FilterListRules { get; set; }
|
||||
public DbSet<Fork> Forks { get; set; }
|
||||
public DbSet<Merge> Merges { get; set; }
|
||||
public DbSet<SnapshotRule> SnapshotRules { get; set; }
|
||||
public DbSet<SoftwareSyntax> SoftwareSyntaxes { get; set; }
|
||||
|
||||
#endregion
|
||||
|
|
|
|||
|
|
@ -75,7 +75,6 @@ private static List<TEntityType> GetSeedRows<TEntityType>(string dataPath)
|
|||
|
||||
private static string CreateRowValues<TEntityType>(IEnumerable<IProperty> properties, TEntityType row)
|
||||
{
|
||||
//TODO: use EF nav entity rather than foreign key so foreign key doesn't need to be explicitly in model (ie, License rather than LicenseId)
|
||||
return (from property in properties
|
||||
let value = row.GetType().GetProperty(property.Name).GetValue(row)
|
||||
select FormatDataForMySql(property, value)).Aggregate("",
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ public static void AddFilterListsServices(this IServiceCollection services, ICon
|
|||
b => b.MigrationsAssembly("FilterLists.Api")));
|
||||
services.TryAddScoped<FilterListService.FilterListService>();
|
||||
services.TryAddScoped<SeedService.SeedService>();
|
||||
services.TryAddScoped<ScrapeService.ScrapeService>();
|
||||
services.TryAddScoped<SnapshotService.SnapshotService>();
|
||||
services.AddAutoMapper();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,105 +0,0 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutoMapper.QueryableExtensions;
|
||||
using FilterLists.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace FilterLists.Services.ScrapeService
|
||||
{
|
||||
public class ScrapeService
|
||||
{
|
||||
private readonly FilterListsDbContext dbContext;
|
||||
|
||||
public ScrapeService(FilterListsDbContext dbContext)
|
||||
{
|
||||
this.dbContext = dbContext;
|
||||
}
|
||||
|
||||
//TODO: call via scheduled job
|
||||
public async Task ScrapeAsync(int batchSize)
|
||||
{
|
||||
var lists = await GetNextListsToScrape(batchSize);
|
||||
var snapshots = await GetSnapshots(lists);
|
||||
await SaveSnapshots(snapshots);
|
||||
}
|
||||
|
||||
private async Task<List<FilterListViewUrlDto>> GetNextListsToScrape(int batchSize)
|
||||
{
|
||||
var neverScraped = await GetNeverScrapedLists(batchSize);
|
||||
if (neverScraped.Count == batchSize)
|
||||
return neverScraped;
|
||||
var leastRecentlyScraped = await GetLeastRecentlyScrapedLists(batchSize - neverScraped.Count);
|
||||
return neverScraped.Concat(leastRecentlyScraped).ToList();
|
||||
}
|
||||
|
||||
private async Task<List<FilterListViewUrlDto>> GetNeverScrapedLists(int batchSize)
|
||||
{
|
||||
return await dbContext.FilterLists
|
||||
.Where(x => x.Scrapes.Count == 0)
|
||||
.Take(batchSize)
|
||||
.ProjectTo<FilterListViewUrlDto>()
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
private async Task<List<FilterListViewUrlDto>> GetLeastRecentlyScrapedLists(int batchSize)
|
||||
{
|
||||
return await dbContext.Scrapes
|
||||
.GroupBy(x => x.FilterList)
|
||||
.Select(x => x.OrderByDescending(y => y.CreatedDateUtc).First())
|
||||
.OrderBy(x => x.CreatedDateUtc)
|
||||
.Select(x => x.FilterList)
|
||||
.Take(batchSize)
|
||||
.ProjectTo<FilterListViewUrlDto>()
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
private async Task<IEnumerable<Snapshot>> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)
|
||||
{
|
||||
return await Task.WhenAll(lists
|
||||
.Select(async list => new Snapshot(dbContext, list.Id, await TryGetContent(list.ViewUrl)))
|
||||
.Where(x => x.Result.HasRules));
|
||||
}
|
||||
|
||||
private static async Task<string> TryGetContent(string url)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await GetHttpResponseMessageContent(url);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
//TODO: log exception
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<string> GetHttpResponseMessageContent(string url)
|
||||
{
|
||||
using (var httpClient = new HttpClient())
|
||||
using (var httpResponseMessage = await httpClient.GetAsync(url))
|
||||
{
|
||||
if (httpResponseMessage.IsSuccessStatusCode)
|
||||
return await httpResponseMessage.Content.ReadAsStringAsync();
|
||||
}
|
||||
|
||||
//TODO: log httpResponseMessage.StatusCode
|
||||
return null;
|
||||
}
|
||||
|
||||
private static async Task SaveSnapshots(IEnumerable<Snapshot> snapshots)
|
||||
{
|
||||
foreach (var snapshot in snapshots)
|
||||
try
|
||||
{
|
||||
await snapshot.AddOrUpdateRules();
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
//TODO: log exception
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using FilterLists.Data;
|
||||
using FilterLists.Data.Entities;
|
||||
using FilterLists.Data.Entities.Junctions;
|
||||
|
||||
namespace FilterLists.Services.ScrapeService
|
||||
{
|
||||
public class Snapshot
|
||||
{
|
||||
private readonly FilterListsDbContext dbContext;
|
||||
|
||||
private readonly int filterListId;
|
||||
|
||||
private List<string> rawRules;
|
||||
|
||||
public Snapshot(FilterListsDbContext dbContext, int filterListId, string content)
|
||||
{
|
||||
this.dbContext = dbContext;
|
||||
this.filterListId = filterListId;
|
||||
ParseContent(content);
|
||||
}
|
||||
|
||||
public bool HasRules => rawRules != null;
|
||||
|
||||
private void ParseContent(string content)
|
||||
{
|
||||
if (content == null) return;
|
||||
rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList();
|
||||
for (var i = 0; i < rawRules.Count; i++)
|
||||
rawRules[i] = rawRules[i].LintStringForMySql();
|
||||
rawRules = rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
|
||||
}
|
||||
|
||||
//TODO: split large lists into batches
|
||||
public async Task AddOrUpdateRules()
|
||||
{
|
||||
dbContext.ChangeTracker.AutoDetectChangesEnabled = false;
|
||||
|
||||
// add new Rules
|
||||
var preExistingSnapshotRules = dbContext.Rules.Where(x => rawRules.Contains(x.Raw));
|
||||
var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw));
|
||||
var newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule}).ToList();
|
||||
await dbContext.Rules.AddRangeAsync(newSnapshotRules);
|
||||
|
||||
// remove deleted FilterListRules
|
||||
var preExistingFilterListRules = dbContext.FilterListRules.Where(x => x.FilterListId == filterListId);
|
||||
var deletedFilterListRules = preExistingFilterListRules.Where(x => !preExistingSnapshotRules.Select(y => y.Id).Contains(x.RuleId));
|
||||
dbContext.FilterListRules.RemoveRange(deletedFilterListRules);
|
||||
|
||||
// add FilterListRules for pre-existing Rules
|
||||
var preExistingSnapshotFilterListRules = preExistingSnapshotRules.Select(newSnapshotRule =>
|
||||
new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList()
|
||||
.Except(preExistingFilterListRules).ToList();
|
||||
await dbContext.FilterListRules.AddRangeAsync(preExistingSnapshotFilterListRules);
|
||||
|
||||
// add new FilterListRules
|
||||
var newFilterListRules = newSnapshotRules.Select(newSnapshotRule =>
|
||||
new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList();
|
||||
await dbContext.FilterListRules.AddRangeAsync(newFilterListRules);
|
||||
|
||||
// update scrapes
|
||||
|
||||
await dbContext.SaveChangesAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
namespace FilterLists.Services.ScrapeService
|
||||
namespace FilterLists.Services.SnapshotService
|
||||
{
|
||||
public class FilterListViewUrlDto
|
||||
{
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
namespace FilterLists.Services.ScrapeService
|
||||
namespace FilterLists.Services.SnapshotService
|
||||
{
|
||||
public static class RawRuleLinterExtensions
|
||||
{
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using FilterLists.Data;
|
||||
using FilterLists.Data.Entities;
|
||||
using FilterLists.Data.Entities.Junctions;
|
||||
|
||||
namespace FilterLists.Services.SnapshotService
|
||||
{
|
||||
public class SnapshotDomainEntity
|
||||
{
|
||||
private readonly FilterListsDbContext dbContext;
|
||||
private readonly FilterListViewUrlDto list;
|
||||
private IEnumerable<Rule> newSnapshotRules;
|
||||
private IEnumerable<Rule> preExistingSnapshotRules;
|
||||
private List<string> rawRules;
|
||||
private Snapshot snapshot;
|
||||
|
||||
public SnapshotDomainEntity(FilterListsDbContext dbContext, FilterListViewUrlDto list)
|
||||
{
|
||||
this.dbContext = dbContext;
|
||||
this.list = list;
|
||||
}
|
||||
|
||||
public async Task SaveSnapshot()
|
||||
{
|
||||
await AddSnapshot();
|
||||
var content = await TryGetContent();
|
||||
if (content != null)
|
||||
{
|
||||
ParseRawRules(content);
|
||||
await AddNewRules();
|
||||
AddSnapshotRules();
|
||||
}
|
||||
|
||||
await dbContext.SaveChangesAsync();
|
||||
}
|
||||
|
||||
private async Task AddSnapshot()
|
||||
{
|
||||
snapshot = new Snapshot {FilterList = await dbContext.FilterLists.FindAsync(list.Id)};
|
||||
dbContext.Snapshots.Add(snapshot);
|
||||
}
|
||||
|
||||
private async Task<string> TryGetContent()
|
||||
{
|
||||
try
|
||||
{
|
||||
return await GetContent();
|
||||
}
|
||||
catch (WebException we)
|
||||
{
|
||||
snapshot.HttpStatusCode = (int)((HttpWebResponse)we.Response).StatusCode;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> GetContent()
|
||||
{
|
||||
using (var httpClient = new HttpClient())
|
||||
using (var httpResponseMessage = await httpClient.GetAsync(list.ViewUrl))
|
||||
{
|
||||
snapshot.HttpStatusCode = (int)httpResponseMessage.StatusCode;
|
||||
if (httpResponseMessage.IsSuccessStatusCode)
|
||||
return await httpResponseMessage.Content.ReadAsStringAsync();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private void ParseRawRules(string content)
|
||||
{
|
||||
rawRules = content.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.RemoveEmptyEntries).ToList();
|
||||
for (var i = 0; i < rawRules.Count; i++)
|
||||
rawRules[i] = rawRules[i].LintStringForMySql();
|
||||
rawRules = rawRules.Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
|
||||
}
|
||||
|
||||
private async Task AddNewRules()
|
||||
{
|
||||
preExistingSnapshotRules = dbContext.Rules.Where(rule => rawRules.Contains(rule.Raw));
|
||||
var newSnapshotRawRules = rawRules.Except(preExistingSnapshotRules.Select(x => x.Raw));
|
||||
newSnapshotRules = newSnapshotRawRules.Select(newSnapshotRawRule => new Rule {Raw = newSnapshotRawRule})
|
||||
.ToList();
|
||||
await dbContext.Rules.AddRangeAsync(newSnapshotRules);
|
||||
}
|
||||
|
||||
private void AddSnapshotRules()
|
||||
{
|
||||
snapshot.SnapshotRules = preExistingSnapshotRules
|
||||
.Concat(newSnapshotRules)
|
||||
.Select(rule => new SnapshotRule {Rule = rule, Snapshot = snapshot})
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
}
|
||||
68
src/FilterLists.Services/SnapshotService/SnapshotService.cs
Normal file
68
src/FilterLists.Services/SnapshotService/SnapshotService.cs
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using AutoMapper.QueryableExtensions;
|
||||
using FilterLists.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace FilterLists.Services.SnapshotService
|
||||
{
|
||||
public class SnapshotService
|
||||
{
|
||||
private readonly FilterListsDbContext dbContext;
|
||||
|
||||
public SnapshotService(FilterListsDbContext dbContext)
|
||||
{
|
||||
this.dbContext = dbContext;
|
||||
}
|
||||
|
||||
//TODO: call via scheduled job
|
||||
public async Task CaptureSnapshotsAsync(int batchSize)
|
||||
{
|
||||
var lists = await GetNextListsToCapture(batchSize);
|
||||
var snapshots = GetSnapshots(lists);
|
||||
await SaveSnapshots(snapshots);
|
||||
}
|
||||
|
||||
private async Task<IEnumerable<FilterListViewUrlDto>> GetNextListsToCapture(int batchSize)
|
||||
{
|
||||
var neverCapturedLists = await GetNeverCapturedLists(batchSize);
|
||||
if (neverCapturedLists.Count == batchSize)
|
||||
return neverCapturedLists;
|
||||
var leastRecentlyCapturedLists = await GetLeastRecentlyCapturedLists(batchSize - neverCapturedLists.Count);
|
||||
return neverCapturedLists.Concat(leastRecentlyCapturedLists);
|
||||
}
|
||||
|
||||
private async Task<List<FilterListViewUrlDto>> GetNeverCapturedLists(int batchSize)
|
||||
{
|
||||
return await dbContext.FilterLists
|
||||
.Where(x => x.Snapshots.Count == 0)
|
||||
.Take(batchSize)
|
||||
.ProjectTo<FilterListViewUrlDto>()
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
private async Task<List<FilterListViewUrlDto>> GetLeastRecentlyCapturedLists(int batchSize)
|
||||
{
|
||||
return await dbContext.Snapshots
|
||||
.GroupBy(x => x.FilterList)
|
||||
.Select(x => x.OrderByDescending(y => y.CreatedDateUtc).First())
|
||||
.OrderBy(x => x.CreatedDateUtc)
|
||||
.Select(x => x.FilterList)
|
||||
.Take(batchSize)
|
||||
.ProjectTo<FilterListViewUrlDto>()
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
private IEnumerable<SnapshotDomainEntity> GetSnapshots(IEnumerable<FilterListViewUrlDto> lists)
|
||||
{
|
||||
return lists.Select(list => new SnapshotDomainEntity(dbContext, list));
|
||||
}
|
||||
|
||||
private static async Task SaveSnapshots(IEnumerable<SnapshotDomainEntity> snapshots)
|
||||
{
|
||||
foreach (var snapshot in snapshots)
|
||||
await snapshot.SaveSnapshot();
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue