prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'snap-algo'

Browse source
This commit is contained in:
Collin M. Barrett 2018-08-23 13:19:59 -05:00
commit 3da275a2e7
14 changed files with 823 additions and 189 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

645
src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.Designer.cs generated Normal file
View file

@ -0,0 +1,645 @@
// <auto-generated />
using System;
using FilterLists.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Migrations;
namespace FilterLists.Api.Migrations
{
[DbContext(typeof(FilterListsDbContext))]
[Migration("20180823174733_SimplifySnapshotRuleModel")]
partial class SimplifySnapshotRuleModel
{
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "2.1.2-rtm-30932")
.HasAnnotation("Relational:MaxIdentifierLength", 64);
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
{
b.Property<ushort>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("SMALLINT UNSIGNED")
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
b.Property<bool>("CantSnapshot");
b.Property<string>("ChatUrl")
.HasColumnType("TEXT");
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.Property<string>("Description")
.HasColumnType("TEXT");
b.Property<string>("DescriptionSourceUrl")
.HasColumnType("TEXT");
b.Property<DateTime?>("DiscontinuedDate")
.ValueGeneratedOnAdd()
.HasColumnType("DATE")
.HasDefaultValueSql("NULL");
b.Property<string>("DonateUrl")
.HasColumnType("TEXT");
b.Property<string>("EmailAddress")
.ValueGeneratedOnAdd()
.HasColumnType("VARCHAR(126)")
.HasDefaultValueSql("NULL");
b.Property<string>("ForumUrl")
.HasColumnType("TEXT");
b.Property<string>("HomeUrl")
.HasColumnType("TEXT");
b.Property<string>("IssuesUrl")
.HasColumnType("TEXT");
b.Property<byte>("LicenseId")
.ValueGeneratedOnAdd()
.HasDefaultValue((byte)5);
b.Property<DateTime?>("ModifiedDateUtc")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("VARCHAR(126)");
b.Property<string>("PolicyUrl")
.HasColumnType("TEXT");
b.Property<DateTime?>("PublishedDate")
.ValueGeneratedOnAdd()
.HasColumnType("DATE")
.HasDefaultValueSql("NULL");
b.Property<string>("SubmissionUrl")
.HasColumnType("TEXT");
b.Property<byte?>("SyntaxId");
b.Property<string>("ViewUrl")
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("LicenseId");
b.HasIndex("SyntaxId");
b.ToTable("filterlists");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
{
b.Property<ushort>("FilterListId");
b.Property<ushort>("LanguageId");
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.HasKey("FilterListId", "LanguageId");
b.HasIndex("LanguageId");
b.ToTable("filterlists_languages");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
{
b.Property<ushort>("FilterListId");
b.Property<ushort>("MaintainerId");
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.HasKey("FilterListId", "MaintainerId");
b.HasIndex("MaintainerId");
b.ToTable("filterlists_maintainers");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
{
b.Property<ushort>("FilterListId");
b.Property<byte>("TagId");
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.HasKey("FilterListId", "TagId");
b.HasIndex("TagId");
b.ToTable("filterlists_tags");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
{
b.Property<ushort>("ForkFilterListId");
b.Property<ushort>("UpstreamFilterListId");
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.HasKey("ForkFilterListId", "UpstreamFilterListId");
b.HasIndex("UpstreamFilterListId");
b.ToTable("forks");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
{
b.Property<ushort>("MergeFilterListId");
b.Property<ushort>("UpstreamFilterListId");
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.HasKey("MergeFilterListId", "UpstreamFilterListId");
b.HasIndex("UpstreamFilterListId");
b.ToTable("merges");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
{
b.Property<uint>("SnapshotId");
b.Property<uint>("RuleId");
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.HasKey("SnapshotId", "RuleId");
b.HasIndex("RuleId");
b.ToTable("snapshots_rules");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
{
b.Property<byte>("SyntaxId");
b.Property<byte>("SoftwareId");
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.HasKey("SyntaxId", "SoftwareId");
b.HasIndex("SoftwareId");
b.ToTable("software_syntaxes");
});
modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
{
b.Property<ushort>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("SMALLINT UNSIGNED")
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.Property<string>("Iso6391")
.ValueGeneratedOnAdd()
.HasColumnType("VARCHAR(2)")
.HasDefaultValueSql("NULL");
b.Property<string>("Iso6392")
.ValueGeneratedOnAdd()
.HasColumnType("VARCHAR(3)")
.HasDefaultValueSql("NULL");
b.Property<string>("Iso6392B")
.ValueGeneratedOnAdd()
.HasColumnType("VARCHAR(3)")
.HasDefaultValueSql("NULL");
b.Property<string>("Iso6392T")
.ValueGeneratedOnAdd()
.HasColumnType("VARCHAR(3)")
.HasDefaultValueSql("NULL");
b.Property<string>("Iso6393")
.ValueGeneratedOnAdd()
.HasColumnType("VARCHAR(3)")
.HasDefaultValueSql("NULL");
b.Property<string>("LocalName")
.ValueGeneratedOnAdd()
.HasColumnType("VARCHAR(126)")
.HasDefaultValueSql("NULL");
b.Property<DateTime?>("ModifiedDateUtc")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
b.Property<string>("Name")
.ValueGeneratedOnAdd()
.HasColumnType("VARCHAR(126)")
.HasDefaultValueSql("NULL");
b.HasKey("Id");
b.ToTable("languages");
});
modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
{
b.Property<byte>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("TINYINT UNSIGNED")
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.Property<string>("DescriptionUrl")
.HasColumnType("TEXT");
b.Property<DateTime?>("ModifiedDateUtc")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("VARCHAR(126)");
b.Property<bool>("PermissiveAdaptation");
b.Property<bool>("PermissiveCommercial");
b.HasKey("Id");
b.ToTable("licenses");
});
modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
{
b.Property<ushort>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("SMALLINT UNSIGNED")
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.Property<string>("EmailAddress")
.ValueGeneratedOnAdd()
.HasColumnType("VARCHAR(126)")
.HasDefaultValueSql("NULL");
b.Property<string>("HomeUrl")
.HasColumnType("TEXT");
b.Property<DateTime?>("ModifiedDateUtc")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("VARCHAR(126)");
b.Property<string>("TwitterHandle")
.ValueGeneratedOnAdd()
.HasColumnType("VARCHAR(126)")
.HasDefaultValueSql("NULL");
b.HasKey("Id");
b.ToTable("maintainers");
});
modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
{
b.Property<uint>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INT UNSIGNED")
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.Property<string>("Raw")
.IsRequired()
.HasColumnType("VARCHAR(768)");
b.HasKey("Id");
b.HasIndex("Raw")
.IsUnique();
b.ToTable("rules");
});
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
{
b.Property<uint>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("MEDIUMINT UNSIGNED")
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.Property<ushort>("FilterListId");
b.Property<ushort?>("HttpStatusCode")
.ValueGeneratedOnAdd()
.HasColumnType("SMALLINT UNSIGNED")
.HasDefaultValueSql("NULL");
b.Property<DateTime?>("ModifiedDateUtc")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
b.Property<bool>("WasSuccessful");
b.HasKey("Id");
b.HasIndex("FilterListId");
b.ToTable("snapshots");
});
modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
{
b.Property<byte>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("TINYINT UNSIGNED")
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.Property<string>("DownloadUrl")
.HasColumnType("TEXT");
b.Property<string>("HomeUrl")
.HasColumnType("TEXT");
b.Property<DateTime?>("ModifiedDateUtc")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("VARCHAR(126)");
b.HasKey("Id");
b.ToTable("software");
});
modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
{
b.Property<byte>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("TINYINT UNSIGNED")
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.Property<string>("DefinitionUrl")
.HasColumnType("TEXT");
b.Property<DateTime?>("ModifiedDateUtc")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("VARCHAR(126)");
b.HasKey("Id");
b.ToTable("syntaxes");
});
modelBuilder.Entity("FilterLists.Data.Entities.Tag", b =>
{
b.Property<byte>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("TINYINT UNSIGNED")
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
b.Property<DateTime?>("CreatedDateUtc")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.Property<string>("Description")
.HasColumnType("TEXT");
b.Property<DateTime?>("ModifiedDateUtc")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("VARCHAR(126)");
b.HasKey("Id");
b.ToTable("tags");
});
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
{
b.HasOne("FilterLists.Data.Entities.License", "License")
.WithMany("FilterLists")
.HasForeignKey("LicenseId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
.WithMany("FilterLists")
.HasForeignKey("SyntaxId");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
.WithMany("FilterListLanguages")
.HasForeignKey("FilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Language", "Language")
.WithMany("FilterListLanguages")
.HasForeignKey("LanguageId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
.WithMany("FilterListMaintainers")
.HasForeignKey("FilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
.WithMany("FilterListMaintainers")
.HasForeignKey("MaintainerId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
.WithMany("FilterListTags")
.HasForeignKey("FilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Tag", "Tag")
.WithMany("FilterListTags")
.HasForeignKey("TagId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
.WithMany("ForkFilterLists")
.HasForeignKey("ForkFilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
.WithMany("UpstreamForkFilterLists")
.HasForeignKey("UpstreamFilterListId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
.WithMany("MergeFilterLists")
.HasForeignKey("MergeFilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
.WithMany("UpstreamMergeFilterLists")
.HasForeignKey("UpstreamFilterListId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
{
b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
.WithMany("SnapshotRules")
.HasForeignKey("RuleId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
.WithMany("SnapshotRules")
.HasForeignKey("SnapshotId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
{
b.HasOne("FilterLists.Data.Entities.Software", "Software")
.WithMany("SoftwareSyntaxes")
.HasForeignKey("SoftwareId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
.WithMany("SoftwareSyntaxes")
.HasForeignKey("SyntaxId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
.WithMany("Snapshots")
.HasForeignKey("FilterListId")
.OnDelete(DeleteBehavior.Cascade);
});
#pragma warning restore 612, 618
}
}
}

111
src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.cs Normal file
View file

@ -0,0 +1,111 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
namespace FilterLists.Api.Migrations
{
public partial class SimplifySnapshotRuleModel : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
"FK_snapshots_rules_snapshots_AddedBySnapshotId",
"snapshots_rules");
migrationBuilder.DropForeignKey(
"FK_snapshots_rules_snapshots_RemovedBySnapshotId",
"snapshots_rules");
migrationBuilder.DropIndex(
"IX_snapshots_rules_RemovedBySnapshotId",
"snapshots_rules");
migrationBuilder.DropColumn(
"ModifiedDateUtc",
"snapshots_rules");
migrationBuilder.DropColumn(
"RemovedBySnapshotId",
"snapshots_rules");
migrationBuilder.RenameColumn(
"AddedBySnapshotId",
"snapshots_rules",
"SnapshotId");
migrationBuilder.AlterColumn<ushort>(
"HttpStatusCode",
"snapshots",
"SMALLINT UNSIGNED",
nullable: true,
defaultValueSql: "NULL",
oldClrType: typeof(string),
oldType: "VARCHAR(3)",
oldNullable: true,
oldDefaultValueSql: "NULL");
migrationBuilder.AddForeignKey(
"FK_snapshots_rules_snapshots_SnapshotId",
"snapshots_rules",
"SnapshotId",
"snapshots",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
"FK_snapshots_rules_snapshots_SnapshotId",
"snapshots_rules");
migrationBuilder.RenameColumn(
"SnapshotId",
"snapshots_rules",
"AddedBySnapshotId");
migrationBuilder.AddColumn<DateTime>(
"ModifiedDateUtc",
"snapshots_rules",
"TIMESTAMP",
nullable: false,
defaultValueSql: "current_timestamp() ON UPDATE current_timestamp()");
migrationBuilder.AddColumn<uint>(
"RemovedBySnapshotId",
"snapshots_rules",
nullable: true);
migrationBuilder.AlterColumn<string>(
"HttpStatusCode",
"snapshots",
"VARCHAR(3)",
nullable: true,
defaultValueSql: "NULL",
oldClrType: typeof(ushort),
oldType: "SMALLINT UNSIGNED",
oldNullable: true,
oldDefaultValueSql: "NULL");
migrationBuilder.CreateIndex(
"IX_snapshots_rules_RemovedBySnapshotId",
"snapshots_rules",
"RemovedBySnapshotId");
migrationBuilder.AddForeignKey(
"FK_snapshots_rules_snapshots_AddedBySnapshotId",
"snapshots_rules",
"AddedBySnapshotId",
"snapshots",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
migrationBuilder.AddForeignKey(
"FK_snapshots_rules_snapshots_RemovedBySnapshotId",
"snapshots_rules",
"RemovedBySnapshotId",
"snapshots",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
}
}
}

33
src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
View file

@ -200,7 +200,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
{
b.Property<uint>("AddedBySnapshotId");
b.Property<uint>("SnapshotId");
b.Property<uint>("RuleId");
@ -210,17 +210,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp()");
b.Property<DateTime?>("ModifiedDateUtc")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP")
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
b.Property<uint?>("RemovedBySnapshotId");
b.HasKey("AddedBySnapshotId", "RuleId");
b.HasIndex("RemovedBySnapshotId");
b.HasKey("SnapshotId", "RuleId");
b.HasIndex("RuleId");
@ -421,9 +411,9 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property<ushort>("FilterListId");
b.Property<string>("HttpStatusCode")
b.Property<ushort?>("HttpStatusCode")
.ValueGeneratedOnAdd()
.HasColumnType("VARCHAR(3)")
.HasColumnType("SMALLINT UNSIGNED")
.HasDefaultValueSql("NULL");
b.Property<DateTime?>("ModifiedDateUtc")
@ -616,20 +606,15 @@ protected override void BuildModel(ModelBuilder modelBuilder)
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
{
b.HasOne("FilterLists.Data.Entities.Snapshot", "AddedBySnapshot")
.WithMany("AddedSnapshotRules")
.HasForeignKey("AddedBySnapshotId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Snapshot", "RemovedBySnapshot")
.WithMany("RemovedSnapshotRules")
.HasForeignKey("RemovedBySnapshotId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
.WithMany("SnapshotRules")
.HasForeignKey("RuleId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
.WithMany("SnapshotRules")
.HasForeignKey("SnapshotId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>

11
src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs
View file

@ -1,14 +1,9 @@
using System;
namespace FilterLists.Data.Entities.Junctions
namespace FilterLists.Data.Entities.Junctions
{
public class SnapshotRule : BaseJunctionEntity
{
public DateTime? ModifiedDateUtc { get; set; }
public uint AddedBySnapshotId { get; set; }
public Snapshot AddedBySnapshot { get; set; }
public uint? RemovedBySnapshotId { get; set; }
public Snapshot RemovedBySnapshot { get; set; }
public uint SnapshotId { get; set; }
public Snapshot Snapshot { get; set; }
public uint RuleId { get; set; }
public Rule Rule { get; set; }
}

5
src/FilterLists.Data/Entities/Snapshot.cs
View file

@ -7,9 +7,8 @@ public class Snapshot : BaseEntity
{
public uint FilterListId { get; set; }
public FilterList FilterList { get; set; }
public string HttpStatusCode { get; set; }
public uint? HttpStatusCode { get; set; }
public bool WasSuccessful { get; set; }
public ICollection<SnapshotRule> AddedSnapshotRules { get; set; }
public ICollection<SnapshotRule> RemovedSnapshotRules { get; set; }
public ICollection<SnapshotRule> SnapshotRules { get; set; }
}
}

17
src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs
View file

@ -10,19 +10,10 @@ public override void Configure(EntityTypeBuilder<SnapshotRule> entityTypeBuilder
{
base.Configure(entityTypeBuilder);
entityTypeBuilder.ToTable("snapshots_rules");
entityTypeBuilder.HasKey(x => new {x.AddedBySnapshotId, x.RuleId});
entityTypeBuilder.Property(x => x.ModifiedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAddOrUpdate()
.IsRequired()
.HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
entityTypeBuilder.HasOne(x => x.AddedBySnapshot)
.WithMany(x => x.AddedSnapshotRules)
.HasForeignKey(x => x.AddedBySnapshotId);
entityTypeBuilder.HasOne(x => x.RemovedBySnapshot)
.WithMany(x => x.RemovedSnapshotRules)
.HasForeignKey(x => x.RemovedBySnapshotId)
.OnDelete(DeleteBehavior.Cascade);
entityTypeBuilder.HasKey(x => new {x.SnapshotId, x.RuleId});
entityTypeBuilder.HasOne(x => x.Snapshot)
.WithMany(x => x.SnapshotRules)
.HasForeignKey(x => x.SnapshotId);
entityTypeBuilder.HasOne(x => x.Rule)
.WithMany(x => x.SnapshotRules)
.HasForeignKey(x => x.RuleId);

2
src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
View file

@ -13,7 +13,7 @@ public override void Configure(EntityTypeBuilder<Snapshot> entityTypeBuilder)
entityTypeBuilder.Property(x => x.Id)
.HasColumnType("MEDIUMINT UNSIGNED");
entityTypeBuilder.Property(x => x.HttpStatusCode)
.HasColumnType("VARCHAR(3)")
.HasColumnType("SMALLINT UNSIGNED")
.HasDefaultValueSql("NULL");
}
}

1
src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
View file

@ -33,7 +33,6 @@ public static void AddFilterListsAgentServices(this IServiceCollection services,
o.UseMySql(config.GetConnectionString("FilterListsConnection"),
m => m.MigrationsAssembly("FilterLists.Api")));
services.TryAddScoped<SnapshotService>();
services.TryAddSingleton<EmailService>();
services.AddAutoMapper();
}
}

10
src/FilterLists.Services/Extensions/CollectionExtensions.cs
View file

@ -4,16 +4,6 @@ namespace FilterLists.Services.Extensions
{
public static class CollectionExtensions
{
//https://stackoverflow.com/a/26360010/2343739
public static void AddRange<T>(this ICollection<T> destination, IEnumerable<T> source)
{
if (destination is List<T> list)
list.AddRange(source);
else
foreach (var item in source)
destination.Add(item);
}
public static void AddIfNotNullOrEmpty(this ICollection<string> set, string item)
{
if (!string.IsNullOrEmpty(item))

14
src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs
View file

@ -8,6 +8,7 @@ namespace FilterLists.Services.FilterList.MappingProfiles
[UsedImplicitly]
public class ListDetailsDtoMappingProfile : Profile
{
//TODO: fix UpdatedDate to look at most recent snapshot with changes
public ListDetailsDtoMappingProfile() =>
CreateMap<Data.Entities.FilterList, ListDetailsDto>()
.ForMember(d => d.Languages, c => c.MapFrom(l => l.FilterListLanguages.Select(la => la.Language.Name)))
@ -16,17 +17,14 @@ public ListDetailsDtoMappingProfile() =>
.ForMember(d => d.RuleCount,
c => c.MapFrom(l =>
l.Snapshots.Where(s => s.WasSuccessful)
.SelectMany(sr => sr.AddedSnapshotRules)
.Count() -
l.Snapshots.Where(s => s.WasSuccessful)
.SelectMany(sr => sr.RemovedSnapshotRules)
.Count()))
.OrderByDescending(s => s.CreatedDateUtc)
.FirstOrDefault()
.SnapshotRules.Count))
.ForMember(d => d.UpdatedDate,
c => c.MapFrom(l =>
l.Snapshots.Where(s => s.WasSuccessful)
.Where(s => s.AddedSnapshotRules.Count > 0 || s.RemovedSnapshotRules.Count > 0)
.OrderByDescending(s => s.CreatedDateUtc)
.Select(s => s.CreatedDateUtc)
.FirstOrDefault()));
.FirstOrDefault()
.CreatedDateUtc));
}
}

6
src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs
View file

@ -8,6 +8,7 @@ namespace FilterLists.Services.FilterList.MappingProfiles
[UsedImplicitly]
public class ListSummaryDtoMappingProfile : Profile
{
//TODO: fix UpdatedDate to look at most recent snapshot with changes
public ListSummaryDtoMappingProfile() =>
CreateMap<Data.Entities.FilterList, ListSummaryDto>()
.ForMember(d => d.Languages, c => c.MapFrom(l => l.FilterListLanguages.Select(la => la.Language)))
@ -15,9 +16,8 @@ public ListSummaryDtoMappingProfile() =>
.ForMember(d => d.UpdatedDate,
c => c.MapFrom(l =>
l.Snapshots.Where(s => s.WasSuccessful)
.Where(s => s.AddedSnapshotRules.Count > 0 || s.RemovedSnapshotRules.Count > 0)
.OrderByDescending(s => s.CreatedDateUtc)
.Select(s => s.CreatedDateUtc)
.FirstOrDefault()));
.FirstOrDefault()
.CreatedDateUtc));
}
}

118
src/FilterLists.Services/Snapshot/Snapshot.cs
View file

@ -4,11 +4,9 @@
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Data.Entities.Junctions;
using FilterLists.Services.Extensions;
using FilterLists.Services.Snapshot.Models;
using Microsoft.ApplicationInsights;
@ -20,28 +18,24 @@ public class Snapshot
{
private const int BatchSize = 500;
private readonly FilterListsDbContext dbContext;
private readonly EmailService emailService;
private readonly FilterListViewUrlDto list;
private readonly Data.Entities.Snapshot snapEntity;
private readonly TelemetryClient telemetryClient;
private readonly string userAgentString;
private readonly string uaString;
private HashSet<string> lines;
public Snapshot(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list,
string userAgentString)
public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, string uaString)
{
this.dbContext = dbContext;
this.emailService = emailService;
this.list = list;
this.uaString = uaString;
snapEntity = new Data.Entities.Snapshot
{
FilterListId = list.Id,
AddedSnapshotRules = new List<SnapshotRule>()
FilterListId = list.Id
};
this.userAgentString = userAgentString;
telemetryClient = new TelemetryClient();
}
//TODO: add better compliance with Try/Parse pattern (https://stackoverflow.com/q/37810660/2343739)
public async Task TrySaveAsync()
{
await AddSnapEntity();
@ -53,7 +47,7 @@ public async Task TrySaveAsync()
}
catch (Exception e)
{
await TrackException(e);
TrackException(e);
}
}
@ -65,50 +59,42 @@ private async Task AddSnapEntity()
private async Task SaveAsync()
{
using (var transaction = dbContext.Database.BeginTransaction())
await TryGetLines();
if (lines != null)
{
var lines = await TryGetLines();
if (lines != null)
{
await SaveInBatches(lines);
await DedupSnapshotRules();
await SetSuccessful();
}
transaction.Commit();
await SaveInBatches();
await SetSuccessful();
}
}
private async Task<IEnumerable<string>> TryGetLines()
private async Task TryGetLines()
{
try
{
return await GetLines();
await GetLines();
}
catch (HttpRequestException hre)
{
await dbContext.SaveChangesAsync();
await TrackException(hre);
return null;
TrackException(hre);
}
catch (WebException we)
{
snapEntity.HttpStatusCode = ((int)((HttpWebResponse)we.Response).StatusCode).ToString();
snapEntity.HttpStatusCode = (uint)((HttpWebResponse)we.Response).StatusCode;
await dbContext.SaveChangesAsync();
await TrackException(we);
return null;
TrackException(we);
}
}
private async Task<IEnumerable<string>> GetLines()
private async Task GetLines()
{
var lines = new HashSet<string>();
using (var httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgentString);
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(uaString);
var response = await httpClient.GetAsync(list.ViewUrl, HttpCompletionOption.ResponseHeadersRead);
snapEntity.HttpStatusCode = ((int)response.StatusCode).ToString();
snapEntity.HttpStatusCode = (uint)response.StatusCode;
response.EnsureSuccessStatusCode();
lines = new HashSet<string>();
using (var stream = await response.Content.ReadAsStreamAsync())
using (var streamReader = new StreamReader(stream))
{
@ -117,17 +103,15 @@ private async Task<IEnumerable<string>> GetLines()
lines.AddIfNotNullOrEmpty(line.LintLine());
}
}
return lines;
}
private async Task SaveInBatches(IEnumerable<string> lines)
private async Task SaveInBatches()
{
var snapshotBatches = CreateBatches(lines);
await SaveBatches(snapshotBatches);
var snapBatches = CreateBatches();
await SaveBatches(snapBatches);
}
private IEnumerable<SnapshotBatch> CreateBatches(IEnumerable<string> lines) =>
private IEnumerable<SnapshotBatch> CreateBatches() =>
lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity));
private static async Task SaveBatches(IEnumerable<SnapshotBatch> batches)
@ -136,67 +120,13 @@ private static async Task SaveBatches(IEnumerable<SnapshotBatch> batches)
await batch.SaveAsync();
}
private async Task DedupSnapshotRules()
{
var existingSnapshotRules = GetExistingSnapshotRules();
AddRemovedBySnapshots(existingSnapshotRules);
RemoveDuplicateSnapshotRules(existingSnapshotRules);
await dbContext.SaveChangesAsync();
}
private IQueryable<SnapshotRule> GetExistingSnapshotRules() =>
dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot.FilterListId == list.Id &&
sr.AddedBySnapshot != snapEntity &&
sr.RemovedBySnapshot == null);
private void AddRemovedBySnapshots(IQueryable<SnapshotRule> existingSnapshotRules)
{
var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapEntity);
var removedSnapshotRules = existingSnapshotRules.Where(sr => !newSnapshotRules.Any(n => n.Rule == sr.Rule));
removedSnapshotRules.ForEach(sr => sr.RemovedBySnapshot = snapEntity);
}
private void RemoveDuplicateSnapshotRules(IQueryable<SnapshotRule> existingSnapshotRules)
{
var duplicateSnapshotRules = dbContext.SnapshotRules.Where(sr =>
sr.AddedBySnapshot == snapEntity &&
existingSnapshotRules.Any(e =>
e.Rule == sr.Rule && e.RemovedBySnapshot == null));
dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules);
}
private async Task SetSuccessful()
{
snapEntity.WasSuccessful = true;
await dbContext.SaveChangesAsync();
}
private async Task TrackException(Exception e)
{
await SendExceptionEmail(e);
TrackExceptionInApplicationInsights(e);
}
private async Task SendExceptionEmail(Exception e)
{
if (!IsDeploymentInterrupted(e))
{
var msg = new StringBuilder();
msg.AppendLine("FilterListId: " + list.Id);
msg.AppendLine("Exception:");
msg.AppendLine(e.Message);
msg.AppendLine(e.StackTrace);
msg.AppendLine(e.InnerException?.Message);
msg.AppendLine(e.InnerException?.StackTrace);
await emailService.SendEmailAsync("Snapshot Exception", msg.ToString());
}
}
private static bool IsDeploymentInterrupted(Exception e) =>
e.Message.Contains("Failed to read the result set.");
private void TrackExceptionInApplicationInsights(Exception e)
private void TrackException(Exception e)
{
telemetryClient.TrackException(e);
telemetryClient.Flush();

17
src/FilterLists.Services/Snapshot/SnapshotBatch.cs
View file

@ -4,7 +4,6 @@
using FilterLists.Data;
using FilterLists.Data.Entities;
using FilterLists.Data.Entities.Junctions;
using FilterLists.Services.Extensions;
namespace FilterLists.Services.Snapshot
{
@ -24,21 +23,13 @@ public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable<string> lines,
public async Task SaveAsync()
{
var existingRules = GetExistingRules();
var newRules = CreateNewRules(existingRules);
var existingRules = dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule);
var newRules = lines.Except(existingRules.Select(r => r.Raw)).Select(l => new Rule { Raw = l }).ToList();
dbContext.Rules.AddRange(newRules);
var rules = existingRules.Concat(newRules);
AddSnapshotRules(rules);
var snapshotRules = rules.Select(r => new SnapshotRule { Rule = r });
snapEntity.SnapshotRules = snapshotRules.ToList();
await dbContext.SaveChangesAsync();
}
private IQueryable<Rule> GetExistingRules() =>
dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule);
private List<Rule> CreateNewRules(IQueryable<Rule> existingRules) =>
lines.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList();
private void AddSnapshotRules(IQueryable<Rule> rules) =>
snapEntity.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r}));
}
}

22
src/FilterLists.Services/Snapshot/SnapshotService.cs
View file

@ -12,19 +12,19 @@ namespace FilterLists.Services.Snapshot
{
public class SnapshotService : Service
{
private readonly EmailService emailService;
private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1);
public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig,
EmailService emailService)
: base(dbContext, mapConfig) => this.emailService = emailService;
public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig)
: base(dbContext, mapConfig)
{
}
public async Task CaptureAsync(int batchSize)
{
var lists = await GetListsToCapture(batchSize);
var uaString = await UserAgentService.GetMostPopularString();
var snapshots = CreateSnapshots(lists, uaString);
await SaveSnapshots(snapshots);
var snaps = CreateSnaps(lists, uaString);
await SaveSnaps(snaps);
}
private async Task<IEnumerable<FilterListViewUrlDto>> GetListsToCapture(int batchSize) =>
@ -49,13 +49,13 @@ await DbContext
.ProjectTo<FilterListViewUrlDto>(MapConfig)
.ToListAsync();
private IEnumerable<Snapshot> CreateSnapshots(IEnumerable<FilterListViewUrlDto> lists, string uaString) =>
lists.Select(l => new Snapshot(DbContext, emailService, l, uaString));
private IEnumerable<Snapshot> CreateSnaps(IEnumerable<FilterListViewUrlDto> lists, string uaString) =>
lists.Select(l => new Snapshot(DbContext, l, uaString));
private static async Task SaveSnapshots(IEnumerable<Snapshot> snapshots)
private static async Task SaveSnaps(IEnumerable<Snapshot> snaps)
{
foreach (var snapshot in snapshots)
await snapshot.TrySaveAsync();
foreach (var snap in snaps)
await snap.TrySaveAsync();
}
}
}