diff --git a/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.Designer.cs b/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.Designer.cs new file mode 100644 index 000000000..ef2b3968e --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.Designer.cs @@ -0,0 +1,645 @@ +// + +using System; +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + [DbContext(typeof(FilterListsDbContext))] + [Migration("20180823174733_SimplifySnapshotRuleModel")] + partial class SimplifySnapshotRuleModel + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "2.1.2-rtm-30932") + .HasAnnotation("Relational:MaxIdentifierLength", 64); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CantSnapshot"); + + b.Property("ChatUrl") + .HasColumnType("TEXT"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("DescriptionSourceUrl") + .HasColumnType("TEXT"); + + b.Property("DiscontinuedDate") + .ValueGeneratedOnAdd() + .HasColumnType("DATE") + .HasDefaultValueSql("NULL"); + + b.Property("DonateUrl") + .HasColumnType("TEXT"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ForumUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("IssuesUrl") + .HasColumnType("TEXT"); + + b.Property("LicenseId") + .ValueGeneratedOnAdd() + .HasDefaultValue((byte)5); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PolicyUrl") + .HasColumnType("TEXT"); + + b.Property("PublishedDate") + .ValueGeneratedOnAdd() + .HasColumnType("DATE") + .HasDefaultValueSql("NULL"); + + b.Property("SubmissionUrl") + .HasColumnType("TEXT"); + + b.Property("SyntaxId"); + + b.Property("ViewUrl") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("LicenseId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("filterlists"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.Property("FilterListId"); + + b.Property("LanguageId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "LanguageId"); + + b.HasIndex("LanguageId"); + + b.ToTable("filterlists_languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.Property("FilterListId"); + + b.Property("MaintainerId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "MaintainerId"); + + b.HasIndex("MaintainerId"); + + b.ToTable("filterlists_maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b => + { + b.Property("FilterListId"); + + b.Property("TagId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "TagId"); + + b.HasIndex("TagId"); + + b.ToTable("filterlists_tags"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.Property("ForkFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("ForkFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("forks"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.Property("MergeFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("MergeFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("merges"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.Property("SnapshotId"); + + b.Property("RuleId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("SnapshotId", "RuleId"); + + b.HasIndex("RuleId"); + + b.ToTable("snapshots_rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.Property("SyntaxId"); + + b.Property("SoftwareId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("SyntaxId", "SoftwareId"); + + b.HasIndex("SoftwareId"); + + b.ToTable("software_syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Language", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Iso6391") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(2)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392B") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392T") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6393") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("LocalName") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.License", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DescriptionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PermissiveAdaptation"); + + b.Property("PermissiveCommercial"); + + b.HasKey("Id"); + + b.ToTable("licenses"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("TwitterHandle") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Rule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Raw") + .IsRequired() + .HasColumnType("VARCHAR(768)"); + + b.HasKey("Id"); + + b.HasIndex("Raw") + .IsUnique(); + + b.ToTable("rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("MEDIUMINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("FilterListId"); + + b.Property("HttpStatusCode") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("WasSuccessful"); + + b.HasKey("Id"); + + b.HasIndex("FilterListId"); + + b.ToTable("snapshots"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DownloadUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("software"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DefinitionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Tag", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("tags"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.HasOne("FilterLists.Data.Entities.License", "License") + .WithMany("FilterLists") + .HasForeignKey("LicenseId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("FilterLists") + .HasForeignKey("SyntaxId"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListLanguages") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Language", "Language") + .WithMany("FilterListLanguages") + .HasForeignKey("LanguageId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListMaintainers") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer") + .WithMany("FilterListMaintainers") + .HasForeignKey("MaintainerId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListTags") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Tag", "Tag") + .WithMany("FilterListTags") + .HasForeignKey("TagId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") + .WithMany("ForkFilterLists") + .HasForeignKey("ForkFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamForkFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList") + .WithMany("MergeFilterLists") + .HasForeignKey("MergeFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamMergeFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.HasOne("FilterLists.Data.Entities.Rule", "Rule") + .WithMany("SnapshotRules") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot") + .WithMany("SnapshotRules") + .HasForeignKey("SnapshotId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.HasOne("FilterLists.Data.Entities.Software", "Software") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SoftwareId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SyntaxId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("Snapshots") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.cs b/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.cs new file mode 100644 index 000000000..17ca42463 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180823174733_SimplifySnapshotRuleModel.cs @@ -0,0 +1,111 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + public partial class SimplifySnapshotRuleModel : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + "FK_snapshots_rules_snapshots_AddedBySnapshotId", + "snapshots_rules"); + + migrationBuilder.DropForeignKey( + "FK_snapshots_rules_snapshots_RemovedBySnapshotId", + "snapshots_rules"); + + migrationBuilder.DropIndex( + "IX_snapshots_rules_RemovedBySnapshotId", + "snapshots_rules"); + + migrationBuilder.DropColumn( + "ModifiedDateUtc", + "snapshots_rules"); + + migrationBuilder.DropColumn( + "RemovedBySnapshotId", + "snapshots_rules"); + + migrationBuilder.RenameColumn( + "AddedBySnapshotId", + "snapshots_rules", + "SnapshotId"); + + migrationBuilder.AlterColumn( + "HttpStatusCode", + "snapshots", + "SMALLINT UNSIGNED", + nullable: true, + defaultValueSql: "NULL", + oldClrType: typeof(string), + oldType: "VARCHAR(3)", + oldNullable: true, + oldDefaultValueSql: "NULL"); + + migrationBuilder.AddForeignKey( + "FK_snapshots_rules_snapshots_SnapshotId", + "snapshots_rules", + "SnapshotId", + "snapshots", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + "FK_snapshots_rules_snapshots_SnapshotId", + "snapshots_rules"); + + migrationBuilder.RenameColumn( + "SnapshotId", + "snapshots_rules", + "AddedBySnapshotId"); + + migrationBuilder.AddColumn( + "ModifiedDateUtc", + "snapshots_rules", + "TIMESTAMP", + nullable: false, + defaultValueSql: "current_timestamp() ON UPDATE current_timestamp()"); + + migrationBuilder.AddColumn( + "RemovedBySnapshotId", + "snapshots_rules", + nullable: true); + + migrationBuilder.AlterColumn( + "HttpStatusCode", + "snapshots", + "VARCHAR(3)", + nullable: true, + defaultValueSql: "NULL", + oldClrType: typeof(ushort), + oldType: "SMALLINT UNSIGNED", + oldNullable: true, + oldDefaultValueSql: "NULL"); + + migrationBuilder.CreateIndex( + "IX_snapshots_rules_RemovedBySnapshotId", + "snapshots_rules", + "RemovedBySnapshotId"); + + migrationBuilder.AddForeignKey( + "FK_snapshots_rules_snapshots_AddedBySnapshotId", + "snapshots_rules", + "AddedBySnapshotId", + "snapshots", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + + migrationBuilder.AddForeignKey( + "FK_snapshots_rules_snapshots_RemovedBySnapshotId", + "snapshots_rules", + "RemovedBySnapshotId", + "snapshots", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs index da65dc322..971c63ce9 100644 --- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs +++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs @@ -200,7 +200,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => { - b.Property("AddedBySnapshotId"); + b.Property("SnapshotId"); b.Property("RuleId"); @@ -210,17 +210,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("TIMESTAMP") .HasDefaultValueSql("current_timestamp()"); - b.Property("ModifiedDateUtc") - .IsRequired() - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("TIMESTAMP") - .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); - - b.Property("RemovedBySnapshotId"); - - b.HasKey("AddedBySnapshotId", "RuleId"); - - b.HasIndex("RemovedBySnapshotId"); + b.HasKey("SnapshotId", "RuleId"); b.HasIndex("RuleId"); @@ -421,9 +411,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("FilterListId"); - b.Property("HttpStatusCode") + b.Property("HttpStatusCode") .ValueGeneratedOnAdd() - .HasColumnType("VARCHAR(3)") + .HasColumnType("SMALLINT UNSIGNED") .HasDefaultValueSql("NULL"); b.Property("ModifiedDateUtc") @@ -616,20 +606,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => { - b.HasOne("FilterLists.Data.Entities.Snapshot", "AddedBySnapshot") - .WithMany("AddedSnapshotRules") - .HasForeignKey("AddedBySnapshotId") - .OnDelete(DeleteBehavior.Cascade); - - b.HasOne("FilterLists.Data.Entities.Snapshot", "RemovedBySnapshot") - .WithMany("RemovedSnapshotRules") - .HasForeignKey("RemovedBySnapshotId") - .OnDelete(DeleteBehavior.Cascade); - b.HasOne("FilterLists.Data.Entities.Rule", "Rule") .WithMany("SnapshotRules") .HasForeignKey("RuleId") .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot") + .WithMany("SnapshotRules") + .HasForeignKey("SnapshotId") + .OnDelete(DeleteBehavior.Cascade); }); modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => diff --git a/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs b/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs index 4bc330ecf..088226aa7 100644 --- a/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs +++ b/src/FilterLists.Data/Entities/Junctions/SnapshotRule.cs @@ -1,14 +1,9 @@ -using System; - -namespace FilterLists.Data.Entities.Junctions +namespace FilterLists.Data.Entities.Junctions { public class SnapshotRule : BaseJunctionEntity { - public DateTime? ModifiedDateUtc { get; set; } - public uint AddedBySnapshotId { get; set; } - public Snapshot AddedBySnapshot { get; set; } - public uint? RemovedBySnapshotId { get; set; } - public Snapshot RemovedBySnapshot { get; set; } + public uint SnapshotId { get; set; } + public Snapshot Snapshot { get; set; } public uint RuleId { get; set; } public Rule Rule { get; set; } } diff --git a/src/FilterLists.Data/Entities/Snapshot.cs b/src/FilterLists.Data/Entities/Snapshot.cs index 74702a8cb..b151b2c23 100644 --- a/src/FilterLists.Data/Entities/Snapshot.cs +++ b/src/FilterLists.Data/Entities/Snapshot.cs @@ -7,9 +7,8 @@ public class Snapshot : BaseEntity { public uint FilterListId { get; set; } public FilterList FilterList { get; set; } - public string HttpStatusCode { get; set; } + public uint? HttpStatusCode { get; set; } public bool WasSuccessful { get; set; } - public ICollection AddedSnapshotRules { get; set; } - public ICollection RemovedSnapshotRules { get; set; } + public ICollection SnapshotRules { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs index 68088f440..52df04c6a 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/Junctions/SnapshotRuleTypeConfiguration.cs @@ -10,19 +10,10 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder { base.Configure(entityTypeBuilder); entityTypeBuilder.ToTable("snapshots_rules"); - entityTypeBuilder.HasKey(x => new {x.AddedBySnapshotId, x.RuleId}); - entityTypeBuilder.Property(x => x.ModifiedDateUtc) - .HasColumnType("TIMESTAMP") - .ValueGeneratedOnAddOrUpdate() - .IsRequired() - .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); - entityTypeBuilder.HasOne(x => x.AddedBySnapshot) - .WithMany(x => x.AddedSnapshotRules) - .HasForeignKey(x => x.AddedBySnapshotId); - entityTypeBuilder.HasOne(x => x.RemovedBySnapshot) - .WithMany(x => x.RemovedSnapshotRules) - .HasForeignKey(x => x.RemovedBySnapshotId) - .OnDelete(DeleteBehavior.Cascade); + entityTypeBuilder.HasKey(x => new {x.SnapshotId, x.RuleId}); + entityTypeBuilder.HasOne(x => x.Snapshot) + .WithMany(x => x.SnapshotRules) + .HasForeignKey(x => x.SnapshotId); entityTypeBuilder.HasOne(x => x.Rule) .WithMany(x => x.SnapshotRules) .HasForeignKey(x => x.RuleId); diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs index 946c4a730..924b1ca02 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs @@ -13,7 +13,7 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) entityTypeBuilder.Property(x => x.Id) .HasColumnType("MEDIUMINT UNSIGNED"); entityTypeBuilder.Property(x => x.HttpStatusCode) - .HasColumnType("VARCHAR(3)") + .HasColumnType("SMALLINT UNSIGNED") .HasDefaultValueSql("NULL"); } } diff --git a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs index 769134cb7..ee93937d8 100644 --- a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs +++ b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs @@ -33,7 +33,6 @@ public static void AddFilterListsAgentServices(this IServiceCollection services, o.UseMySql(config.GetConnectionString("FilterListsConnection"), m => m.MigrationsAssembly("FilterLists.Api"))); services.TryAddScoped(); - services.TryAddSingleton(); services.AddAutoMapper(); } } diff --git a/src/FilterLists.Services/Extensions/CollectionExtensions.cs b/src/FilterLists.Services/Extensions/CollectionExtensions.cs index 68c6042a1..2c65a12c0 100644 --- a/src/FilterLists.Services/Extensions/CollectionExtensions.cs +++ b/src/FilterLists.Services/Extensions/CollectionExtensions.cs @@ -4,16 +4,6 @@ namespace FilterLists.Services.Extensions { public static class CollectionExtensions { - //https://stackoverflow.com/a/26360010/2343739 - public static void AddRange(this ICollection destination, IEnumerable source) - { - if (destination is List list) - list.AddRange(source); - else - foreach (var item in source) - destination.Add(item); - } - public static void AddIfNotNullOrEmpty(this ICollection set, string item) { if (!string.IsNullOrEmpty(item)) diff --git a/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs b/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs index ea54d1e0f..12b92acf0 100644 --- a/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs +++ b/src/FilterLists.Services/FilterList/MappingProfiles/ListDetailsDtoMappingProfile.cs @@ -8,6 +8,7 @@ namespace FilterLists.Services.FilterList.MappingProfiles [UsedImplicitly] public class ListDetailsDtoMappingProfile : Profile { + //TODO: fix UpdatedDate to look at most recent snapshot with changes public ListDetailsDtoMappingProfile() => CreateMap() .ForMember(d => d.Languages, c => c.MapFrom(l => l.FilterListLanguages.Select(la => la.Language.Name))) @@ -16,17 +17,14 @@ public ListDetailsDtoMappingProfile() => .ForMember(d => d.RuleCount, c => c.MapFrom(l => l.Snapshots.Where(s => s.WasSuccessful) - .SelectMany(sr => sr.AddedSnapshotRules) - .Count() - - l.Snapshots.Where(s => s.WasSuccessful) - .SelectMany(sr => sr.RemovedSnapshotRules) - .Count())) + .OrderByDescending(s => s.CreatedDateUtc) + .FirstOrDefault() + .SnapshotRules.Count)) .ForMember(d => d.UpdatedDate, c => c.MapFrom(l => l.Snapshots.Where(s => s.WasSuccessful) - .Where(s => s.AddedSnapshotRules.Count > 0 || s.RemovedSnapshotRules.Count > 0) .OrderByDescending(s => s.CreatedDateUtc) - .Select(s => s.CreatedDateUtc) - .FirstOrDefault())); + .FirstOrDefault() + .CreatedDateUtc)); } } \ No newline at end of file diff --git a/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs b/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs index 0c9918aaf..9c1a40a1d 100644 --- a/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs +++ b/src/FilterLists.Services/FilterList/MappingProfiles/ListSummaryDtoMappingProfile.cs @@ -8,6 +8,7 @@ namespace FilterLists.Services.FilterList.MappingProfiles [UsedImplicitly] public class ListSummaryDtoMappingProfile : Profile { + //TODO: fix UpdatedDate to look at most recent snapshot with changes public ListSummaryDtoMappingProfile() => CreateMap() .ForMember(d => d.Languages, c => c.MapFrom(l => l.FilterListLanguages.Select(la => la.Language))) @@ -15,9 +16,8 @@ public ListSummaryDtoMappingProfile() => .ForMember(d => d.UpdatedDate, c => c.MapFrom(l => l.Snapshots.Where(s => s.WasSuccessful) - .Where(s => s.AddedSnapshotRules.Count > 0 || s.RemovedSnapshotRules.Count > 0) .OrderByDescending(s => s.CreatedDateUtc) - .Select(s => s.CreatedDateUtc) - .FirstOrDefault())); + .FirstOrDefault() + .CreatedDateUtc)); } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 8b1f16923..6340f5079 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -4,11 +4,9 @@ using System.Linq; using System.Net; using System.Net.Http; -using System.Text; using System.Threading; using System.Threading.Tasks; using FilterLists.Data; -using FilterLists.Data.Entities.Junctions; using FilterLists.Services.Extensions; using FilterLists.Services.Snapshot.Models; using Microsoft.ApplicationInsights; @@ -20,28 +18,24 @@ public class Snapshot { private const int BatchSize = 500; private readonly FilterListsDbContext dbContext; - private readonly EmailService emailService; private readonly FilterListViewUrlDto list; private readonly Data.Entities.Snapshot snapEntity; private readonly TelemetryClient telemetryClient; - private readonly string userAgentString; + private readonly string uaString; + private HashSet lines; - public Snapshot(FilterListsDbContext dbContext, EmailService emailService, FilterListViewUrlDto list, - string userAgentString) + public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, string uaString) { this.dbContext = dbContext; - this.emailService = emailService; this.list = list; + this.uaString = uaString; snapEntity = new Data.Entities.Snapshot { - FilterListId = list.Id, - AddedSnapshotRules = new List() + FilterListId = list.Id }; - this.userAgentString = userAgentString; telemetryClient = new TelemetryClient(); } - //TODO: add better compliance with Try/Parse pattern (https://stackoverflow.com/q/37810660/2343739) public async Task TrySaveAsync() { await AddSnapEntity(); @@ -53,7 +47,7 @@ public async Task TrySaveAsync() } catch (Exception e) { - await TrackException(e); + TrackException(e); } } @@ -65,50 +59,42 @@ private async Task AddSnapEntity() private async Task SaveAsync() { - using (var transaction = dbContext.Database.BeginTransaction()) + await TryGetLines(); + if (lines != null) { - var lines = await TryGetLines(); - if (lines != null) - { - await SaveInBatches(lines); - await DedupSnapshotRules(); - await SetSuccessful(); - } - - transaction.Commit(); + await SaveInBatches(); + await SetSuccessful(); } } - private async Task> TryGetLines() + private async Task TryGetLines() { try { - return await GetLines(); + await GetLines(); } catch (HttpRequestException hre) { await dbContext.SaveChangesAsync(); - await TrackException(hre); - return null; + TrackException(hre); } catch (WebException we) { - snapEntity.HttpStatusCode = ((int)((HttpWebResponse)we.Response).StatusCode).ToString(); + snapEntity.HttpStatusCode = (uint)((HttpWebResponse)we.Response).StatusCode; await dbContext.SaveChangesAsync(); - await TrackException(we); - return null; + TrackException(we); } } - private async Task> GetLines() + private async Task GetLines() { - var lines = new HashSet(); using (var httpClient = new HttpClient()) { - httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgentString); + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(uaString); var response = await httpClient.GetAsync(list.ViewUrl, HttpCompletionOption.ResponseHeadersRead); - snapEntity.HttpStatusCode = ((int)response.StatusCode).ToString(); + snapEntity.HttpStatusCode = (uint)response.StatusCode; response.EnsureSuccessStatusCode(); + lines = new HashSet(); using (var stream = await response.Content.ReadAsStreamAsync()) using (var streamReader = new StreamReader(stream)) { @@ -117,17 +103,15 @@ private async Task> GetLines() lines.AddIfNotNullOrEmpty(line.LintLine()); } } - - return lines; } - private async Task SaveInBatches(IEnumerable lines) + private async Task SaveInBatches() { - var snapshotBatches = CreateBatches(lines); - await SaveBatches(snapshotBatches); + var snapBatches = CreateBatches(); + await SaveBatches(snapBatches); } - private IEnumerable CreateBatches(IEnumerable lines) => + private IEnumerable CreateBatches() => lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity)); private static async Task SaveBatches(IEnumerable batches) @@ -136,67 +120,13 @@ private static async Task SaveBatches(IEnumerable batches) await batch.SaveAsync(); } - private async Task DedupSnapshotRules() - { - var existingSnapshotRules = GetExistingSnapshotRules(); - AddRemovedBySnapshots(existingSnapshotRules); - RemoveDuplicateSnapshotRules(existingSnapshotRules); - await dbContext.SaveChangesAsync(); - } - - private IQueryable GetExistingSnapshotRules() => - dbContext.SnapshotRules.Where(sr => - sr.AddedBySnapshot.FilterListId == list.Id && - sr.AddedBySnapshot != snapEntity && - sr.RemovedBySnapshot == null); - - private void AddRemovedBySnapshots(IQueryable existingSnapshotRules) - { - var newSnapshotRules = dbContext.SnapshotRules.Where(sr => sr.AddedBySnapshot == snapEntity); - var removedSnapshotRules = existingSnapshotRules.Where(sr => !newSnapshotRules.Any(n => n.Rule == sr.Rule)); - removedSnapshotRules.ForEach(sr => sr.RemovedBySnapshot = snapEntity); - } - - private void RemoveDuplicateSnapshotRules(IQueryable existingSnapshotRules) - { - var duplicateSnapshotRules = dbContext.SnapshotRules.Where(sr => - sr.AddedBySnapshot == snapEntity && - existingSnapshotRules.Any(e => - e.Rule == sr.Rule && e.RemovedBySnapshot == null)); - dbContext.SnapshotRules.RemoveRange(duplicateSnapshotRules); - } - private async Task SetSuccessful() { snapEntity.WasSuccessful = true; await dbContext.SaveChangesAsync(); } - private async Task TrackException(Exception e) - { - await SendExceptionEmail(e); - TrackExceptionInApplicationInsights(e); - } - - private async Task SendExceptionEmail(Exception e) - { - if (!IsDeploymentInterrupted(e)) - { - var msg = new StringBuilder(); - msg.AppendLine("FilterListId: " + list.Id); - msg.AppendLine("Exception:"); - msg.AppendLine(e.Message); - msg.AppendLine(e.StackTrace); - msg.AppendLine(e.InnerException?.Message); - msg.AppendLine(e.InnerException?.StackTrace); - await emailService.SendEmailAsync("Snapshot Exception", msg.ToString()); - } - } - - private static bool IsDeploymentInterrupted(Exception e) => - e.Message.Contains("Failed to read the result set."); - - private void TrackExceptionInApplicationInsights(Exception e) + private void TrackException(Exception e) { telemetryClient.TrackException(e); telemetryClient.Flush(); diff --git a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs index a1af7099e..2581932f1 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotBatch.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotBatch.cs @@ -4,7 +4,6 @@ using FilterLists.Data; using FilterLists.Data.Entities; using FilterLists.Data.Entities.Junctions; -using FilterLists.Services.Extensions; namespace FilterLists.Services.Snapshot { @@ -24,21 +23,13 @@ public SnapshotBatch(FilterListsDbContext dbContext, IEnumerable lines, public async Task SaveAsync() { - var existingRules = GetExistingRules(); - var newRules = CreateNewRules(existingRules); + var existingRules = dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule); + var newRules = lines.Except(existingRules.Select(r => r.Raw)).Select(l => new Rule { Raw = l }).ToList(); dbContext.Rules.AddRange(newRules); var rules = existingRules.Concat(newRules); - AddSnapshotRules(rules); + var snapshotRules = rules.Select(r => new SnapshotRule { Rule = r }); + snapEntity.SnapshotRules = snapshotRules.ToList(); await dbContext.SaveChangesAsync(); } - - private IQueryable GetExistingRules() => - dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule); - - private List CreateNewRules(IQueryable existingRules) => - lines.Except(existingRules.Select(r => r.Raw)).Select(r => new Rule {Raw = r}).ToList(); - - private void AddSnapshotRules(IQueryable rules) => - snapEntity.AddedSnapshotRules.AddRange(rules.Select(r => new SnapshotRule {Rule = r})); } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs index 87f43102b..18700a7e1 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotService.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs @@ -12,19 +12,19 @@ namespace FilterLists.Services.Snapshot { public class SnapshotService : Service { - private readonly EmailService emailService; private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1); - public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig, - EmailService emailService) - : base(dbContext, mapConfig) => this.emailService = emailService; + public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig) + : base(dbContext, mapConfig) + { + } public async Task CaptureAsync(int batchSize) { var lists = await GetListsToCapture(batchSize); var uaString = await UserAgentService.GetMostPopularString(); - var snapshots = CreateSnapshots(lists, uaString); - await SaveSnapshots(snapshots); + var snaps = CreateSnaps(lists, uaString); + await SaveSnaps(snaps); } private async Task> GetListsToCapture(int batchSize) => @@ -49,13 +49,13 @@ await DbContext .ProjectTo(MapConfig) .ToListAsync(); - private IEnumerable CreateSnapshots(IEnumerable lists, string uaString) => - lists.Select(l => new Snapshot(DbContext, emailService, l, uaString)); + private IEnumerable CreateSnaps(IEnumerable lists, string uaString) => + lists.Select(l => new Snapshot(DbContext, l, uaString)); - private static async Task SaveSnapshots(IEnumerable snapshots) + private static async Task SaveSnaps(IEnumerable snaps) { - foreach (var snapshot in snapshots) - await snapshot.TrySaveAsync(); + foreach (var snap in snaps) + await snap.TrySaveAsync(); } } } \ No newline at end of file