diff --git a/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.Designer.cs b/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.Designer.cs new file mode 100644 index 000000000..e267a3983 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.Designer.cs @@ -0,0 +1,477 @@ +// +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.EntityFrameworkCore.Storage.Internal; +using System; + +namespace FilterLists.Api.Migrations +{ + [DbContext(typeof(FilterListsDbContext))] + [Migration("20180127014225_partialRemoveModDateAndMore")] + partial class partialRemoveModDateAndMore + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn) + .HasAnnotation("ProductVersion", "2.0.1-rtm-125"); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("DescriptionSourceUrl") + .HasColumnType("TEXT"); + + b.Property("DiscontinuedDate"); + + b.Property("DonateUrl") + .HasColumnType("TEXT"); + + b.Property("EmailAddress") + .HasColumnType("NVARCHAR(126)"); + + b.Property("ForumUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("IssuesUrl") + .HasColumnType("TEXT"); + + b.Property("LicenseId"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("NVARCHAR(126)"); + + b.Property("SubmissionUrl") + .HasColumnType("TEXT"); + + b.Property("SyntaxId"); + + b.Property("ViewUrl") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("LicenseId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("filterlists"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.Property("FilterListId"); + + b.Property("LanguageId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "LanguageId"); + + b.HasIndex("LanguageId"); + + b.ToTable("filterlists_languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.Property("FilterListId"); + + b.Property("MaintainerId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "MaintainerId"); + + b.HasIndex("MaintainerId"); + + b.ToTable("filterlists_maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b => + { + b.Property("FilterListId"); + + b.Property("RuleId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "RuleId"); + + b.HasIndex("RuleId"); + + b.ToTable("filterlists_rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.Property("ForkFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("ForkFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("forks"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.Property("MergeFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("MergeFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("merges"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.Property("SoftwareId"); + + b.Property("SyntaxId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("SoftwareId", "SyntaxId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("software_syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Language", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Iso6391") + .HasColumnType("NVARCHAR(2)"); + + b.Property("Iso6392") + .HasColumnType("NVARCHAR(3)"); + + b.Property("Iso6392B") + .HasColumnType("NVARCHAR(3)"); + + b.Property("Iso6392T") + .HasColumnType("NVARCHAR(3)"); + + b.Property("Iso6393") + .HasColumnType("NVARCHAR(3)"); + + b.Property("LocalName") + .HasColumnType("NVARCHAR(126)"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .HasColumnType("NVARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.License", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DescriptionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("NVARCHAR(126)"); + + b.Property("PermissiveAdaptation"); + + b.Property("PermissiveCommercial"); + + b.HasKey("Id"); + + b.ToTable("licenses"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("EmailAddress") + .HasColumnType("NVARCHAR(126)"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("NVARCHAR(126)"); + + b.Property("TwitterHandle") + .HasColumnType("NVARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Rule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Raw") + .IsRequired() + .HasColumnType("NVARCHAR(2083)"); + + b.HasKey("Id"); + + b.ToTable("rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("Content") + .IsRequired() + .HasColumnType("MEDIUMTEXT"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("FilterListId"); + + b.HasKey("Id"); + + b.HasIndex("FilterListId"); + + b.ToTable("snapshots"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DownloadUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("NVARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("software"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b => + { + b.Property("Id") + .ValueGeneratedOnAdd(); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DefinitionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("NVARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.HasOne("FilterLists.Data.Entities.License") + .WithMany("FilterLists") + .HasForeignKey("LicenseId"); + + b.HasOne("FilterLists.Data.Entities.Syntax") + .WithMany("FilterLists") + .HasForeignKey("SyntaxId"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListLanguages") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Language", "Language") + .WithMany("FilterListLanguages") + .HasForeignKey("LanguageId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListMaintainers") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer") + .WithMany("FilterListMaintainers") + .HasForeignKey("MaintainerId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListRules") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Rule", "Rule") + .WithMany("FilterListRules") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") + .WithMany() + .HasForeignKey("ForkFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany() + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList") + .WithMany() + .HasForeignKey("MergeFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany() + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.HasOne("FilterLists.Data.Entities.Software", "Software") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SoftwareId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SyntaxId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList") + .WithMany("Snapshots") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.cs b/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.cs new file mode 100644 index 000000000..a69c9b180 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.cs @@ -0,0 +1,137 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using System; +using System.Collections.Generic; + +namespace FilterLists.Api.Migrations +{ + public partial class partialRemoveModDateAndMore : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "ModifiedDateUtc", + table: "software_syntaxes"); + + migrationBuilder.DropColumn( + name: "ModifiedDateUtc", + table: "snapshots"); + + migrationBuilder.DropColumn( + name: "ModifiedDateUtc", + table: "rules"); + + migrationBuilder.DropColumn( + name: "ModifiedDateUtc", + table: "merges"); + + migrationBuilder.DropColumn( + name: "ModifiedDateUtc", + table: "forks"); + + migrationBuilder.DropColumn( + name: "ModifiedDateUtc", + table: "filterlists_rules"); + + migrationBuilder.DropColumn( + name: "ModifiedDateUtc", + table: "filterlists_maintainers"); + + migrationBuilder.DropColumn( + name: "ModifiedDateUtc", + table: "filterlists_languages"); + + migrationBuilder.AlterColumn( + name: "Raw", + table: "rules", + type: "NVARCHAR(2083)", + nullable: false, + oldClrType: typeof(string), + oldType: "NVARCHAR(2083)", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "ViewUrl", + table: "filterlists", + type: "TEXT", + nullable: false, + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "ModifiedDateUtc", + table: "software_syntaxes", + type: "TIMESTAMP", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AddColumn( + name: "ModifiedDateUtc", + table: "snapshots", + type: "TIMESTAMP", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AlterColumn( + name: "Raw", + table: "rules", + type: "NVARCHAR(2083)", + nullable: true, + oldClrType: typeof(string), + oldType: "NVARCHAR(2083)"); + + migrationBuilder.AddColumn( + name: "ModifiedDateUtc", + table: "rules", + type: "TIMESTAMP", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AddColumn( + name: "ModifiedDateUtc", + table: "merges", + type: "TIMESTAMP", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AddColumn( + name: "ModifiedDateUtc", + table: "forks", + type: "TIMESTAMP", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AddColumn( + name: "ModifiedDateUtc", + table: "filterlists_rules", + type: "TIMESTAMP", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AddColumn( + name: "ModifiedDateUtc", + table: "filterlists_maintainers", + type: "TIMESTAMP", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AddColumn( + name: "ModifiedDateUtc", + table: "filterlists_languages", + type: "TIMESTAMP", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AlterColumn( + name: "ViewUrl", + table: "filterlists", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + } + } +} diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs index a903bd3f5..fd633c29d 100644 --- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs +++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs @@ -68,6 +68,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("SyntaxId"); b.Property("ViewUrl") + .IsRequired() .HasColumnType("TEXT"); b.HasKey("Id"); @@ -89,10 +90,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .ValueGeneratedOnAdd() .HasColumnType("TIMESTAMP"); - b.Property("ModifiedDateUtc") - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("TIMESTAMP"); - b.HasKey("FilterListId", "LanguageId"); b.HasIndex("LanguageId"); @@ -110,10 +107,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .ValueGeneratedOnAdd() .HasColumnType("TIMESTAMP"); - b.Property("ModifiedDateUtc") - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("TIMESTAMP"); - b.HasKey("FilterListId", "MaintainerId"); b.HasIndex("MaintainerId"); @@ -131,10 +124,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .ValueGeneratedOnAdd() .HasColumnType("TIMESTAMP"); - b.Property("ModifiedDateUtc") - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("TIMESTAMP"); - b.HasKey("FilterListId", "RuleId"); b.HasIndex("RuleId"); @@ -152,10 +141,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .ValueGeneratedOnAdd() .HasColumnType("TIMESTAMP"); - b.Property("ModifiedDateUtc") - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("TIMESTAMP"); - b.HasKey("ForkFilterListId", "UpstreamFilterListId"); b.HasIndex("UpstreamFilterListId"); @@ -173,10 +158,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .ValueGeneratedOnAdd() .HasColumnType("TIMESTAMP"); - b.Property("ModifiedDateUtc") - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("TIMESTAMP"); - b.HasKey("MergeFilterListId", "UpstreamFilterListId"); b.HasIndex("UpstreamFilterListId"); @@ -194,10 +175,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .ValueGeneratedOnAdd() .HasColumnType("TIMESTAMP"); - b.Property("ModifiedDateUtc") - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("TIMESTAMP"); - b.HasKey("SoftwareId", "SyntaxId"); b.HasIndex("SyntaxId"); @@ -313,11 +290,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) .ValueGeneratedOnAdd() .HasColumnType("TIMESTAMP"); - b.Property("ModifiedDateUtc") - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("TIMESTAMP"); - b.Property("Raw") + .IsRequired() .HasColumnType("NVARCHAR(2083)"); b.HasKey("Id"); @@ -340,10 +314,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("FilterListId"); - b.Property("ModifiedDateUtc") - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("TIMESTAMP"); - b.HasKey("Id"); b.HasIndex("FilterListId"); diff --git a/src/FilterLists.Data/Entities/BaseEntity.cs b/src/FilterLists.Data/Entities/BaseEntity.cs index c7bfba6e6..10c6c1e5c 100644 --- a/src/FilterLists.Data/Entities/BaseEntity.cs +++ b/src/FilterLists.Data/Entities/BaseEntity.cs @@ -6,6 +6,5 @@ public abstract class BaseEntity { public int Id { get; set; } public DateTime CreatedDateUtc { get; set; } - public DateTime ModifiedDateUtc { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/Entities/FilterList.cs b/src/FilterLists.Data/Entities/FilterList.cs index 6453a477b..3d43c279a 100644 --- a/src/FilterLists.Data/Entities/FilterList.cs +++ b/src/FilterLists.Data/Entities/FilterList.cs @@ -6,6 +6,7 @@ namespace FilterLists.Data.Entities { public class FilterList : BaseEntity { + public DateTime ModifiedDateUtc { get; set; } public string Description { get; set; } public string DescriptionSourceUrl { get; set; } public DateTime? DiscontinuedDate { get; set; } diff --git a/src/FilterLists.Data/Entities/Language.cs b/src/FilterLists.Data/Entities/Language.cs index 178aca1ae..d8b0760f7 100644 --- a/src/FilterLists.Data/Entities/Language.cs +++ b/src/FilterLists.Data/Entities/Language.cs @@ -1,10 +1,12 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using FilterLists.Data.Entities.Junctions; namespace FilterLists.Data.Entities { public class Language : BaseEntity { + public DateTime ModifiedDateUtc { get; set; } public ICollection FilterListLanguages { get; set; } public string Iso6391 { get; set; } public string Iso6392 { get; set; } diff --git a/src/FilterLists.Data/Entities/License.cs b/src/FilterLists.Data/Entities/License.cs index fecc22fce..22a16d8b4 100644 --- a/src/FilterLists.Data/Entities/License.cs +++ b/src/FilterLists.Data/Entities/License.cs @@ -1,9 +1,11 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; namespace FilterLists.Data.Entities { public class License : BaseEntity { + public DateTime ModifiedDateUtc { get; set; } public string DescriptionUrl { get; set; } public ICollection FilterLists { get; set; } public string Name { get; set; } diff --git a/src/FilterLists.Data/Entities/Maintainer.cs b/src/FilterLists.Data/Entities/Maintainer.cs index 64b6f46e9..5db1cd6b0 100644 --- a/src/FilterLists.Data/Entities/Maintainer.cs +++ b/src/FilterLists.Data/Entities/Maintainer.cs @@ -1,10 +1,12 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using FilterLists.Data.Entities.Junctions; namespace FilterLists.Data.Entities { public class Maintainer : BaseEntity { + public DateTime ModifiedDateUtc { get; set; } public string EmailAddress { get; set; } public ICollection FilterListMaintainers { get; set; } public string HomeUrl { get; set; } diff --git a/src/FilterLists.Data/Entities/Software.cs b/src/FilterLists.Data/Entities/Software.cs index d3d918697..380297260 100644 --- a/src/FilterLists.Data/Entities/Software.cs +++ b/src/FilterLists.Data/Entities/Software.cs @@ -1,10 +1,12 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using FilterLists.Data.Entities.Junctions; namespace FilterLists.Data.Entities { public class Software : BaseEntity { + public DateTime ModifiedDateUtc { get; set; } public string DownloadUrl { get; set; } public string HomeUrl { get; set; } public string Name { get; set; } diff --git a/src/FilterLists.Data/Entities/Syntax.cs b/src/FilterLists.Data/Entities/Syntax.cs index 7a25dabe3..d7c5e86e4 100644 --- a/src/FilterLists.Data/Entities/Syntax.cs +++ b/src/FilterLists.Data/Entities/Syntax.cs @@ -1,10 +1,12 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using FilterLists.Data.Entities.Junctions; namespace FilterLists.Data.Entities { public class Syntax : BaseEntity { + public DateTime ModifiedDateUtc { get; set; } public string DefinitionUrl { get; set; } public ICollection FilterLists { get; set; } public string Name { get; set; } diff --git a/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityConfiguration.cs index 5e469454f..0fb6e84b6 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityConfiguration.cs @@ -11,9 +11,6 @@ public virtual void Configure(EntityTypeBuilder entityTypeBuilder) entityTypeBuilder.Property(x => x.CreatedDateUtc) .HasColumnType("TIMESTAMP") .ValueGeneratedOnAdd(); - entityTypeBuilder.Property(x => x.ModifiedDateUtc) - .HasColumnType("TIMESTAMP") - .ValueGeneratedOnAddOrUpdate(); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs index 185ca5bd6..8009db6cf 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs @@ -9,6 +9,9 @@ public class FilterListTypeConfiguration : BaseEntityTypeConfiguration entityTypeBuilder) { entityTypeBuilder.ToTable("filterlists"); + entityTypeBuilder.Property(x => x.ModifiedDateUtc) + .HasColumnType("TIMESTAMP") + .ValueGeneratedOnAddOrUpdate(); entityTypeBuilder.Property(x => x.Description) .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.DescriptionSourceUrl) @@ -29,7 +32,8 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) entityTypeBuilder.Property(x => x.SubmissionUrl) .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.ViewUrl) - .HasColumnType("TEXT"); + .HasColumnType("TEXT") + .IsRequired(); base.Configure(entityTypeBuilder); } } diff --git a/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs index a37450983..c2a74ae00 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs @@ -9,6 +9,9 @@ public class LanguageTypeConfiguration : BaseEntityTypeConfiguration public override void Configure(EntityTypeBuilder entityTypeBuilder) { entityTypeBuilder.ToTable("languages"); + entityTypeBuilder.Property(x => x.ModifiedDateUtc) + .HasColumnType("TIMESTAMP") + .ValueGeneratedOnAddOrUpdate(); entityTypeBuilder.Property(x => x.Iso6391) .HasColumnType("NVARCHAR(2)"); entityTypeBuilder.Property(x => x.Iso6392) diff --git a/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs index d12b6b64b..e76bad580 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs @@ -9,6 +9,9 @@ public class LicenseTypeConfiguration : BaseEntityTypeConfiguration public override void Configure(EntityTypeBuilder entityTypeBuilder) { entityTypeBuilder.ToTable("licenses"); + entityTypeBuilder.Property(x => x.ModifiedDateUtc) + .HasColumnType("TIMESTAMP") + .ValueGeneratedOnAddOrUpdate(); entityTypeBuilder.Property(x => x.DescriptionUrl) .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.Name) diff --git a/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs index 95239c3a9..fa5278675 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs @@ -9,6 +9,9 @@ public class MaintainerTypeConfiguration : BaseEntityTypeConfiguration entityTypeBuilder) { entityTypeBuilder.ToTable("maintainers"); + entityTypeBuilder.Property(x => x.ModifiedDateUtc) + .HasColumnType("TIMESTAMP") + .ValueGeneratedOnAddOrUpdate(); entityTypeBuilder.Property(x => x.EmailAddress) .HasColumnType("NVARCHAR(126)"); entityTypeBuilder.Property(x => x.HomeUrl) diff --git a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs index 6af964a04..3f6670de3 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs @@ -10,7 +10,8 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) { entityTypeBuilder.ToTable("rules"); entityTypeBuilder.Property(x => x.Raw) - .HasColumnType("NVARCHAR(2083)"); + .HasColumnType("NVARCHAR(2083)") + .IsRequired(); base.Configure(entityTypeBuilder); } } diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs index 65e50fde2..2582c927e 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs @@ -12,7 +12,6 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) entityTypeBuilder.Property(x => x.Content) .HasColumnType("MEDIUMTEXT") .IsRequired(); - entityTypeBuilder.Ignore(x => x.ModifiedDateUtc); base.Configure(entityTypeBuilder); } } diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs index fdef874a2..f0026e115 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs @@ -9,6 +9,9 @@ public class SoftwareTypeConfiguration : BaseEntityTypeConfiguration public override void Configure(EntityTypeBuilder entityTypeBuilder) { entityTypeBuilder.ToTable("software"); + entityTypeBuilder.Property(x => x.ModifiedDateUtc) + .HasColumnType("TIMESTAMP") + .ValueGeneratedOnAddOrUpdate(); entityTypeBuilder.Property(x => x.DownloadUrl) .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.HomeUrl) diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs index 7be00ce5c..6ca7c03d6 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs @@ -9,6 +9,9 @@ public class SyntaxTypeConfiguration : BaseEntityTypeConfiguration public override void Configure(EntityTypeBuilder entityTypeBuilder) { entityTypeBuilder.ToTable("syntaxes"); + entityTypeBuilder.Property(x => x.ModifiedDateUtc) + .HasColumnType("TIMESTAMP") + .ValueGeneratedOnAddOrUpdate(); entityTypeBuilder.Property(x => x.DefinitionUrl) .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.Name) diff --git a/src/FilterLists.Services/Services/ScrapeService.cs b/src/FilterLists.Services/Services/ScrapeService.cs index 1085e641a..65f5bbcef 100644 --- a/src/FilterLists.Services/Services/ScrapeService.cs +++ b/src/FilterLists.Services/Services/ScrapeService.cs @@ -1,8 +1,11 @@ using System; +using System.Linq; using System.Net.Http; using System.Threading.Tasks; using FilterLists.Data; using FilterLists.Data.Entities; +using FilterLists.Data.Entities.Junctions; +using Microsoft.EntityFrameworkCore; namespace FilterLists.Services.Services { @@ -17,32 +20,25 @@ public ScrapeService(FilterListsDbContext filterListsDbContext) public async Task Scrape() { - var lists = filterListsDbContext.FilterLists; - foreach (var list in lists) await AddSnapshot(list); - await filterListsDbContext.SaveChangesAsync(); + var lists = await filterListsDbContext.FilterLists.Take(1).ToListAsync(); + foreach (var list in lists) await ScrapeList(list); } - private async Task AddSnapshot(FilterList list) + private async Task ScrapeList(FilterList list) { var snapshot = await GetContent(list.ViewUrl); try { if (snapshot == null) return; } - catch (AggregateException) - { - return; - } - catch (HttpRequestException) + catch (Exception) { + //TODO: log exception return; } - filterListsDbContext.Snapshots.Add(new Snapshot - { - Content = snapshot, - FilterListId = list.Id - }); + await AddSnapshot(list.Id, snapshot); + await AddRules(list, snapshot); } private static async Task GetContent(string url) @@ -58,5 +54,50 @@ private static async Task GetContent(string url) return null; } + + private async Task AddSnapshot(int listId, string snapshot) + { + filterListsDbContext.Snapshots.Add(new Snapshot + { + Content = snapshot, + FilterListId = listId + }); + + try + { + await filterListsDbContext.SaveChangesAsync(); + } + catch (Exception) + { + //TODO: log exception + } + } + + private async Task AddRules(FilterList list, string snapshot) + { + var lines = snapshot.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.None); + + foreach (var line in lines) + { + if (string.IsNullOrWhiteSpace(line)) continue; + var rule = new Rule {Raw = line}; + if (!await filterListsDbContext.Rules.AnyAsync(x => x.Raw == rule.Raw)) + filterListsDbContext.Rules.Add(rule); + var filterListRule = new FilterListRule {FilterList = list, Rule = rule}; + if (!await filterListsDbContext.FilterListRules.AnyAsync(x => + x.RuleId == rule.Id && x.FilterListId == list.Id)) + filterListsDbContext.FilterListRules.Add(filterListRule); + filterListsDbContext.FilterListRules.Add(filterListRule); + } + + try + { + await filterListsDbContext.SaveChangesAsync(); + } + catch (Exception) + { + //TODO: log exception + } + } } } \ No newline at end of file