diff --git a/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.Designer.cs b/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.Designer.cs
new file mode 100644
index 000000000..e267a3983
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.Designer.cs
@@ -0,0 +1,477 @@
+//
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage;
+using Microsoft.EntityFrameworkCore.Storage.Internal;
+using System;
+
+namespace FilterLists.Api.Migrations
+{
+ [DbContext(typeof(FilterListsDbContext))]
+ [Migration("20180127014225_partialRemoveModDateAndMore")]
+ partial class partialRemoveModDateAndMore
+ {
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
+ .HasAnnotation("ProductVersion", "2.0.1-rtm-125");
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd();
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Description")
+ .HasColumnType("TEXT");
+
+ b.Property("DescriptionSourceUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("DiscontinuedDate");
+
+ b.Property("DonateUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("EmailAddress")
+ .HasColumnType("NVARCHAR(126)");
+
+ b.Property("ForumUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("IssuesUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("LicenseId");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("NVARCHAR(126)");
+
+ b.Property("SubmissionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("SyntaxId");
+
+ b.Property("ViewUrl")
+ .IsRequired()
+ .HasColumnType("TEXT");
+
+ b.HasKey("Id");
+
+ b.HasIndex("LicenseId");
+
+ b.HasIndex("SyntaxId");
+
+ b.ToTable("filterlists");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("LanguageId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("FilterListId", "LanguageId");
+
+ b.HasIndex("LanguageId");
+
+ b.ToTable("filterlists_languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("MaintainerId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("FilterListId", "MaintainerId");
+
+ b.HasIndex("MaintainerId");
+
+ b.ToTable("filterlists_maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("RuleId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("FilterListId", "RuleId");
+
+ b.HasIndex("RuleId");
+
+ b.ToTable("filterlists_rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.Property("ForkFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("forks");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.Property("MergeFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("merges");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.Property("SoftwareId");
+
+ b.Property("SyntaxId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("SoftwareId", "SyntaxId");
+
+ b.HasIndex("SyntaxId");
+
+ b.ToTable("software_syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd();
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Iso6391")
+ .HasColumnType("NVARCHAR(2)");
+
+ b.Property("Iso6392")
+ .HasColumnType("NVARCHAR(3)");
+
+ b.Property("Iso6392B")
+ .HasColumnType("NVARCHAR(3)");
+
+ b.Property("Iso6392T")
+ .HasColumnType("NVARCHAR(3)");
+
+ b.Property("Iso6393")
+ .HasColumnType("NVARCHAR(3)");
+
+ b.Property("LocalName")
+ .HasColumnType("NVARCHAR(126)");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .HasColumnType("NVARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd();
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("DescriptionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("NVARCHAR(126)");
+
+ b.Property("PermissiveAdaptation");
+
+ b.Property("PermissiveCommercial");
+
+ b.HasKey("Id");
+
+ b.ToTable("licenses");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd();
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("EmailAddress")
+ .HasColumnType("NVARCHAR(126)");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("NVARCHAR(126)");
+
+ b.Property("TwitterHandle")
+ .HasColumnType("NVARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd();
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Raw")
+ .IsRequired()
+ .HasColumnType("NVARCHAR(2083)");
+
+ b.HasKey("Id");
+
+ b.ToTable("rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd();
+
+ b.Property("Content")
+ .IsRequired()
+ .HasColumnType("MEDIUMTEXT");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("FilterListId");
+
+ b.HasKey("Id");
+
+ b.HasIndex("FilterListId");
+
+ b.ToTable("snapshots");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd();
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("DownloadUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("NVARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("software");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd();
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("DefinitionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("NVARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.License")
+ .WithMany("FilterLists")
+ .HasForeignKey("LicenseId");
+
+ b.HasOne("FilterLists.Data.Entities.Syntax")
+ .WithMany("FilterLists")
+ .HasForeignKey("SyntaxId");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Language", "Language")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("LanguageId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("MaintainerId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListRules")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+ .WithMany("FilterListRules")
+ .HasForeignKey("RuleId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+ .WithMany()
+ .HasForeignKey("ForkFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany()
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+ .WithMany()
+ .HasForeignKey("MergeFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany()
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Software", "Software")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SoftwareId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SyntaxId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList")
+ .WithMany("Snapshots")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+#pragma warning restore 612, 618
+ }
+ }
+}
diff --git a/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.cs b/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.cs
new file mode 100644
index 000000000..a69c9b180
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.cs
@@ -0,0 +1,137 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+using System;
+using System.Collections.Generic;
+
+namespace FilterLists.Api.Migrations
+{
+ public partial class partialRemoveModDateAndMore : Migration
+ {
+ protected override void Up(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropColumn(
+ name: "ModifiedDateUtc",
+ table: "software_syntaxes");
+
+ migrationBuilder.DropColumn(
+ name: "ModifiedDateUtc",
+ table: "snapshots");
+
+ migrationBuilder.DropColumn(
+ name: "ModifiedDateUtc",
+ table: "rules");
+
+ migrationBuilder.DropColumn(
+ name: "ModifiedDateUtc",
+ table: "merges");
+
+ migrationBuilder.DropColumn(
+ name: "ModifiedDateUtc",
+ table: "forks");
+
+ migrationBuilder.DropColumn(
+ name: "ModifiedDateUtc",
+ table: "filterlists_rules");
+
+ migrationBuilder.DropColumn(
+ name: "ModifiedDateUtc",
+ table: "filterlists_maintainers");
+
+ migrationBuilder.DropColumn(
+ name: "ModifiedDateUtc",
+ table: "filterlists_languages");
+
+ migrationBuilder.AlterColumn(
+ name: "Raw",
+ table: "rules",
+ type: "NVARCHAR(2083)",
+ nullable: false,
+ oldClrType: typeof(string),
+ oldType: "NVARCHAR(2083)",
+ oldNullable: true);
+
+ migrationBuilder.AlterColumn(
+ name: "ViewUrl",
+ table: "filterlists",
+ type: "TEXT",
+ nullable: false,
+ oldClrType: typeof(string),
+ oldType: "TEXT",
+ oldNullable: true);
+ }
+
+ protected override void Down(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.AddColumn(
+ name: "ModifiedDateUtc",
+ table: "software_syntaxes",
+ type: "TIMESTAMP",
+ nullable: false,
+ defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
+ migrationBuilder.AddColumn(
+ name: "ModifiedDateUtc",
+ table: "snapshots",
+ type: "TIMESTAMP",
+ nullable: false,
+ defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
+ migrationBuilder.AlterColumn(
+ name: "Raw",
+ table: "rules",
+ type: "NVARCHAR(2083)",
+ nullable: true,
+ oldClrType: typeof(string),
+ oldType: "NVARCHAR(2083)");
+
+ migrationBuilder.AddColumn(
+ name: "ModifiedDateUtc",
+ table: "rules",
+ type: "TIMESTAMP",
+ nullable: false,
+ defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
+ migrationBuilder.AddColumn(
+ name: "ModifiedDateUtc",
+ table: "merges",
+ type: "TIMESTAMP",
+ nullable: false,
+ defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
+ migrationBuilder.AddColumn(
+ name: "ModifiedDateUtc",
+ table: "forks",
+ type: "TIMESTAMP",
+ nullable: false,
+ defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
+ migrationBuilder.AddColumn(
+ name: "ModifiedDateUtc",
+ table: "filterlists_rules",
+ type: "TIMESTAMP",
+ nullable: false,
+ defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
+ migrationBuilder.AddColumn(
+ name: "ModifiedDateUtc",
+ table: "filterlists_maintainers",
+ type: "TIMESTAMP",
+ nullable: false,
+ defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
+ migrationBuilder.AddColumn(
+ name: "ModifiedDateUtc",
+ table: "filterlists_languages",
+ type: "TIMESTAMP",
+ nullable: false,
+ defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
+ migrationBuilder.AlterColumn(
+ name: "ViewUrl",
+ table: "filterlists",
+ type: "TEXT",
+ nullable: true,
+ oldClrType: typeof(string),
+ oldType: "TEXT");
+ }
+ }
+}
diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
index a903bd3f5..fd633c29d 100644
--- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
+++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
@@ -68,6 +68,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property("SyntaxId");
b.Property("ViewUrl")
+ .IsRequired()
.HasColumnType("TEXT");
b.HasKey("Id");
@@ -89,10 +90,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
- b.Property("ModifiedDateUtc")
- .ValueGeneratedOnAddOrUpdate()
- .HasColumnType("TIMESTAMP");
-
b.HasKey("FilterListId", "LanguageId");
b.HasIndex("LanguageId");
@@ -110,10 +107,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
- b.Property("ModifiedDateUtc")
- .ValueGeneratedOnAddOrUpdate()
- .HasColumnType("TIMESTAMP");
-
b.HasKey("FilterListId", "MaintainerId");
b.HasIndex("MaintainerId");
@@ -131,10 +124,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
- b.Property("ModifiedDateUtc")
- .ValueGeneratedOnAddOrUpdate()
- .HasColumnType("TIMESTAMP");
-
b.HasKey("FilterListId", "RuleId");
b.HasIndex("RuleId");
@@ -152,10 +141,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
- b.Property("ModifiedDateUtc")
- .ValueGeneratedOnAddOrUpdate()
- .HasColumnType("TIMESTAMP");
-
b.HasKey("ForkFilterListId", "UpstreamFilterListId");
b.HasIndex("UpstreamFilterListId");
@@ -173,10 +158,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
- b.Property("ModifiedDateUtc")
- .ValueGeneratedOnAddOrUpdate()
- .HasColumnType("TIMESTAMP");
-
b.HasKey("MergeFilterListId", "UpstreamFilterListId");
b.HasIndex("UpstreamFilterListId");
@@ -194,10 +175,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
- b.Property("ModifiedDateUtc")
- .ValueGeneratedOnAddOrUpdate()
- .HasColumnType("TIMESTAMP");
-
b.HasKey("SoftwareId", "SyntaxId");
b.HasIndex("SyntaxId");
@@ -313,11 +290,8 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
- b.Property("ModifiedDateUtc")
- .ValueGeneratedOnAddOrUpdate()
- .HasColumnType("TIMESTAMP");
-
b.Property("Raw")
+ .IsRequired()
.HasColumnType("NVARCHAR(2083)");
b.HasKey("Id");
@@ -340,10 +314,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property("FilterListId");
- b.Property("ModifiedDateUtc")
- .ValueGeneratedOnAddOrUpdate()
- .HasColumnType("TIMESTAMP");
-
b.HasKey("Id");
b.HasIndex("FilterListId");
diff --git a/src/FilterLists.Data/Entities/BaseEntity.cs b/src/FilterLists.Data/Entities/BaseEntity.cs
index c7bfba6e6..10c6c1e5c 100644
--- a/src/FilterLists.Data/Entities/BaseEntity.cs
+++ b/src/FilterLists.Data/Entities/BaseEntity.cs
@@ -6,6 +6,5 @@ public abstract class BaseEntity
{
public int Id { get; set; }
public DateTime CreatedDateUtc { get; set; }
- public DateTime ModifiedDateUtc { get; set; }
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/Entities/FilterList.cs b/src/FilterLists.Data/Entities/FilterList.cs
index 6453a477b..3d43c279a 100644
--- a/src/FilterLists.Data/Entities/FilterList.cs
+++ b/src/FilterLists.Data/Entities/FilterList.cs
@@ -6,6 +6,7 @@ namespace FilterLists.Data.Entities
{
public class FilterList : BaseEntity
{
+ public DateTime ModifiedDateUtc { get; set; }
public string Description { get; set; }
public string DescriptionSourceUrl { get; set; }
public DateTime? DiscontinuedDate { get; set; }
diff --git a/src/FilterLists.Data/Entities/Language.cs b/src/FilterLists.Data/Entities/Language.cs
index 178aca1ae..d8b0760f7 100644
--- a/src/FilterLists.Data/Entities/Language.cs
+++ b/src/FilterLists.Data/Entities/Language.cs
@@ -1,10 +1,12 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
using FilterLists.Data.Entities.Junctions;
namespace FilterLists.Data.Entities
{
public class Language : BaseEntity
{
+ public DateTime ModifiedDateUtc { get; set; }
public ICollection FilterListLanguages { get; set; }
public string Iso6391 { get; set; }
public string Iso6392 { get; set; }
diff --git a/src/FilterLists.Data/Entities/License.cs b/src/FilterLists.Data/Entities/License.cs
index fecc22fce..22a16d8b4 100644
--- a/src/FilterLists.Data/Entities/License.cs
+++ b/src/FilterLists.Data/Entities/License.cs
@@ -1,9 +1,11 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
namespace FilterLists.Data.Entities
{
public class License : BaseEntity
{
+ public DateTime ModifiedDateUtc { get; set; }
public string DescriptionUrl { get; set; }
public ICollection FilterLists { get; set; }
public string Name { get; set; }
diff --git a/src/FilterLists.Data/Entities/Maintainer.cs b/src/FilterLists.Data/Entities/Maintainer.cs
index 64b6f46e9..5db1cd6b0 100644
--- a/src/FilterLists.Data/Entities/Maintainer.cs
+++ b/src/FilterLists.Data/Entities/Maintainer.cs
@@ -1,10 +1,12 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
using FilterLists.Data.Entities.Junctions;
namespace FilterLists.Data.Entities
{
public class Maintainer : BaseEntity
{
+ public DateTime ModifiedDateUtc { get; set; }
public string EmailAddress { get; set; }
public ICollection FilterListMaintainers { get; set; }
public string HomeUrl { get; set; }
diff --git a/src/FilterLists.Data/Entities/Software.cs b/src/FilterLists.Data/Entities/Software.cs
index d3d918697..380297260 100644
--- a/src/FilterLists.Data/Entities/Software.cs
+++ b/src/FilterLists.Data/Entities/Software.cs
@@ -1,10 +1,12 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
using FilterLists.Data.Entities.Junctions;
namespace FilterLists.Data.Entities
{
public class Software : BaseEntity
{
+ public DateTime ModifiedDateUtc { get; set; }
public string DownloadUrl { get; set; }
public string HomeUrl { get; set; }
public string Name { get; set; }
diff --git a/src/FilterLists.Data/Entities/Syntax.cs b/src/FilterLists.Data/Entities/Syntax.cs
index 7a25dabe3..d7c5e86e4 100644
--- a/src/FilterLists.Data/Entities/Syntax.cs
+++ b/src/FilterLists.Data/Entities/Syntax.cs
@@ -1,10 +1,12 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
using FilterLists.Data.Entities.Junctions;
namespace FilterLists.Data.Entities
{
public class Syntax : BaseEntity
{
+ public DateTime ModifiedDateUtc { get; set; }
public string DefinitionUrl { get; set; }
public ICollection FilterLists { get; set; }
public string Name { get; set; }
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityConfiguration.cs
index 5e469454f..0fb6e84b6 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/BaseEntityConfiguration.cs
@@ -11,9 +11,6 @@ public virtual void Configure(EntityTypeBuilder entityTypeBuilder)
entityTypeBuilder.Property(x => x.CreatedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAdd();
- entityTypeBuilder.Property(x => x.ModifiedDateUtc)
- .HasColumnType("TIMESTAMP")
- .ValueGeneratedOnAddOrUpdate();
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs
index 185ca5bd6..8009db6cf 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs
@@ -9,6 +9,9 @@ public class FilterListTypeConfiguration : BaseEntityTypeConfiguration entityTypeBuilder)
{
entityTypeBuilder.ToTable("filterlists");
+ entityTypeBuilder.Property(x => x.ModifiedDateUtc)
+ .HasColumnType("TIMESTAMP")
+ .ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.Description)
.HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.DescriptionSourceUrl)
@@ -29,7 +32,8 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder)
entityTypeBuilder.Property(x => x.SubmissionUrl)
.HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.ViewUrl)
- .HasColumnType("TEXT");
+ .HasColumnType("TEXT")
+ .IsRequired();
base.Configure(entityTypeBuilder);
}
}
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs
index a37450983..c2a74ae00 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs
@@ -9,6 +9,9 @@ public class LanguageTypeConfiguration : BaseEntityTypeConfiguration
public override void Configure(EntityTypeBuilder entityTypeBuilder)
{
entityTypeBuilder.ToTable("languages");
+ entityTypeBuilder.Property(x => x.ModifiedDateUtc)
+ .HasColumnType("TIMESTAMP")
+ .ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.Iso6391)
.HasColumnType("NVARCHAR(2)");
entityTypeBuilder.Property(x => x.Iso6392)
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs
index d12b6b64b..e76bad580 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs
@@ -9,6 +9,9 @@ public class LicenseTypeConfiguration : BaseEntityTypeConfiguration
public override void Configure(EntityTypeBuilder entityTypeBuilder)
{
entityTypeBuilder.ToTable("licenses");
+ entityTypeBuilder.Property(x => x.ModifiedDateUtc)
+ .HasColumnType("TIMESTAMP")
+ .ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.DescriptionUrl)
.HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.Name)
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs
index 95239c3a9..fa5278675 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs
@@ -9,6 +9,9 @@ public class MaintainerTypeConfiguration : BaseEntityTypeConfiguration entityTypeBuilder)
{
entityTypeBuilder.ToTable("maintainers");
+ entityTypeBuilder.Property(x => x.ModifiedDateUtc)
+ .HasColumnType("TIMESTAMP")
+ .ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.EmailAddress)
.HasColumnType("NVARCHAR(126)");
entityTypeBuilder.Property(x => x.HomeUrl)
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs
index 6af964a04..3f6670de3 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs
@@ -10,7 +10,8 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder)
{
entityTypeBuilder.ToTable("rules");
entityTypeBuilder.Property(x => x.Raw)
- .HasColumnType("NVARCHAR(2083)");
+ .HasColumnType("NVARCHAR(2083)")
+ .IsRequired();
base.Configure(entityTypeBuilder);
}
}
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
index 65e50fde2..2582c927e 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
@@ -12,7 +12,6 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder)
entityTypeBuilder.Property(x => x.Content)
.HasColumnType("MEDIUMTEXT")
.IsRequired();
- entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
base.Configure(entityTypeBuilder);
}
}
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs
index fdef874a2..f0026e115 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs
@@ -9,6 +9,9 @@ public class SoftwareTypeConfiguration : BaseEntityTypeConfiguration
public override void Configure(EntityTypeBuilder entityTypeBuilder)
{
entityTypeBuilder.ToTable("software");
+ entityTypeBuilder.Property(x => x.ModifiedDateUtc)
+ .HasColumnType("TIMESTAMP")
+ .ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.DownloadUrl)
.HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.HomeUrl)
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs
index 7be00ce5c..6ca7c03d6 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs
@@ -9,6 +9,9 @@ public class SyntaxTypeConfiguration : BaseEntityTypeConfiguration
public override void Configure(EntityTypeBuilder entityTypeBuilder)
{
entityTypeBuilder.ToTable("syntaxes");
+ entityTypeBuilder.Property(x => x.ModifiedDateUtc)
+ .HasColumnType("TIMESTAMP")
+ .ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.DefinitionUrl)
.HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.Name)
diff --git a/src/FilterLists.Services/Services/ScrapeService.cs b/src/FilterLists.Services/Services/ScrapeService.cs
index 1085e641a..65f5bbcef 100644
--- a/src/FilterLists.Services/Services/ScrapeService.cs
+++ b/src/FilterLists.Services/Services/ScrapeService.cs
@@ -1,8 +1,11 @@
using System;
+using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Data.Entities;
+using FilterLists.Data.Entities.Junctions;
+using Microsoft.EntityFrameworkCore;
namespace FilterLists.Services.Services
{
@@ -17,32 +20,25 @@ public ScrapeService(FilterListsDbContext filterListsDbContext)
public async Task Scrape()
{
- var lists = filterListsDbContext.FilterLists;
- foreach (var list in lists) await AddSnapshot(list);
- await filterListsDbContext.SaveChangesAsync();
+ var lists = await filterListsDbContext.FilterLists.Take(1).ToListAsync();
+ foreach (var list in lists) await ScrapeList(list);
}
- private async Task AddSnapshot(FilterList list)
+ private async Task ScrapeList(FilterList list)
{
var snapshot = await GetContent(list.ViewUrl);
try
{
if (snapshot == null) return;
}
- catch (AggregateException)
- {
- return;
- }
- catch (HttpRequestException)
+ catch (Exception)
{
+ //TODO: log exception
return;
}
- filterListsDbContext.Snapshots.Add(new Snapshot
- {
- Content = snapshot,
- FilterListId = list.Id
- });
+ await AddSnapshot(list.Id, snapshot);
+ await AddRules(list, snapshot);
}
private static async Task GetContent(string url)
@@ -58,5 +54,50 @@ private static async Task GetContent(string url)
return null;
}
+
+ private async Task AddSnapshot(int listId, string snapshot)
+ {
+ filterListsDbContext.Snapshots.Add(new Snapshot
+ {
+ Content = snapshot,
+ FilterListId = listId
+ });
+
+ try
+ {
+ await filterListsDbContext.SaveChangesAsync();
+ }
+ catch (Exception)
+ {
+ //TODO: log exception
+ }
+ }
+
+ private async Task AddRules(FilterList list, string snapshot)
+ {
+ var lines = snapshot.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.None);
+
+ foreach (var line in lines)
+ {
+ if (string.IsNullOrWhiteSpace(line)) continue;
+ var rule = new Rule {Raw = line};
+ if (!await filterListsDbContext.Rules.AnyAsync(x => x.Raw == rule.Raw))
+ filterListsDbContext.Rules.Add(rule);
+ var filterListRule = new FilterListRule {FilterList = list, Rule = rule};
+ if (!await filterListsDbContext.FilterListRules.AnyAsync(x =>
+ x.RuleId == rule.Id && x.FilterListId == list.Id))
+ filterListsDbContext.FilterListRules.Add(filterListRule);
+ filterListsDbContext.FilterListRules.Add(filterListRule);
+ }
+
+ try
+ {
+ await filterListsDbContext.SaveChangesAsync();
+ }
+ catch (Exception)
+ {
+ //TODO: log exception
+ }
+ }
}
}
\ No newline at end of file