diff --git a/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.Designer.cs b/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.Designer.cs
new file mode 100644
index 000000000..bb8e3a8d6
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.Designer.cs
@@ -0,0 +1,503 @@
+//
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage;
+using System;
+
+namespace FilterLists.Api.Migrations
+{
+ [DbContext(typeof(FilterListsDbContext))]
+ [Migration("20180202235529_AddScrapes")]
+ partial class AddScrapes
+ {
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
+ .HasAnnotation("ProductVersion", "2.0.1-rtm-125");
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Description")
+ .HasColumnType("TEXT");
+
+ b.Property("DescriptionSourceUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("DiscontinuedDate");
+
+ b.Property("DonateUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ForumUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("IssuesUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("LicenseId");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PolicyUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("SubmissionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("SyntaxId");
+
+ b.Property("ViewUrl")
+ .IsRequired()
+ .HasColumnType("TEXT");
+
+ b.HasKey("Id");
+
+ b.HasIndex("LicenseId");
+
+ b.HasIndex("SyntaxId");
+
+ b.ToTable("filterlists");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("LanguageId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("FilterListId", "LanguageId");
+
+ b.HasIndex("LanguageId");
+
+ b.ToTable("filterlists_languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("MaintainerId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("FilterListId", "MaintainerId");
+
+ b.HasIndex("MaintainerId");
+
+ b.ToTable("filterlists_maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("RuleId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("FilterListId", "RuleId");
+
+ b.HasIndex("RuleId");
+
+ b.ToTable("filterlists_rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.Property("ForkFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("forks");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.Property("MergeFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("merges");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.Property("SoftwareId");
+
+ b.Property("SyntaxId");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("SoftwareId", "SyntaxId");
+
+ b.HasIndex("SyntaxId");
+
+ b.ToTable("software_syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Iso6391")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(2)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392B")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392T")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6393")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("LocalName")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("DescriptionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PermissiveAdaptation");
+
+ b.Property("PermissiveCommercial");
+
+ b.HasKey("Id");
+
+ b.ToTable("licenses");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("TwitterHandle")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("BIGINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Raw")
+ .IsRequired()
+ .HasColumnType("VARCHAR(8192)");
+
+ b.HasKey("Id");
+
+ b.ToTable("rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("FilterListId");
+
+ b.HasKey("Id");
+
+ b.HasIndex("FilterListId");
+
+ b.ToTable("scrapes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("DownloadUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("software");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("DefinitionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.License")
+ .WithMany("FilterLists")
+ .HasForeignKey("LicenseId");
+
+ b.HasOne("FilterLists.Data.Entities.Syntax")
+ .WithMany("FilterLists")
+ .HasForeignKey("SyntaxId");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Language", "Language")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("LanguageId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("MaintainerId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListRules")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+ .WithMany("FilterListRules")
+ .HasForeignKey("RuleId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+ .WithMany("ForkFilterLists")
+ .HasForeignKey("ForkFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamForkFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+ .WithMany("MergeFilterLists")
+ .HasForeignKey("MergeFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamMergeFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Software", "Software")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SoftwareId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SyntaxId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList")
+ .WithMany("Scrapes")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+#pragma warning restore 612, 618
+ }
+ }
+}
diff --git a/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.cs b/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.cs
new file mode 100644
index 000000000..c0dd2fc37
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.cs
@@ -0,0 +1,90 @@
+using System;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+ public partial class AddScrapes : Migration
+ {
+ protected override void Up(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropColumn(
+ "ModifiedDateUtc",
+ "rules");
+
+ migrationBuilder.DropColumn(
+ "ScrapedDateUtc",
+ "filterlists");
+
+ migrationBuilder.DropColumn(
+ "UpdatedDateUtc",
+ "filterlists");
+
+ migrationBuilder.AlterColumn(
+ "ViewUrl",
+ "filterlists",
+ "TEXT",
+ nullable: false,
+ oldClrType: typeof(string),
+ oldType: "TEXT NOT NULL",
+ oldNullable: true);
+
+ migrationBuilder.CreateTable(
+ "scrapes",
+ table => new
+ {
+ Id = table.Column("SMALLINT UNSIGNED", nullable: false)
+ .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
+ CreatedDateUtc = table.Column("TIMESTAMP", nullable: false)
+ .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
+ FilterListId = table.Column(nullable: false)
+ },
+ constraints: table =>
+ {
+ table.PrimaryKey("PK_scrapes", x => x.Id);
+ table.ForeignKey(
+ "FK_scrapes_filterlists_FilterListId",
+ x => x.FilterListId,
+ "filterlists",
+ "Id",
+ onDelete: ReferentialAction.Cascade);
+ });
+
+ migrationBuilder.CreateIndex(
+ "IX_scrapes_FilterListId",
+ "scrapes",
+ "FilterListId");
+ }
+
+ protected override void Down(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropTable(
+ "scrapes");
+
+ migrationBuilder.AddColumn(
+ "ModifiedDateUtc",
+ "rules",
+ "TIMESTAMP",
+ nullable: false,
+ defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
+ migrationBuilder.AlterColumn(
+ "ViewUrl",
+ "filterlists",
+ "TEXT NOT NULL",
+ nullable: true,
+ oldClrType: typeof(string),
+ oldType: "TEXT");
+
+ migrationBuilder.AddColumn(
+ "ScrapedDateUtc",
+ "filterlists",
+ nullable: true);
+
+ migrationBuilder.AddColumn(
+ "UpdatedDateUtc",
+ "filterlists",
+ nullable: true);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
index 19269c4db..53bf8f848 100644
--- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
+++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
@@ -66,17 +66,14 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property("PolicyUrl")
.HasColumnType("TEXT");
- b.Property("ScrapedDateUtc");
-
b.Property("SubmissionUrl")
.HasColumnType("TEXT");
b.Property("SyntaxId");
- b.Property("UpdatedDateUtc");
-
b.Property("ViewUrl")
- .HasColumnType("TEXT NOT NULL");
+ .IsRequired()
+ .HasColumnType("TEXT");
b.HasKey("Id");
@@ -319,10 +316,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
- b.Property("ModifiedDateUtc")
- .ValueGeneratedOnAddOrUpdate()
- .HasColumnType("TIMESTAMP");
-
b.Property("Raw")
.IsRequired()
.HasColumnType("VARCHAR(8192)");
@@ -332,6 +325,25 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.ToTable("rules");
});
+ modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED");
+
+ b.Property("CreatedDateUtc")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("FilterListId");
+
+ b.HasKey("Id");
+
+ b.HasIndex("FilterListId");
+
+ b.ToTable("scrapes");
+ });
+
modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
{
b.Property("Id")
@@ -440,12 +452,12 @@ protected override void BuildModel(ModelBuilder modelBuilder)
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
- .WithMany()
+ .WithMany("ForkFilterLists")
.HasForeignKey("ForkFilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
- .WithMany()
+ .WithMany("UpstreamForkFilterLists")
.HasForeignKey("UpstreamFilterListId")
.OnDelete(DeleteBehavior.Cascade);
});
@@ -453,12 +465,12 @@ protected override void BuildModel(ModelBuilder modelBuilder)
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
- .WithMany()
+ .WithMany("MergeFilterLists")
.HasForeignKey("MergeFilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
- .WithMany()
+ .WithMany("UpstreamMergeFilterLists")
.HasForeignKey("UpstreamFilterListId")
.OnDelete(DeleteBehavior.Cascade);
});
@@ -475,6 +487,14 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.HasForeignKey("SyntaxId")
.OnDelete(DeleteBehavior.Cascade);
});
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList")
+ .WithMany("Scrapes")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
#pragma warning restore 612, 618
}
}
diff --git a/src/FilterLists.Data/Entities/FilterList.cs b/src/FilterLists.Data/Entities/FilterList.cs
index b5117ec6b..b65e3ed8f 100644
--- a/src/FilterLists.Data/Entities/FilterList.cs
+++ b/src/FilterLists.Data/Entities/FilterList.cs
@@ -11,20 +11,22 @@ public class FilterList : BaseEntity
public DateTime? DiscontinuedDate { get; set; }
public string DonateUrl { get; set; }
public string EmailAddress { get; set; }
+ public ICollection ForkFilterLists { get; set; }
+ public ICollection UpstreamForkFilterLists { get; set; }
public string ForumUrl { get; set; }
public string HomeUrl { get; set; }
public string IssuesUrl { get; set; }
+ public ICollection FilterListLanguages { get; set; }
public int? LicenseId { get; set; }
+ public ICollection FilterListMaintainers { get; set; }
+ public ICollection MergeFilterLists { get; set; }
+ public ICollection UpstreamMergeFilterLists { get; set; }
public string Name { get; set; }
public string PolicyUrl { get; set; }
- public DateTime? ScrapedDateUtc { get; set; }
+ public ICollection FilterListRules { get; set; }
+ public ICollection Scrapes { get; set; }
public string SubmissionUrl { get; set; }
public int? SyntaxId { get; set; }
- public DateTime? UpdatedDateUtc { get; set; }
public string ViewUrl { get; set; }
-
- public ICollection FilterListLanguages { get; set; }
- public ICollection FilterListMaintainers { get; set; }
- public ICollection FilterListRules { get; set; }
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/Entities/Language.cs b/src/FilterLists.Data/Entities/Language.cs
index 518344660..178aca1ae 100644
--- a/src/FilterLists.Data/Entities/Language.cs
+++ b/src/FilterLists.Data/Entities/Language.cs
@@ -5,6 +5,7 @@ namespace FilterLists.Data.Entities
{
public class Language : BaseEntity
{
+ public ICollection FilterListLanguages { get; set; }
public string Iso6391 { get; set; }
public string Iso6392 { get; set; }
public string Iso6392B { get; set; }
@@ -12,7 +13,5 @@ public class Language : BaseEntity
public string Iso6393 { get; set; }
public string LocalName { get; set; }
public string Name { get; set; }
-
- public ICollection FilterListLanguages { get; set; }
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/Entities/Maintainer.cs b/src/FilterLists.Data/Entities/Maintainer.cs
index 344cb9bd6..64b6f46e9 100644
--- a/src/FilterLists.Data/Entities/Maintainer.cs
+++ b/src/FilterLists.Data/Entities/Maintainer.cs
@@ -6,10 +6,9 @@ namespace FilterLists.Data.Entities
public class Maintainer : BaseEntity
{
public string EmailAddress { get; set; }
+ public ICollection FilterListMaintainers { get; set; }
public string HomeUrl { get; set; }
public string Name { get; set; }
public string TwitterHandle { get; set; }
-
- public ICollection FilterListMaintainers { get; set; }
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/Entities/Rule.cs b/src/FilterLists.Data/Entities/Rule.cs
index f18f7e5c0..41b15b313 100644
--- a/src/FilterLists.Data/Entities/Rule.cs
+++ b/src/FilterLists.Data/Entities/Rule.cs
@@ -5,8 +5,7 @@ namespace FilterLists.Data.Entities
{
public class Rule : BaseEntity
{
- public string Raw { get; set; }
-
public ICollection FilterListRules { get; set; }
+ public string Raw { get; set; }
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/Entities/Scrape.cs b/src/FilterLists.Data/Entities/Scrape.cs
new file mode 100644
index 000000000..b22ff191a
--- /dev/null
+++ b/src/FilterLists.Data/Entities/Scrape.cs
@@ -0,0 +1,7 @@
+namespace FilterLists.Data.Entities
+{
+ public class Scrape : BaseEntity
+ {
+ public int FilterListId { get; set; }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Data/Entities/Software.cs b/src/FilterLists.Data/Entities/Software.cs
index a4d75f64a..d3d918697 100644
--- a/src/FilterLists.Data/Entities/Software.cs
+++ b/src/FilterLists.Data/Entities/Software.cs
@@ -8,7 +8,6 @@ public class Software : BaseEntity
public string DownloadUrl { get; set; }
public string HomeUrl { get; set; }
public string Name { get; set; }
-
public ICollection SoftwareSyntaxes { get; set; }
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/Entities/Syntax.cs b/src/FilterLists.Data/Entities/Syntax.cs
index 89ab38e2b..7a25dabe3 100644
--- a/src/FilterLists.Data/Entities/Syntax.cs
+++ b/src/FilterLists.Data/Entities/Syntax.cs
@@ -8,7 +8,6 @@ public class Syntax : BaseEntity
public string DefinitionUrl { get; set; }
public ICollection FilterLists { get; set; }
public string Name { get; set; }
-
public ICollection SoftwareSyntaxes { get; set; }
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs
index 2dfb4e47d..dd7dad9a6 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs
@@ -35,7 +35,8 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder)
entityTypeBuilder.Property(x => x.SubmissionUrl)
.HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.ViewUrl)
- .HasColumnType("TEXT NOT NULL");
+ .HasColumnType("TEXT")
+ .IsRequired();
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs
index 629edf7fb..f921ac24e 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs
@@ -15,6 +15,8 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder)
entityTypeBuilder.Property(x => x.Id)
.HasColumnType("BIGINT UNSIGNED");
+ entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
+
entityTypeBuilder.Property(x => x.Raw)
.HasColumnType("VARCHAR(8192)")
.IsRequired();
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs
new file mode 100644
index 000000000..b2d851ebf
--- /dev/null
+++ b/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs
@@ -0,0 +1,18 @@
+using FilterLists.Data.Entities;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace FilterLists.Data.EntityTypeConfigurations
+{
+ public class ScrapeTypeConfiguration : BaseEntityTypeConfiguration
+ {
+ public override void Configure(EntityTypeBuilder entityTypeBuilder)
+ {
+ base.Configure(entityTypeBuilder);
+
+ entityTypeBuilder.ToTable("scrapes");
+
+ entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Data/FilterListsDbContext.cs b/src/FilterLists.Data/FilterListsDbContext.cs
index 7e86403ea..64fc89969 100644
--- a/src/FilterLists.Data/FilterListsDbContext.cs
+++ b/src/FilterLists.Data/FilterListsDbContext.cs
@@ -27,6 +27,7 @@ private static void ApplyConfigurationsEntities(ModelBuilder modelBuilder)
modelBuilder.ApplyConfiguration(new LicenseTypeConfiguration());
modelBuilder.ApplyConfiguration(new MaintainerTypeConfiguration());
modelBuilder.ApplyConfiguration(new RuleTypeConfiguration());
+ modelBuilder.ApplyConfiguration(new ScrapeTypeConfiguration());
modelBuilder.ApplyConfiguration(new SoftwareTypeConfiguration());
modelBuilder.ApplyConfiguration(new SyntaxTypeConfiguration());
}
@@ -37,10 +38,36 @@ private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder)
modelBuilder.ApplyConfiguration(new FilterListMaintainerTypeConfiguration());
modelBuilder.ApplyConfiguration(new FilterListRuleTypeConfiguration());
modelBuilder.ApplyConfiguration(new ForkTypeConfiguration());
+ ConfigureForkSelfReferencing(modelBuilder);
modelBuilder.ApplyConfiguration(new MergeTypeConfiguration());
+ ConfigureMergeSelfReferencing(modelBuilder);
modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration());
}
+ private static void ConfigureForkSelfReferencing(ModelBuilder modelBuilder)
+ {
+ modelBuilder.Entity()
+ .HasOne(x => x.ForkFilterList)
+ .WithMany(x => x.ForkFilterLists)
+ .HasForeignKey(x => x.ForkFilterListId);
+ modelBuilder.Entity()
+ .HasOne(x => x.UpstreamFilterList)
+ .WithMany(x => x.UpstreamForkFilterLists)
+ .HasForeignKey(x => x.UpstreamFilterListId);
+ }
+
+ private static void ConfigureMergeSelfReferencing(ModelBuilder modelBuilder)
+ {
+ modelBuilder.Entity()
+ .HasOne(x => x.MergeFilterList)
+ .WithMany(x => x.MergeFilterLists)
+ .HasForeignKey(x => x.MergeFilterListId);
+ modelBuilder.Entity()
+ .HasOne(x => x.UpstreamFilterList)
+ .WithMany(x => x.UpstreamMergeFilterLists)
+ .HasForeignKey(x => x.UpstreamFilterListId);
+ }
+
#region Entities
public DbSet FilterLists { get; set; }
@@ -48,6 +75,7 @@ private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder)
public DbSet Licenses { get; set; }
public DbSet Maintainers { get; set; }
public DbSet Rules { get; set; }
+ public DbSet Scrapes { get; set; }
public DbSet Software { get; set; }
public DbSet Syntaxes { get; set; }
diff --git a/src/FilterLists.Services/ScrapeService/ScrapeService.cs b/src/FilterLists.Services/ScrapeService/ScrapeService.cs
index 54722870c..903677908 100644
--- a/src/FilterLists.Services/ScrapeService/ScrapeService.cs
+++ b/src/FilterLists.Services/ScrapeService/ScrapeService.cs
@@ -28,8 +28,8 @@ public async Task ScrapeAsync(int batchSize)
private async Task> GetNextListsToScrape(int batchSize)
{
- return await dbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(batchSize)
- .ProjectTo().ToListAsync();
+ //TODO: order by least recently scraped
+ return await dbContext.FilterLists.Take(batchSize).ProjectTo().ToListAsync();
}
private async Task> GetSnapshots(IEnumerable lists)
diff --git a/src/FilterLists.Services/ScrapeService/Snapshot.cs b/src/FilterLists.Services/ScrapeService/Snapshot.cs
index d97c61891..3da339b91 100644
--- a/src/FilterLists.Services/ScrapeService/Snapshot.cs
+++ b/src/FilterLists.Services/ScrapeService/Snapshot.cs
@@ -61,15 +61,7 @@ public async Task AddOrUpdateRules()
new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList();
await dbContext.FilterListRules.AddRangeAsync(newFilterListRules);
- // update UpdatedDateUtc
- var list = dbContext.FilterLists.Find(filterListId);
- if (preExistingSnapshotFilterListRules.Any() || newFilterListRules.Any() || deletedFilterListRules.Any())
- list.UpdatedDateUtc = DateTime.UtcNow;
-
- // update ScrapedDateUtc
- list.ScrapedDateUtc = DateTime.UtcNow;
-
- dbContext.FilterLists.Update(list);
+ // update scrapes
await dbContext.SaveChangesAsync();
}