diff --git a/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.Designer.cs b/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.Designer.cs new file mode 100644 index 000000000..bb8e3a8d6 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.Designer.cs @@ -0,0 +1,503 @@ +// +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage; +using System; + +namespace FilterLists.Api.Migrations +{ + [DbContext(typeof(FilterListsDbContext))] + [Migration("20180202235529_AddScrapes")] + partial class AddScrapes + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn) + .HasAnnotation("ProductVersion", "2.0.1-rtm-125"); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("DescriptionSourceUrl") + .HasColumnType("TEXT"); + + b.Property("DiscontinuedDate"); + + b.Property("DonateUrl") + .HasColumnType("TEXT"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ForumUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("IssuesUrl") + .HasColumnType("TEXT"); + + b.Property("LicenseId"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PolicyUrl") + .HasColumnType("TEXT"); + + b.Property("SubmissionUrl") + .HasColumnType("TEXT"); + + b.Property("SyntaxId"); + + b.Property("ViewUrl") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("LicenseId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("filterlists"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.Property("FilterListId"); + + b.Property("LanguageId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "LanguageId"); + + b.HasIndex("LanguageId"); + + b.ToTable("filterlists_languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.Property("FilterListId"); + + b.Property("MaintainerId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "MaintainerId"); + + b.HasIndex("MaintainerId"); + + b.ToTable("filterlists_maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b => + { + b.Property("FilterListId"); + + b.Property("RuleId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("FilterListId", "RuleId"); + + b.HasIndex("RuleId"); + + b.ToTable("filterlists_rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.Property("ForkFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("ForkFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("forks"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.Property("MergeFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("MergeFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("merges"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.Property("SoftwareId"); + + b.Property("SyntaxId"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.HasKey("SoftwareId", "SyntaxId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("software_syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Language", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Iso6391") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(2)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392B") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392T") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6393") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("LocalName") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.License", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DescriptionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PermissiveAdaptation"); + + b.Property("PermissiveCommercial"); + + b.HasKey("Id"); + + b.ToTable("licenses"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("TwitterHandle") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Rule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("BIGINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("Raw") + .IsRequired() + .HasColumnType("VARCHAR(8192)"); + + b.HasKey("Id"); + + b.ToTable("rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("FilterListId"); + + b.HasKey("Id"); + + b.HasIndex("FilterListId"); + + b.ToTable("scrapes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DownloadUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("software"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("DefinitionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.HasOne("FilterLists.Data.Entities.License") + .WithMany("FilterLists") + .HasForeignKey("LicenseId"); + + b.HasOne("FilterLists.Data.Entities.Syntax") + .WithMany("FilterLists") + .HasForeignKey("SyntaxId"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListLanguages") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Language", "Language") + .WithMany("FilterListLanguages") + .HasForeignKey("LanguageId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListMaintainers") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer") + .WithMany("FilterListMaintainers") + .HasForeignKey("MaintainerId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListRules") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Rule", "Rule") + .WithMany("FilterListRules") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") + .WithMany("ForkFilterLists") + .HasForeignKey("ForkFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamForkFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList") + .WithMany("MergeFilterLists") + .HasForeignKey("MergeFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamMergeFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.HasOne("FilterLists.Data.Entities.Software", "Software") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SoftwareId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SyntaxId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList") + .WithMany("Scrapes") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.cs b/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.cs new file mode 100644 index 000000000..c0dd2fc37 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180202235529_AddScrapes.cs @@ -0,0 +1,90 @@ +using System; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + public partial class AddScrapes : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + "ModifiedDateUtc", + "rules"); + + migrationBuilder.DropColumn( + "ScrapedDateUtc", + "filterlists"); + + migrationBuilder.DropColumn( + "UpdatedDateUtc", + "filterlists"); + + migrationBuilder.AlterColumn( + "ViewUrl", + "filterlists", + "TEXT", + nullable: false, + oldClrType: typeof(string), + oldType: "TEXT NOT NULL", + oldNullable: true); + + migrationBuilder.CreateTable( + "scrapes", + table => new + { + Id = table.Column("SMALLINT UNSIGNED", nullable: false) + .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn), + CreatedDateUtc = table.Column("TIMESTAMP", nullable: false) + .Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn), + FilterListId = table.Column(nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_scrapes", x => x.Id); + table.ForeignKey( + "FK_scrapes_filterlists_FilterListId", + x => x.FilterListId, + "filterlists", + "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + "IX_scrapes_FilterListId", + "scrapes", + "FilterListId"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + "scrapes"); + + migrationBuilder.AddColumn( + "ModifiedDateUtc", + "rules", + "TIMESTAMP", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AlterColumn( + "ViewUrl", + "filterlists", + "TEXT NOT NULL", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AddColumn( + "ScrapedDateUtc", + "filterlists", + nullable: true); + + migrationBuilder.AddColumn( + "UpdatedDateUtc", + "filterlists", + nullable: true); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs index 19269c4db..53bf8f848 100644 --- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs +++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs @@ -66,17 +66,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("PolicyUrl") .HasColumnType("TEXT"); - b.Property("ScrapedDateUtc"); - b.Property("SubmissionUrl") .HasColumnType("TEXT"); b.Property("SyntaxId"); - b.Property("UpdatedDateUtc"); - b.Property("ViewUrl") - .HasColumnType("TEXT NOT NULL"); + .IsRequired() + .HasColumnType("TEXT"); b.HasKey("Id"); @@ -319,10 +316,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .ValueGeneratedOnAdd() .HasColumnType("TIMESTAMP"); - b.Property("ModifiedDateUtc") - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("TIMESTAMP"); - b.Property("Raw") .IsRequired() .HasColumnType("VARCHAR(8192)"); @@ -332,6 +325,25 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("rules"); }); + modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED"); + + b.Property("CreatedDateUtc") + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP"); + + b.Property("FilterListId"); + + b.HasKey("Id"); + + b.HasIndex("FilterListId"); + + b.ToTable("scrapes"); + }); + modelBuilder.Entity("FilterLists.Data.Entities.Software", b => { b.Property("Id") @@ -440,12 +452,12 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => { b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") - .WithMany() + .WithMany("ForkFilterLists") .HasForeignKey("ForkFilterListId") .OnDelete(DeleteBehavior.Cascade); b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") - .WithMany() + .WithMany("UpstreamForkFilterLists") .HasForeignKey("UpstreamFilterListId") .OnDelete(DeleteBehavior.Cascade); }); @@ -453,12 +465,12 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => { b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList") - .WithMany() + .WithMany("MergeFilterLists") .HasForeignKey("MergeFilterListId") .OnDelete(DeleteBehavior.Cascade); b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") - .WithMany() + .WithMany("UpstreamMergeFilterLists") .HasForeignKey("UpstreamFilterListId") .OnDelete(DeleteBehavior.Cascade); }); @@ -475,6 +487,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasForeignKey("SyntaxId") .OnDelete(DeleteBehavior.Cascade); }); + + modelBuilder.Entity("FilterLists.Data.Entities.Scrape", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList") + .WithMany("Scrapes") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); #pragma warning restore 612, 618 } } diff --git a/src/FilterLists.Data/Entities/FilterList.cs b/src/FilterLists.Data/Entities/FilterList.cs index b5117ec6b..b65e3ed8f 100644 --- a/src/FilterLists.Data/Entities/FilterList.cs +++ b/src/FilterLists.Data/Entities/FilterList.cs @@ -11,20 +11,22 @@ public class FilterList : BaseEntity public DateTime? DiscontinuedDate { get; set; } public string DonateUrl { get; set; } public string EmailAddress { get; set; } + public ICollection ForkFilterLists { get; set; } + public ICollection UpstreamForkFilterLists { get; set; } public string ForumUrl { get; set; } public string HomeUrl { get; set; } public string IssuesUrl { get; set; } + public ICollection FilterListLanguages { get; set; } public int? LicenseId { get; set; } + public ICollection FilterListMaintainers { get; set; } + public ICollection MergeFilterLists { get; set; } + public ICollection UpstreamMergeFilterLists { get; set; } public string Name { get; set; } public string PolicyUrl { get; set; } - public DateTime? ScrapedDateUtc { get; set; } + public ICollection FilterListRules { get; set; } + public ICollection Scrapes { get; set; } public string SubmissionUrl { get; set; } public int? SyntaxId { get; set; } - public DateTime? UpdatedDateUtc { get; set; } public string ViewUrl { get; set; } - - public ICollection FilterListLanguages { get; set; } - public ICollection FilterListMaintainers { get; set; } - public ICollection FilterListRules { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/Entities/Language.cs b/src/FilterLists.Data/Entities/Language.cs index 518344660..178aca1ae 100644 --- a/src/FilterLists.Data/Entities/Language.cs +++ b/src/FilterLists.Data/Entities/Language.cs @@ -5,6 +5,7 @@ namespace FilterLists.Data.Entities { public class Language : BaseEntity { + public ICollection FilterListLanguages { get; set; } public string Iso6391 { get; set; } public string Iso6392 { get; set; } public string Iso6392B { get; set; } @@ -12,7 +13,5 @@ public class Language : BaseEntity public string Iso6393 { get; set; } public string LocalName { get; set; } public string Name { get; set; } - - public ICollection FilterListLanguages { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/Entities/Maintainer.cs b/src/FilterLists.Data/Entities/Maintainer.cs index 344cb9bd6..64b6f46e9 100644 --- a/src/FilterLists.Data/Entities/Maintainer.cs +++ b/src/FilterLists.Data/Entities/Maintainer.cs @@ -6,10 +6,9 @@ namespace FilterLists.Data.Entities public class Maintainer : BaseEntity { public string EmailAddress { get; set; } + public ICollection FilterListMaintainers { get; set; } public string HomeUrl { get; set; } public string Name { get; set; } public string TwitterHandle { get; set; } - - public ICollection FilterListMaintainers { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/Entities/Rule.cs b/src/FilterLists.Data/Entities/Rule.cs index f18f7e5c0..41b15b313 100644 --- a/src/FilterLists.Data/Entities/Rule.cs +++ b/src/FilterLists.Data/Entities/Rule.cs @@ -5,8 +5,7 @@ namespace FilterLists.Data.Entities { public class Rule : BaseEntity { - public string Raw { get; set; } - public ICollection FilterListRules { get; set; } + public string Raw { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/Entities/Scrape.cs b/src/FilterLists.Data/Entities/Scrape.cs new file mode 100644 index 000000000..b22ff191a --- /dev/null +++ b/src/FilterLists.Data/Entities/Scrape.cs @@ -0,0 +1,7 @@ +namespace FilterLists.Data.Entities +{ + public class Scrape : BaseEntity + { + public int FilterListId { get; set; } + } +} \ No newline at end of file diff --git a/src/FilterLists.Data/Entities/Software.cs b/src/FilterLists.Data/Entities/Software.cs index a4d75f64a..d3d918697 100644 --- a/src/FilterLists.Data/Entities/Software.cs +++ b/src/FilterLists.Data/Entities/Software.cs @@ -8,7 +8,6 @@ public class Software : BaseEntity public string DownloadUrl { get; set; } public string HomeUrl { get; set; } public string Name { get; set; } - public ICollection SoftwareSyntaxes { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/Entities/Syntax.cs b/src/FilterLists.Data/Entities/Syntax.cs index 89ab38e2b..7a25dabe3 100644 --- a/src/FilterLists.Data/Entities/Syntax.cs +++ b/src/FilterLists.Data/Entities/Syntax.cs @@ -8,7 +8,6 @@ public class Syntax : BaseEntity public string DefinitionUrl { get; set; } public ICollection FilterLists { get; set; } public string Name { get; set; } - public ICollection SoftwareSyntaxes { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs index 2dfb4e47d..dd7dad9a6 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs @@ -35,7 +35,8 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) entityTypeBuilder.Property(x => x.SubmissionUrl) .HasColumnType("TEXT"); entityTypeBuilder.Property(x => x.ViewUrl) - .HasColumnType("TEXT NOT NULL"); + .HasColumnType("TEXT") + .IsRequired(); } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs index 629edf7fb..f921ac24e 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs @@ -15,6 +15,8 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) entityTypeBuilder.Property(x => x.Id) .HasColumnType("BIGINT UNSIGNED"); + entityTypeBuilder.Ignore(x => x.ModifiedDateUtc); + entityTypeBuilder.Property(x => x.Raw) .HasColumnType("VARCHAR(8192)") .IsRequired(); diff --git a/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs new file mode 100644 index 000000000..b2d851ebf --- /dev/null +++ b/src/FilterLists.Data/EntityTypeConfigurations/ScrapeTypeConfiguration.cs @@ -0,0 +1,18 @@ +using FilterLists.Data.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace FilterLists.Data.EntityTypeConfigurations +{ + public class ScrapeTypeConfiguration : BaseEntityTypeConfiguration + { + public override void Configure(EntityTypeBuilder entityTypeBuilder) + { + base.Configure(entityTypeBuilder); + + entityTypeBuilder.ToTable("scrapes"); + + entityTypeBuilder.Ignore(x => x.ModifiedDateUtc); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Data/FilterListsDbContext.cs b/src/FilterLists.Data/FilterListsDbContext.cs index 7e86403ea..64fc89969 100644 --- a/src/FilterLists.Data/FilterListsDbContext.cs +++ b/src/FilterLists.Data/FilterListsDbContext.cs @@ -27,6 +27,7 @@ private static void ApplyConfigurationsEntities(ModelBuilder modelBuilder) modelBuilder.ApplyConfiguration(new LicenseTypeConfiguration()); modelBuilder.ApplyConfiguration(new MaintainerTypeConfiguration()); modelBuilder.ApplyConfiguration(new RuleTypeConfiguration()); + modelBuilder.ApplyConfiguration(new ScrapeTypeConfiguration()); modelBuilder.ApplyConfiguration(new SoftwareTypeConfiguration()); modelBuilder.ApplyConfiguration(new SyntaxTypeConfiguration()); } @@ -37,10 +38,36 @@ private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder) modelBuilder.ApplyConfiguration(new FilterListMaintainerTypeConfiguration()); modelBuilder.ApplyConfiguration(new FilterListRuleTypeConfiguration()); modelBuilder.ApplyConfiguration(new ForkTypeConfiguration()); + ConfigureForkSelfReferencing(modelBuilder); modelBuilder.ApplyConfiguration(new MergeTypeConfiguration()); + ConfigureMergeSelfReferencing(modelBuilder); modelBuilder.ApplyConfiguration(new SoftwareSyntaxTypeConfiguration()); } + private static void ConfigureForkSelfReferencing(ModelBuilder modelBuilder) + { + modelBuilder.Entity() + .HasOne(x => x.ForkFilterList) + .WithMany(x => x.ForkFilterLists) + .HasForeignKey(x => x.ForkFilterListId); + modelBuilder.Entity() + .HasOne(x => x.UpstreamFilterList) + .WithMany(x => x.UpstreamForkFilterLists) + .HasForeignKey(x => x.UpstreamFilterListId); + } + + private static void ConfigureMergeSelfReferencing(ModelBuilder modelBuilder) + { + modelBuilder.Entity() + .HasOne(x => x.MergeFilterList) + .WithMany(x => x.MergeFilterLists) + .HasForeignKey(x => x.MergeFilterListId); + modelBuilder.Entity() + .HasOne(x => x.UpstreamFilterList) + .WithMany(x => x.UpstreamMergeFilterLists) + .HasForeignKey(x => x.UpstreamFilterListId); + } + #region Entities public DbSet FilterLists { get; set; } @@ -48,6 +75,7 @@ private static void ApplyConfigurationsJunctions(ModelBuilder modelBuilder) public DbSet Licenses { get; set; } public DbSet Maintainers { get; set; } public DbSet Rules { get; set; } + public DbSet Scrapes { get; set; } public DbSet Software { get; set; } public DbSet Syntaxes { get; set; } diff --git a/src/FilterLists.Services/ScrapeService/ScrapeService.cs b/src/FilterLists.Services/ScrapeService/ScrapeService.cs index 54722870c..903677908 100644 --- a/src/FilterLists.Services/ScrapeService/ScrapeService.cs +++ b/src/FilterLists.Services/ScrapeService/ScrapeService.cs @@ -28,8 +28,8 @@ public async Task ScrapeAsync(int batchSize) private async Task> GetNextListsToScrape(int batchSize) { - return await dbContext.FilterLists.OrderBy(x => x.ScrapedDateUtc).Take(batchSize) - .ProjectTo().ToListAsync(); + //TODO: order by least recently scraped + return await dbContext.FilterLists.Take(batchSize).ProjectTo().ToListAsync(); } private async Task> GetSnapshots(IEnumerable lists) diff --git a/src/FilterLists.Services/ScrapeService/Snapshot.cs b/src/FilterLists.Services/ScrapeService/Snapshot.cs index d97c61891..3da339b91 100644 --- a/src/FilterLists.Services/ScrapeService/Snapshot.cs +++ b/src/FilterLists.Services/ScrapeService/Snapshot.cs @@ -61,15 +61,7 @@ public async Task AddOrUpdateRules() new FilterListRule {FilterListId = filterListId, Rule = newSnapshotRule}).ToList(); await dbContext.FilterListRules.AddRangeAsync(newFilterListRules); - // update UpdatedDateUtc - var list = dbContext.FilterLists.Find(filterListId); - if (preExistingSnapshotFilterListRules.Any() || newFilterListRules.Any() || deletedFilterListRules.Any()) - list.UpdatedDateUtc = DateTime.UtcNow; - - // update ScrapedDateUtc - list.ScrapedDateUtc = DateTime.UtcNow; - - dbContext.FilterLists.Update(list); + // update scrapes await dbContext.SaveChangesAsync(); }