mirror of
https://github.com/collinbarrett/FilterLists.git
synced 2026-03-11 09:04:27 +00:00
further work on scraper
This commit is contained in:
Collin M. Barrett
parent
12e5c4e2ca
commit
d2532067ca
5 changed files with 659 additions and 56 deletions
507
src/FilterLists.Api/Migrations/20180126190529_addSnapshot.Designer.cs
generated
Normal file
507
src/FilterLists.Api/Migrations/20180126190529_addSnapshot.Designer.cs
generated
Normal file
|
|
@ -0,0 +1,507 @@
|
|||
// <auto-generated />
|
||||
using FilterLists.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Metadata;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Storage;
|
||||
using Microsoft.EntityFrameworkCore.Storage.Internal;
|
||||
using System;
|
||||
|
||||
namespace FilterLists.Api.Migrations
|
||||
{
|
||||
[DbContext(typeof(FilterListsDbContext))]
|
||||
[Migration("20180126190529_addSnapshot")]
|
||||
partial class addSnapshot
|
||||
{
|
||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
|
||||
.HasAnnotation("ProductVersion", "2.0.1-rtm-125");
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd();
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Description")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("DescriptionSourceUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime?>("DiscontinuedDate");
|
||||
|
||||
b.Property<string>("DonateUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("EmailAddress")
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.Property<string>("ForumUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("HomeUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("IssuesUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<int?>("LicenseId");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.Property<string>("SubmissionUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<int?>("SyntaxId");
|
||||
|
||||
b.Property<string>("ViewUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("LicenseId");
|
||||
|
||||
b.HasIndex("SyntaxId");
|
||||
|
||||
b.ToTable("filterlists");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<int>("LanguageId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("FilterListId", "LanguageId");
|
||||
|
||||
b.HasIndex("LanguageId");
|
||||
|
||||
b.ToTable("filterlists_languages");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<int>("MaintainerId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("FilterListId", "MaintainerId");
|
||||
|
||||
b.HasIndex("MaintainerId");
|
||||
|
||||
b.ToTable("filterlists_maintainers");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<int>("RuleId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("FilterListId", "RuleId");
|
||||
|
||||
b.HasIndex("RuleId");
|
||||
|
||||
b.ToTable("filterlists_rules");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
|
||||
{
|
||||
b.Property<int>("ForkFilterListId");
|
||||
|
||||
b.Property<int>("UpstreamFilterListId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("ForkFilterListId", "UpstreamFilterListId");
|
||||
|
||||
b.HasIndex("UpstreamFilterListId");
|
||||
|
||||
b.ToTable("forks");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
|
||||
{
|
||||
b.Property<int>("MergeFilterListId");
|
||||
|
||||
b.Property<int>("UpstreamFilterListId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("MergeFilterListId", "UpstreamFilterListId");
|
||||
|
||||
b.HasIndex("UpstreamFilterListId");
|
||||
|
||||
b.ToTable("merges");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
|
||||
{
|
||||
b.Property<int>("SoftwareId");
|
||||
|
||||
b.Property<int>("SyntaxId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("SoftwareId", "SyntaxId");
|
||||
|
||||
b.HasIndex("SyntaxId");
|
||||
|
||||
b.ToTable("software_syntaxes");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd();
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Iso6391")
|
||||
.HasColumnType("NVARCHAR(2)");
|
||||
|
||||
b.Property<string>("Iso6392")
|
||||
.HasColumnType("NVARCHAR(3)");
|
||||
|
||||
b.Property<string>("Iso6392B")
|
||||
.HasColumnType("NVARCHAR(3)");
|
||||
|
||||
b.Property<string>("Iso6392T")
|
||||
.HasColumnType("NVARCHAR(3)");
|
||||
|
||||
b.Property<string>("Iso6393")
|
||||
.HasColumnType("NVARCHAR(3)");
|
||||
|
||||
b.Property<string>("LocalName")
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("languages");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd();
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("DescriptionUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.Property<bool>("PermissiveAdaptation");
|
||||
|
||||
b.Property<bool>("PermissiveCommercial");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("licenses");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd();
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("EmailAddress")
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.Property<string>("HomeUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.Property<string>("TwitterHandle")
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("maintainers");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd();
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Raw")
|
||||
.HasColumnType("NVARCHAR(2083)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("rules");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd();
|
||||
|
||||
b.Property<string>("Content")
|
||||
.IsRequired()
|
||||
.HasColumnType("MEDIUMTEXT");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("FilterListId");
|
||||
|
||||
b.ToTable("snapshots");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd();
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("DownloadUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<string>("HomeUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("software");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd();
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("DefinitionUrl")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("NVARCHAR(126)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("syntaxes");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.License")
|
||||
.WithMany("FilterLists")
|
||||
.HasForeignKey("LicenseId");
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Syntax")
|
||||
.WithMany("FilterLists")
|
||||
.HasForeignKey("SyntaxId");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListLanguages")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Language", "Language")
|
||||
.WithMany("FilterListLanguages")
|
||||
.HasForeignKey("LanguageId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListMaintainers")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
|
||||
.WithMany("FilterListMaintainers")
|
||||
.HasForeignKey("MaintainerId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListRules")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
|
||||
.WithMany("FilterListRules")
|
||||
.HasForeignKey("RuleId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
|
||||
.WithMany()
|
||||
.HasForeignKey("ForkFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
|
||||
.WithMany()
|
||||
.HasForeignKey("UpstreamFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
|
||||
.WithMany()
|
||||
.HasForeignKey("MergeFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
|
||||
.WithMany()
|
||||
.HasForeignKey("UpstreamFilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.Software", "Software")
|
||||
.WithMany("SoftwareSyntaxes")
|
||||
.HasForeignKey("SoftwareId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
|
||||
.WithMany("SoftwareSyntaxes")
|
||||
.HasForeignKey("SyntaxId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList")
|
||||
.WithMany("Snapshots")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
#pragma warning restore 612, 618
|
||||
}
|
||||
}
|
||||
}
|
||||
47
src/FilterLists.Api/Migrations/20180126190529_addSnapshot.cs
Normal file
47
src/FilterLists.Api/Migrations/20180126190529_addSnapshot.cs
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
using Microsoft.EntityFrameworkCore.Metadata;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace FilterLists.Api.Migrations
|
||||
{
|
||||
public partial class addSnapshot : Migration
|
||||
{
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.CreateTable(
|
||||
name: "snapshots",
|
||||
columns: table => new
|
||||
{
|
||||
Id = table.Column<int>(nullable: false)
|
||||
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
|
||||
Content = table.Column<string>(type: "MEDIUMTEXT", nullable: false),
|
||||
CreatedDateUtc = table.Column<DateTime>(type: "TIMESTAMP", nullable: false)
|
||||
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
|
||||
FilterListId = table.Column<int>(nullable: false),
|
||||
ModifiedDateUtc = table.Column<DateTime>(type: "TIMESTAMP", nullable: false)
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_snapshots", x => x.Id);
|
||||
table.ForeignKey(
|
||||
name: "FK_snapshots_filterlists_FilterListId",
|
||||
column: x => x.FilterListId,
|
||||
principalTable: "filterlists",
|
||||
principalColumn: "Id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_snapshots_FilterListId",
|
||||
table: "snapshots",
|
||||
column: "FilterListId");
|
||||
}
|
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropTable(
|
||||
name: "snapshots");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -79,7 +79,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.ToTable("filterlists");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterListLanguage", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
|
|
@ -100,7 +100,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.ToTable("filterlists_languages");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterListMaintainer", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
|
|
@ -121,7 +121,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.ToTable("filterlists_maintainers");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterListRule", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
|
||||
{
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
|
|
@ -142,7 +142,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.ToTable("filterlists_rules");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Fork", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
|
||||
{
|
||||
b.Property<int>("ForkFilterListId");
|
||||
|
||||
|
|
@ -163,6 +163,48 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.ToTable("forks");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
|
||||
{
|
||||
b.Property<int>("MergeFilterListId");
|
||||
|
||||
b.Property<int>("UpstreamFilterListId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("MergeFilterListId", "UpstreamFilterListId");
|
||||
|
||||
b.HasIndex("UpstreamFilterListId");
|
||||
|
||||
b.ToTable("merges");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
|
||||
{
|
||||
b.Property<int>("SoftwareId");
|
||||
|
||||
b.Property<int>("SyntaxId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("SoftwareId", "SyntaxId");
|
||||
|
||||
b.HasIndex("SyntaxId");
|
||||
|
||||
b.ToTable("software_syntaxes");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
|
|
@ -262,27 +304,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.ToTable("maintainers");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Merge", b =>
|
||||
{
|
||||
b.Property<int>("MergeFilterListId");
|
||||
|
||||
b.Property<int>("UpstreamFilterListId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("MergeFilterListId", "UpstreamFilterListId");
|
||||
|
||||
b.HasIndex("UpstreamFilterListId");
|
||||
|
||||
b.ToTable("merges");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
|
|
@ -304,6 +325,32 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.ToTable("rules");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd();
|
||||
|
||||
b.Property<string>("Content")
|
||||
.IsRequired()
|
||||
.HasColumnType("MEDIUMTEXT");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<int>("FilterListId");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("FilterListId");
|
||||
|
||||
b.ToTable("snapshots");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
|
|
@ -332,27 +379,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
b.ToTable("software");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.SoftwareSyntax", b =>
|
||||
{
|
||||
b.Property<int>("SoftwareId");
|
||||
|
||||
b.Property<int>("SyntaxId");
|
||||
|
||||
b.Property<DateTime>("CreatedDateUtc")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.Property<DateTime>("ModifiedDateUtc")
|
||||
.ValueGeneratedOnAddOrUpdate()
|
||||
.HasColumnType("TIMESTAMP");
|
||||
|
||||
b.HasKey("SoftwareId", "SyntaxId");
|
||||
|
||||
b.HasIndex("SyntaxId");
|
||||
|
||||
b.ToTable("software_syntaxes");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
|
|
@ -389,7 +415,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
.HasForeignKey("SyntaxId");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterListLanguage", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListLanguages")
|
||||
|
|
@ -402,7 +428,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterListMaintainer", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListMaintainers")
|
||||
|
|
@ -415,7 +441,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.FilterListRule", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
|
||||
.WithMany("FilterListRules")
|
||||
|
|
@ -428,7 +454,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Fork", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
|
||||
.WithMany()
|
||||
|
|
@ -441,7 +467,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Merge", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
|
||||
.WithMany()
|
||||
|
|
@ -454,7 +480,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.SoftwareSyntax", b =>
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.Software", "Software")
|
||||
.WithMany("SoftwareSyntaxes")
|
||||
|
|
@ -466,6 +492,14 @@ protected override void BuildModel(ModelBuilder modelBuilder)
|
|||
.HasForeignKey("SyntaxId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
|
||||
{
|
||||
b.HasOne("FilterLists.Data.Entities.FilterList")
|
||||
.WithMany("Snapshots")
|
||||
.HasForeignKey("FilterListId")
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
#pragma warning restore 612, 618
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,10 +13,11 @@ public ScrapeController(ScrapeService scrapeService)
|
|||
}
|
||||
|
||||
[HttpPost]
|
||||
public void Index()
|
||||
public IActionResult Index()
|
||||
{
|
||||
//TODO: Convert endpoint into scheduled job, don't expose on prod
|
||||
//TODO: convert endpoint into scheduled job, don't expose on prod
|
||||
//scrapeService.Scrape();
|
||||
return Ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -18,14 +18,28 @@ public ScrapeService(FilterListsDbContext filterListsDbContext)
|
|||
|
||||
public void Scrape()
|
||||
{
|
||||
var list = filterListsDbContext.FilterLists.First();
|
||||
var lists = filterListsDbContext.FilterLists.Take(5);
|
||||
foreach (var list in lists) AddSnapshot(list);
|
||||
filterListsDbContext.SaveChangesAsync();
|
||||
}
|
||||
|
||||
private void AddSnapshot(FilterList list)
|
||||
{
|
||||
var snapshot = GetContent(list.ViewUrl);
|
||||
try
|
||||
{
|
||||
if (snapshot.Result == null) return;
|
||||
}
|
||||
catch (AggregateException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
filterListsDbContext.Snapshots.Add(new Snapshot
|
||||
{
|
||||
Content = snapshot.Result,
|
||||
FilterListId = list.Id
|
||||
});
|
||||
filterListsDbContext.SaveChangesAsync();
|
||||
}
|
||||
|
||||
private static async Task<string> GetContent(string url)
|
||||
|
|
|
|||
Loading…
Reference in a new issue