prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

scraper and db updates

Browse source
This commit is contained in:
Collin M. Barrett 2018-01-26 19:44:10 -06:00
parent 6caef4c4d6
commit fa0eeacfc0
20 changed files with 709 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

477
src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.Designer.cs generated Normal file
View file

@ -0,0 +1,477 @@
// <auto-generated />
using FilterLists.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.EntityFrameworkCore.Storage.Internal;
using System;
namespace FilterLists.Api.Migrations
{
[DbContext(typeof(FilterListsDbContext))]
[Migration("20180127014225_partialRemoveModDateAndMore")]
partial class partialRemoveModDateAndMore
{
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn)
.HasAnnotation("ProductVersion", "2.0.1-rtm-125");
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd();
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<string>("Description")
.HasColumnType("TEXT");
b.Property<string>("DescriptionSourceUrl")
.HasColumnType("TEXT");
b.Property<DateTime?>("DiscontinuedDate");
b.Property<string>("DonateUrl")
.HasColumnType("TEXT");
b.Property<string>("EmailAddress")
.HasColumnType("NVARCHAR(126)");
b.Property<string>("ForumUrl")
.HasColumnType("TEXT");
b.Property<string>("HomeUrl")
.HasColumnType("TEXT");
b.Property<string>("IssuesUrl")
.HasColumnType("TEXT");
b.Property<int?>("LicenseId");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("NVARCHAR(126)");
b.Property<string>("SubmissionUrl")
.HasColumnType("TEXT");
b.Property<int?>("SyntaxId");
b.Property<string>("ViewUrl")
.IsRequired()
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("LicenseId");
b.HasIndex("SyntaxId");
b.ToTable("filterlists");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
{
b.Property<int>("FilterListId");
b.Property<int>("LanguageId");
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.HasKey("FilterListId", "LanguageId");
b.HasIndex("LanguageId");
b.ToTable("filterlists_languages");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
{
b.Property<int>("FilterListId");
b.Property<int>("MaintainerId");
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.HasKey("FilterListId", "MaintainerId");
b.HasIndex("MaintainerId");
b.ToTable("filterlists_maintainers");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
{
b.Property<int>("FilterListId");
b.Property<int>("RuleId");
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.HasKey("FilterListId", "RuleId");
b.HasIndex("RuleId");
b.ToTable("filterlists_rules");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
{
b.Property<int>("ForkFilterListId");
b.Property<int>("UpstreamFilterListId");
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.HasKey("ForkFilterListId", "UpstreamFilterListId");
b.HasIndex("UpstreamFilterListId");
b.ToTable("forks");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
{
b.Property<int>("MergeFilterListId");
b.Property<int>("UpstreamFilterListId");
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.HasKey("MergeFilterListId", "UpstreamFilterListId");
b.HasIndex("UpstreamFilterListId");
b.ToTable("merges");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
{
b.Property<int>("SoftwareId");
b.Property<int>("SyntaxId");
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.HasKey("SoftwareId", "SyntaxId");
b.HasIndex("SyntaxId");
b.ToTable("software_syntaxes");
});
modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd();
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<string>("Iso6391")
.HasColumnType("NVARCHAR(2)");
b.Property<string>("Iso6392")
.HasColumnType("NVARCHAR(3)");
b.Property<string>("Iso6392B")
.HasColumnType("NVARCHAR(3)");
b.Property<string>("Iso6392T")
.HasColumnType("NVARCHAR(3)");
b.Property<string>("Iso6393")
.HasColumnType("NVARCHAR(3)");
b.Property<string>("LocalName")
.HasColumnType("NVARCHAR(126)");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.Property<string>("Name")
.HasColumnType("NVARCHAR(126)");
b.HasKey("Id");
b.ToTable("languages");
});
modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd();
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<string>("DescriptionUrl")
.HasColumnType("TEXT");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("NVARCHAR(126)");
b.Property<bool>("PermissiveAdaptation");
b.Property<bool>("PermissiveCommercial");
b.HasKey("Id");
b.ToTable("licenses");
});
modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd();
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<string>("EmailAddress")
.HasColumnType("NVARCHAR(126)");
b.Property<string>("HomeUrl")
.HasColumnType("TEXT");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("NVARCHAR(126)");
b.Property<string>("TwitterHandle")
.HasColumnType("NVARCHAR(126)");
b.HasKey("Id");
b.ToTable("maintainers");
});
modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd();
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<string>("Raw")
.IsRequired()
.HasColumnType("NVARCHAR(2083)");
b.HasKey("Id");
b.ToTable("rules");
});
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd();
b.Property<string>("Content")
.IsRequired()
.HasColumnType("MEDIUMTEXT");
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<int>("FilterListId");
b.HasKey("Id");
b.HasIndex("FilterListId");
b.ToTable("snapshots");
});
modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd();
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<string>("DownloadUrl")
.HasColumnType("TEXT");
b.Property<string>("HomeUrl")
.HasColumnType("TEXT");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("NVARCHAR(126)");
b.HasKey("Id");
b.ToTable("software");
});
modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd();
b.Property<DateTime>("CreatedDateUtc")
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<string>("DefinitionUrl")
.HasColumnType("TEXT");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("NVARCHAR(126)");
b.HasKey("Id");
b.ToTable("syntaxes");
});
modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
{
b.HasOne("FilterLists.Data.Entities.License")
.WithMany("FilterLists")
.HasForeignKey("LicenseId");
b.HasOne("FilterLists.Data.Entities.Syntax")
.WithMany("FilterLists")
.HasForeignKey("SyntaxId");
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
.WithMany("FilterListLanguages")
.HasForeignKey("FilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Language", "Language")
.WithMany("FilterListLanguages")
.HasForeignKey("LanguageId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
.WithMany("FilterListMaintainers")
.HasForeignKey("FilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
.WithMany("FilterListMaintainers")
.HasForeignKey("MaintainerId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListRule", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
.WithMany("FilterListRules")
.HasForeignKey("FilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
.WithMany("FilterListRules")
.HasForeignKey("RuleId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
.WithMany()
.HasForeignKey("ForkFilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
.WithMany()
.HasForeignKey("UpstreamFilterListId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
.WithMany()
.HasForeignKey("MergeFilterListId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
.WithMany()
.HasForeignKey("UpstreamFilterListId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
{
b.HasOne("FilterLists.Data.Entities.Software", "Software")
.WithMany("SoftwareSyntaxes")
.HasForeignKey("SoftwareId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
.WithMany("SoftwareSyntaxes")
.HasForeignKey("SyntaxId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
{
b.HasOne("FilterLists.Data.Entities.FilterList")
.WithMany("Snapshots")
.HasForeignKey("FilterListId")
.OnDelete(DeleteBehavior.Cascade);
});
#pragma warning restore 612, 618
}
}
}

137
src/FilterLists.Api/Migrations/20180127014225_partialRemoveModDateAndMore.cs Normal file
View file

@ -0,0 +1,137 @@
using Microsoft.EntityFrameworkCore.Migrations;
using System;
using System.Collections.Generic;
namespace FilterLists.Api.Migrations
{
public partial class partialRemoveModDateAndMore : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "ModifiedDateUtc",
table: "software_syntaxes");
migrationBuilder.DropColumn(
name: "ModifiedDateUtc",
table: "snapshots");
migrationBuilder.DropColumn(
name: "ModifiedDateUtc",
table: "rules");
migrationBuilder.DropColumn(
name: "ModifiedDateUtc",
table: "merges");
migrationBuilder.DropColumn(
name: "ModifiedDateUtc",
table: "forks");
migrationBuilder.DropColumn(
name: "ModifiedDateUtc",
table: "filterlists_rules");
migrationBuilder.DropColumn(
name: "ModifiedDateUtc",
table: "filterlists_maintainers");
migrationBuilder.DropColumn(
name: "ModifiedDateUtc",
table: "filterlists_languages");
migrationBuilder.AlterColumn<string>(
name: "Raw",
table: "rules",
type: "NVARCHAR(2083)",
nullable: false,
oldClrType: typeof(string),
oldType: "NVARCHAR(2083)",
oldNullable: true);
migrationBuilder.AlterColumn<string>(
name: "ViewUrl",
table: "filterlists",
type: "TEXT",
nullable: false,
oldClrType: typeof(string),
oldType: "TEXT",
oldNullable: true);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<DateTime>(
name: "ModifiedDateUtc",
table: "software_syntaxes",
type: "TIMESTAMP",
nullable: false,
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
migrationBuilder.AddColumn<DateTime>(
name: "ModifiedDateUtc",
table: "snapshots",
type: "TIMESTAMP",
nullable: false,
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
migrationBuilder.AlterColumn<string>(
name: "Raw",
table: "rules",
type: "NVARCHAR(2083)",
nullable: true,
oldClrType: typeof(string),
oldType: "NVARCHAR(2083)");
migrationBuilder.AddColumn<DateTime>(
name: "ModifiedDateUtc",
table: "rules",
type: "TIMESTAMP",
nullable: false,
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
migrationBuilder.AddColumn<DateTime>(
name: "ModifiedDateUtc",
table: "merges",
type: "TIMESTAMP",
nullable: false,
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
migrationBuilder.AddColumn<DateTime>(
name: "ModifiedDateUtc",
table: "forks",
type: "TIMESTAMP",
nullable: false,
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
migrationBuilder.AddColumn<DateTime>(
name: "ModifiedDateUtc",
table: "filterlists_rules",
type: "TIMESTAMP",
nullable: false,
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
migrationBuilder.AddColumn<DateTime>(
name: "ModifiedDateUtc",
table: "filterlists_maintainers",
type: "TIMESTAMP",
nullable: false,
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
migrationBuilder.AddColumn<DateTime>(
name: "ModifiedDateUtc",
table: "filterlists_languages",
type: "TIMESTAMP",
nullable: false,
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
migrationBuilder.AlterColumn<string>(
name: "ViewUrl",
table: "filterlists",
type: "TEXT",
nullable: true,
oldClrType: typeof(string),
oldType: "TEXT");
}
}
}

34
src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
View file

@ -68,6 +68,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property<int?>("SyntaxId");
b.Property<string>("ViewUrl")
.IsRequired()
.HasColumnType("TEXT");
b.HasKey("Id");
@ -89,10 +90,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.HasKey("FilterListId", "LanguageId");
b.HasIndex("LanguageId");
@ -110,10 +107,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.HasKey("FilterListId", "MaintainerId");
b.HasIndex("MaintainerId");
@ -131,10 +124,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.HasKey("FilterListId", "RuleId");
b.HasIndex("RuleId");
@ -152,10 +141,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.HasKey("ForkFilterListId", "UpstreamFilterListId");
b.HasIndex("UpstreamFilterListId");
@ -173,10 +158,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.HasKey("MergeFilterListId", "UpstreamFilterListId");
b.HasIndex("UpstreamFilterListId");
@ -194,10 +175,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.HasKey("SoftwareId", "SyntaxId");
b.HasIndex("SyntaxId");
@ -313,11 +290,8 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("TIMESTAMP");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.Property<string>("Raw")
.IsRequired()
.HasColumnType("NVARCHAR(2083)");
b.HasKey("Id");
@ -340,10 +314,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property<int>("FilterListId");
b.Property<DateTime>("ModifiedDateUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("TIMESTAMP");
b.HasKey("Id");
b.HasIndex("FilterListId");

1
src/FilterLists.Data/Entities/BaseEntity.cs
View file

@ -6,6 +6,5 @@ public abstract class BaseEntity
{
public int Id { get; set; }
public DateTime CreatedDateUtc { get; set; }
public DateTime ModifiedDateUtc { get; set; }
}
}

1
src/FilterLists.Data/Entities/FilterList.cs
View file

@ -6,6 +6,7 @@ namespace FilterLists.Data.Entities
{
public class FilterList : BaseEntity
{
public DateTime ModifiedDateUtc { get; set; }
public string Description { get; set; }
public string DescriptionSourceUrl { get; set; }
public DateTime? DiscontinuedDate { get; set; }

4
src/FilterLists.Data/Entities/Language.cs
View file

@ -1,10 +1,12 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using FilterLists.Data.Entities.Junctions;
namespace FilterLists.Data.Entities
{
public class Language : BaseEntity
{
public DateTime ModifiedDateUtc { get; set; }
public ICollection<FilterListLanguage> FilterListLanguages { get; set; }
public string Iso6391 { get; set; }
public string Iso6392 { get; set; }

4
src/FilterLists.Data/Entities/License.cs
View file

@ -1,9 +1,11 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
namespace FilterLists.Data.Entities
{
public class License : BaseEntity
{
public DateTime ModifiedDateUtc { get; set; }
public string DescriptionUrl { get; set; }
public ICollection<FilterList> FilterLists { get; set; }
public string Name { get; set; }

4
src/FilterLists.Data/Entities/Maintainer.cs
View file

@ -1,10 +1,12 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using FilterLists.Data.Entities.Junctions;
namespace FilterLists.Data.Entities
{
public class Maintainer : BaseEntity
{
public DateTime ModifiedDateUtc { get; set; }
public string EmailAddress { get; set; }
public ICollection<FilterListMaintainer> FilterListMaintainers { get; set; }
public string HomeUrl { get; set; }

4
src/FilterLists.Data/Entities/Software.cs
View file

@ -1,10 +1,12 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using FilterLists.Data.Entities.Junctions;
namespace FilterLists.Data.Entities
{
public class Software : BaseEntity
{
public DateTime ModifiedDateUtc { get; set; }
public string DownloadUrl { get; set; }
public string HomeUrl { get; set; }
public string Name { get; set; }

4
src/FilterLists.Data/Entities/Syntax.cs
View file

@ -1,10 +1,12 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using FilterLists.Data.Entities.Junctions;
namespace FilterLists.Data.Entities
{
public class Syntax : BaseEntity
{
public DateTime ModifiedDateUtc { get; set; }
public string DefinitionUrl { get; set; }
public ICollection<FilterList> FilterLists { get; set; }
public string Name { get; set; }

3
src/FilterLists.Data/EntityTypeConfigurations/BaseEntityConfiguration.cs
View file

@ -11,9 +11,6 @@ public virtual void Configure(EntityTypeBuilder<TBase> entityTypeBuilder)
entityTypeBuilder.Property(x => x.CreatedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAdd();
entityTypeBuilder.Property(x => x.ModifiedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAddOrUpdate();
}
}
}

6
src/FilterLists.Data/EntityTypeConfigurations/FilterListTypeConfiguration.cs
View file

@ -9,6 +9,9 @@ public class FilterListTypeConfiguration : BaseEntityTypeConfiguration<FilterLis
public override void Configure(EntityTypeBuilder<FilterList> entityTypeBuilder)
{
entityTypeBuilder.ToTable("filterlists");
entityTypeBuilder.Property(x => x.ModifiedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.Description)
.HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.DescriptionSourceUrl)
@ -29,7 +32,8 @@ public override void Configure(EntityTypeBuilder<FilterList> entityTypeBuilder)
entityTypeBuilder.Property(x => x.SubmissionUrl)
.HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.ViewUrl)
.HasColumnType("TEXT");
.HasColumnType("TEXT")
.IsRequired();
base.Configure(entityTypeBuilder);
}
}

3
src/FilterLists.Data/EntityTypeConfigurations/LanguageTypeConfiguration.cs
View file

@ -9,6 +9,9 @@ public class LanguageTypeConfiguration : BaseEntityTypeConfiguration<Language>
public override void Configure(EntityTypeBuilder<Language> entityTypeBuilder)
{
entityTypeBuilder.ToTable("languages");
entityTypeBuilder.Property(x => x.ModifiedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.Iso6391)
.HasColumnType("NVARCHAR(2)");
entityTypeBuilder.Property(x => x.Iso6392)

3
src/FilterLists.Data/EntityTypeConfigurations/LicenseTypeConfiguration.cs
View file

@ -9,6 +9,9 @@ public class LicenseTypeConfiguration : BaseEntityTypeConfiguration<License>
public override void Configure(EntityTypeBuilder<License> entityTypeBuilder)
{
entityTypeBuilder.ToTable("licenses");
entityTypeBuilder.Property(x => x.ModifiedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.DescriptionUrl)
.HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.Name)

3
src/FilterLists.Data/EntityTypeConfigurations/MaintainerTypeConfiguration.cs
View file

@ -9,6 +9,9 @@ public class MaintainerTypeConfiguration : BaseEntityTypeConfiguration<Maintaine
public override void Configure(EntityTypeBuilder<Maintainer> entityTypeBuilder)
{
entityTypeBuilder.ToTable("maintainers");
entityTypeBuilder.Property(x => x.ModifiedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.EmailAddress)
.HasColumnType("NVARCHAR(126)");
entityTypeBuilder.Property(x => x.HomeUrl)

3
src/FilterLists.Data/EntityTypeConfigurations/RuleTypeConfiguration.cs
View file

@ -10,7 +10,8 @@ public override void Configure(EntityTypeBuilder<Rule> entityTypeBuilder)
{
entityTypeBuilder.ToTable("rules");
entityTypeBuilder.Property(x => x.Raw)
.HasColumnType("NVARCHAR(2083)");
.HasColumnType("NVARCHAR(2083)")
.IsRequired();
base.Configure(entityTypeBuilder);
}
}

1
src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
View file

@ -12,7 +12,6 @@ public override void Configure(EntityTypeBuilder<Snapshot> entityTypeBuilder)
entityTypeBuilder.Property(x => x.Content)
.HasColumnType("MEDIUMTEXT")
.IsRequired();
entityTypeBuilder.Ignore(x => x.ModifiedDateUtc);
base.Configure(entityTypeBuilder);
}
}

3
src/FilterLists.Data/EntityTypeConfigurations/SoftwareTypeConfiguration.cs
View file

@ -9,6 +9,9 @@ public class SoftwareTypeConfiguration : BaseEntityTypeConfiguration<Software>
public override void Configure(EntityTypeBuilder<Software> entityTypeBuilder)
{
entityTypeBuilder.ToTable("software");
entityTypeBuilder.Property(x => x.ModifiedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.DownloadUrl)
.HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.HomeUrl)

3
src/FilterLists.Data/EntityTypeConfigurations/SyntaxTypeConfiguration.cs
View file

@ -9,6 +9,9 @@ public class SyntaxTypeConfiguration : BaseEntityTypeConfiguration<Syntax>
public override void Configure(EntityTypeBuilder<Syntax> entityTypeBuilder)
{
entityTypeBuilder.ToTable("syntaxes");
entityTypeBuilder.Property(x => x.ModifiedDateUtc)
.HasColumnType("TIMESTAMP")
.ValueGeneratedOnAddOrUpdate();
entityTypeBuilder.Property(x => x.DefinitionUrl)
.HasColumnType("TEXT");
entityTypeBuilder.Property(x => x.Name)

69
src/FilterLists.Services/Services/ScrapeService.cs
View file

@ -1,8 +1,11 @@
using System;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Data.Entities;
using FilterLists.Data.Entities.Junctions;
using Microsoft.EntityFrameworkCore;
namespace FilterLists.Services.Services
{
@ -17,32 +20,25 @@ public ScrapeService(FilterListsDbContext filterListsDbContext)
public async Task Scrape()
{
var lists = filterListsDbContext.FilterLists;
foreach (var list in lists) await AddSnapshot(list);
await filterListsDbContext.SaveChangesAsync();
var lists = await filterListsDbContext.FilterLists.Take(1).ToListAsync();
foreach (var list in lists) await ScrapeList(list);
}
private async Task AddSnapshot(FilterList list)
private async Task ScrapeList(FilterList list)
{
var snapshot = await GetContent(list.ViewUrl);
try
{
if (snapshot == null) return;
}
catch (AggregateException)
{
return;
}
catch (HttpRequestException)
catch (Exception)
{
//TODO: log exception
return;
}
filterListsDbContext.Snapshots.Add(new Snapshot
{
Content = snapshot,
FilterListId = list.Id
});
await AddSnapshot(list.Id, snapshot);
await AddRules(list, snapshot);
}
private static async Task<string> GetContent(string url)
@ -58,5 +54,50 @@ private static async Task<string> GetContent(string url)
return null;
}
private async Task AddSnapshot(int listId, string snapshot)
{
filterListsDbContext.Snapshots.Add(new Snapshot
{
Content = snapshot,
FilterListId = listId
});
try
{
await filterListsDbContext.SaveChangesAsync();
}
catch (Exception)
{
//TODO: log exception
}
}
private async Task AddRules(FilterList list, string snapshot)
{
var lines = snapshot.Split(new[] {"\r\n", "\r", "\n"}, StringSplitOptions.None);
foreach (var line in lines)
{
if (string.IsNullOrWhiteSpace(line)) continue;
var rule = new Rule {Raw = line};
if (!await filterListsDbContext.Rules.AnyAsync(x => x.Raw == rule.Raw))
filterListsDbContext.Rules.Add(rule);
var filterListRule = new FilterListRule {FilterList = list, Rule = rule};
if (!await filterListsDbContext.FilterListRules.AnyAsync(x =>
x.RuleId == rule.Id && x.FilterListId == list.Id))
filterListsDbContext.FilterListRules.Add(filterListRule);
filterListsDbContext.FilterListRules.Add(filterListRule);
}
try
{
await filterListsDbContext.SaveChangesAsync();
}
catch (Exception)
{
//TODO: log exception
}
}
}
}