From e8c60bc24bf55fe180838bbdd5efb7474965ad5f Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Wed, 29 Aug 2018 10:30:14 -0500 Subject: [PATCH] support capturing .7z lists closes #201 --- data/FilterList.json | 4 +- ...0180829152737_DropCantSnapshot.Designer.cs | 640 ++++++++++++++++++ .../20180829152737_DropCantSnapshot.cs | 23 + .../FilterListsDbContextModelSnapshot.cs | 2 - src/FilterLists.Data/Entities/FilterList.cs | 1 - .../Extensions/SeedFilterListsDbContext.cs | 9 - .../FilterLists.Services.csproj | 1 + src/FilterLists.Services/Snapshot/Snapshot.cs | 37 +- src/FilterLists.Web/npm-shrinkwrap.json | 47 +- 9 files changed, 733 insertions(+), 31 deletions(-) create mode 100644 src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.Designer.cs create mode 100644 src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.cs diff --git a/data/FilterList.json b/data/FilterList.json index 4e7559382..eb21ec4ea 100644 --- a/data/FilterList.json +++ b/data/FilterList.json @@ -3512,7 +3512,7 @@ "publishedDate": null, "submissionUrl": null, "syntaxId": null, - "viewUrl": "https://v.firebog.net/hosts/Airelle-hrsk.txt" + "viewUrl": "http://rlwpx.free.fr/WPFF/hrsk.7z" }, { "id": 187, @@ -3531,7 +3531,7 @@ "publishedDate": null, "submissionUrl": null, "syntaxId": null, - "viewUrl": "https://v.firebog.net/hosts/Airelle-trc.txt" + "viewUrl": "http://rlwpx.free.fr/WPFF/htrc.7z" }, { "id": 188, diff --git a/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.Designer.cs b/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.Designer.cs new file mode 100644 index 000000000..c9e666ebd --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.Designer.cs @@ -0,0 +1,640 @@ +// + +using System; +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + [DbContext(typeof(FilterListsDbContext))] + [Migration("20180829152737_DropCantSnapshot")] + partial class DropCantSnapshot + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "2.1.2-rtm-30932") + .HasAnnotation("Relational:MaxIdentifierLength", 64); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("ChatUrl") + .HasColumnType("TEXT"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("DescriptionSourceUrl") + .HasColumnType("TEXT"); + + b.Property("DiscontinuedDate") + .ValueGeneratedOnAdd() + .HasColumnType("DATE") + .HasDefaultValueSql("NULL"); + + b.Property("DonateUrl") + .HasColumnType("TEXT"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ForumUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("IssuesUrl") + .HasColumnType("TEXT"); + + b.Property("LicenseId") + .ValueGeneratedOnAdd() + .HasDefaultValue((byte)5); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PolicyUrl") + .HasColumnType("TEXT"); + + b.Property("PublishedDate") + .ValueGeneratedOnAdd() + .HasColumnType("DATE") + .HasDefaultValueSql("NULL"); + + b.Property("SubmissionUrl") + .HasColumnType("TEXT"); + + b.Property("SyntaxId"); + + b.Property("ViewUrl") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("LicenseId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("filterlists"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.Property("FilterListId"); + + b.Property("LanguageId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "LanguageId"); + + b.HasIndex("LanguageId"); + + b.ToTable("filterlists_languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.Property("FilterListId"); + + b.Property("MaintainerId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "MaintainerId"); + + b.HasIndex("MaintainerId"); + + b.ToTable("filterlists_maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b => + { + b.Property("FilterListId"); + + b.Property("TagId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "TagId"); + + b.HasIndex("TagId"); + + b.ToTable("filterlists_tags"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.Property("ForkFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("ForkFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("forks"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.Property("MergeFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("MergeFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("merges"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.Property("SnapshotId"); + + b.Property("RuleId"); + + b.HasKey("SnapshotId", "RuleId"); + + b.HasIndex("RuleId"); + + b.ToTable("snapshots_rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.Property("SyntaxId"); + + b.Property("SoftwareId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("SyntaxId", "SoftwareId"); + + b.HasIndex("SoftwareId"); + + b.ToTable("software_syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Language", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Iso6391") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(2)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392B") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392T") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6393") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("LocalName") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.License", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DescriptionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PermissiveAdaptation"); + + b.Property("PermissiveCommercial"); + + b.HasKey("Id"); + + b.ToTable("licenses"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("TwitterHandle") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Rule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Raw") + .IsRequired() + .HasColumnType("LONGTEXT"); + + b.HasKey("Id"); + + b.ToTable("rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("MEDIUMINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("FilterListId"); + + b.Property("HttpStatusCode") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("WasSuccessful"); + + b.Property("WaybackTimestamp") + .HasColumnType("TIMESTAMP"); + + b.Property("WaybackUrl") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("FilterListId"); + + b.ToTable("snapshots"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DownloadUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("software"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DefinitionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Tag", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("tags"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.HasOne("FilterLists.Data.Entities.License", "License") + .WithMany("FilterLists") + .HasForeignKey("LicenseId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("FilterLists") + .HasForeignKey("SyntaxId"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListLanguages") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Language", "Language") + .WithMany("FilterListLanguages") + .HasForeignKey("LanguageId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListMaintainers") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer") + .WithMany("FilterListMaintainers") + .HasForeignKey("MaintainerId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListTags") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Tag", "Tag") + .WithMany("FilterListTags") + .HasForeignKey("TagId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") + .WithMany("ForkFilterLists") + .HasForeignKey("ForkFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamForkFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList") + .WithMany("MergeFilterLists") + .HasForeignKey("MergeFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamMergeFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.HasOne("FilterLists.Data.Entities.Rule", "Rule") + .WithMany("SnapshotRules") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot") + .WithMany("SnapshotRules") + .HasForeignKey("SnapshotId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.HasOne("FilterLists.Data.Entities.Software", "Software") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SoftwareId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SyntaxId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("Snapshots") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.cs b/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.cs new file mode 100644 index 000000000..13a1bd523 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.cs @@ -0,0 +1,23 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + public partial class DropCantSnapshot : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + "CantSnapshot", + "filterlists"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + "CantSnapshot", + "filterlists", + nullable: false, + defaultValue: false); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs index 815b8c144..7263e6578 100644 --- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs +++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs @@ -25,8 +25,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("SMALLINT UNSIGNED") .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); - b.Property("CantSnapshot"); - b.Property("ChatUrl") .HasColumnType("TEXT"); diff --git a/src/FilterLists.Data/Entities/FilterList.cs b/src/FilterLists.Data/Entities/FilterList.cs index 88672314f..48d30207a 100644 --- a/src/FilterLists.Data/Entities/FilterList.cs +++ b/src/FilterLists.Data/Entities/FilterList.cs @@ -8,7 +8,6 @@ namespace FilterLists.Data.Entities [UsedImplicitly] public class FilterList : BaseEntity { - public bool CantSnapshot { get; set; } public string ChatUrl { get; set; } public string Description { get; set; } public string DescriptionSourceUrl { get; set; } diff --git a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs index af7d1fce0..0bc4b116d 100644 --- a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs +++ b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs @@ -27,7 +27,6 @@ public static void SeedOrUpdate(this FilterListsDbContext dbContext, string data dbContext.InsertOnDuplicateKeyUpdate(dataPath); dbContext.InsertOnDuplicateKeyUpdate(dataPath); dbContext.InsertOnDuplicateKeyUpdate(dataPath); - SetCantSnapshot(dbContext); } private static void InsertOnDuplicateKeyUpdate(this DbContext dbContext, string dataPath) @@ -107,13 +106,5 @@ private static string GetUpdateUnchangedColumnHack(IEnumerable proper var firstId = properties.First(x => x.IsPrimaryKey()).Name; return firstId + " = VALUES(" + firstId + ")"; } - - //TODO: https://github.com/collinbarrett/FilterLists/issues/201 - private static void SetCantSnapshot(DbContext dbContext) - { - const string rawSqlString = "UPDATE filterlists SET CantSnapshot = 1 WHERE id IN(173, 185, 188, 189)"; - dbContext.Database.ExecuteSqlCommand(rawSqlString); - dbContext.SaveChanges(); - } } } \ No newline at end of file diff --git a/src/FilterLists.Services/FilterLists.Services.csproj b/src/FilterLists.Services/FilterLists.Services.csproj index f7851f75f..9afe5858e 100644 --- a/src/FilterLists.Services/FilterLists.Services.csproj +++ b/src/FilterLists.Services/FilterLists.Services.csproj @@ -24,6 +24,7 @@ + diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 30a70bd8f..f2b59eab9 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -13,6 +13,7 @@ using JetBrains.Annotations; using Microsoft.ApplicationInsights; using MoreLinq; +using SharpCompress.Archives.SevenZip; namespace FilterLists.Services.Snapshot { @@ -72,7 +73,7 @@ private async Task AddSnapEntity() private async Task SaveAsync() { await TryGetLines(); - if (lines != null) + if (lines != null && lines.Count > 0) { await SaveInBatches(); await SetSuccessful(); @@ -110,15 +111,41 @@ private async Task GetLines() response.EnsureSuccessStatusCode(); lines = new HashSet(); using (var stream = await response.Content.ReadAsStreamAsync()) - using (var streamReader = new StreamReader(stream)) { - string line; - while ((line = await streamReader.ReadLineAsync()) != null) - lines.AddIfNotNullOrEmpty(line.Trim()); + if (ListUrl.EndsWith(".7z")) + await GetLinesFrom7Zip(stream); + else + await GetLinesFromStream(stream); } } } + private async Task GetLinesFrom7Zip(Stream stream) + { + using (var memoryStream = new MemoryStream()) + { + stream.CopyTo(memoryStream); + using (var archive = SevenZipArchive.Open(memoryStream)) + { + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + using (var entryMemoryStream = entry.OpenEntryStream()) + { + await GetLinesFromStream(entryMemoryStream); + } + } + } + } + + private async Task GetLinesFromStream(Stream stream) + { + using (var streamReader = new StreamReader(stream)) + { + string line; + while ((line = await streamReader.ReadLineAsync()) != null) + lines.AddIfNotNullOrEmpty(line.Trim()); + } + } + private async Task SaveInBatches() { var snapBatches = CreateBatches(); diff --git a/src/FilterLists.Web/npm-shrinkwrap.json b/src/FilterLists.Web/npm-shrinkwrap.json index 66dba29ec..3d5ea1b05 100644 --- a/src/FilterLists.Web/npm-shrinkwrap.json +++ b/src/FilterLists.Web/npm-shrinkwrap.json @@ -2049,6 +2049,7 @@ "version": "0.0.9", "bundled": true, "dev": true, + "optional": true, "requires": { "inherits": "~2.0.0" } @@ -2073,7 +2074,8 @@ "buffer-shims": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "caseless": { "version": "0.12.0", @@ -2090,12 +2092,14 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "combined-stream": { "version": "1.0.5", "bundled": true, "dev": true, + "optional": true, "requires": { "delayed-stream": "~1.0.0" } @@ -2108,17 +2112,20 @@ "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "cryptiles": { "version": "2.0.5", "bundled": true, "dev": true, + "optional": true, "requires": { "boom": "2.x.x" } @@ -2158,7 +2165,8 @@ "delayed-stream": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "delegates": { "version": "1.0.0", @@ -2190,7 +2198,8 @@ "extsprintf": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "forever-agent": { "version": "0.6.1", @@ -2313,6 +2322,7 @@ "version": "3.1.3", "bundled": true, "dev": true, + "optional": true, "requires": { "boom": "2.x.x", "cryptiles": "2.x.x", @@ -2360,6 +2370,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -2373,7 +2384,8 @@ "isarray": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "isstream": { "version": "0.1.2", @@ -2446,12 +2458,14 @@ "mime-db": { "version": "1.27.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "mime-types": { "version": "2.1.15", "bundled": true, "dev": true, + "optional": true, "requires": { "mime-db": "~1.27.0" } @@ -2527,7 +2541,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "oauth-sign": { "version": "0.8.2", @@ -2585,7 +2600,8 @@ "process-nextick-args": { "version": "1.0.7", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "punycode": { "version": "1.4.1", @@ -2623,6 +2639,7 @@ "version": "2.2.9", "bundled": true, "dev": true, + "optional": true, "requires": { "buffer-shims": "~1.0.0", "core-util-is": "~1.0.0", @@ -2674,7 +2691,8 @@ "safe-buffer": { "version": "5.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "semver": { "version": "5.3.0", @@ -2698,6 +2716,7 @@ "version": "1.0.9", "bundled": true, "dev": true, + "optional": true, "requires": { "hoek": "2.x.x" } @@ -2731,6 +2750,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -2741,6 +2761,7 @@ "version": "1.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.0.1" } @@ -2769,6 +2790,7 @@ "version": "2.2.1", "bundled": true, "dev": true, + "optional": true, "requires": { "block-stream": "*", "fstream": "^1.0.2", @@ -2824,7 +2846,8 @@ "util-deprecate": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "uuid": { "version": "3.0.1",