diff --git a/data/FilterList.json b/data/FilterList.json
index 4e7559382..eb21ec4ea 100644
--- a/data/FilterList.json
+++ b/data/FilterList.json
@@ -3512,7 +3512,7 @@
"publishedDate": null,
"submissionUrl": null,
"syntaxId": null,
- "viewUrl": "https://v.firebog.net/hosts/Airelle-hrsk.txt"
+ "viewUrl": "http://rlwpx.free.fr/WPFF/hrsk.7z"
},
{
"id": 187,
@@ -3531,7 +3531,7 @@
"publishedDate": null,
"submissionUrl": null,
"syntaxId": null,
- "viewUrl": "https://v.firebog.net/hosts/Airelle-trc.txt"
+ "viewUrl": "http://rlwpx.free.fr/WPFF/htrc.7z"
},
{
"id": 188,
diff --git a/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.Designer.cs b/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.Designer.cs
new file mode 100644
index 000000000..c9e666ebd
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.Designer.cs
@@ -0,0 +1,640 @@
+//
+
+using System;
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+ [DbContext(typeof(FilterListsDbContext))]
+ [Migration("20180829152737_DropCantSnapshot")]
+ partial class DropCantSnapshot
+ {
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasAnnotation("ProductVersion", "2.1.2-rtm-30932")
+ .HasAnnotation("Relational:MaxIdentifierLength", 64);
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("ChatUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Description")
+ .HasColumnType("TEXT");
+
+ b.Property("DescriptionSourceUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("DiscontinuedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("DATE")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("DonateUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ForumUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("IssuesUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("LicenseId")
+ .ValueGeneratedOnAdd()
+ .HasDefaultValue((byte)5);
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PolicyUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("PublishedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("DATE")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("SubmissionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("SyntaxId");
+
+ b.Property("ViewUrl")
+ .HasColumnType("TEXT");
+
+ b.HasKey("Id");
+
+ b.HasIndex("LicenseId");
+
+ b.HasIndex("SyntaxId");
+
+ b.ToTable("filterlists");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("LanguageId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "LanguageId");
+
+ b.HasIndex("LanguageId");
+
+ b.ToTable("filterlists_languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("MaintainerId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "MaintainerId");
+
+ b.HasIndex("MaintainerId");
+
+ b.ToTable("filterlists_maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("TagId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "TagId");
+
+ b.HasIndex("TagId");
+
+ b.ToTable("filterlists_tags");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.Property("ForkFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("forks");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.Property("MergeFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("merges");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.Property("SnapshotId");
+
+ b.Property("RuleId");
+
+ b.HasKey("SnapshotId", "RuleId");
+
+ b.HasIndex("RuleId");
+
+ b.ToTable("snapshots_rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.Property("SyntaxId");
+
+ b.Property("SoftwareId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("SyntaxId", "SoftwareId");
+
+ b.HasIndex("SoftwareId");
+
+ b.ToTable("software_syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Iso6391")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(2)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392B")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392T")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6393")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("LocalName")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DescriptionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PermissiveAdaptation");
+
+ b.Property("PermissiveCommercial");
+
+ b.HasKey("Id");
+
+ b.ToTable("licenses");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("TwitterHandle")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("INT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Raw")
+ .IsRequired()
+ .HasColumnType("LONGTEXT");
+
+ b.HasKey("Id");
+
+ b.ToTable("rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("MEDIUMINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("FilterListId");
+
+ b.Property("HttpStatusCode")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("WasSuccessful");
+
+ b.Property("WaybackTimestamp")
+ .HasColumnType("TIMESTAMP");
+
+ b.Property("WaybackUrl")
+ .HasColumnType("TEXT");
+
+ b.HasKey("Id");
+
+ b.HasIndex("FilterListId");
+
+ b.ToTable("snapshots");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DownloadUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("software");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DefinitionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Tag", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Description")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("tags");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.License", "License")
+ .WithMany("FilterLists")
+ .HasForeignKey("LicenseId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("FilterLists")
+ .HasForeignKey("SyntaxId");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Language", "Language")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("LanguageId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("MaintainerId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListTags")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Tag", "Tag")
+ .WithMany("FilterListTags")
+ .HasForeignKey("TagId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+ .WithMany("ForkFilterLists")
+ .HasForeignKey("ForkFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamForkFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+ .WithMany("MergeFilterLists")
+ .HasForeignKey("MergeFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamMergeFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("RuleId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("SnapshotId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Software", "Software")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SoftwareId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SyntaxId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("Snapshots")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+#pragma warning restore 612, 618
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.cs b/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.cs
new file mode 100644
index 000000000..13a1bd523
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180829152737_DropCantSnapshot.cs
@@ -0,0 +1,23 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+ public partial class DropCantSnapshot : Migration
+ {
+ protected override void Up(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropColumn(
+ "CantSnapshot",
+ "filterlists");
+ }
+
+ protected override void Down(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.AddColumn(
+ "CantSnapshot",
+ "filterlists",
+ nullable: false,
+ defaultValue: false);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
index 815b8c144..7263e6578 100644
--- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
+++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
@@ -25,8 +25,6 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.HasColumnType("SMALLINT UNSIGNED")
.HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
- b.Property("CantSnapshot");
-
b.Property("ChatUrl")
.HasColumnType("TEXT");
diff --git a/src/FilterLists.Data/Entities/FilterList.cs b/src/FilterLists.Data/Entities/FilterList.cs
index 88672314f..48d30207a 100644
--- a/src/FilterLists.Data/Entities/FilterList.cs
+++ b/src/FilterLists.Data/Entities/FilterList.cs
@@ -8,7 +8,6 @@ namespace FilterLists.Data.Entities
[UsedImplicitly]
public class FilterList : BaseEntity
{
- public bool CantSnapshot { get; set; }
public string ChatUrl { get; set; }
public string Description { get; set; }
public string DescriptionSourceUrl { get; set; }
diff --git a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs
index af7d1fce0..0bc4b116d 100644
--- a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs
+++ b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs
@@ -27,7 +27,6 @@ public static void SeedOrUpdate(this FilterListsDbContext dbContext, string data
dbContext.InsertOnDuplicateKeyUpdate(dataPath);
dbContext.InsertOnDuplicateKeyUpdate(dataPath);
dbContext.InsertOnDuplicateKeyUpdate(dataPath);
- SetCantSnapshot(dbContext);
}
private static void InsertOnDuplicateKeyUpdate(this DbContext dbContext, string dataPath)
@@ -107,13 +106,5 @@ private static string GetUpdateUnchangedColumnHack(IEnumerable proper
var firstId = properties.First(x => x.IsPrimaryKey()).Name;
return firstId + " = VALUES(" + firstId + ")";
}
-
- //TODO: https://github.com/collinbarrett/FilterLists/issues/201
- private static void SetCantSnapshot(DbContext dbContext)
- {
- const string rawSqlString = "UPDATE filterlists SET CantSnapshot = 1 WHERE id IN(173, 185, 188, 189)";
- dbContext.Database.ExecuteSqlCommand(rawSqlString);
- dbContext.SaveChanges();
- }
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Services/FilterLists.Services.csproj b/src/FilterLists.Services/FilterLists.Services.csproj
index f7851f75f..9afe5858e 100644
--- a/src/FilterLists.Services/FilterLists.Services.csproj
+++ b/src/FilterLists.Services/FilterLists.Services.csproj
@@ -24,6 +24,7 @@
+
diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs
index 30a70bd8f..f2b59eab9 100644
--- a/src/FilterLists.Services/Snapshot/Snapshot.cs
+++ b/src/FilterLists.Services/Snapshot/Snapshot.cs
@@ -13,6 +13,7 @@
using JetBrains.Annotations;
using Microsoft.ApplicationInsights;
using MoreLinq;
+using SharpCompress.Archives.SevenZip;
namespace FilterLists.Services.Snapshot
{
@@ -72,7 +73,7 @@ private async Task AddSnapEntity()
private async Task SaveAsync()
{
await TryGetLines();
- if (lines != null)
+ if (lines != null && lines.Count > 0)
{
await SaveInBatches();
await SetSuccessful();
@@ -110,15 +111,41 @@ private async Task GetLines()
response.EnsureSuccessStatusCode();
lines = new HashSet();
using (var stream = await response.Content.ReadAsStreamAsync())
- using (var streamReader = new StreamReader(stream))
{
- string line;
- while ((line = await streamReader.ReadLineAsync()) != null)
- lines.AddIfNotNullOrEmpty(line.Trim());
+ if (ListUrl.EndsWith(".7z"))
+ await GetLinesFrom7Zip(stream);
+ else
+ await GetLinesFromStream(stream);
}
}
}
+ private async Task GetLinesFrom7Zip(Stream stream)
+ {
+ using (var memoryStream = new MemoryStream())
+ {
+ stream.CopyTo(memoryStream);
+ using (var archive = SevenZipArchive.Open(memoryStream))
+ {
+ foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory))
+ using (var entryMemoryStream = entry.OpenEntryStream())
+ {
+ await GetLinesFromStream(entryMemoryStream);
+ }
+ }
+ }
+ }
+
+ private async Task GetLinesFromStream(Stream stream)
+ {
+ using (var streamReader = new StreamReader(stream))
+ {
+ string line;
+ while ((line = await streamReader.ReadLineAsync()) != null)
+ lines.AddIfNotNullOrEmpty(line.Trim());
+ }
+ }
+
private async Task SaveInBatches()
{
var snapBatches = CreateBatches();
diff --git a/src/FilterLists.Web/npm-shrinkwrap.json b/src/FilterLists.Web/npm-shrinkwrap.json
index 66dba29ec..3d5ea1b05 100644
--- a/src/FilterLists.Web/npm-shrinkwrap.json
+++ b/src/FilterLists.Web/npm-shrinkwrap.json
@@ -2049,6 +2049,7 @@
"version": "0.0.9",
"bundled": true,
"dev": true,
+ "optional": true,
"requires": {
"inherits": "~2.0.0"
}
@@ -2073,7 +2074,8 @@
"buffer-shims": {
"version": "1.0.0",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"caseless": {
"version": "0.12.0",
@@ -2090,12 +2092,14 @@
"code-point-at": {
"version": "1.1.0",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"combined-stream": {
"version": "1.0.5",
"bundled": true,
"dev": true,
+ "optional": true,
"requires": {
"delayed-stream": "~1.0.0"
}
@@ -2108,17 +2112,20 @@
"console-control-strings": {
"version": "1.1.0",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"core-util-is": {
"version": "1.0.2",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"cryptiles": {
"version": "2.0.5",
"bundled": true,
"dev": true,
+ "optional": true,
"requires": {
"boom": "2.x.x"
}
@@ -2158,7 +2165,8 @@
"delayed-stream": {
"version": "1.0.0",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"delegates": {
"version": "1.0.0",
@@ -2190,7 +2198,8 @@
"extsprintf": {
"version": "1.0.2",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"forever-agent": {
"version": "0.6.1",
@@ -2313,6 +2322,7 @@
"version": "3.1.3",
"bundled": true,
"dev": true,
+ "optional": true,
"requires": {
"boom": "2.x.x",
"cryptiles": "2.x.x",
@@ -2360,6 +2370,7 @@
"version": "1.0.0",
"bundled": true,
"dev": true,
+ "optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@@ -2373,7 +2384,8 @@
"isarray": {
"version": "1.0.0",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"isstream": {
"version": "0.1.2",
@@ -2446,12 +2458,14 @@
"mime-db": {
"version": "1.27.0",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"mime-types": {
"version": "2.1.15",
"bundled": true,
"dev": true,
+ "optional": true,
"requires": {
"mime-db": "~1.27.0"
}
@@ -2527,7 +2541,8 @@
"number-is-nan": {
"version": "1.0.1",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"oauth-sign": {
"version": "0.8.2",
@@ -2585,7 +2600,8 @@
"process-nextick-args": {
"version": "1.0.7",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"punycode": {
"version": "1.4.1",
@@ -2623,6 +2639,7 @@
"version": "2.2.9",
"bundled": true,
"dev": true,
+ "optional": true,
"requires": {
"buffer-shims": "~1.0.0",
"core-util-is": "~1.0.0",
@@ -2674,7 +2691,8 @@
"safe-buffer": {
"version": "5.0.1",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"semver": {
"version": "5.3.0",
@@ -2698,6 +2716,7 @@
"version": "1.0.9",
"bundled": true,
"dev": true,
+ "optional": true,
"requires": {
"hoek": "2.x.x"
}
@@ -2731,6 +2750,7 @@
"version": "1.0.2",
"bundled": true,
"dev": true,
+ "optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@@ -2741,6 +2761,7 @@
"version": "1.0.1",
"bundled": true,
"dev": true,
+ "optional": true,
"requires": {
"safe-buffer": "^5.0.1"
}
@@ -2769,6 +2790,7 @@
"version": "2.2.1",
"bundled": true,
"dev": true,
+ "optional": true,
"requires": {
"block-stream": "*",
"fstream": "^1.0.2",
@@ -2824,7 +2846,8 @@
"util-deprecate": {
"version": "1.0.2",
"bundled": true,
- "dev": true
+ "dev": true,
+ "optional": true
},
"uuid": {
"version": "3.0.1",