diff --git a/FilterLists.sln.DotSettings b/FilterLists.sln.DotSettings
index a74afe9ba..17fba0e90 100644
--- a/FilterLists.sln.DotSettings
+++ b/FilterLists.sln.DotSettings
@@ -16,4 +16,6 @@
<Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" />
<Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" />
True
+ True
+ True
\ No newline at end of file
diff --git a/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.Designer.cs b/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.Designer.cs
new file mode 100644
index 000000000..e8bf7c1e3
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.Designer.cs
@@ -0,0 +1,645 @@
+//
+
+using System;
+using FilterLists.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+ [DbContext(typeof(FilterListsDbContext))]
+ [Migration("20180824155224_AddWaybackTimestamp")]
+ partial class AddWaybackTimestamp
+ {
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasAnnotation("ProductVersion", "2.1.2-rtm-30932")
+ .HasAnnotation("Relational:MaxIdentifierLength", 64);
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CantSnapshot");
+
+ b.Property("ChatUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Description")
+ .HasColumnType("TEXT");
+
+ b.Property("DescriptionSourceUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("DiscontinuedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("DATE")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("DonateUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ForumUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("IssuesUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("LicenseId")
+ .ValueGeneratedOnAdd()
+ .HasDefaultValue((byte)5);
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PolicyUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("PublishedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("DATE")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("SubmissionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("SyntaxId");
+
+ b.Property("ViewUrl")
+ .HasColumnType("TEXT");
+
+ b.HasKey("Id");
+
+ b.HasIndex("LicenseId");
+
+ b.HasIndex("SyntaxId");
+
+ b.ToTable("filterlists");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("LanguageId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "LanguageId");
+
+ b.HasIndex("LanguageId");
+
+ b.ToTable("filterlists_languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("MaintainerId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "MaintainerId");
+
+ b.HasIndex("MaintainerId");
+
+ b.ToTable("filterlists_maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+ {
+ b.Property("FilterListId");
+
+ b.Property("TagId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("FilterListId", "TagId");
+
+ b.HasIndex("TagId");
+
+ b.ToTable("filterlists_tags");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.Property("ForkFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("ForkFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("forks");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.Property("MergeFilterListId");
+
+ b.Property("UpstreamFilterListId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("MergeFilterListId", "UpstreamFilterListId");
+
+ b.HasIndex("UpstreamFilterListId");
+
+ b.ToTable("merges");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.Property("SnapshotId");
+
+ b.Property("RuleId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("SnapshotId", "RuleId");
+
+ b.HasIndex("RuleId");
+
+ b.ToTable("snapshots_rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.Property("SyntaxId");
+
+ b.Property("SoftwareId");
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.HasKey("SyntaxId", "SoftwareId");
+
+ b.HasIndex("SoftwareId");
+
+ b.ToTable("software_syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Language", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Iso6391")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(2)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392B")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6392T")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("Iso6393")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(3)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("LocalName")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("languages");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.License", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DescriptionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("PermissiveAdaptation");
+
+ b.Property("PermissiveCommercial");
+
+ b.HasKey("Id");
+
+ b.ToTable("licenses");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("EmailAddress")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.Property("TwitterHandle")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("VARCHAR(126)")
+ .HasDefaultValueSql("NULL");
+
+ b.HasKey("Id");
+
+ b.ToTable("maintainers");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Rule", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("INT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Raw")
+ .IsRequired()
+ .HasColumnType("LONGTEXT");
+
+ b.HasKey("Id");
+
+ b.ToTable("rules");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("MEDIUMINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("FilterListId");
+
+ b.Property("HttpStatusCode")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("SMALLINT UNSIGNED")
+ .HasDefaultValueSql("NULL");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("WasSuccessful");
+
+ b.Property("WaybackTimestamp")
+ .HasColumnType("TIMESTAMP");
+
+ b.HasKey("Id");
+
+ b.HasIndex("FilterListId");
+
+ b.ToTable("snapshots");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Software", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DownloadUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("HomeUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("software");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("DefinitionUrl")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("syntaxes");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Tag", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TINYINT UNSIGNED")
+ .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn);
+
+ b.Property("CreatedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp()");
+
+ b.Property("Description")
+ .HasColumnType("TEXT");
+
+ b.Property("ModifiedDateUtc")
+ .IsRequired()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("TIMESTAMP")
+ .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("VARCHAR(126)");
+
+ b.HasKey("Id");
+
+ b.ToTable("tags");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.License", "License")
+ .WithMany("FilterLists")
+ .HasForeignKey("LicenseId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("FilterLists")
+ .HasForeignKey("SyntaxId");
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Language", "Language")
+ .WithMany("FilterListLanguages")
+ .HasForeignKey("LanguageId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer")
+ .WithMany("FilterListMaintainers")
+ .HasForeignKey("MaintainerId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("FilterListTags")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Tag", "Tag")
+ .WithMany("FilterListTags")
+ .HasForeignKey("TagId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList")
+ .WithMany("ForkFilterLists")
+ .HasForeignKey("ForkFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamForkFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList")
+ .WithMany("MergeFilterLists")
+ .HasForeignKey("MergeFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList")
+ .WithMany("UpstreamMergeFilterLists")
+ .HasForeignKey("UpstreamFilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Rule", "Rule")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("RuleId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot")
+ .WithMany("SnapshotRules")
+ .HasForeignKey("SnapshotId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.Software", "Software")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SoftwareId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax")
+ .WithMany("SoftwareSyntaxes")
+ .HasForeignKey("SyntaxId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+
+ modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b =>
+ {
+ b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList")
+ .WithMany("Snapshots")
+ .HasForeignKey("FilterListId")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+#pragma warning restore 612, 618
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.cs b/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.cs
new file mode 100644
index 000000000..274a054b8
--- /dev/null
+++ b/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.cs
@@ -0,0 +1,24 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace FilterLists.Api.Migrations
+{
+ public partial class AddWaybackTimestamp : Migration
+ {
+ protected override void Up(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.AddColumn(
+ "WaybackTimestamp",
+ "snapshots",
+ "TIMESTAMP",
+ nullable: true);
+ }
+
+ protected override void Down(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropColumn(
+ "WaybackTimestamp",
+ "snapshots");
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
index d5d8b6dac..33e8ef9fa 100644
--- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
+++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs
@@ -421,6 +421,9 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property("WasSuccessful");
+ b.Property("WaybackTimestamp")
+ .HasColumnType("TIMESTAMP");
+
b.HasKey("Id");
b.HasIndex("FilterListId");
diff --git a/src/FilterLists.Data/Entities/Snapshot.cs b/src/FilterLists.Data/Entities/Snapshot.cs
index b151b2c23..785223479 100644
--- a/src/FilterLists.Data/Entities/Snapshot.cs
+++ b/src/FilterLists.Data/Entities/Snapshot.cs
@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
using FilterLists.Data.Entities.Junctions;
namespace FilterLists.Data.Entities
@@ -10,5 +11,6 @@ public class Snapshot : BaseEntity
public uint? HttpStatusCode { get; set; }
public bool WasSuccessful { get; set; }
public ICollection SnapshotRules { get; set; }
+ public DateTime? WaybackTimestamp { get; set; }
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
index 924b1ca02..a9e33a8ef 100644
--- a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
+++ b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs
@@ -15,6 +15,8 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder)
entityTypeBuilder.Property(x => x.HttpStatusCode)
.HasColumnType("SMALLINT UNSIGNED")
.HasDefaultValueSql("NULL");
+ entityTypeBuilder.Property(x => x.WaybackTimestamp)
+ .HasColumnType("TIMESTAMP");
}
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Services/Extensions/StringExtensions.cs b/src/FilterLists.Services/Extensions/StringExtensions.cs
index bf25db4ca..5374618be 100644
--- a/src/FilterLists.Services/Extensions/StringExtensions.cs
+++ b/src/FilterLists.Services/Extensions/StringExtensions.cs
@@ -8,5 +8,8 @@ public static class StringExtensions
public static bool IsValidHttpOrHttpsUrl(this string source) =>
Uri.TryCreate(source, UriKind.Absolute, out var uriResult) &&
new[] {Uri.UriSchemeHttps, Uri.UriSchemeHttp}.Contains(uriResult.Scheme);
+
+ public static int GetNthIndexOfChar(this string s, int n, char t) =>
+ s.TakeWhile(c => (n -= c == t ? 1 : 0) > 0).Count();
}
}
\ No newline at end of file
diff --git a/src/FilterLists.Services/Snapshot/Models/Wayback/ArchivedSnapshots.cs b/src/FilterLists.Services/Snapshot/Models/Wayback/ArchivedSnapshots.cs
new file mode 100644
index 000000000..57f1f9ebc
--- /dev/null
+++ b/src/FilterLists.Services/Snapshot/Models/Wayback/ArchivedSnapshots.cs
@@ -0,0 +1,12 @@
+using JetBrains.Annotations;
+using Newtonsoft.Json;
+
+namespace FilterLists.Services.Snapshot.Models.Wayback
+{
+ [UsedImplicitly]
+ public class ArchivedSnapshots
+ {
+ [JsonProperty("closest")]
+ public Closest Closest { get; set; }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Services/Snapshot/Models/Wayback/Availability.cs b/src/FilterLists.Services/Snapshot/Models/Wayback/Availability.cs
new file mode 100644
index 000000000..5b9b076ff
--- /dev/null
+++ b/src/FilterLists.Services/Snapshot/Models/Wayback/Availability.cs
@@ -0,0 +1,15 @@
+using JetBrains.Annotations;
+using Newtonsoft.Json;
+
+namespace FilterLists.Services.Snapshot.Models.Wayback
+{
+ [UsedImplicitly]
+ public class Availability
+ {
+ [JsonProperty("url")]
+ public string Url { get; set; }
+
+ [JsonProperty("archived_snapshots")]
+ public ArchivedSnapshots ArchivedSnapshots { get; set; }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Services/Snapshot/Models/Wayback/Closest.cs b/src/FilterLists.Services/Snapshot/Models/Wayback/Closest.cs
new file mode 100644
index 000000000..4ed418fd4
--- /dev/null
+++ b/src/FilterLists.Services/Snapshot/Models/Wayback/Closest.cs
@@ -0,0 +1,21 @@
+using JetBrains.Annotations;
+using Newtonsoft.Json;
+
+namespace FilterLists.Services.Snapshot.Models.Wayback
+{
+ [UsedImplicitly]
+ public class Closest
+ {
+ [JsonProperty("status")]
+ public string Status { get; set; }
+
+ [JsonProperty("available")]
+ public bool Available { get; set; }
+
+ [JsonProperty("url")]
+ public string Url { get; set; }
+
+ [JsonProperty("timestamp")]
+ public string Timestamp { get; set; }
+ }
+}
\ No newline at end of file
diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs
index 881890961..be0f94c3d 100644
--- a/src/FilterLists.Services/Snapshot/Snapshot.cs
+++ b/src/FilterLists.Services/Snapshot/Snapshot.cs
@@ -9,6 +9,7 @@
using FilterLists.Data;
using FilterLists.Services.Extensions;
using FilterLists.Services.Snapshot.Models;
+using JetBrains.Annotations;
using Microsoft.ApplicationInsights;
using MoreLinq;
@@ -18,25 +19,36 @@ public class Snapshot
{
private const int BatchSize = 500;
private readonly FilterListsDbContext dbContext;
- private readonly string listViewUrl;
- private readonly Data.Entities.Snapshot snapEntity;
+ public readonly FilterListViewUrlDto List;
+ protected readonly Data.Entities.Snapshot SnapEntity;
private readonly TelemetryClient telemetryClient;
private readonly string uaString;
private HashSet lines;
+ protected string ListUrl;
+ public Snapshot()
+ {
+ }
+
+ [UsedImplicitly]
public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, string uaString)
{
this.dbContext = dbContext;
- listViewUrl = list.ViewUrl;
- snapEntity = new Data.Entities.Snapshot {FilterListId = list.Id};
+ List = list;
+ ListUrl = list.ViewUrl;
+ SnapEntity = new Data.Entities.Snapshot {FilterListId = list.Id};
this.uaString = uaString;
telemetryClient = new TelemetryClient();
}
- public async Task TrySaveAsync()
+ public bool WasSuccessful => SnapEntity.WasSuccessful;
+
+ public virtual async Task TrySaveAsync() => await TrySaveAsyncBase();
+
+ protected async Task TrySaveAsyncBase()
{
await AddSnapEntity();
- if (!listViewUrl.IsValidHttpOrHttpsUrl())
+ if (!ListUrl.IsValidHttpOrHttpsUrl())
return;
try
{
@@ -50,7 +62,7 @@ public async Task TrySaveAsync()
private async Task AddSnapEntity()
{
- dbContext.Snapshots.Add(snapEntity);
+ dbContext.Snapshots.Add(SnapEntity);
await dbContext.SaveChangesAsync();
}
@@ -77,7 +89,7 @@ private async Task TryGetLines()
}
catch (WebException we)
{
- snapEntity.HttpStatusCode = (uint)((HttpWebResponse)we.Response).StatusCode;
+ SnapEntity.HttpStatusCode = (uint)((HttpWebResponse)we.Response).StatusCode;
await dbContext.SaveChangesAsync();
TrackException(we);
}
@@ -88,8 +100,8 @@ private async Task GetLines()
using (var httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(uaString);
- var response = await httpClient.GetAsync(listViewUrl, HttpCompletionOption.ResponseHeadersRead);
- snapEntity.HttpStatusCode = (uint)response.StatusCode;
+ var response = await httpClient.GetAsync(ListUrl, HttpCompletionOption.ResponseHeadersRead);
+ SnapEntity.HttpStatusCode = (uint)response.StatusCode;
response.EnsureSuccessStatusCode();
lines = new HashSet();
using (var stream = await response.Content.ReadAsStreamAsync())
@@ -109,7 +121,7 @@ private async Task SaveInBatches()
}
private IEnumerable CreateBatches() =>
- lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity));
+ lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, SnapEntity));
private static async Task SaveBatches(IEnumerable batches)
{
@@ -119,7 +131,7 @@ private static async Task SaveBatches(IEnumerable batches)
private async Task SetSuccessful()
{
- snapEntity.WasSuccessful = true;
+ SnapEntity.WasSuccessful = true;
await dbContext.SaveChangesAsync();
}
diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs
index 18700a7e1..328f8b295 100644
--- a/src/FilterLists.Services/Snapshot/SnapshotService.cs
+++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs
@@ -13,6 +13,7 @@ namespace FilterLists.Services.Snapshot
public class SnapshotService : Service
{
private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1);
+ private string uaString;
public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig)
: base(dbContext, mapConfig)
@@ -21,10 +22,11 @@ public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider ma
public async Task CaptureAsync(int batchSize)
{
+ uaString = await UserAgentService.GetMostPopularString();
var lists = await GetListsToCapture(batchSize);
- var uaString = await UserAgentService.GetMostPopularString();
- var snaps = CreateSnaps(lists, uaString);
- await SaveSnaps(snaps);
+ var snaps = await CreateAndSaveSnaps(lists);
+ var listsToRetry = snaps.Where(s => !s.WasSuccessful).Select(s => s.List);
+ await CreateAndSaveSnaps(listsToRetry);
}
private async Task> GetListsToCapture(int batchSize) =>
@@ -49,8 +51,17 @@ await DbContext
.ProjectTo(MapConfig)
.ToListAsync();
- private IEnumerable CreateSnaps(IEnumerable lists, string uaString) =>
- lists.Select(l => new Snapshot(DbContext, l, uaString));
+ private async Task> CreateAndSaveSnaps(IEnumerable lists)
+ where TSnap : Snapshot, new()
+ {
+ var snaps = CreateSnaps(lists).ToList();
+ await SaveSnaps(snaps);
+ return snaps;
+ }
+
+ private IEnumerable CreateSnaps(IEnumerable lists)
+ where TSnap : Snapshot, new() =>
+ lists.Select(l => Activator.CreateInstance(typeof(TSnap), DbContext, l, uaString) as TSnap);
private static async Task SaveSnaps(IEnumerable snaps)
{
diff --git a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs
new file mode 100644
index 000000000..36d413d02
--- /dev/null
+++ b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs
@@ -0,0 +1,72 @@
+using System;
+using System.Net.Http;
+using System.Threading.Tasks;
+using FilterLists.Data;
+using FilterLists.Services.Extensions;
+using FilterLists.Services.Snapshot.Models;
+using FilterLists.Services.Snapshot.Models.Wayback;
+using JetBrains.Annotations;
+using Newtonsoft.Json;
+
+namespace FilterLists.Services.Snapshot
+{
+ public class SnapshotWayback : Snapshot
+ {
+ private const string WaybackAvailabilityApiUrlPrefix = "https://archive.org/wayback/available?url=";
+
+ public SnapshotWayback()
+ {
+ }
+
+ [UsedImplicitly]
+ public SnapshotWayback(FilterListsDbContext dbContext, FilterListViewUrlDto list, string uaString)
+ : base(dbContext, list, uaString)
+ {
+ }
+
+ public override async Task TrySaveAsync()
+ {
+ await UpdateWaybackData();
+ await TrySaveAsyncBase();
+ }
+
+ private async Task UpdateWaybackData()
+ {
+ var availability = await GetWaybackAvailability();
+ var closest = availability.ArchivedSnapshots?.Closest;
+ if (closest != null)
+ {
+ UpdateViewUrl(closest);
+ UpdateTimestamp(closest);
+ }
+ else
+ {
+ ListUrl = null;
+ }
+ }
+
+ private async Task GetWaybackAvailability()
+ {
+ using (var client = new HttpClient())
+ {
+ var url = WaybackAvailabilityApiUrlPrefix + ListUrl;
+ var response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
+ var jsonString = await response.Content.ReadAsStringAsync();
+ return JsonConvert.DeserializeObject(jsonString);
+ }
+ }
+
+ private void UpdateViewUrl(Closest closest)
+ {
+ var waybackUrl = closest.Url;
+ var indexOfUrlModification = waybackUrl.GetNthIndexOfChar(5, '/');
+ const string urlModification = "if_";
+ ListUrl = waybackUrl.Substring(0, indexOfUrlModification) + urlModification +
+ waybackUrl.Substring(indexOfUrlModification);
+ }
+
+ private void UpdateTimestamp(Closest closest) =>
+ SnapEntity.WaybackTimestamp =
+ DateTimeOffset.ParseExact(closest.Timestamp, "yyyyMMddHHmmss", null).UtcDateTime;
+ }
+}
\ No newline at end of file