From 209bd9a0b84acc3db5389e59cfa07987ad95ffde Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Fri, 24 Aug 2018 11:30:17 -0500 Subject: [PATCH] add WaybackMachine backup snapshot ref #325 --- FilterLists.sln.DotSettings | 2 + ...0824155224_AddWaybackTimestamp.Designer.cs | 645 ++++++++++++++++++ .../20180824155224_AddWaybackTimestamp.cs | 24 + .../FilterListsDbContextModelSnapshot.cs | 3 + src/FilterLists.Data/Entities/Snapshot.cs | 4 +- .../SnapshotTypeConfiguration.cs | 2 + .../Extensions/StringExtensions.cs | 3 + .../Models/Wayback/ArchivedSnapshots.cs | 12 + .../Snapshot/Models/Wayback/Availability.cs | 15 + .../Snapshot/Models/Wayback/Closest.cs | 21 + src/FilterLists.Services/Snapshot/Snapshot.cs | 36 +- .../Snapshot/SnapshotService.cs | 21 +- .../Snapshot/SnapshotWayback.cs | 72 ++ 13 files changed, 842 insertions(+), 18 deletions(-) create mode 100644 src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.Designer.cs create mode 100644 src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.cs create mode 100644 src/FilterLists.Services/Snapshot/Models/Wayback/ArchivedSnapshots.cs create mode 100644 src/FilterLists.Services/Snapshot/Models/Wayback/Availability.cs create mode 100644 src/FilterLists.Services/Snapshot/Models/Wayback/Closest.cs create mode 100644 src/FilterLists.Services/Snapshot/SnapshotWayback.cs diff --git a/FilterLists.sln.DotSettings b/FilterLists.sln.DotSettings index a74afe9ba..17fba0e90 100644 --- a/FilterLists.sln.DotSettings +++ b/FilterLists.sln.DotSettings @@ -16,4 +16,6 @@ <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> True + True + True \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.Designer.cs b/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.Designer.cs new file mode 100644 index 000000000..e8bf7c1e3 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.Designer.cs @@ -0,0 +1,645 @@ +// + +using System; +using FilterLists.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + [DbContext(typeof(FilterListsDbContext))] + [Migration("20180824155224_AddWaybackTimestamp")] + partial class AddWaybackTimestamp + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "2.1.2-rtm-30932") + .HasAnnotation("Relational:MaxIdentifierLength", 64); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CantSnapshot"); + + b.Property("ChatUrl") + .HasColumnType("TEXT"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("DescriptionSourceUrl") + .HasColumnType("TEXT"); + + b.Property("DiscontinuedDate") + .ValueGeneratedOnAdd() + .HasColumnType("DATE") + .HasDefaultValueSql("NULL"); + + b.Property("DonateUrl") + .HasColumnType("TEXT"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ForumUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("IssuesUrl") + .HasColumnType("TEXT"); + + b.Property("LicenseId") + .ValueGeneratedOnAdd() + .HasDefaultValue((byte)5); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PolicyUrl") + .HasColumnType("TEXT"); + + b.Property("PublishedDate") + .ValueGeneratedOnAdd() + .HasColumnType("DATE") + .HasDefaultValueSql("NULL"); + + b.Property("SubmissionUrl") + .HasColumnType("TEXT"); + + b.Property("SyntaxId"); + + b.Property("ViewUrl") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("LicenseId"); + + b.HasIndex("SyntaxId"); + + b.ToTable("filterlists"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.Property("FilterListId"); + + b.Property("LanguageId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "LanguageId"); + + b.HasIndex("LanguageId"); + + b.ToTable("filterlists_languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.Property("FilterListId"); + + b.Property("MaintainerId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "MaintainerId"); + + b.HasIndex("MaintainerId"); + + b.ToTable("filterlists_maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b => + { + b.Property("FilterListId"); + + b.Property("TagId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("FilterListId", "TagId"); + + b.HasIndex("TagId"); + + b.ToTable("filterlists_tags"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.Property("ForkFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("ForkFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("forks"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.Property("MergeFilterListId"); + + b.Property("UpstreamFilterListId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("MergeFilterListId", "UpstreamFilterListId"); + + b.HasIndex("UpstreamFilterListId"); + + b.ToTable("merges"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.Property("SnapshotId"); + + b.Property("RuleId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("SnapshotId", "RuleId"); + + b.HasIndex("RuleId"); + + b.ToTable("snapshots_rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.Property("SyntaxId"); + + b.Property("SoftwareId"); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.HasKey("SyntaxId", "SoftwareId"); + + b.HasIndex("SoftwareId"); + + b.ToTable("software_syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Language", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Iso6391") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(2)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392B") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6392T") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("Iso6393") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(3)") + .HasDefaultValueSql("NULL"); + + b.Property("LocalName") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("languages"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.License", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DescriptionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("PermissiveAdaptation"); + + b.Property("PermissiveCommercial"); + + b.HasKey("Id"); + + b.ToTable("licenses"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Maintainer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("EmailAddress") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.Property("TwitterHandle") + .ValueGeneratedOnAdd() + .HasColumnType("VARCHAR(126)") + .HasDefaultValueSql("NULL"); + + b.HasKey("Id"); + + b.ToTable("maintainers"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Rule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Raw") + .IsRequired() + .HasColumnType("LONGTEXT"); + + b.HasKey("Id"); + + b.ToTable("rules"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("MEDIUMINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("FilterListId"); + + b.Property("HttpStatusCode") + .ValueGeneratedOnAdd() + .HasColumnType("SMALLINT UNSIGNED") + .HasDefaultValueSql("NULL"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("WasSuccessful"); + + b.Property("WaybackTimestamp") + .HasColumnType("TIMESTAMP"); + + b.HasKey("Id"); + + b.HasIndex("FilterListId"); + + b.ToTable("snapshots"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DownloadUrl") + .HasColumnType("TEXT"); + + b.Property("HomeUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("software"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Syntax", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("DefinitionUrl") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("syntaxes"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Tag", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TINYINT UNSIGNED") + .HasAnnotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn); + + b.Property("CreatedDateUtc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp()"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("ModifiedDateUtc") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("TIMESTAMP") + .HasDefaultValueSql("current_timestamp() ON UPDATE current_timestamp()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("VARCHAR(126)"); + + b.HasKey("Id"); + + b.ToTable("tags"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.FilterList", b => + { + b.HasOne("FilterLists.Data.Entities.License", "License") + .WithMany("FilterLists") + .HasForeignKey("LicenseId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("FilterLists") + .HasForeignKey("SyntaxId"); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListLanguage", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListLanguages") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Language", "Language") + .WithMany("FilterListLanguages") + .HasForeignKey("LanguageId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListMaintainer", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListMaintainers") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Maintainer", "Maintainer") + .WithMany("FilterListMaintainers") + .HasForeignKey("MaintainerId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.FilterListTag", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("FilterListTags") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Tag", "Tag") + .WithMany("FilterListTags") + .HasForeignKey("TagId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Fork", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "ForkFilterList") + .WithMany("ForkFilterLists") + .HasForeignKey("ForkFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamForkFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.Merge", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "MergeFilterList") + .WithMany("MergeFilterLists") + .HasForeignKey("MergeFilterListId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.FilterList", "UpstreamFilterList") + .WithMany("UpstreamMergeFilterLists") + .HasForeignKey("UpstreamFilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SnapshotRule", b => + { + b.HasOne("FilterLists.Data.Entities.Rule", "Rule") + .WithMany("SnapshotRules") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Snapshot", "Snapshot") + .WithMany("SnapshotRules") + .HasForeignKey("SnapshotId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Junctions.SoftwareSyntax", b => + { + b.HasOne("FilterLists.Data.Entities.Software", "Software") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SoftwareId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("FilterLists.Data.Entities.Syntax", "Syntax") + .WithMany("SoftwareSyntaxes") + .HasForeignKey("SyntaxId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FilterLists.Data.Entities.Snapshot", b => + { + b.HasOne("FilterLists.Data.Entities.FilterList", "FilterList") + .WithMany("Snapshots") + .HasForeignKey("FilterListId") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.cs b/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.cs new file mode 100644 index 000000000..274a054b8 --- /dev/null +++ b/src/FilterLists.Api/Migrations/20180824155224_AddWaybackTimestamp.cs @@ -0,0 +1,24 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace FilterLists.Api.Migrations +{ + public partial class AddWaybackTimestamp : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + "WaybackTimestamp", + "snapshots", + "TIMESTAMP", + nullable: true); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + "WaybackTimestamp", + "snapshots"); + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs index d5d8b6dac..33e8ef9fa 100644 --- a/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs +++ b/src/FilterLists.Api/Migrations/FilterListsDbContextModelSnapshot.cs @@ -421,6 +421,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("WasSuccessful"); + b.Property("WaybackTimestamp") + .HasColumnType("TIMESTAMP"); + b.HasKey("Id"); b.HasIndex("FilterListId"); diff --git a/src/FilterLists.Data/Entities/Snapshot.cs b/src/FilterLists.Data/Entities/Snapshot.cs index b151b2c23..785223479 100644 --- a/src/FilterLists.Data/Entities/Snapshot.cs +++ b/src/FilterLists.Data/Entities/Snapshot.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using FilterLists.Data.Entities.Junctions; namespace FilterLists.Data.Entities @@ -10,5 +11,6 @@ public class Snapshot : BaseEntity public uint? HttpStatusCode { get; set; } public bool WasSuccessful { get; set; } public ICollection SnapshotRules { get; set; } + public DateTime? WaybackTimestamp { get; set; } } } \ No newline at end of file diff --git a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs index 924b1ca02..a9e33a8ef 100644 --- a/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs +++ b/src/FilterLists.Data/EntityTypeConfigurations/SnapshotTypeConfiguration.cs @@ -15,6 +15,8 @@ public override void Configure(EntityTypeBuilder entityTypeBuilder) entityTypeBuilder.Property(x => x.HttpStatusCode) .HasColumnType("SMALLINT UNSIGNED") .HasDefaultValueSql("NULL"); + entityTypeBuilder.Property(x => x.WaybackTimestamp) + .HasColumnType("TIMESTAMP"); } } } \ No newline at end of file diff --git a/src/FilterLists.Services/Extensions/StringExtensions.cs b/src/FilterLists.Services/Extensions/StringExtensions.cs index bf25db4ca..5374618be 100644 --- a/src/FilterLists.Services/Extensions/StringExtensions.cs +++ b/src/FilterLists.Services/Extensions/StringExtensions.cs @@ -8,5 +8,8 @@ public static class StringExtensions public static bool IsValidHttpOrHttpsUrl(this string source) => Uri.TryCreate(source, UriKind.Absolute, out var uriResult) && new[] {Uri.UriSchemeHttps, Uri.UriSchemeHttp}.Contains(uriResult.Scheme); + + public static int GetNthIndexOfChar(this string s, int n, char t) => + s.TakeWhile(c => (n -= c == t ? 1 : 0) > 0).Count(); } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Models/Wayback/ArchivedSnapshots.cs b/src/FilterLists.Services/Snapshot/Models/Wayback/ArchivedSnapshots.cs new file mode 100644 index 000000000..57f1f9ebc --- /dev/null +++ b/src/FilterLists.Services/Snapshot/Models/Wayback/ArchivedSnapshots.cs @@ -0,0 +1,12 @@ +using JetBrains.Annotations; +using Newtonsoft.Json; + +namespace FilterLists.Services.Snapshot.Models.Wayback +{ + [UsedImplicitly] + public class ArchivedSnapshots + { + [JsonProperty("closest")] + public Closest Closest { get; set; } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Models/Wayback/Availability.cs b/src/FilterLists.Services/Snapshot/Models/Wayback/Availability.cs new file mode 100644 index 000000000..5b9b076ff --- /dev/null +++ b/src/FilterLists.Services/Snapshot/Models/Wayback/Availability.cs @@ -0,0 +1,15 @@ +using JetBrains.Annotations; +using Newtonsoft.Json; + +namespace FilterLists.Services.Snapshot.Models.Wayback +{ + [UsedImplicitly] + public class Availability + { + [JsonProperty("url")] + public string Url { get; set; } + + [JsonProperty("archived_snapshots")] + public ArchivedSnapshots ArchivedSnapshots { get; set; } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Models/Wayback/Closest.cs b/src/FilterLists.Services/Snapshot/Models/Wayback/Closest.cs new file mode 100644 index 000000000..4ed418fd4 --- /dev/null +++ b/src/FilterLists.Services/Snapshot/Models/Wayback/Closest.cs @@ -0,0 +1,21 @@ +using JetBrains.Annotations; +using Newtonsoft.Json; + +namespace FilterLists.Services.Snapshot.Models.Wayback +{ + [UsedImplicitly] + public class Closest + { + [JsonProperty("status")] + public string Status { get; set; } + + [JsonProperty("available")] + public bool Available { get; set; } + + [JsonProperty("url")] + public string Url { get; set; } + + [JsonProperty("timestamp")] + public string Timestamp { get; set; } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 881890961..be0f94c3d 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -9,6 +9,7 @@ using FilterLists.Data; using FilterLists.Services.Extensions; using FilterLists.Services.Snapshot.Models; +using JetBrains.Annotations; using Microsoft.ApplicationInsights; using MoreLinq; @@ -18,25 +19,36 @@ public class Snapshot { private const int BatchSize = 500; private readonly FilterListsDbContext dbContext; - private readonly string listViewUrl; - private readonly Data.Entities.Snapshot snapEntity; + public readonly FilterListViewUrlDto List; + protected readonly Data.Entities.Snapshot SnapEntity; private readonly TelemetryClient telemetryClient; private readonly string uaString; private HashSet lines; + protected string ListUrl; + public Snapshot() + { + } + + [UsedImplicitly] public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, string uaString) { this.dbContext = dbContext; - listViewUrl = list.ViewUrl; - snapEntity = new Data.Entities.Snapshot {FilterListId = list.Id}; + List = list; + ListUrl = list.ViewUrl; + SnapEntity = new Data.Entities.Snapshot {FilterListId = list.Id}; this.uaString = uaString; telemetryClient = new TelemetryClient(); } - public async Task TrySaveAsync() + public bool WasSuccessful => SnapEntity.WasSuccessful; + + public virtual async Task TrySaveAsync() => await TrySaveAsyncBase(); + + protected async Task TrySaveAsyncBase() { await AddSnapEntity(); - if (!listViewUrl.IsValidHttpOrHttpsUrl()) + if (!ListUrl.IsValidHttpOrHttpsUrl()) return; try { @@ -50,7 +62,7 @@ public async Task TrySaveAsync() private async Task AddSnapEntity() { - dbContext.Snapshots.Add(snapEntity); + dbContext.Snapshots.Add(SnapEntity); await dbContext.SaveChangesAsync(); } @@ -77,7 +89,7 @@ private async Task TryGetLines() } catch (WebException we) { - snapEntity.HttpStatusCode = (uint)((HttpWebResponse)we.Response).StatusCode; + SnapEntity.HttpStatusCode = (uint)((HttpWebResponse)we.Response).StatusCode; await dbContext.SaveChangesAsync(); TrackException(we); } @@ -88,8 +100,8 @@ private async Task GetLines() using (var httpClient = new HttpClient()) { httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(uaString); - var response = await httpClient.GetAsync(listViewUrl, HttpCompletionOption.ResponseHeadersRead); - snapEntity.HttpStatusCode = (uint)response.StatusCode; + var response = await httpClient.GetAsync(ListUrl, HttpCompletionOption.ResponseHeadersRead); + SnapEntity.HttpStatusCode = (uint)response.StatusCode; response.EnsureSuccessStatusCode(); lines = new HashSet(); using (var stream = await response.Content.ReadAsStreamAsync()) @@ -109,7 +121,7 @@ private async Task SaveInBatches() } private IEnumerable CreateBatches() => - lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, snapEntity)); + lines.Batch(BatchSize).Select(b => new SnapshotBatch(dbContext, b, SnapEntity)); private static async Task SaveBatches(IEnumerable batches) { @@ -119,7 +131,7 @@ private static async Task SaveBatches(IEnumerable batches) private async Task SetSuccessful() { - snapEntity.WasSuccessful = true; + SnapEntity.WasSuccessful = true; await dbContext.SaveChangesAsync(); } diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs index 18700a7e1..328f8b295 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotService.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs @@ -13,6 +13,7 @@ namespace FilterLists.Services.Snapshot public class SnapshotService : Service { private readonly DateTime yesterday = DateTime.UtcNow.AddDays(-1); + private string uaString; public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig) : base(dbContext, mapConfig) @@ -21,10 +22,11 @@ public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider ma public async Task CaptureAsync(int batchSize) { + uaString = await UserAgentService.GetMostPopularString(); var lists = await GetListsToCapture(batchSize); - var uaString = await UserAgentService.GetMostPopularString(); - var snaps = CreateSnaps(lists, uaString); - await SaveSnaps(snaps); + var snaps = await CreateAndSaveSnaps(lists); + var listsToRetry = snaps.Where(s => !s.WasSuccessful).Select(s => s.List); + await CreateAndSaveSnaps(listsToRetry); } private async Task> GetListsToCapture(int batchSize) => @@ -49,8 +51,17 @@ await DbContext .ProjectTo(MapConfig) .ToListAsync(); - private IEnumerable CreateSnaps(IEnumerable lists, string uaString) => - lists.Select(l => new Snapshot(DbContext, l, uaString)); + private async Task> CreateAndSaveSnaps(IEnumerable lists) + where TSnap : Snapshot, new() + { + var snaps = CreateSnaps(lists).ToList(); + await SaveSnaps(snaps); + return snaps; + } + + private IEnumerable CreateSnaps(IEnumerable lists) + where TSnap : Snapshot, new() => + lists.Select(l => Activator.CreateInstance(typeof(TSnap), DbContext, l, uaString) as TSnap); private static async Task SaveSnaps(IEnumerable snaps) { diff --git a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs new file mode 100644 index 000000000..36d413d02 --- /dev/null +++ b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs @@ -0,0 +1,72 @@ +using System; +using System.Net.Http; +using System.Threading.Tasks; +using FilterLists.Data; +using FilterLists.Services.Extensions; +using FilterLists.Services.Snapshot.Models; +using FilterLists.Services.Snapshot.Models.Wayback; +using JetBrains.Annotations; +using Newtonsoft.Json; + +namespace FilterLists.Services.Snapshot +{ + public class SnapshotWayback : Snapshot + { + private const string WaybackAvailabilityApiUrlPrefix = "https://archive.org/wayback/available?url="; + + public SnapshotWayback() + { + } + + [UsedImplicitly] + public SnapshotWayback(FilterListsDbContext dbContext, FilterListViewUrlDto list, string uaString) + : base(dbContext, list, uaString) + { + } + + public override async Task TrySaveAsync() + { + await UpdateWaybackData(); + await TrySaveAsyncBase(); + } + + private async Task UpdateWaybackData() + { + var availability = await GetWaybackAvailability(); + var closest = availability.ArchivedSnapshots?.Closest; + if (closest != null) + { + UpdateViewUrl(closest); + UpdateTimestamp(closest); + } + else + { + ListUrl = null; + } + } + + private async Task GetWaybackAvailability() + { + using (var client = new HttpClient()) + { + var url = WaybackAvailabilityApiUrlPrefix + ListUrl; + var response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); + var jsonString = await response.Content.ReadAsStringAsync(); + return JsonConvert.DeserializeObject(jsonString); + } + } + + private void UpdateViewUrl(Closest closest) + { + var waybackUrl = closest.Url; + var indexOfUrlModification = waybackUrl.GetNthIndexOfChar(5, '/'); + const string urlModification = "if_"; + ListUrl = waybackUrl.Substring(0, indexOfUrlModification) + urlModification + + waybackUrl.Substring(indexOfUrlModification); + } + + private void UpdateTimestamp(Closest closest) => + SnapEntity.WaybackTimestamp = + DateTimeOffset.ParseExact(closest.Timestamp, "yyyyMMddHHmmss", null).UtcDateTime; + } +} \ No newline at end of file