From 46be9ff6bb76a4a81a93bbfedc1a8011d72d0448 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Fri, 24 Aug 2018 12:41:28 -0500 Subject: [PATCH] extract WaybackService for re-use ref #325 --- .../Snapshot/SnapshotWayback.cs | 43 +++------------ .../Models}/ArchivedSnapshots.cs | 2 +- .../Models}/Availability.cs | 2 +- .../Wayback => Wayback/Models}/Closest.cs | 2 +- .../Wayback/Models/SnapshotMetaDto.cs | 10 ++++ .../Wayback/WaybackService.cs | 52 +++++++++++++++++++ 6 files changed, 71 insertions(+), 40 deletions(-) rename src/FilterLists.Services/{Snapshot/Models/Wayback => Wayback/Models}/ArchivedSnapshots.cs (78%) rename src/FilterLists.Services/{Snapshot/Models/Wayback => Wayback/Models}/Availability.cs (84%) rename src/FilterLists.Services/{Snapshot/Models/Wayback => Wayback/Models}/Closest.cs (88%) create mode 100644 src/FilterLists.Services/Wayback/Models/SnapshotMetaDto.cs create mode 100644 src/FilterLists.Services/Wayback/WaybackService.cs diff --git a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs index 36d413d02..f9914aeb9 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs @@ -1,19 +1,13 @@ -using System; -using System.Net.Http; -using System.Threading.Tasks; +using System.Threading.Tasks; using FilterLists.Data; -using FilterLists.Services.Extensions; using FilterLists.Services.Snapshot.Models; -using FilterLists.Services.Snapshot.Models.Wayback; +using FilterLists.Services.Wayback; using JetBrains.Annotations; -using Newtonsoft.Json; namespace FilterLists.Services.Snapshot { public class SnapshotWayback : Snapshot { - private const string WaybackAvailabilityApiUrlPrefix = "https://archive.org/wayback/available?url="; - public SnapshotWayback() { } @@ -32,41 +26,16 @@ public override async Task TrySaveAsync() private async Task UpdateWaybackData() { - var availability = await GetWaybackAvailability(); - var closest = availability.ArchivedSnapshots?.Closest; - if (closest != null) + var snapshotMeta = await WaybackService.GetMostRecentSnapshotMeta(ListUrl); + if (snapshotMeta != null) { - UpdateViewUrl(closest); - UpdateTimestamp(closest); + ListUrl = snapshotMeta.UrlRaw; + SnapEntity.WaybackTimestamp = snapshotMeta.TimestampUtc; } else { ListUrl = null; } } - - private async Task GetWaybackAvailability() - { - using (var client = new HttpClient()) - { - var url = WaybackAvailabilityApiUrlPrefix + ListUrl; - var response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); - var jsonString = await response.Content.ReadAsStringAsync(); - return JsonConvert.DeserializeObject(jsonString); - } - } - - private void UpdateViewUrl(Closest closest) - { - var waybackUrl = closest.Url; - var indexOfUrlModification = waybackUrl.GetNthIndexOfChar(5, '/'); - const string urlModification = "if_"; - ListUrl = waybackUrl.Substring(0, indexOfUrlModification) + urlModification + - waybackUrl.Substring(indexOfUrlModification); - } - - private void UpdateTimestamp(Closest closest) => - SnapEntity.WaybackTimestamp = - DateTimeOffset.ParseExact(closest.Timestamp, "yyyyMMddHHmmss", null).UtcDateTime; } } \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Models/Wayback/ArchivedSnapshots.cs b/src/FilterLists.Services/Wayback/Models/ArchivedSnapshots.cs similarity index 78% rename from src/FilterLists.Services/Snapshot/Models/Wayback/ArchivedSnapshots.cs rename to src/FilterLists.Services/Wayback/Models/ArchivedSnapshots.cs index 57f1f9ebc..7e6de8ad0 100644 --- a/src/FilterLists.Services/Snapshot/Models/Wayback/ArchivedSnapshots.cs +++ b/src/FilterLists.Services/Wayback/Models/ArchivedSnapshots.cs @@ -1,7 +1,7 @@ using JetBrains.Annotations; using Newtonsoft.Json; -namespace FilterLists.Services.Snapshot.Models.Wayback +namespace FilterLists.Services.Wayback.Models { [UsedImplicitly] public class ArchivedSnapshots diff --git a/src/FilterLists.Services/Snapshot/Models/Wayback/Availability.cs b/src/FilterLists.Services/Wayback/Models/Availability.cs similarity index 84% rename from src/FilterLists.Services/Snapshot/Models/Wayback/Availability.cs rename to src/FilterLists.Services/Wayback/Models/Availability.cs index 5b9b076ff..9df58d4b3 100644 --- a/src/FilterLists.Services/Snapshot/Models/Wayback/Availability.cs +++ b/src/FilterLists.Services/Wayback/Models/Availability.cs @@ -1,7 +1,7 @@ using JetBrains.Annotations; using Newtonsoft.Json; -namespace FilterLists.Services.Snapshot.Models.Wayback +namespace FilterLists.Services.Wayback.Models { [UsedImplicitly] public class Availability diff --git a/src/FilterLists.Services/Snapshot/Models/Wayback/Closest.cs b/src/FilterLists.Services/Wayback/Models/Closest.cs similarity index 88% rename from src/FilterLists.Services/Snapshot/Models/Wayback/Closest.cs rename to src/FilterLists.Services/Wayback/Models/Closest.cs index 4ed418fd4..f77b1f1d6 100644 --- a/src/FilterLists.Services/Snapshot/Models/Wayback/Closest.cs +++ b/src/FilterLists.Services/Wayback/Models/Closest.cs @@ -1,7 +1,7 @@ using JetBrains.Annotations; using Newtonsoft.Json; -namespace FilterLists.Services.Snapshot.Models.Wayback +namespace FilterLists.Services.Wayback.Models { [UsedImplicitly] public class Closest diff --git a/src/FilterLists.Services/Wayback/Models/SnapshotMetaDto.cs b/src/FilterLists.Services/Wayback/Models/SnapshotMetaDto.cs new file mode 100644 index 000000000..8ebceb370 --- /dev/null +++ b/src/FilterLists.Services/Wayback/Models/SnapshotMetaDto.cs @@ -0,0 +1,10 @@ +using System; + +namespace FilterLists.Services.Wayback.Models +{ + public class SnapshotMetaDto + { + public string UrlRaw { get; set; } + public DateTime TimestampUtc { get; set; } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/Wayback/WaybackService.cs b/src/FilterLists.Services/Wayback/WaybackService.cs new file mode 100644 index 000000000..12fbfe99b --- /dev/null +++ b/src/FilterLists.Services/Wayback/WaybackService.cs @@ -0,0 +1,52 @@ +using System; +using System.Net.Http; +using System.Threading.Tasks; +using FilterLists.Services.Extensions; +using FilterLists.Services.Wayback.Models; +using Newtonsoft.Json; + +namespace FilterLists.Services.Wayback +{ + public static class WaybackService + { + private const string WaybackAvailabilityApiUrlPrefix = "https://archive.org/wayback/available?url="; + + public static async Task GetMostRecentSnapshotMeta(string url) + { + var closest = (await GetWaybackAvailability(url))?.ArchivedSnapshots?.Closest; + return closest != null + ? new SnapshotMetaDto {TimestampUtc = ParseTimestampUtc(closest), UrlRaw = ParseUrlRaw(closest)} + : null; + } + + private static async Task GetWaybackAvailability(string url) + { + using (var client = new HttpClient()) + { + var fullUrl = WaybackAvailabilityApiUrlPrefix + url; + var response = await client.GetAsync(fullUrl, HttpCompletionOption.ResponseHeadersRead); + var jsonString = await response.Content.ReadAsStringAsync(); + try + { + return JsonConvert.DeserializeObject(jsonString); + } + catch (JsonReaderException) + { + return null; + } + } + } + + private static string ParseUrlRaw(Closest closest) + { + var waybackUrl = closest.Url; + var indexOfUrlModification = waybackUrl.GetNthIndexOfChar(5, '/'); + const string urlModification = "if_"; + return waybackUrl.Substring(0, indexOfUrlModification) + urlModification + + waybackUrl.Substring(indexOfUrlModification); + } + + private static DateTime ParseTimestampUtc(Closest closest) => + DateTimeOffset.ParseExact(closest.Timestamp, "yyyyMMddHHmmss", null).UtcDateTime; + } +} \ No newline at end of file