prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

extract WaybackService for re-use

Browse source 
ref #325
This commit is contained in:
Collin M. Barrett 2018-08-24 12:41:28 -05:00
parent aeb5ed38c1
commit 46be9ff6bb
6 changed files with 71 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
src/FilterLists.Services/Snapshot/SnapshotWayback.cs
View file

@ -1,19 +1,13 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Services.Extensions;
using FilterLists.Services.Snapshot.Models;
using FilterLists.Services.Snapshot.Models.Wayback;
using FilterLists.Services.Wayback;
using JetBrains.Annotations;
using Newtonsoft.Json;
namespace FilterLists.Services.Snapshot
{
public class SnapshotWayback : Snapshot
{
private const string WaybackAvailabilityApiUrlPrefix = "https://archive.org/wayback/available?url=";
public SnapshotWayback()
{
}
@ -32,41 +26,16 @@ public override async Task TrySaveAsync()
private async Task UpdateWaybackData()
{
var availability = await GetWaybackAvailability();
var closest = availability.ArchivedSnapshots?.Closest;
if (closest != null)
var snapshotMeta = await WaybackService.GetMostRecentSnapshotMeta(ListUrl);
if (snapshotMeta != null)
{
UpdateViewUrl(closest);
UpdateTimestamp(closest);
ListUrl = snapshotMeta.UrlRaw;
SnapEntity.WaybackTimestamp = snapshotMeta.TimestampUtc;
}
else
{
ListUrl = null;
}
}
private async Task<Availability> GetWaybackAvailability()
{
using (var client = new HttpClient())
{
var url = WaybackAvailabilityApiUrlPrefix + ListUrl;
var response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
var jsonString = await response.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<Availability>(jsonString);
}
}
private void UpdateViewUrl(Closest closest)
{
var waybackUrl = closest.Url;
var indexOfUrlModification = waybackUrl.GetNthIndexOfChar(5, '/');
const string urlModification = "if_";
ListUrl = waybackUrl.Substring(0, indexOfUrlModification) + urlModification +
waybackUrl.Substring(indexOfUrlModification);
}
private void UpdateTimestamp(Closest closest) =>
SnapEntity.WaybackTimestamp =
DateTimeOffset.ParseExact(closest.Timestamp, "yyyyMMddHHmmss", null).UtcDateTime;
}
}

2
src/FilterLists.Services/Snapshot/Models/Wayback/ArchivedSnapshots.cs → src/FilterLists.Services/Wayback/Models/ArchivedSnapshots.cs
View file

@ -1,7 +1,7 @@
using JetBrains.Annotations;
using Newtonsoft.Json;
namespace FilterLists.Services.Snapshot.Models.Wayback
namespace FilterLists.Services.Wayback.Models
{
[UsedImplicitly]
public class ArchivedSnapshots

2
src/FilterLists.Services/Snapshot/Models/Wayback/Availability.cs → src/FilterLists.Services/Wayback/Models/Availability.cs
View file

@ -1,7 +1,7 @@
using JetBrains.Annotations;
using Newtonsoft.Json;
namespace FilterLists.Services.Snapshot.Models.Wayback
namespace FilterLists.Services.Wayback.Models
{
[UsedImplicitly]
public class Availability

2
src/FilterLists.Services/Snapshot/Models/Wayback/Closest.cs → src/FilterLists.Services/Wayback/Models/Closest.cs
View file

@ -1,7 +1,7 @@
using JetBrains.Annotations;
using Newtonsoft.Json;
namespace FilterLists.Services.Snapshot.Models.Wayback
namespace FilterLists.Services.Wayback.Models
{
[UsedImplicitly]
public class Closest

10
src/FilterLists.Services/Wayback/Models/SnapshotMetaDto.cs Normal file
View file

@ -0,0 +1,10 @@
using System;
namespace FilterLists.Services.Wayback.Models
{
public class SnapshotMetaDto
{
public string UrlRaw { get; set; }
public DateTime TimestampUtc { get; set; }
}
}

52
src/FilterLists.Services/Wayback/WaybackService.cs Normal file
View file

@ -0,0 +1,52 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using FilterLists.Services.Extensions;
using FilterLists.Services.Wayback.Models;
using Newtonsoft.Json;
namespace FilterLists.Services.Wayback
{
public static class WaybackService
{
private const string WaybackAvailabilityApiUrlPrefix = "https://archive.org/wayback/available?url=";
public static async Task<SnapshotMetaDto> GetMostRecentSnapshotMeta(string url)
{
var closest = (await GetWaybackAvailability(url))?.ArchivedSnapshots?.Closest;
return closest != null
? new SnapshotMetaDto {TimestampUtc = ParseTimestampUtc(closest), UrlRaw = ParseUrlRaw(closest)}
: null;
}
private static async Task<Availability> GetWaybackAvailability(string url)
{
using (var client = new HttpClient())
{
var fullUrl = WaybackAvailabilityApiUrlPrefix + url;
var response = await client.GetAsync(fullUrl, HttpCompletionOption.ResponseHeadersRead);
var jsonString = await response.Content.ReadAsStringAsync();
try
{
return JsonConvert.DeserializeObject<Availability>(jsonString);
}
catch (JsonReaderException)
{
return null;
}
}
}
private static string ParseUrlRaw(Closest closest)
{
var waybackUrl = closest.Url;
var indexOfUrlModification = waybackUrl.GetNthIndexOfChar(5, '/');
const string urlModification = "if_";
return waybackUrl.Substring(0, indexOfUrlModification) + urlModification +
waybackUrl.Substring(indexOfUrlModification);
}
private static DateTime ParseTimestampUtc(Closest closest) =>
DateTimeOffset.ParseExact(closest.Timestamp, "yyyyMMddHHmmss", null).UtcDateTime;
}
}