prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

still snap wayback urls if url changed since added

Browse source 
rely on md5 hash for perf gains of no-change in list rather than excluding hardcoded wayback urls altogether
This commit is contained in:
Collin M. Barrett 2018-09-03 08:39:27 -05:00
parent 38dfffdbc7
commit 6ecd9d8109
2 changed files with 0 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/FilterLists.Services/Snapshot/SnapshotService.cs
View file

@ -7,7 +7,6 @@
using AutoMapper.QueryableExtensions;
using FilterLists.Data;
using FilterLists.Services.Snapshot.Models;
using FilterLists.Services.Wayback;
using Microsoft.EntityFrameworkCore;
namespace FilterLists.Services.Snapshot
@ -20,10 +19,6 @@ public class SnapshotService : Service
.Select(s => s.WasUpdated && !s.WasSuccessful)
.FirstOrDefault();
private readonly Expression<Func<Data.Entities.FilterList, bool>> isNotWaybackViewUrlWithSuccessfulSnap =
l => !(l.ViewUrl.StartsWith(WaybackService.WaybackMachineUrlPrefix) &&
l.Snapshots.Any(s => s.WasSuccessful));
private readonly Expression<Func<Data.Entities.FilterList, DateTime?>> lastSnapTimestamp =
l => l.Snapshots
.Select(s => s.CreatedDateUtc)
@ -55,7 +50,6 @@ private async Task CleanupFailedSnapshots()
private async Task<IEnumerable<FilterListViewUrlDto>> GetListsToCapture(int batchSize) =>
await DbContext
.FilterLists
.Where(isNotWaybackViewUrlWithSuccessfulSnap)
.OrderBy(l => l.Snapshots.Any())
.ThenByDescending(ifLastSnapFailed)
.ThenBy(lastSnapTimestamp)

1
src/FilterLists.Services/Wayback/WaybackService.cs
View file

@ -9,7 +9,6 @@ namespace FilterLists.Services.Wayback
{
public static class WaybackService
{
public const string WaybackMachineUrlPrefix = "https://web.archive.org/web/";
private const string WaybackAvailabilityApiUrlPrefix = "https://archive.org/wayback/available?url=";
public static async Task<SnapshotMetaDto> GetMostRecentSnapshotMeta(string url)