mirror of
https://github.com/collinbarrett/FilterLists.git
synced 2026-03-11 09:04:27 +00:00
rm legacy snapshot service
This commit is contained in:
Collin M. Barrett
parent
6c140d1fb3
commit
8f5efc93ff
4 changed files with 0 additions and 416 deletions
|
|
@ -1,45 +0,0 @@
|
|||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using FilterLists.Data;
|
||||
using FilterLists.Data.Entities;
|
||||
using FilterLists.Data.Entities.Junctions;
|
||||
using FilterLists.Services.Extensions;
|
||||
|
||||
namespace FilterLists.Services.Snapshot
|
||||
{
|
||||
public class Batch
|
||||
{
|
||||
private readonly FilterListsDbContext dbContext;
|
||||
private readonly IEnumerable<string> lines;
|
||||
private readonly Data.Entities.Snapshot snapEntity;
|
||||
|
||||
public Batch(FilterListsDbContext dbContext, IEnumerable<string> lines, Data.Entities.Snapshot snapEntity)
|
||||
{
|
||||
this.dbContext = dbContext;
|
||||
this.lines = lines;
|
||||
this.snapEntity = snapEntity;
|
||||
}
|
||||
|
||||
public async Task SaveAsync()
|
||||
{
|
||||
var rules = GetOrCreateRules();
|
||||
CreateSnapRules(rules);
|
||||
await dbContext.SaveChangesAsync();
|
||||
}
|
||||
|
||||
private IQueryable<Rule> GetOrCreateRules()
|
||||
{
|
||||
var existingRules = dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule);
|
||||
var newRules = lines.Except(existingRules.Select(r => r.Raw)).Select(l => new Rule {Raw = l}).ToList();
|
||||
dbContext.Rules.AddRange(newRules);
|
||||
return existingRules.Concat(newRules);
|
||||
}
|
||||
|
||||
private void CreateSnapRules(IQueryable<Rule> rules)
|
||||
{
|
||||
var snapRules = rules.Select(r => new SnapshotRule {Rule = r}).ToList();
|
||||
snapEntity.SnapshotRules.AddRange(snapRules);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,251 +0,0 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Security.Cryptography;
|
||||
using System.Threading.Tasks;
|
||||
using FilterLists.Data;
|
||||
using FilterLists.Data.Entities.Junctions;
|
||||
using FilterLists.Services.Extensions;
|
||||
using FilterLists.Services.GitHub;
|
||||
using JetBrains.Annotations;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using MoreLinq;
|
||||
using SharpCompress.Archives.SevenZip;
|
||||
|
||||
namespace FilterLists.Services.Snapshot
|
||||
{
|
||||
public class Snapshot
|
||||
{
|
||||
private const int BatchSize = 25000;
|
||||
public readonly Data.Entities.FilterList List;
|
||||
protected readonly Data.Entities.Snapshot SnapEntity;
|
||||
private readonly FilterListsDbContext dbContext;
|
||||
private readonly GitHubService gitHubService;
|
||||
private readonly Logger logger;
|
||||
private readonly string uaString;
|
||||
protected string ListUrl;
|
||||
private bool isUpdatedDateFromGitHub;
|
||||
private HashSet<string> lines;
|
||||
|
||||
public Snapshot()
|
||||
{
|
||||
}
|
||||
|
||||
[UsedImplicitly]
|
||||
public Snapshot(FilterListsDbContext dbContext, Data.Entities.FilterList list, GitHubService gitHubService,
|
||||
Logger logger, string uaString)
|
||||
{
|
||||
this.dbContext = dbContext;
|
||||
List = list;
|
||||
ListUrl = list.ViewUrl;
|
||||
this.gitHubService = gitHubService;
|
||||
this.logger = logger;
|
||||
SnapEntity = new Data.Entities.Snapshot {FilterListId = list.Id, SnapshotRules = new List<SnapshotRule>()};
|
||||
this.uaString = uaString;
|
||||
}
|
||||
|
||||
public bool WebExcepted { get; private set; }
|
||||
|
||||
public virtual async Task TrySaveAsync() => await TrySaveAsyncBase();
|
||||
|
||||
protected async Task TrySaveAsyncBase()
|
||||
{
|
||||
await AddSnapEntity();
|
||||
if (!ListUrl.IsValidHttpOrHttpsUrl())
|
||||
return;
|
||||
await UpdateDatesFromGitHub();
|
||||
try
|
||||
{
|
||||
await SaveAsync();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
logger.Log(e);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AddSnapEntity()
|
||||
{
|
||||
dbContext.Snapshots.Add(SnapEntity);
|
||||
await dbContext.SaveChangesAsync();
|
||||
}
|
||||
|
||||
private async Task UpdateDatesFromGitHub()
|
||||
{
|
||||
var dates = await gitHubService.GetCommitDatesAsync(ListUrl);
|
||||
if (dates is null)
|
||||
return;
|
||||
UpdatePublishedDateFromGitHub(dates.First);
|
||||
UpdateUpdatedDateFromGitHub(dates.Last);
|
||||
await dbContext.SaveChangesAsync();
|
||||
}
|
||||
|
||||
private void UpdatePublishedDateFromGitHub(DateTime? date)
|
||||
{
|
||||
if (date is DateTime firstDate && (List.PublishedDate is null || firstDate < List.PublishedDate))
|
||||
List.PublishedDate = firstDate;
|
||||
}
|
||||
|
||||
private void UpdateUpdatedDateFromGitHub(DateTime? date)
|
||||
{
|
||||
if (date is DateTime lastDate && (List.UpdatedDate is null || lastDate > List.UpdatedDate))
|
||||
{
|
||||
List.UpdatedDate = lastDate;
|
||||
isUpdatedDateFromGitHub = true;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SaveAsync()
|
||||
{
|
||||
await TryGetLines();
|
||||
if (lines != null)
|
||||
{
|
||||
await SaveInBatches();
|
||||
await SetSuccessful();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task TryGetLines()
|
||||
{
|
||||
try
|
||||
{
|
||||
await GetLines();
|
||||
}
|
||||
catch (HttpRequestException hre)
|
||||
{
|
||||
WebExcepted = true;
|
||||
await dbContext.SaveChangesAsync();
|
||||
logger.Log(hre);
|
||||
}
|
||||
catch (WebException we)
|
||||
{
|
||||
WebExcepted = true;
|
||||
SnapEntity.HttpStatusCode = (int)((HttpWebResponse)we.Response).StatusCode;
|
||||
await dbContext.SaveChangesAsync();
|
||||
logger.Log(we);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task GetLines()
|
||||
{
|
||||
using (var httpClient = new HttpClient())
|
||||
{
|
||||
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(uaString);
|
||||
var response = await httpClient.GetAsync(ListUrl, HttpCompletionOption.ResponseHeadersRead);
|
||||
SnapEntity.HttpStatusCode = (int)response.StatusCode;
|
||||
response.EnsureSuccessStatusCode();
|
||||
await GetLinesFromSource(response);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task GetLinesFromSource(HttpResponseMessage response)
|
||||
{
|
||||
lines = new HashSet<string>();
|
||||
using (var stream = await response.Content.ReadAsStreamAsync())
|
||||
using (var memoryStream = new MemoryStream())
|
||||
{
|
||||
await stream.CopyToAsync(memoryStream);
|
||||
memoryStream.Position = 0;
|
||||
SetChecksum(memoryStream);
|
||||
await SetWasUpdated();
|
||||
if (SnapEntity.WasUpdated)
|
||||
{
|
||||
UpdateUpdatedDate();
|
||||
memoryStream.Position = 0;
|
||||
if (ListUrl.EndsWith(".7z"))
|
||||
await GetLinesFrom7Zip(memoryStream);
|
||||
else
|
||||
await GetLinesFromStream(memoryStream);
|
||||
}
|
||||
|
||||
await dbContext.SaveChangesAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private void SetChecksum(Stream stream)
|
||||
{
|
||||
using (var md5 = MD5.Create())
|
||||
{
|
||||
SnapEntity.Md5Checksum = md5.ComputeHash(stream);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SetWasUpdated() =>
|
||||
SnapEntity.WasUpdated = !SnapEntity.Md5Checksum.SequenceEqual(await GetPreviousChecksum());
|
||||
|
||||
private async Task<byte[]> GetPreviousChecksum() =>
|
||||
await dbContext.Snapshots
|
||||
.Where(s => s.WasSuccessful && s.FilterListId == List.Id)
|
||||
.OrderByDescending(s => s.CreatedDateUtc)
|
||||
.Select(s => s.Md5Checksum)
|
||||
.FirstOrDefaultAsync() ?? Array.Empty<byte>();
|
||||
|
||||
private void UpdateUpdatedDate()
|
||||
{
|
||||
if (isUpdatedDateFromGitHub)
|
||||
return;
|
||||
if (List.Snapshots.Any(s => s.WasUpdated && s.WasSuccessful))
|
||||
List.UpdatedDate = DateTime.Now;
|
||||
}
|
||||
|
||||
private async Task GetLinesFrom7Zip(Stream stream)
|
||||
{
|
||||
using (var archive = SevenZipArchive.Open(stream))
|
||||
{
|
||||
var archiveEntries = archive.Entries.Where(entry => !entry.IsDirectory);
|
||||
foreach (var entry in archiveEntries)
|
||||
using (var entryStream = entry.OpenEntryStream())
|
||||
{
|
||||
await GetLinesFromStream(entryStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task GetLinesFromStream(Stream stream)
|
||||
{
|
||||
using (var streamReader = new StreamReader(stream))
|
||||
{
|
||||
string line;
|
||||
while ((line = await streamReader.ReadLineAsync()) != null)
|
||||
lines.AddIfNotNullOrEmpty(line.Trim());
|
||||
stream.Position = 0;
|
||||
await SaveOriginalFile(stream);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SaveOriginalFile(Stream stream)
|
||||
{
|
||||
var fileName = Path.Combine("snapshots", List.Id.ToString(),
|
||||
BitConverter.ToString(SnapEntity.Md5Checksum).Replace("-", "") + ".txt");
|
||||
new FileInfo(fileName).Directory?.Create();
|
||||
using (var fileStream = File.Create(fileName))
|
||||
{
|
||||
await stream.CopyToAsync(fileStream);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SaveInBatches()
|
||||
{
|
||||
var snapBatches = CreateBatches();
|
||||
await SaveBatches(snapBatches);
|
||||
}
|
||||
|
||||
private IEnumerable<Batch> CreateBatches() =>
|
||||
lines.Batch(BatchSize).Select(b => new Batch(dbContext, b, SnapEntity));
|
||||
|
||||
private static async Task SaveBatches(IEnumerable<Batch> batches)
|
||||
{
|
||||
foreach (var batch in batches)
|
||||
await batch.SaveAsync();
|
||||
}
|
||||
|
||||
private async Task SetSuccessful()
|
||||
{
|
||||
SnapEntity.WasSuccessful = true;
|
||||
await dbContext.SaveChangesAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Threading.Tasks;
|
||||
using AutoMapper;
|
||||
using FilterLists.Data;
|
||||
using FilterLists.Services.GitHub;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace FilterLists.Services.Snapshot
|
||||
{
|
||||
public class SnapshotService : Service
|
||||
{
|
||||
private readonly Expression<Func<Data.Entities.FilterList, DateTime?>> lastSnapTimestamp =
|
||||
l => l.Snapshots
|
||||
.Select(s => s.CreatedDateUtc)
|
||||
.OrderByDescending(d => d)
|
||||
.FirstOrDefault();
|
||||
|
||||
private readonly GitHubService gitHubService;
|
||||
private readonly Logger logger;
|
||||
private string uaString;
|
||||
|
||||
public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig,
|
||||
GitHubService gitHubService, Logger logger) : base(dbContext, mapConfig)
|
||||
{
|
||||
this.gitHubService = gitHubService;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task CaptureAsync(int batchSize)
|
||||
{
|
||||
await CleanupFailedSnapshots();
|
||||
uaString = await UserAgentService.GetMostPopularStringAsync();
|
||||
var lists = await GetListsToCapture(batchSize);
|
||||
var snaps = await CreateAndSaveSnaps<Snapshot>(lists);
|
||||
var listsToRetry = snaps.Where(s => s.WebExcepted).Select(s => s.List);
|
||||
await CreateAndSaveSnaps<SnapshotWayback>(listsToRetry);
|
||||
}
|
||||
|
||||
private async Task CleanupFailedSnapshots()
|
||||
{
|
||||
const string command =
|
||||
@"DELETE snapshots_rules
|
||||
FROM snapshots_rules
|
||||
JOIN snapshots ON snapshots.Id = snapshots_rules.SnapshotId
|
||||
WHERE snapshots.WasSuccessful = 0;";
|
||||
await DbContext.Database.ExecuteSqlCommandAsync(command);
|
||||
}
|
||||
|
||||
private async Task<IEnumerable<Data.Entities.FilterList>> GetListsToCapture(int batchSize) =>
|
||||
await DbContext.FilterLists
|
||||
.OrderBy(l => l.Snapshots.Any())
|
||||
.ThenBy(lastSnapTimestamp)
|
||||
.Take(batchSize)
|
||||
.Include(l => l.Snapshots)
|
||||
.ToListAsync();
|
||||
|
||||
private async Task<List<TSnap>> CreateAndSaveSnaps<TSnap>(IEnumerable<Data.Entities.FilterList> lists)
|
||||
where TSnap : Snapshot, new()
|
||||
{
|
||||
var snaps = CreateSnaps<TSnap>(lists).ToList();
|
||||
await SaveSnaps(snaps);
|
||||
return snaps;
|
||||
}
|
||||
|
||||
private IEnumerable<TSnap> CreateSnaps<TSnap>(IEnumerable<Data.Entities.FilterList> lists)
|
||||
where TSnap : Snapshot, new() => lists.Select(l =>
|
||||
Activator.CreateInstance(typeof(TSnap), DbContext, l, gitHubService, logger, uaString) as TSnap);
|
||||
|
||||
private static async Task SaveSnaps(IEnumerable<Snapshot> snaps)
|
||||
{
|
||||
foreach (var snap in snaps)
|
||||
await snap.TrySaveAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
using System.Threading.Tasks;
|
||||
using FilterLists.Data;
|
||||
using FilterLists.Services.GitHub;
|
||||
using FilterLists.Services.Wayback;
|
||||
using JetBrains.Annotations;
|
||||
|
||||
namespace FilterLists.Services.Snapshot
|
||||
{
|
||||
public class SnapshotWayback : Snapshot
|
||||
{
|
||||
public SnapshotWayback()
|
||||
{
|
||||
}
|
||||
|
||||
[UsedImplicitly]
|
||||
public SnapshotWayback(FilterListsDbContext dbContext, Data.Entities.FilterList list,
|
||||
GitHubService gitHubService, Logger logger, string uaString) :
|
||||
base(dbContext, list, gitHubService, logger, uaString)
|
||||
{
|
||||
}
|
||||
|
||||
public override async Task TrySaveAsync()
|
||||
{
|
||||
await UpdateWaybackData();
|
||||
await TrySaveAsyncBase();
|
||||
}
|
||||
|
||||
private async Task UpdateWaybackData()
|
||||
{
|
||||
var snapshotMeta = await WaybackService.GetMostRecentSnapshotMetaAsync(ListUrl);
|
||||
if (snapshotMeta != null)
|
||||
{
|
||||
ListUrl = SnapEntity.WaybackUrl = snapshotMeta.RawUrl;
|
||||
SnapEntity.WaybackTimestamp = snapshotMeta.TimestampUtc;
|
||||
}
|
||||
else
|
||||
{
|
||||
ListUrl = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue