diff --git a/src/FilterLists.Services/Snapshot/Batch.cs b/src/FilterLists.Services/Snapshot/Batch.cs deleted file mode 100644 index deb710de7..000000000 --- a/src/FilterLists.Services/Snapshot/Batch.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using FilterLists.Data; -using FilterLists.Data.Entities; -using FilterLists.Data.Entities.Junctions; -using FilterLists.Services.Extensions; - -namespace FilterLists.Services.Snapshot -{ - public class Batch - { - private readonly FilterListsDbContext dbContext; - private readonly IEnumerable lines; - private readonly Data.Entities.Snapshot snapEntity; - - public Batch(FilterListsDbContext dbContext, IEnumerable lines, Data.Entities.Snapshot snapEntity) - { - this.dbContext = dbContext; - this.lines = lines; - this.snapEntity = snapEntity; - } - - public async Task SaveAsync() - { - var rules = GetOrCreateRules(); - CreateSnapRules(rules); - await dbContext.SaveChangesAsync(); - } - - private IQueryable GetOrCreateRules() - { - var existingRules = dbContext.Rules.Join(lines, rule => rule.Raw, line => line, (rule, line) => rule); - var newRules = lines.Except(existingRules.Select(r => r.Raw)).Select(l => new Rule {Raw = l}).ToList(); - dbContext.Rules.AddRange(newRules); - return existingRules.Concat(newRules); - } - - private void CreateSnapRules(IQueryable rules) - { - var snapRules = rules.Select(r => new SnapshotRule {Rule = r}).ToList(); - snapEntity.SnapshotRules.AddRange(snapRules); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs deleted file mode 100644 index 7a471bea3..000000000 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ /dev/null @@ -1,251 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Security.Cryptography; -using System.Threading.Tasks; -using FilterLists.Data; -using FilterLists.Data.Entities.Junctions; -using FilterLists.Services.Extensions; -using FilterLists.Services.GitHub; -using JetBrains.Annotations; -using Microsoft.EntityFrameworkCore; -using MoreLinq; -using SharpCompress.Archives.SevenZip; - -namespace FilterLists.Services.Snapshot -{ - public class Snapshot - { - private const int BatchSize = 25000; - public readonly Data.Entities.FilterList List; - protected readonly Data.Entities.Snapshot SnapEntity; - private readonly FilterListsDbContext dbContext; - private readonly GitHubService gitHubService; - private readonly Logger logger; - private readonly string uaString; - protected string ListUrl; - private bool isUpdatedDateFromGitHub; - private HashSet lines; - - public Snapshot() - { - } - - [UsedImplicitly] - public Snapshot(FilterListsDbContext dbContext, Data.Entities.FilterList list, GitHubService gitHubService, - Logger logger, string uaString) - { - this.dbContext = dbContext; - List = list; - ListUrl = list.ViewUrl; - this.gitHubService = gitHubService; - this.logger = logger; - SnapEntity = new Data.Entities.Snapshot {FilterListId = list.Id, SnapshotRules = new List()}; - this.uaString = uaString; - } - - public bool WebExcepted { get; private set; } - - public virtual async Task TrySaveAsync() => await TrySaveAsyncBase(); - - protected async Task TrySaveAsyncBase() - { - await AddSnapEntity(); - if (!ListUrl.IsValidHttpOrHttpsUrl()) - return; - await UpdateDatesFromGitHub(); - try - { - await SaveAsync(); - } - catch (Exception e) - { - logger.Log(e); - } - } - - private async Task AddSnapEntity() - { - dbContext.Snapshots.Add(SnapEntity); - await dbContext.SaveChangesAsync(); - } - - private async Task UpdateDatesFromGitHub() - { - var dates = await gitHubService.GetCommitDatesAsync(ListUrl); - if (dates is null) - return; - UpdatePublishedDateFromGitHub(dates.First); - UpdateUpdatedDateFromGitHub(dates.Last); - await dbContext.SaveChangesAsync(); - } - - private void UpdatePublishedDateFromGitHub(DateTime? date) - { - if (date is DateTime firstDate && (List.PublishedDate is null || firstDate < List.PublishedDate)) - List.PublishedDate = firstDate; - } - - private void UpdateUpdatedDateFromGitHub(DateTime? date) - { - if (date is DateTime lastDate && (List.UpdatedDate is null || lastDate > List.UpdatedDate)) - { - List.UpdatedDate = lastDate; - isUpdatedDateFromGitHub = true; - } - } - - private async Task SaveAsync() - { - await TryGetLines(); - if (lines != null) - { - await SaveInBatches(); - await SetSuccessful(); - } - } - - private async Task TryGetLines() - { - try - { - await GetLines(); - } - catch (HttpRequestException hre) - { - WebExcepted = true; - await dbContext.SaveChangesAsync(); - logger.Log(hre); - } - catch (WebException we) - { - WebExcepted = true; - SnapEntity.HttpStatusCode = (int)((HttpWebResponse)we.Response).StatusCode; - await dbContext.SaveChangesAsync(); - logger.Log(we); - } - } - - private async Task GetLines() - { - using (var httpClient = new HttpClient()) - { - httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(uaString); - var response = await httpClient.GetAsync(ListUrl, HttpCompletionOption.ResponseHeadersRead); - SnapEntity.HttpStatusCode = (int)response.StatusCode; - response.EnsureSuccessStatusCode(); - await GetLinesFromSource(response); - } - } - - private async Task GetLinesFromSource(HttpResponseMessage response) - { - lines = new HashSet(); - using (var stream = await response.Content.ReadAsStreamAsync()) - using (var memoryStream = new MemoryStream()) - { - await stream.CopyToAsync(memoryStream); - memoryStream.Position = 0; - SetChecksum(memoryStream); - await SetWasUpdated(); - if (SnapEntity.WasUpdated) - { - UpdateUpdatedDate(); - memoryStream.Position = 0; - if (ListUrl.EndsWith(".7z")) - await GetLinesFrom7Zip(memoryStream); - else - await GetLinesFromStream(memoryStream); - } - - await dbContext.SaveChangesAsync(); - } - } - - private void SetChecksum(Stream stream) - { - using (var md5 = MD5.Create()) - { - SnapEntity.Md5Checksum = md5.ComputeHash(stream); - } - } - - private async Task SetWasUpdated() => - SnapEntity.WasUpdated = !SnapEntity.Md5Checksum.SequenceEqual(await GetPreviousChecksum()); - - private async Task GetPreviousChecksum() => - await dbContext.Snapshots - .Where(s => s.WasSuccessful && s.FilterListId == List.Id) - .OrderByDescending(s => s.CreatedDateUtc) - .Select(s => s.Md5Checksum) - .FirstOrDefaultAsync() ?? Array.Empty(); - - private void UpdateUpdatedDate() - { - if (isUpdatedDateFromGitHub) - return; - if (List.Snapshots.Any(s => s.WasUpdated && s.WasSuccessful)) - List.UpdatedDate = DateTime.Now; - } - - private async Task GetLinesFrom7Zip(Stream stream) - { - using (var archive = SevenZipArchive.Open(stream)) - { - var archiveEntries = archive.Entries.Where(entry => !entry.IsDirectory); - foreach (var entry in archiveEntries) - using (var entryStream = entry.OpenEntryStream()) - { - await GetLinesFromStream(entryStream); - } - } - } - - private async Task GetLinesFromStream(Stream stream) - { - using (var streamReader = new StreamReader(stream)) - { - string line; - while ((line = await streamReader.ReadLineAsync()) != null) - lines.AddIfNotNullOrEmpty(line.Trim()); - stream.Position = 0; - await SaveOriginalFile(stream); - } - } - - private async Task SaveOriginalFile(Stream stream) - { - var fileName = Path.Combine("snapshots", List.Id.ToString(), - BitConverter.ToString(SnapEntity.Md5Checksum).Replace("-", "") + ".txt"); - new FileInfo(fileName).Directory?.Create(); - using (var fileStream = File.Create(fileName)) - { - await stream.CopyToAsync(fileStream); - } - } - - private async Task SaveInBatches() - { - var snapBatches = CreateBatches(); - await SaveBatches(snapBatches); - } - - private IEnumerable CreateBatches() => - lines.Batch(BatchSize).Select(b => new Batch(dbContext, b, SnapEntity)); - - private static async Task SaveBatches(IEnumerable batches) - { - foreach (var batch in batches) - await batch.SaveAsync(); - } - - private async Task SetSuccessful() - { - SnapEntity.WasSuccessful = true; - await dbContext.SaveChangesAsync(); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs deleted file mode 100644 index 23c56737c..000000000 --- a/src/FilterLists.Services/Snapshot/SnapshotService.cs +++ /dev/null @@ -1,78 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Linq.Expressions; -using System.Threading.Tasks; -using AutoMapper; -using FilterLists.Data; -using FilterLists.Services.GitHub; -using Microsoft.EntityFrameworkCore; - -namespace FilterLists.Services.Snapshot -{ - public class SnapshotService : Service - { - private readonly Expression> lastSnapTimestamp = - l => l.Snapshots - .Select(s => s.CreatedDateUtc) - .OrderByDescending(d => d) - .FirstOrDefault(); - - private readonly GitHubService gitHubService; - private readonly Logger logger; - private string uaString; - - public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig, - GitHubService gitHubService, Logger logger) : base(dbContext, mapConfig) - { - this.gitHubService = gitHubService; - this.logger = logger; - } - - public async Task CaptureAsync(int batchSize) - { - await CleanupFailedSnapshots(); - uaString = await UserAgentService.GetMostPopularStringAsync(); - var lists = await GetListsToCapture(batchSize); - var snaps = await CreateAndSaveSnaps(lists); - var listsToRetry = snaps.Where(s => s.WebExcepted).Select(s => s.List); - await CreateAndSaveSnaps(listsToRetry); - } - - private async Task CleanupFailedSnapshots() - { - const string command = - @"DELETE snapshots_rules - FROM snapshots_rules - JOIN snapshots ON snapshots.Id = snapshots_rules.SnapshotId - WHERE snapshots.WasSuccessful = 0;"; - await DbContext.Database.ExecuteSqlCommandAsync(command); - } - - private async Task> GetListsToCapture(int batchSize) => - await DbContext.FilterLists - .OrderBy(l => l.Snapshots.Any()) - .ThenBy(lastSnapTimestamp) - .Take(batchSize) - .Include(l => l.Snapshots) - .ToListAsync(); - - private async Task> CreateAndSaveSnaps(IEnumerable lists) - where TSnap : Snapshot, new() - { - var snaps = CreateSnaps(lists).ToList(); - await SaveSnaps(snaps); - return snaps; - } - - private IEnumerable CreateSnaps(IEnumerable lists) - where TSnap : Snapshot, new() => lists.Select(l => - Activator.CreateInstance(typeof(TSnap), DbContext, l, gitHubService, logger, uaString) as TSnap); - - private static async Task SaveSnaps(IEnumerable snaps) - { - foreach (var snap in snaps) - await snap.TrySaveAsync(); - } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs deleted file mode 100644 index 04b61a017..000000000 --- a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System.Threading.Tasks; -using FilterLists.Data; -using FilterLists.Services.GitHub; -using FilterLists.Services.Wayback; -using JetBrains.Annotations; - -namespace FilterLists.Services.Snapshot -{ - public class SnapshotWayback : Snapshot - { - public SnapshotWayback() - { - } - - [UsedImplicitly] - public SnapshotWayback(FilterListsDbContext dbContext, Data.Entities.FilterList list, - GitHubService gitHubService, Logger logger, string uaString) : - base(dbContext, list, gitHubService, logger, uaString) - { - } - - public override async Task TrySaveAsync() - { - await UpdateWaybackData(); - await TrySaveAsyncBase(); - } - - private async Task UpdateWaybackData() - { - var snapshotMeta = await WaybackService.GetMostRecentSnapshotMetaAsync(ListUrl); - if (snapshotMeta != null) - { - ListUrl = SnapEntity.WaybackUrl = snapshotMeta.RawUrl; - SnapEntity.WaybackTimestamp = snapshotMeta.TimestampUtc; - } - else - { - ListUrl = null; - } - } - } -} \ No newline at end of file