From 4576ab9ea03d5d9cb21e8e95b66c608d45f8ddca Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Fri, 5 Oct 2018 14:22:00 -0500 Subject: [PATCH] pass FilterList entity in SnapshotService rather than dto for simplicity --- .../Snapshot/Models/FilterListViewUrlDto.cs | 11 ------ src/FilterLists.Services/Snapshot/Snapshot.cs | 38 +++++++------------ .../Snapshot/SnapshotService.cs | 9 ++--- .../Snapshot/SnapshotWayback.cs | 6 +-- 4 files changed, 20 insertions(+), 44 deletions(-) delete mode 100644 src/FilterLists.Services/Snapshot/Models/FilterListViewUrlDto.cs diff --git a/src/FilterLists.Services/Snapshot/Models/FilterListViewUrlDto.cs b/src/FilterLists.Services/Snapshot/Models/FilterListViewUrlDto.cs deleted file mode 100644 index 19ff0a5a7..000000000 --- a/src/FilterLists.Services/Snapshot/Models/FilterListViewUrlDto.cs +++ /dev/null @@ -1,11 +0,0 @@ -using JetBrains.Annotations; - -namespace FilterLists.Services.Snapshot.Models -{ - [UsedImplicitly] - public class FilterListViewUrlDto - { - public uint Id { get; set; } - public string ViewUrl { get; set; } - } -} \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 6289ba916..5c57106ac 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -10,7 +10,6 @@ using FilterLists.Data.Entities.Junctions; using FilterLists.Services.Extensions; using FilterLists.Services.GitHub; -using FilterLists.Services.Snapshot.Models; using JetBrains.Annotations; using Microsoft.EntityFrameworkCore; using MoreLinq; @@ -21,23 +20,22 @@ namespace FilterLists.Services.Snapshot public class Snapshot { private const int BatchSize = 25000; - public readonly FilterListViewUrlDto List; + public readonly Data.Entities.FilterList List; protected readonly Data.Entities.Snapshot SnapEntity; private readonly FilterListsDbContext dbContext; private readonly GitHubService gitHubService; private readonly Logger logger; private readonly string uaString; protected string ListUrl; - private Data.Entities.FilterList filterList; private bool isUpdatedDateFromGitHub; private HashSet lines; - + public Snapshot() { } [UsedImplicitly] - public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, GitHubService gitHubService, + public Snapshot(FilterListsDbContext dbContext, Data.Entities.FilterList list, GitHubService gitHubService, Logger logger, string uaString) { this.dbContext = dbContext; @@ -80,29 +78,22 @@ private async Task UpdateDatesFromGitHub() var dates = await gitHubService.GetCommitDatesAsync(ListUrl); if (dates is null) return; - var list = await GetList(); - UpdatePublishedDateFromGitHub(dates.First, list); - UpdateUpdatedDateFromGitHub(dates.Last, list); + UpdatePublishedDateFromGitHub(dates.First); + UpdateUpdatedDateFromGitHub(dates.Last); await dbContext.SaveChangesAsync(); } - private async Task GetList() + private void UpdatePublishedDateFromGitHub(DateTime? date) { - filterList = filterList ?? await dbContext.FilterLists.FirstOrDefaultAsync(l => l.Id == List.Id); - return filterList; + if (date is DateTime firstDate && (List.PublishedDate is null || firstDate < List.PublishedDate)) + List.PublishedDate = firstDate; } - private static void UpdatePublishedDateFromGitHub(DateTime? date, Data.Entities.FilterList list) + private void UpdateUpdatedDateFromGitHub(DateTime? date) { - if (date is DateTime firstDate && (list.PublishedDate is null || firstDate < list.PublishedDate)) - list.PublishedDate = firstDate; - } - - private void UpdateUpdatedDateFromGitHub(DateTime? date, Data.Entities.FilterList list) - { - if (date is DateTime lastDate && (list.UpdatedDate is null || lastDate > list.UpdatedDate)) + if (date is DateTime lastDate && (List.UpdatedDate is null || lastDate > List.UpdatedDate)) { - list.UpdatedDate = lastDate; + List.UpdatedDate = lastDate; isUpdatedDateFromGitHub = true; } } @@ -162,7 +153,7 @@ private async Task GetLinesFromSource(HttpResponseMessage response) await SetWasUpdated(); if (SnapEntity.WasUpdated) { - await UpdateUpdatedDate(); + UpdateUpdatedDate(); memoryStream.Position = 0; if (ListUrl.EndsWith(".7z")) await GetLinesFrom7Zip(memoryStream); @@ -192,12 +183,11 @@ await dbContext.Snapshots .Select(s => s.Md5Checksum) .FirstOrDefaultAsync() ?? Array.Empty(); - private async Task UpdateUpdatedDate() + private void UpdateUpdatedDate() { if (isUpdatedDateFromGitHub) return; - var list = await GetList(); - list.UpdatedDate = DateTime.Now; + List.UpdatedDate = DateTime.Now; } private async Task GetLinesFrom7Zip(Stream stream) diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs index 577381243..4182461c0 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotService.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs @@ -4,10 +4,8 @@ using System.Linq.Expressions; using System.Threading.Tasks; using AutoMapper; -using AutoMapper.QueryableExtensions; using FilterLists.Data; using FilterLists.Services.GitHub; -using FilterLists.Services.Snapshot.Models; using Microsoft.EntityFrameworkCore; namespace FilterLists.Services.Snapshot @@ -51,15 +49,14 @@ FROM snapshots_rules await DbContext.Database.ExecuteSqlCommandAsync(command); } - private async Task> GetListsToCapture(int batchSize) => + private async Task> GetListsToCapture(int batchSize) => await DbContext.FilterLists .OrderBy(l => l.Snapshots.Any()) .ThenBy(lastSnapTimestamp) .Take(batchSize) - .ProjectTo(MapConfig) .ToListAsync(); - private async Task> CreateAndSaveSnaps(IEnumerable lists) + private async Task> CreateAndSaveSnaps(IEnumerable lists) where TSnap : Snapshot, new() { var snaps = CreateSnaps(lists).ToList(); @@ -67,7 +64,7 @@ private async Task> CreateAndSaveSnaps(IEnumerable CreateSnaps(IEnumerable lists) + private IEnumerable CreateSnaps(IEnumerable lists) where TSnap : Snapshot, new() => lists.Select(l => Activator.CreateInstance(typeof(TSnap), DbContext, l, gitHubService, logger, uaString) as TSnap); diff --git a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs index 36a20a6e9..6543e10b8 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs @@ -1,7 +1,6 @@ using System.Threading.Tasks; using FilterLists.Data; using FilterLists.Services.GitHub; -using FilterLists.Services.Snapshot.Models; using FilterLists.Services.Wayback; using JetBrains.Annotations; @@ -14,8 +13,9 @@ public SnapshotWayback() } [UsedImplicitly] - public SnapshotWayback(FilterListsDbContext dbContext, FilterListViewUrlDto list, GitHubService gitHubService, - Logger logger, string uaString) : base(dbContext, list, gitHubService, logger, uaString) + public SnapshotWayback(FilterListsDbContext dbContext, Data.Entities.FilterList list, + GitHubService gitHubService, Logger logger, string uaString) : + base(dbContext, list, gitHubService, logger, uaString) { }