prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

pass FilterList entity in SnapshotService rather than dto for simplicity

Browse source
This commit is contained in:
Collin M. Barrett 2018-10-05 14:22:00 -05:00
parent 8c22569622
commit 4576ab9ea0
4 changed files with 20 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/FilterLists.Services/Snapshot/Models/FilterListViewUrlDto.cs
View file

@ -1,11 +0,0 @@
using JetBrains.Annotations;
namespace FilterLists.Services.Snapshot.Models
{
[UsedImplicitly]
public class FilterListViewUrlDto
{
public uint Id { get; set; }
public string ViewUrl { get; set; }
}
}

38
src/FilterLists.Services/Snapshot/Snapshot.cs
View file

@ -10,7 +10,6 @@
using FilterLists.Data.Entities.Junctions;
using FilterLists.Services.Extensions;
using FilterLists.Services.GitHub;
using FilterLists.Services.Snapshot.Models;
using JetBrains.Annotations;
using Microsoft.EntityFrameworkCore;
using MoreLinq;
@ -21,23 +20,22 @@ namespace FilterLists.Services.Snapshot
public class Snapshot
{
private const int BatchSize = 25000;
public readonly FilterListViewUrlDto List;
public readonly Data.Entities.FilterList List;
protected readonly Data.Entities.Snapshot SnapEntity;
private readonly FilterListsDbContext dbContext;
private readonly GitHubService gitHubService;
private readonly Logger logger;
private readonly string uaString;
protected string ListUrl;
private Data.Entities.FilterList filterList;
private bool isUpdatedDateFromGitHub;
private HashSet<string> lines;
public Snapshot()
{
}
[UsedImplicitly]
public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, GitHubService gitHubService,
public Snapshot(FilterListsDbContext dbContext, Data.Entities.FilterList list, GitHubService gitHubService,
Logger logger, string uaString)
{
this.dbContext = dbContext;
@ -80,29 +78,22 @@ private async Task UpdateDatesFromGitHub()
var dates = await gitHubService.GetCommitDatesAsync(ListUrl);
if (dates is null)
return;
var list = await GetList();
UpdatePublishedDateFromGitHub(dates.First, list);
UpdateUpdatedDateFromGitHub(dates.Last, list);
UpdatePublishedDateFromGitHub(dates.First);
UpdateUpdatedDateFromGitHub(dates.Last);
await dbContext.SaveChangesAsync();
}
private async Task<Data.Entities.FilterList> GetList()
private void UpdatePublishedDateFromGitHub(DateTime? date)
{
filterList = filterList ?? await dbContext.FilterLists.FirstOrDefaultAsync(l => l.Id == List.Id);
return filterList;
if (date is DateTime firstDate && (List.PublishedDate is null || firstDate < List.PublishedDate))
List.PublishedDate = firstDate;
}
private static void UpdatePublishedDateFromGitHub(DateTime? date, Data.Entities.FilterList list)
private void UpdateUpdatedDateFromGitHub(DateTime? date)
{
if (date is DateTime firstDate && (list.PublishedDate is null || firstDate < list.PublishedDate))
list.PublishedDate = firstDate;
}
private void UpdateUpdatedDateFromGitHub(DateTime? date, Data.Entities.FilterList list)
{
if (date is DateTime lastDate && (list.UpdatedDate is null || lastDate > list.UpdatedDate))
if (date is DateTime lastDate && (List.UpdatedDate is null || lastDate > List.UpdatedDate))
{
list.UpdatedDate = lastDate;
List.UpdatedDate = lastDate;
isUpdatedDateFromGitHub = true;
}
}
@ -162,7 +153,7 @@ private async Task GetLinesFromSource(HttpResponseMessage response)
await SetWasUpdated();
if (SnapEntity.WasUpdated)
{
await UpdateUpdatedDate();
UpdateUpdatedDate();
memoryStream.Position = 0;
if (ListUrl.EndsWith(".7z"))
await GetLinesFrom7Zip(memoryStream);
@ -192,12 +183,11 @@ await dbContext.Snapshots
.Select(s => s.Md5Checksum)
.FirstOrDefaultAsync() ?? Array.Empty<byte>();
private async Task UpdateUpdatedDate()
private void UpdateUpdatedDate()
{
if (isUpdatedDateFromGitHub)
return;
var list = await GetList();
list.UpdatedDate = DateTime.Now;
List.UpdatedDate = DateTime.Now;
}
private async Task GetLinesFrom7Zip(Stream stream)

9
src/FilterLists.Services/Snapshot/SnapshotService.cs
View file

@ -4,10 +4,8 @@
using System.Linq.Expressions;
using System.Threading.Tasks;
using AutoMapper;
using AutoMapper.QueryableExtensions;
using FilterLists.Data;
using FilterLists.Services.GitHub;
using FilterLists.Services.Snapshot.Models;
using Microsoft.EntityFrameworkCore;
namespace FilterLists.Services.Snapshot
@ -51,15 +49,14 @@ FROM snapshots_rules
await DbContext.Database.ExecuteSqlCommandAsync(command);
}
private async Task<IEnumerable<FilterListViewUrlDto>> GetListsToCapture(int batchSize) =>
private async Task<IEnumerable<Data.Entities.FilterList>> GetListsToCapture(int batchSize) =>
await DbContext.FilterLists
.OrderBy(l => l.Snapshots.Any())
.ThenBy(lastSnapTimestamp)
.Take(batchSize)
.ProjectTo<FilterListViewUrlDto>(MapConfig)
.ToListAsync();
private async Task<List<TSnap>> CreateAndSaveSnaps<TSnap>(IEnumerable<FilterListViewUrlDto> lists)
private async Task<List<TSnap>> CreateAndSaveSnaps<TSnap>(IEnumerable<Data.Entities.FilterList> lists)
where TSnap : Snapshot, new()
{
var snaps = CreateSnaps<TSnap>(lists).ToList();
@ -67,7 +64,7 @@ private async Task<List<TSnap>> CreateAndSaveSnaps<TSnap>(IEnumerable<FilterList
return snaps;
}
private IEnumerable<TSnap> CreateSnaps<TSnap>(IEnumerable<FilterListViewUrlDto> lists)
private IEnumerable<TSnap> CreateSnaps<TSnap>(IEnumerable<Data.Entities.FilterList> lists)
where TSnap : Snapshot, new() => lists.Select(l =>
Activator.CreateInstance(typeof(TSnap), DbContext, l, gitHubService, logger, uaString) as TSnap);

6
src/FilterLists.Services/Snapshot/SnapshotWayback.cs
View file

@ -1,7 +1,6 @@
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Services.GitHub;
using FilterLists.Services.Snapshot.Models;
using FilterLists.Services.Wayback;
using JetBrains.Annotations;
@ -14,8 +13,9 @@ public SnapshotWayback()
}
[UsedImplicitly]
public SnapshotWayback(FilterListsDbContext dbContext, FilterListViewUrlDto list, GitHubService gitHubService,
Logger logger, string uaString) : base(dbContext, list, gitHubService, logger, uaString)
public SnapshotWayback(FilterListsDbContext dbContext, Data.Entities.FilterList list,
GitHubService gitHubService, Logger logger, string uaString) :
base(dbContext, list, gitHubService, logger, uaString)
{
}