prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update discontinuedDate and publishedDate from GitHub

Browse source 
ref #485
This commit is contained in:
Collin M. Barrett 2018-10-05 09:20:52 -05:00
parent 1960d09aa4
commit 5998137bfc
8 changed files with 131 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs
View file

@ -102,8 +102,13 @@ private static async Task InsertOnDuplicateKeyUpdate<TEntity>(DbContext dbContex
private static List<IProperty> GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType) =>
entityType.GetProperties()
.Where(x =>
!new List<string> {"CreatedDateUtc", "ModifiedDateUtc", "DiscontinuedDate"}.Contains(x.Name))
.Where(x => !new List<string>
{
"CreatedDateUtc",
"ModifiedDateUtc",
"DiscontinuedDate",
"PublishedDate"
}.Contains(x.Name))
.ToList();
private static string CreateValues<TEntity>(IEnumerable<TEntity> seed,

2
src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs
View file

@ -1,6 +1,7 @@
using AutoMapper;
using FilterLists.Data;
using FilterLists.Services.FilterList;
using FilterLists.Services.GitHub;
using FilterLists.Services.Language;
using FilterLists.Services.License;
using FilterLists.Services.Maintainer;
@ -44,6 +45,7 @@ public static void AddFilterListsAgentServices(this IServiceCollection services,
.AddDbContextPool<FilterListsDbContext>(o =>
o.UseMySql(config.GetConnectionString("FilterListsConnection"),
m => m.MigrationsAssembly("FilterLists.Api")));
services.TryAddScoped<GitHubService>();
services.TryAddScoped<SnapshotService>();
services.TryAddScoped<Logger>();
services.AddAutoMapper();

1
src/FilterLists.Services/FilterLists.Services.csproj
View file

@ -23,6 +23,7 @@
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.4.1" />
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha" />
<PackageReference Include="morelinq" Version="3.0.0" />
<PackageReference Include="Octokit" Version="0.32.0" />
<PackageReference Include="SendGrid" Version="9.10.0" />
<PackageReference Include="SharpCompress" Version="0.22.0" />
</ItemGroup>

62
src/FilterLists.Services/GitHub/GitHubService.cs Normal file
View file

@ -0,0 +1,62 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FilterLists.Services.GitHub.Models;
using JetBrains.Annotations;
using Octokit;
namespace FilterLists.Services.GitHub
{
[UsedImplicitly]
public class GitHubService
{
private const string RawFileUrlRoot = "raw.githubusercontent.com";
private readonly GitHubClient client = new GitHubClient(new ProductHeaderValue("FilterLists"));
public async Task<CommitDates> GetCommitDatesAsync(string rawFileUrl)
{
if (!IsValidUrl(rawFileUrl))
return null;
var commits = await GetAllCommits(rawFileUrl);
return new CommitDates
{
First = commits.LastOrDefault()?.Commit.Author.Date.DateTime,
Last = commits.FirstOrDefault()?.Commit.Author.Date.DateTime
};
}
private static bool IsValidUrl(string fileRawUrl) => fileRawUrl.Contains(RawFileUrlRoot);
private async Task<IReadOnlyList<GitHubCommit>> GetAllCommits(string rawFileUrl)
{
var meta = ParseMeta(rawFileUrl);
return await client.Repository.Commit.GetAll(
meta.Owner,
meta.RepositoryName,
new CommitRequest {Path = meta.FilePath});
}
private static FileMeta ParseMeta(string rawFileUrl)
{
var urlPath = rawFileUrl.Split(new[] {RawFileUrlRoot}, StringSplitOptions.None)[1];
var urlPathSlugs = urlPath.Split('/');
var filePath = string.Empty;
for (var i = 4; i < urlPathSlugs.Length; i++)
filePath = filePath + '/' + urlPathSlugs[i];
return new FileMeta
{
Owner = urlPathSlugs[1],
RepositoryName = urlPathSlugs[2],
FilePath = filePath.Substring(1)
};
}
private class FileMeta
{
public string Owner { get; set; }
public string RepositoryName { get; set; }
public string FilePath { get; set; }
}
}
}

10
src/FilterLists.Services/GitHub/Models/CommitDates.cs Normal file
View file

@ -0,0 +1,10 @@
using System;
namespace FilterLists.Services.GitHub.Models
{
public class CommitDates
{
public DateTime? First { get; set; }
public DateTime? Last { get; set; }
}
}

31
src/FilterLists.Services/Snapshot/Snapshot.cs
View file

@ -9,6 +9,7 @@
using FilterLists.Data;
using FilterLists.Data.Entities.Junctions;
using FilterLists.Services.Extensions;
using FilterLists.Services.GitHub;
using FilterLists.Services.Snapshot.Models;
using JetBrains.Annotations;
using Microsoft.EntityFrameworkCore;
@ -23,6 +24,7 @@ public class Snapshot
public readonly FilterListViewUrlDto List;
protected readonly Data.Entities.Snapshot SnapEntity;
private readonly FilterListsDbContext dbContext;
private readonly GitHubService gitHubService;
private readonly Logger logger;
private readonly string uaString;
protected string ListUrl;
@ -33,11 +35,13 @@ public Snapshot()
}
[UsedImplicitly]
public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, Logger logger, string uaString)
public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, GitHubService gitHubService,
Logger logger, string uaString)
{
this.dbContext = dbContext;
List = list;
ListUrl = list.ViewUrl;
this.gitHubService = gitHubService;
this.logger = logger;
SnapEntity = new Data.Entities.Snapshot {FilterListId = list.Id, SnapshotRules = new List<SnapshotRule>()};
this.uaString = uaString;
@ -52,6 +56,7 @@ protected async Task TrySaveAsyncBase()
await AddSnapEntity();
if (!ListUrl.IsValidHttpOrHttpsUrl())
return;
await UpdateDates();
try
{
await SaveAsync();
@ -68,6 +73,30 @@ private async Task AddSnapEntity()
await dbContext.SaveChangesAsync();
}
private async Task UpdateDates()
{
var dates = await gitHubService.GetCommitDatesAsync(ListUrl);
if (dates is null)
return;
var list = await dbContext.FilterLists.FirstOrDefaultAsync(l => l.Id == List.Id);
UpdatePublishedDate(dates.First, list);
UpdateDiscontinuedDate(dates.Last, list);
await dbContext.SaveChangesAsync();
}
private static void UpdatePublishedDate(DateTime? date, Data.Entities.FilterList list)
{
if (date is DateTime firstDate && (list.PublishedDate is null || firstDate < list.PublishedDate))
list.PublishedDate = firstDate;
}
private static void UpdateDiscontinuedDate(DateTime? date, Data.Entities.FilterList list)
{
if (date is DateTime lastDate && lastDate < DateTime.Now.AddYears(-2) &&
(list.DiscontinuedDate is null || lastDate > list.DiscontinuedDate))
list.DiscontinuedDate = lastDate;
}
private async Task SaveAsync()
{
await TryGetLines();

27
src/FilterLists.Services/Snapshot/SnapshotService.cs
View file

@ -6,6 +6,7 @@
using AutoMapper;
using AutoMapper.QueryableExtensions;
using FilterLists.Data;
using FilterLists.Services.GitHub;
using FilterLists.Services.Snapshot.Models;
using Microsoft.EntityFrameworkCore;
@ -19,11 +20,16 @@ public class SnapshotService : Service
.OrderByDescending(d => d)
.FirstOrDefault();
private readonly GitHubService gitHubService;
private readonly Logger logger;
private string uaString;
public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig, Logger logger)
: base(dbContext, mapConfig) => this.logger = logger;
public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig,
GitHubService gitHubService, Logger logger) : base(dbContext, mapConfig)
{
this.gitHubService = gitHubService;
this.logger = logger;
}
public async Task CaptureAsync(int batchSize)
{
@ -46,13 +52,12 @@ FROM snapshots_rules
}
private async Task<IEnumerable<FilterListViewUrlDto>> GetListsToCapture(int batchSize) =>
await DbContext
.FilterLists
.OrderBy(l => l.Snapshots.Any())
.ThenBy(lastSnapTimestamp)
.Take(batchSize)
.ProjectTo<FilterListViewUrlDto>(MapConfig)
.ToListAsync();
await DbContext.FilterLists
.OrderBy(l => l.Snapshots.Any())
.ThenBy(lastSnapTimestamp)
.Take(batchSize)
.ProjectTo<FilterListViewUrlDto>(MapConfig)
.ToListAsync();
private async Task<List<TSnap>> CreateAndSaveSnaps<TSnap>(IEnumerable<FilterListViewUrlDto> lists)
where TSnap : Snapshot, new()
@ -63,8 +68,8 @@ private async Task<List<TSnap>> CreateAndSaveSnaps<TSnap>(IEnumerable<FilterList
}
private IEnumerable<TSnap> CreateSnaps<TSnap>(IEnumerable<FilterListViewUrlDto> lists)
where TSnap : Snapshot, new() =>
lists.Select(l => Activator.CreateInstance(typeof(TSnap), DbContext, l, logger, uaString) as TSnap);
where TSnap : Snapshot, new() => lists.Select(l =>
Activator.CreateInstance(typeof(TSnap), DbContext, l, gitHubService, logger, uaString) as TSnap);
private static async Task SaveSnaps(IEnumerable<Snapshot> snaps)
{

5
src/FilterLists.Services/Snapshot/SnapshotWayback.cs
View file

@ -1,5 +1,6 @@
using System.Threading.Tasks;
using FilterLists.Data;
using FilterLists.Services.GitHub;
using FilterLists.Services.Snapshot.Models;
using FilterLists.Services.Wayback;
using JetBrains.Annotations;
@ -13,8 +14,8 @@ public SnapshotWayback()
}
[UsedImplicitly]
public SnapshotWayback(FilterListsDbContext dbContext, FilterListViewUrlDto list, Logger logger, string uaString)
: base(dbContext, list, logger, uaString)
public SnapshotWayback(FilterListsDbContext dbContext, FilterListViewUrlDto list, GitHubService gitHubService,
Logger logger, string uaString) : base(dbContext, list, gitHubService, logger, uaString)
{
}