diff --git a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs index 810232c5e..96283f234 100644 --- a/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs +++ b/src/FilterLists.Data/Seed/Extensions/SeedFilterListsDbContext.cs @@ -102,8 +102,13 @@ private static async Task InsertOnDuplicateKeyUpdate(DbContext dbContex private static List GetPropertiesLessValueGeneratedTimestamps(IEntityType entityType) => entityType.GetProperties() - .Where(x => - !new List {"CreatedDateUtc", "ModifiedDateUtc", "DiscontinuedDate"}.Contains(x.Name)) + .Where(x => !new List + { + "CreatedDateUtc", + "ModifiedDateUtc", + "DiscontinuedDate", + "PublishedDate" + }.Contains(x.Name)) .ToList(); private static string CreateValues(IEnumerable seed, diff --git a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs index 45a74f08c..f5db60192 100644 --- a/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs +++ b/src/FilterLists.Services/DependencyInjection/Extensions/ConfigureServicesCollection.cs @@ -1,6 +1,7 @@ using AutoMapper; using FilterLists.Data; using FilterLists.Services.FilterList; +using FilterLists.Services.GitHub; using FilterLists.Services.Language; using FilterLists.Services.License; using FilterLists.Services.Maintainer; @@ -44,6 +45,7 @@ public static void AddFilterListsAgentServices(this IServiceCollection services, .AddDbContextPool(o => o.UseMySql(config.GetConnectionString("FilterListsConnection"), m => m.MigrationsAssembly("FilterLists.Api"))); + services.TryAddScoped(); services.TryAddScoped(); services.TryAddScoped(); services.AddAutoMapper(); diff --git a/src/FilterLists.Services/FilterLists.Services.csproj b/src/FilterLists.Services/FilterLists.Services.csproj index a591a0f57..2854ad28c 100644 --- a/src/FilterLists.Services/FilterLists.Services.csproj +++ b/src/FilterLists.Services/FilterLists.Services.csproj @@ -23,6 +23,7 @@ + diff --git a/src/FilterLists.Services/GitHub/GitHubService.cs b/src/FilterLists.Services/GitHub/GitHubService.cs new file mode 100644 index 000000000..0e8afc530 --- /dev/null +++ b/src/FilterLists.Services/GitHub/GitHubService.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FilterLists.Services.GitHub.Models; +using JetBrains.Annotations; +using Octokit; + +namespace FilterLists.Services.GitHub +{ + [UsedImplicitly] + public class GitHubService + { + private const string RawFileUrlRoot = "raw.githubusercontent.com"; + private readonly GitHubClient client = new GitHubClient(new ProductHeaderValue("FilterLists")); + + public async Task GetCommitDatesAsync(string rawFileUrl) + { + if (!IsValidUrl(rawFileUrl)) + return null; + var commits = await GetAllCommits(rawFileUrl); + return new CommitDates + { + First = commits.LastOrDefault()?.Commit.Author.Date.DateTime, + Last = commits.FirstOrDefault()?.Commit.Author.Date.DateTime + }; + } + + private static bool IsValidUrl(string fileRawUrl) => fileRawUrl.Contains(RawFileUrlRoot); + + private async Task> GetAllCommits(string rawFileUrl) + { + var meta = ParseMeta(rawFileUrl); + return await client.Repository.Commit.GetAll( + meta.Owner, + meta.RepositoryName, + new CommitRequest {Path = meta.FilePath}); + } + + private static FileMeta ParseMeta(string rawFileUrl) + { + var urlPath = rawFileUrl.Split(new[] {RawFileUrlRoot}, StringSplitOptions.None)[1]; + var urlPathSlugs = urlPath.Split('/'); + var filePath = string.Empty; + for (var i = 4; i < urlPathSlugs.Length; i++) + filePath = filePath + '/' + urlPathSlugs[i]; + return new FileMeta + { + Owner = urlPathSlugs[1], + RepositoryName = urlPathSlugs[2], + FilePath = filePath.Substring(1) + }; + } + + private class FileMeta + { + public string Owner { get; set; } + public string RepositoryName { get; set; } + public string FilePath { get; set; } + } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/GitHub/Models/CommitDates.cs b/src/FilterLists.Services/GitHub/Models/CommitDates.cs new file mode 100644 index 000000000..421291c38 --- /dev/null +++ b/src/FilterLists.Services/GitHub/Models/CommitDates.cs @@ -0,0 +1,10 @@ +using System; + +namespace FilterLists.Services.GitHub.Models +{ + public class CommitDates + { + public DateTime? First { get; set; } + public DateTime? Last { get; set; } + } +} \ No newline at end of file diff --git a/src/FilterLists.Services/Snapshot/Snapshot.cs b/src/FilterLists.Services/Snapshot/Snapshot.cs index 7bc35c937..50508c58a 100644 --- a/src/FilterLists.Services/Snapshot/Snapshot.cs +++ b/src/FilterLists.Services/Snapshot/Snapshot.cs @@ -9,6 +9,7 @@ using FilterLists.Data; using FilterLists.Data.Entities.Junctions; using FilterLists.Services.Extensions; +using FilterLists.Services.GitHub; using FilterLists.Services.Snapshot.Models; using JetBrains.Annotations; using Microsoft.EntityFrameworkCore; @@ -23,6 +24,7 @@ public class Snapshot public readonly FilterListViewUrlDto List; protected readonly Data.Entities.Snapshot SnapEntity; private readonly FilterListsDbContext dbContext; + private readonly GitHubService gitHubService; private readonly Logger logger; private readonly string uaString; protected string ListUrl; @@ -33,11 +35,13 @@ public Snapshot() } [UsedImplicitly] - public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, Logger logger, string uaString) + public Snapshot(FilterListsDbContext dbContext, FilterListViewUrlDto list, GitHubService gitHubService, + Logger logger, string uaString) { this.dbContext = dbContext; List = list; ListUrl = list.ViewUrl; + this.gitHubService = gitHubService; this.logger = logger; SnapEntity = new Data.Entities.Snapshot {FilterListId = list.Id, SnapshotRules = new List()}; this.uaString = uaString; @@ -52,6 +56,7 @@ protected async Task TrySaveAsyncBase() await AddSnapEntity(); if (!ListUrl.IsValidHttpOrHttpsUrl()) return; + await UpdateDates(); try { await SaveAsync(); @@ -68,6 +73,30 @@ private async Task AddSnapEntity() await dbContext.SaveChangesAsync(); } + private async Task UpdateDates() + { + var dates = await gitHubService.GetCommitDatesAsync(ListUrl); + if (dates is null) + return; + var list = await dbContext.FilterLists.FirstOrDefaultAsync(l => l.Id == List.Id); + UpdatePublishedDate(dates.First, list); + UpdateDiscontinuedDate(dates.Last, list); + await dbContext.SaveChangesAsync(); + } + + private static void UpdatePublishedDate(DateTime? date, Data.Entities.FilterList list) + { + if (date is DateTime firstDate && (list.PublishedDate is null || firstDate < list.PublishedDate)) + list.PublishedDate = firstDate; + } + + private static void UpdateDiscontinuedDate(DateTime? date, Data.Entities.FilterList list) + { + if (date is DateTime lastDate && lastDate < DateTime.Now.AddYears(-2) && + (list.DiscontinuedDate is null || lastDate > list.DiscontinuedDate)) + list.DiscontinuedDate = lastDate; + } + private async Task SaveAsync() { await TryGetLines(); diff --git a/src/FilterLists.Services/Snapshot/SnapshotService.cs b/src/FilterLists.Services/Snapshot/SnapshotService.cs index cd2c9a913..577381243 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotService.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotService.cs @@ -6,6 +6,7 @@ using AutoMapper; using AutoMapper.QueryableExtensions; using FilterLists.Data; +using FilterLists.Services.GitHub; using FilterLists.Services.Snapshot.Models; using Microsoft.EntityFrameworkCore; @@ -19,11 +20,16 @@ public class SnapshotService : Service .OrderByDescending(d => d) .FirstOrDefault(); + private readonly GitHubService gitHubService; private readonly Logger logger; private string uaString; - public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig, Logger logger) - : base(dbContext, mapConfig) => this.logger = logger; + public SnapshotService(FilterListsDbContext dbContext, IConfigurationProvider mapConfig, + GitHubService gitHubService, Logger logger) : base(dbContext, mapConfig) + { + this.gitHubService = gitHubService; + this.logger = logger; + } public async Task CaptureAsync(int batchSize) { @@ -46,13 +52,12 @@ FROM snapshots_rules } private async Task> GetListsToCapture(int batchSize) => - await DbContext - .FilterLists - .OrderBy(l => l.Snapshots.Any()) - .ThenBy(lastSnapTimestamp) - .Take(batchSize) - .ProjectTo(MapConfig) - .ToListAsync(); + await DbContext.FilterLists + .OrderBy(l => l.Snapshots.Any()) + .ThenBy(lastSnapTimestamp) + .Take(batchSize) + .ProjectTo(MapConfig) + .ToListAsync(); private async Task> CreateAndSaveSnaps(IEnumerable lists) where TSnap : Snapshot, new() @@ -63,8 +68,8 @@ private async Task> CreateAndSaveSnaps(IEnumerable CreateSnaps(IEnumerable lists) - where TSnap : Snapshot, new() => - lists.Select(l => Activator.CreateInstance(typeof(TSnap), DbContext, l, logger, uaString) as TSnap); + where TSnap : Snapshot, new() => lists.Select(l => + Activator.CreateInstance(typeof(TSnap), DbContext, l, gitHubService, logger, uaString) as TSnap); private static async Task SaveSnaps(IEnumerable snaps) { diff --git a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs index 2daab965e..36a20a6e9 100644 --- a/src/FilterLists.Services/Snapshot/SnapshotWayback.cs +++ b/src/FilterLists.Services/Snapshot/SnapshotWayback.cs @@ -1,5 +1,6 @@ using System.Threading.Tasks; using FilterLists.Data; +using FilterLists.Services.GitHub; using FilterLists.Services.Snapshot.Models; using FilterLists.Services.Wayback; using JetBrains.Annotations; @@ -13,8 +14,8 @@ public SnapshotWayback() } [UsedImplicitly] - public SnapshotWayback(FilterListsDbContext dbContext, FilterListViewUrlDto list, Logger logger, string uaString) - : base(dbContext, list, logger, uaString) + public SnapshotWayback(FilterListsDbContext dbContext, FilterListViewUrlDto list, GitHubService gitHubService, + Logger logger, string uaString) : base(dbContext, list, gitHubService, logger, uaString) { }