From 8166dec254dd4df0f6ac46bc67727f0a93001c23 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Wed, 16 Sep 2020 08:28:48 -0500 Subject: [PATCH] =?UTF-8?q?feat(archival):=20=E2=9C=A8=20implement=20strat?= =?UTF-8?q?egy=20pattern=20for=20file=20writing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Commands/ArchiveList.cs | 68 ++++++++++++------- .../Models/FileToArchive.cs | 18 +++++ .../FileWriteStrategyFactory.cs | 22 ++++++ .../FileWriteStrategies/IFileWriteStrategy.cs | 10 +++ .../Persistence/FileWriteStrategies/Txt.cs | 20 ++++++ .../Persistence/GitFileArchiver.cs | 52 ++++++++------ .../{IArchiveFiles.cs => IFileArchiver.cs} | 6 +- .../Persistence/IFileToArchive.cs | 11 +++ .../Scheduling/ConfigurationExtensions.cs | 3 +- services/FilterLists.sln.DotSettings | 1 + 10 files changed, 163 insertions(+), 48 deletions(-) create mode 100644 services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs create mode 100644 services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileWriteStrategyFactory.cs create mode 100644 services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IFileWriteStrategy.cs create mode 100644 services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/Txt.cs rename services/Archival/FilterLists.Archival.Infrastructure/Persistence/{IArchiveFiles.cs => IFileArchiver.cs} (72%) create mode 100644 services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs diff --git a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs index 351cb6bc4..74e0846b1 100644 --- a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs +++ b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs @@ -1,12 +1,14 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using FilterLists.Archival.Application.Models; using FilterLists.Archival.Infrastructure.Persistence; using FilterLists.SharedKernel.Apis.Clients; +using FilterLists.SharedKernel.Apis.Contracts.Directory; using MediatR; using Microsoft.Extensions.Logging; @@ -28,12 +30,18 @@ public class Handler : IRequestHandler { private readonly IFileArchiver _archiver; private readonly IDirectoryApi _directory; + private readonly HttpClient _httpClient; private readonly ILogger _logger; - public Handler(IFileArchiver archiver, IDirectoryApi directory, ILogger logger) + public Handler( + IFileArchiver archiver, + IDirectoryApi directory, + IHttpClientFactory httpClientFactory, + ILogger logger) { _archiver = archiver; _directory = directory; + _httpClient = httpClientFactory.CreateClient(); _logger = logger; } @@ -42,37 +50,49 @@ public async Task Handle(Command request, CancellationToken cancellationTo _ = request ?? throw new ArgumentNullException(nameof(request)); _logger.LogDebug("Archiving list {ListId}", request.ListId); - var listDetails = await _directory.GetListDetailsAsync(request.ListId, cancellationToken); - var segments = listDetails.ViewUrls? - .GroupBy(u => u.SegmentNumber) - .Select(g => g.OrderBy(u => u.Primariness)) - .First() - .ToList(); - - if (segments != null) + var segments = await GetSegmentsAsync(request.ListId, cancellationToken); + if (segments.Count > 0) { - var archiveTasks = new List(); - foreach (var segment in segments) - { - Stream contents = null!; // TODO: fetch list file stream - var path = Path.Combine( - listDetails.Id.ToString(CultureInfo.InvariantCulture), - segment.SegmentNumber.ToString(CultureInfo.InvariantCulture)); - _logger.LogDebug("Archiving segment {SegmentNumber} of list {ListId}", segment.SegmentNumber, request.ListId); - archiveTasks.Add(_archiver.ArchiveFileAsync(contents, path, cancellationToken)); - } - - await Task.WhenAll(archiveTasks); + var fetchTasks = segments.Select(s => FetchStreamAsync(s.Url, cancellationToken)); + var streams = await Task.WhenAll(fetchTasks); + var target = new FileInfo(Uri.UnescapeDataString(segments.First().Url.Segments.Last())); + await _archiver.ArchiveFileAsync(new FileToArchive(target, streams), cancellationToken); _archiver.Commit(); - _logger.LogDebug("Archived segments {@SegmentNumbers} of list {ListId}", segments.Select(s => s.SegmentNumber), request.ListId); + + _logger.LogDebug( + "Archived segments {@SegmentNumbers} of list {ListId}", + segments.Select(s => s.SegmentNumber), + request.ListId); } else { - _logger.LogDebug("List {ListId} has no view URLs to archive", request.ListId); + _logger.LogInformation("List {ListId} has no view URLs to archive", request.ListId); } return Unit.Value; } + + private async Task> GetSegmentsAsync( + int listId, + CancellationToken cancellationToken) + { + var listDetails = await _directory.GetListDetailsAsync(listId, cancellationToken); + return listDetails.ViewUrls? + .GroupBy(u => u.SegmentNumber) + .Select(g => g.OrderBy(u => u.Primariness)) + .FirstOrDefault() + ?.ToList() ?? + new List(); + } + + private async Task FetchStreamAsync( + Uri url, + CancellationToken cancellationToken) + { + using var response = await _httpClient.GetAsync(url, cancellationToken); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadAsStreamAsync(); + } } } } diff --git a/services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs b/services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs new file mode 100644 index 000000000..de10459eb --- /dev/null +++ b/services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using System.IO; +using FilterLists.Archival.Infrastructure.Persistence; + +namespace FilterLists.Archival.Application.Models +{ + internal class FileToArchive : IFileToArchive + { + public FileToArchive(FileInfo target, IEnumerable contents) + { + Target = target; + Contents = contents; + } + + public FileInfo Target { get; } + public IEnumerable Contents { get; } + } +} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileWriteStrategyFactory.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileWriteStrategyFactory.cs new file mode 100644 index 000000000..9e2cceed7 --- /dev/null +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileWriteStrategyFactory.cs @@ -0,0 +1,22 @@ +using System; +using System.Linq; +using System.Reflection; + +namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies +{ + internal static class FileWriteStrategyFactory + { + public static TFileWriteStrategy? GetStrategy(this IFileToArchive file) + where TFileWriteStrategy : class, IFileWriteStrategy + { + var strategyType = Assembly.GetExecutingAssembly() + .GetTypes() + .FirstOrDefault(t => + typeof(TFileWriteStrategy).IsAssignableFrom(t) && + string.Equals(t.Name, file.Target.Extension.TrimStart('.'), StringComparison.OrdinalIgnoreCase)); + return strategyType is default(Type) + ? default + : (TFileWriteStrategy)Activator.CreateInstance(strategyType); + } + } +} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IFileWriteStrategy.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IFileWriteStrategy.cs new file mode 100644 index 000000000..5e6ec88e4 --- /dev/null +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IFileWriteStrategy.cs @@ -0,0 +1,10 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies +{ + internal interface IFileWriteStrategy + { + Task WriteAsync(IFileToArchive file, CancellationToken cancellationToken = default); + } +} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/Txt.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/Txt.cs new file mode 100644 index 000000000..840a0cbea --- /dev/null +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/Txt.cs @@ -0,0 +1,20 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies +{ + public class Txt : IFileWriteStrategy + { + public async Task WriteAsync(IFileToArchive file, CancellationToken cancellationToken = default) + { + _ = file ?? throw new ArgumentNullException(nameof(file)); + + await using var target = file.Target.OpenWrite(); + foreach (var source in file.Contents) + { + await source.CopyToAsync(target, cancellationToken); + } + } + } +} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs index eb4ed4a4e..64d2a9763 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs @@ -1,9 +1,11 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using FilterLists.Archival.Infrastructure.Options; +using FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies; using LibGit2Sharp; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -12,56 +14,68 @@ namespace FilterLists.Archival.Infrastructure.Persistence { internal sealed class GitFileArchiver : IFileArchiver { - private readonly IList _filePaths = new List(); private readonly ILogger _logger; private readonly GitOptions _options; - private readonly IRepository _repository; + private readonly IRepository _repo; + private readonly ICollection _writtenFiles = new HashSet(); public GitFileArchiver( ILogger logger, IOptions gitOptions, - IRepository gitRepository) + IRepository repository) { _logger = logger; _options = gitOptions.Value; - _repository = gitRepository; + _repo = repository; } public async Task ArchiveFileAsync( - Stream fileContents, - string filePath, + IFileToArchive file, CancellationToken cancellationToken = default) { - _logger.LogDebug("Writing {FilePath}", filePath); + var strategy = file.GetStrategy(); + if (strategy is default(IFileWriteStrategy)) + { + _logger.LogWarning( + "No write strategy found for {Filename}. Skipping", + file.Target.Name); + } + else + { + _logger.LogDebug( + "Writing {Filename} with strategy {FileWriteStrategy}", + file.Target.Name, + strategy.GetType().Name); - _filePaths.Add(filePath); - await using var output = File.OpenWrite(filePath); - await fileContents.CopyToAsync(output, cancellationToken); + _writtenFiles.Add(file.Target); + await strategy.WriteAsync(file, cancellationToken); - _logger.LogDebug("Finished writing {FilePath}", filePath); + _logger.LogDebug("Finished writing {Filename}", file.Target.Name); + } } public void Commit() { - Commands.Stage(_repository, _filePaths); + var filenames = _writtenFiles.Select(f => f.Name).ToList(); + Commands.Stage(_repo, filenames); var signature = new Signature(_options.UserName, _options.UserEmail, DateTime.UtcNow); - var message = $"feat(archives): archive {_filePaths.Count} files{Environment.NewLine}{string.Join(Environment.NewLine, _filePaths)}"; - _repository.Commit(message, signature, signature); + var message = $"feat(archives): archive {filenames.Count} files{Environment.NewLine}{string.Join(Environment.NewLine, filenames)}"; + _repo.Commit(message, signature, signature); - _logger.LogDebug("Committed {@FilePaths}", _filePaths); + _logger.LogDebug("Committed {@Filenames}", filenames); } public void Dispose() { - foreach (var file in _filePaths) + foreach (var file in _writtenFiles) { - if (File.Exists(file)) + if (File.Exists(file.Name)) { - File.Delete(file); + File.Delete(file.Name); } } - _repository.CheckoutPaths("HEAD", _filePaths); + _repo.CheckoutPaths("HEAD", _writtenFiles.Select(f => f.Name)); } } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IArchiveFiles.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileArchiver.cs similarity index 72% rename from services/Archival/FilterLists.Archival.Infrastructure/Persistence/IArchiveFiles.cs rename to services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileArchiver.cs index 3866f3de1..42a8fa844 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IArchiveFiles.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileArchiver.cs @@ -1,5 +1,4 @@ -using System.IO; -using System.Threading; +using System.Threading; using System.Threading.Tasks; using FilterLists.SharedKernel.SeedWork; @@ -8,8 +7,7 @@ namespace FilterLists.Archival.Infrastructure.Persistence public interface IFileArchiver : IUnitOfWork { Task ArchiveFileAsync( - Stream fileContents, - string filePath, + IFileToArchive file, CancellationToken cancellationToken = default); } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs new file mode 100644 index 000000000..f01fed380 --- /dev/null +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using System.IO; + +namespace FilterLists.Archival.Infrastructure.Persistence +{ + public interface IFileToArchive + { + FileInfo Target { get; } + IEnumerable Contents { get; } + } +} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/ConfigurationExtensions.cs b/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/ConfigurationExtensions.cs index 4b20239c6..4c1f554d9 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/ConfigurationExtensions.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/ConfigurationExtensions.cs @@ -24,7 +24,8 @@ public static void AddSchedulingServices(this IServiceCollection services, IConf public static void UseScheduling(this IApplicationBuilder app) { - app.UseHangfireServer(new BackgroundJobServerOptions {WorkerCount = Environment.ProcessorCount * 1}); + // TODO: re-evaluate git in parallel + app.UseHangfireServer(new BackgroundJobServerOptions {WorkerCount = 1}); } } } diff --git a/services/FilterLists.sln.DotSettings b/services/FilterLists.sln.DotSettings index df1b253f7..cdfa3f3d6 100644 --- a/services/FilterLists.sln.DotSettings +++ b/services/FilterLists.sln.DotSettings @@ -10,6 +10,7 @@ CHOP_IF_LONG CHOP_IF_LONG CHOP_IF_LONG + True True True True