From 2f187cd9da9bdfdcabb2c1b7ccb1404674dcfc9e Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sat, 19 Sep 2020 17:13:52 -0500 Subject: [PATCH] =?UTF-8?q?feat(archival):=20=E2=9C=A8=E2=99=BB=20store=20?= =?UTF-8?q?txt=20files=20in=20configured=20dir,=20ren=20many=20members?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Commands/ArchiveList.cs | 44 +++++++++---------- .../Models/{FileToArchive.cs => File.cs} | 14 +++--- .../Clients/ConfigurationExtensions.cs | 2 +- .../Clients/IFileClient.cs | 12 ----- .../{FileClient.cs => IHttpContentClient.cs} | 13 +++--- .../Persistence/ConfigurationExtensions.cs | 2 +- .../IFileStreamConversionStrategy.cs | 10 ----- .../IStreamToTxtConversionStrategy.cs | 10 +++++ ...> StreamToTxtConversionStrategyFactory.cs} | 11 +++-- .../Persistence/FileWriteStrategies/Txt.cs | 8 ++-- .../{IFileToArchive.cs => IFile.cs} | 8 ++-- .../Persistence/IFileArchiver.cs | 13 ------ ...tFileArchiver.cs => ITxtFileRepository.cs} | 37 +++++++++------- 13 files changed, 82 insertions(+), 102 deletions(-) rename services/Archival/FilterLists.Archival.Application/Models/{FileToArchive.cs => File.cs} (50%) delete mode 100644 services/Archival/FilterLists.Archival.Infrastructure/Clients/IFileClient.cs rename services/Archival/FilterLists.Archival.Infrastructure/Clients/{FileClient.cs => IHttpContentClient.cs} (71%) delete mode 100644 services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IFileStreamConversionStrategy.cs create mode 100644 services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToTxtConversionStrategy.cs rename services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/{FileStreamConversionStrategyFactory.cs => StreamToTxtConversionStrategyFactory.cs} (51%) rename services/Archival/FilterLists.Archival.Infrastructure/Persistence/{IFileToArchive.cs => IFile.cs} (54%) delete mode 100644 services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileArchiver.cs rename services/Archival/FilterLists.Archival.Infrastructure/Persistence/{GitFileArchiver.cs => ITxtFileRepository.cs} (67%) diff --git a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs index 4e8bc1219..597f72a25 100644 --- a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs +++ b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs @@ -13,6 +13,7 @@ using FilterLists.SharedKernel.Apis.Contracts.Directory; using MediatR; using Microsoft.Extensions.Logging; +using File = FilterLists.Archival.Application.Models.File; namespace FilterLists.Archival.Application.Commands { @@ -30,21 +31,21 @@ public Command(int listId) public class Handler : IRequestHandler { - private readonly IFileArchiver _archiver; - private readonly IFileClient _client; + private readonly IHttpContentClient _client; private readonly IDirectoryApi _directory; private readonly ILogger _logger; + private readonly ITxtFileRepository _repo; public Handler( - IFileArchiver archiver, - IFileClient fileClient, - IDirectoryApi directory, - ILogger logger) + IHttpContentClient httpContentClient, + IDirectoryApi directoryApi, + ILogger logger, + ITxtFileRepository txtFileRepository) { - _archiver = archiver; - _client = fileClient; - _directory = directory; + _client = httpContentClient; + _directory = directoryApi; _logger = logger; + _repo = txtFileRepository; } public async Task Handle(Command request, CancellationToken cancellationToken) @@ -52,15 +53,15 @@ public async Task Handle(Command request, CancellationToken cancellationTo _ = request ?? throw new ArgumentNullException(nameof(request)); _logger.LogDebug("Archiving list {ListId}", request.ListId); - var segmentUrls = await GetSegmentUrlsAsync(request.ListId, cancellationToken); + var segmentUrls = (await GetSegmentUrlsAsync(request.ListId, cancellationToken)).ToList(); if (segmentUrls.Count > 0) { var file = GetFileToArchive(request.ListId, segmentUrls, cancellationToken); - await _archiver.ArchiveFileAsync(file, cancellationToken); - _archiver.Commit(); + await _repo.AddFileAsync(file, cancellationToken); + _repo.Commit(); _logger.LogDebug( - "Archived segments {@SegmentNumbers} of list {ListId}", + "Archived segment(s) {@SegmentNumbers} of list {ListId}", segmentUrls.Select(s => s.SegmentNumber), request.ListId); } @@ -72,28 +73,27 @@ public async Task Handle(Command request, CancellationToken cancellationTo return Unit.Value; } - private async Task> GetSegmentUrlsAsync( + private async Task> GetSegmentUrlsAsync( int listId, CancellationToken cancellationToken) { var listDetails = await _directory.GetListDetailsAsync(listId, cancellationToken); return listDetails.ViewUrls? - .GroupBy(u => u.SegmentNumber, (_, g) => g.OrderBy(e => e.Primariness).First()) - ?.ToList() ?? + .GroupBy(u => u.SegmentNumber, (_, ue) => ue.OrderBy(u => u.Primariness).First()) ?? new List(); } - private IFileToArchive GetFileToArchive( + private IFile GetFileToArchive( int listId, IEnumerable segmentUrls, CancellationToken cancellationToken) { var segmentsAsync = GetSegmentsAsync(segmentUrls, cancellationToken); - var target = new FileInfo($"{listId.ToString(CultureInfo.InvariantCulture).PadLeft(5, '0')}.txt"); - return new FileToArchive(segmentsAsync, target); + var target = $"{listId.ToString(CultureInfo.InvariantCulture).PadLeft(5, '0')}.txt"; + return new File(segmentsAsync, target); } - private async IAsyncEnumerable GetSegmentsAsync( + private async IAsyncEnumerable GetSegmentsAsync( IEnumerable segmentUrls, [EnumeratorCancellation] CancellationToken cancellationToken) { @@ -101,9 +101,9 @@ private async IAsyncEnumerable GetSegmentsAsync( { var sourceFileName = Uri.UnescapeDataString(segment.Url.Segments.Last()); var sourceExtension = Path.GetExtension(sourceFileName); - yield return new FileToArchiveSegment( + yield return new FileSegment( sourceExtension, - await _client.DownloadFileAsync(segment.Url, cancellationToken)); + await _client.GetContentAsync(segment.Url, cancellationToken)); } } } diff --git a/services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs b/services/Archival/FilterLists.Archival.Application/Models/File.cs similarity index 50% rename from services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs rename to services/Archival/FilterLists.Archival.Application/Models/File.cs index dc217490b..aec39b45d 100644 --- a/services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs +++ b/services/Archival/FilterLists.Archival.Application/Models/File.cs @@ -4,21 +4,21 @@ namespace FilterLists.Archival.Application.Models { - internal class FileToArchive : IFileToArchive + internal class File : IFile { - public FileToArchive(IAsyncEnumerable segments, FileInfo target) + public File(IAsyncEnumerable segments, string targetFileName) { Segments = segments; - Target = target; + TargetFileName = targetFileName; } - public IAsyncEnumerable Segments { get; } - public FileInfo Target { get; } + public IAsyncEnumerable Segments { get; } + public string TargetFileName { get; } } - internal class FileToArchiveSegment : IFileToArchiveSegment + internal class FileSegment : IFileSegment { - public FileToArchiveSegment(string sourceExtension, Stream contents) + public FileSegment(string sourceExtension, Stream contents) { SourceExtension = sourceExtension; Contents = contents; diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Clients/ConfigurationExtensions.cs b/services/Archival/FilterLists.Archival.Infrastructure/Clients/ConfigurationExtensions.cs index f1119b2a2..9c30d4ed8 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Clients/ConfigurationExtensions.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Clients/ConfigurationExtensions.cs @@ -8,7 +8,7 @@ internal static class ConfigurationExtensions { public static void AddClients(this IServiceCollection services) { - services.AddHttpClient() + services.AddHttpClient() .AddTransientHttpErrorPolicy(b => b.WaitAndRetryAsync(new[] { TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10) diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Clients/IFileClient.cs b/services/Archival/FilterLists.Archival.Infrastructure/Clients/IFileClient.cs deleted file mode 100644 index b4d6a1ceb..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Clients/IFileClient.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.IO; -using System.Threading; -using System.Threading.Tasks; - -namespace FilterLists.Archival.Infrastructure.Clients -{ - public interface IFileClient : IDisposable - { - Task DownloadFileAsync(Uri url, CancellationToken cancellationToken); - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Clients/FileClient.cs b/services/Archival/FilterLists.Archival.Infrastructure/Clients/IHttpContentClient.cs similarity index 71% rename from services/Archival/FilterLists.Archival.Infrastructure/Clients/FileClient.cs rename to services/Archival/FilterLists.Archival.Infrastructure/Clients/IHttpContentClient.cs index 003c5c04f..3c0c32f60 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Clients/FileClient.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Clients/IHttpContentClient.cs @@ -7,17 +7,22 @@ namespace FilterLists.Archival.Infrastructure.Clients { - internal sealed class FileClient : IFileClient + public interface IHttpContentClient : IDisposable + { + Task GetContentAsync(Uri url, CancellationToken cancellationToken); + } + + internal sealed class HttpContentClient : IHttpContentClient { private readonly HttpClient _httpClient; private readonly ICollection _httpResponseMessages = new List(); - public FileClient(HttpClient httpClient) + public HttpContentClient(HttpClient httpClient) { _httpClient = httpClient; } - public async Task DownloadFileAsync(Uri url, CancellationToken cancellationToken) + public async Task GetContentAsync(Uri url, CancellationToken cancellationToken) { var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); _httpResponseMessages.Add(response); @@ -31,8 +36,6 @@ public void Dispose() { message.Dispose(); } - - _httpClient.Dispose(); } } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs index 055998730..d0d39ae8c 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs @@ -20,7 +20,7 @@ public static void AddPersistenceServices(this IServiceCollection services, ICon return new Repository(gitOptions.RepositoryPath); }); - services.AddTransient(); + services.AddTransient(); } } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IFileStreamConversionStrategy.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IFileStreamConversionStrategy.cs deleted file mode 100644 index 5092ce368..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IFileStreamConversionStrategy.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System.IO; -using System.Threading; - -namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies -{ - internal interface IFileStreamConversionStrategy - { - Stream Convert(IFileToArchiveSegment segment, CancellationToken cancellationToken); - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToTxtConversionStrategy.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToTxtConversionStrategy.cs new file mode 100644 index 000000000..78b0be1a2 --- /dev/null +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToTxtConversionStrategy.cs @@ -0,0 +1,10 @@ +using System.IO; +using System.Threading; + +namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies +{ + internal interface IStreamToTxtConversionStrategy + { + Stream Convert(IFileSegment fileSegment, CancellationToken cancellationToken); + } +} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileStreamConversionStrategyFactory.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/StreamToTxtConversionStrategyFactory.cs similarity index 51% rename from services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileStreamConversionStrategyFactory.cs rename to services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/StreamToTxtConversionStrategyFactory.cs index 303a8a1f7..8c8f5e36e 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileStreamConversionStrategyFactory.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/StreamToTxtConversionStrategyFactory.cs @@ -4,20 +4,19 @@ namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies { - internal static class FileStreamConversionStrategyFactory + internal static class StreamToTxtConversionStrategyFactory { - public static TFileStreamConversionStrategy? GetStrategy( - this IFileToArchiveSegment segment) - where TFileStreamConversionStrategy : class, IFileStreamConversionStrategy + public static TStreamToTxtConversionStrategy? GetStrategy( + this IFileSegment segment) where TStreamToTxtConversionStrategy : class, IStreamToTxtConversionStrategy { var strategyType = Assembly.GetExecutingAssembly() .GetTypes() .FirstOrDefault(t => - typeof(TFileStreamConversionStrategy).IsAssignableFrom(t) && + typeof(TStreamToTxtConversionStrategy).IsAssignableFrom(t) && string.Equals(t.Name, segment.SourceExtension.TrimStart('.'), StringComparison.OrdinalIgnoreCase)); return strategyType is default(Type) ? default - : (TFileStreamConversionStrategy)Activator.CreateInstance(strategyType); + : (TStreamToTxtConversionStrategy)Activator.CreateInstance(strategyType); } } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/Txt.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/Txt.cs index 336bdd7f0..63ffeddf0 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/Txt.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/Txt.cs @@ -4,13 +4,13 @@ namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies { - public class Txt : IFileStreamConversionStrategy + public class Txt : IStreamToTxtConversionStrategy { - public Stream Convert(IFileToArchiveSegment segment, CancellationToken cancellationToken) + public Stream Convert(IFileSegment fileSegment, CancellationToken cancellationToken) { - _ = segment ?? throw new ArgumentNullException(nameof(segment)); + _ = fileSegment ?? throw new ArgumentNullException(nameof(fileSegment)); - return segment.Contents; + return fileSegment.Contents; } } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFile.cs similarity index 54% rename from services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs rename to services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFile.cs index 41ee26ea2..1989376c3 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFile.cs @@ -3,13 +3,13 @@ namespace FilterLists.Archival.Infrastructure.Persistence { - public interface IFileToArchive + public interface IFile { - IAsyncEnumerable Segments { get; } - FileInfo Target { get; } + IAsyncEnumerable Segments { get; } + string TargetFileName { get; } } - public interface IFileToArchiveSegment + public interface IFileSegment { string SourceExtension { get; } Stream Contents { get; } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileArchiver.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileArchiver.cs deleted file mode 100644 index 1950c109b..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileArchiver.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System.Threading; -using System.Threading.Tasks; -using FilterLists.SharedKernel.SeedWork; - -namespace FilterLists.Archival.Infrastructure.Persistence -{ - public interface IFileArchiver : IUnitOfWork - { - Task ArchiveFileAsync( - IFileToArchive file, - CancellationToken cancellationToken); - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ITxtFileRepository.cs similarity index 67% rename from services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs rename to services/Archival/FilterLists.Archival.Infrastructure/Persistence/ITxtFileRepository.cs index af88d488b..9b0d76ac1 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ITxtFileRepository.cs @@ -6,21 +6,27 @@ using System.Threading.Tasks; using FilterLists.Archival.Infrastructure.Options; using FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies; +using FilterLists.SharedKernel.SeedWork; using LibGit2Sharp; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace FilterLists.Archival.Infrastructure.Persistence { - internal sealed class GitFileArchiver : IFileArchiver + public interface ITxtFileRepository : IUnitOfWork + { + Task AddFileAsync(IFile file, CancellationToken cancellationToken); + } + + internal sealed class GitTxtFileRepository : ITxtFileRepository { private readonly ILogger _logger; private readonly GitOptions _options; private readonly IRepository _repo; private readonly ICollection _writtenFiles = new HashSet(); - public GitFileArchiver( - ILogger logger, + public GitTxtFileRepository( + ILogger logger, IOptions gitOptions, IRepository repository) { @@ -29,32 +35,29 @@ public GitFileArchiver( _repo = repository; } - public async Task ArchiveFileAsync( - IFileToArchive file, - CancellationToken cancellationToken) + public async Task AddFileAsync(IFile file, CancellationToken cancellationToken) { var textStreams = new List(); await foreach (var segment in file.Segments.WithCancellation(cancellationToken)) { - var strategy = segment.GetStrategy(); - if (strategy is default(IFileStreamConversionStrategy)) + var strategy = segment.GetStrategy(); + if (strategy is default(IStreamToTxtConversionStrategy)) { _logger.LogWarning( - "No file stream conversion strategy found for extension {Extension} for target {Target}. Skipping file", + "No stream to txt conversion strategy found for extension {Extension} for target {Target}. Skipping file", segment.SourceExtension, - file.Target.Name); + file.TargetFileName); return; } textStreams.Add(strategy.Convert(segment, cancellationToken)); } - _logger.LogDebug("Writing {FileName}", file.Target.Name); + _logger.LogDebug("Writing {FileName}", file.TargetFileName); - _writtenFiles.Add(file.Target); - - // TODO: write to _options.RepositoryPath - await using var target = file.Target.OpenWrite(); + var fileInfo = new FileInfo(Path.Combine(_options.RepositoryPath, file.TargetFileName)); + _writtenFiles.Add(fileInfo); + await using var target = fileInfo.OpenWrite(); // TODO: validate multi-segment lists are concatenated correctly and in order foreach (var textStream in textStreams) @@ -62,14 +65,14 @@ public async Task ArchiveFileAsync( await textStream.CopyToAsync(target, cancellationToken); } - _logger.LogDebug("Finished writing {FileName}", file.Target.Name); + _logger.LogDebug("Finished writing {FileName}", file.TargetFileName); } public void Commit() { var fileNames = _writtenFiles.Select(f => f.Name).ToList(); var signature = new Signature(_options.UserName, _options.UserEmail, DateTime.UtcNow); - var message = $"feat(archives): archive {fileNames.Count} files{Environment.NewLine}{string.Join(Environment.NewLine, fileNames)}"; + var message = $"feat(archives): archive {fileNames.Count} file(s){Environment.NewLine}{string.Join(Environment.NewLine, fileNames)}"; Commands.Stage(_repo, fileNames); _repo.Commit(message, signature, signature);