From c4d97dd8b41017603a9655621b2cbca8a498d9a9 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Thu, 17 Sep 2020 20:54:37 -0500 Subject: [PATCH] =?UTF-8?q?feat(archival):=20=E2=9C=A8=20normalize=20targe?= =?UTF-8?q?t=20filenames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Commands/ArchiveList.cs | 39 +++++++++++-------- .../Models/FileToArchive.cs | 8 ++-- .../FileWriteStrategyFactory.cs | 2 +- .../Persistence/IFileToArchive.cs | 4 +- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs index 6377d62f7..ddae8d05f 100644 --- a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs +++ b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs @@ -50,15 +50,16 @@ public async Task Handle(Command request, CancellationToken cancellationTo _ = request ?? throw new ArgumentNullException(nameof(request)); _logger.LogDebug("Archiving list {ListId}", request.ListId); - var segments = await GetSegmentsAsync(request.ListId, cancellationToken); - if (segments.Count > 0) + var segmentUrls = await GetSegmentUrlsAsync(request.ListId, cancellationToken); + if (segmentUrls.Count > 0) { - await DownloadSegments(segments, cancellationToken); + var file = await GetFileAsync(request.ListId, segmentUrls, cancellationToken); + await _archiver.ArchiveFileAsync(file, cancellationToken); _archiver.Commit(); _logger.LogDebug( "Archived segments {@SegmentNumbers} of list {ListId}", - segments.Select(s => s.SegmentNumber), + segmentUrls.Select(s => s.SegmentNumber), request.ListId); } else @@ -69,7 +70,7 @@ public async Task Handle(Command request, CancellationToken cancellationTo return Unit.Value; } - private async Task> GetSegmentsAsync( + private async Task> GetSegmentUrlsAsync( int listId, CancellationToken cancellationToken) { @@ -82,24 +83,30 @@ private async Task> GetSegmentsAsync( new List(); } - private async Task DownloadSegments( - IReadOnlyCollection segments, + private async Task GetFileAsync( + int listId, + IReadOnlyCollection segmentUrls, + CancellationToken cancellationToken) + { + var contentsAsync = GetContentsAsync(segmentUrls, cancellationToken); + var sourceFileName = Uri.UnescapeDataString(segmentUrls.First().Url.Segments.Last()); + var sourceExtension = Path.GetExtension(sourceFileName); + var target = new FileInfo($"{listId}.txt"); + return new FileToArchive(sourceExtension, await contentsAsync, target); + } + + // TODO: IAsyncEnumerable? + private async Task> GetContentsAsync( + IEnumerable segmentUrls, CancellationToken cancellationToken) { var downloads = new List>(); - foreach (var segment in segments) + foreach (var segment in segmentUrls) { downloads.Add(_client.DownloadFileAsync(segment.Url, cancellationToken)); } - var streams = await Task.WhenAll(downloads); - - // TODO: prefix fileName with listId - // TODO: add ".txt" for sources with no extension - var fileName = Uri.UnescapeDataString(segments.First().Url.Segments.Last()); - var target = new FileInfo(fileName); - var file = new FileToArchive(target, streams); - await _archiver.ArchiveFileAsync(file, cancellationToken); + return await Task.WhenAll(downloads); } } } diff --git a/services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs b/services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs index de10459eb..26949b566 100644 --- a/services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs +++ b/services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs @@ -6,13 +6,15 @@ namespace FilterLists.Archival.Application.Models { internal class FileToArchive : IFileToArchive { - public FileToArchive(FileInfo target, IEnumerable contents) + public FileToArchive(string sourceExtension, IEnumerable contents, FileInfo target) { - Target = target; + SourceExtension = sourceExtension; Contents = contents; + Target = target; } - public FileInfo Target { get; } + public string SourceExtension { get; } public IEnumerable Contents { get; } + public FileInfo Target { get; } } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileWriteStrategyFactory.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileWriteStrategyFactory.cs index 9e2cceed7..f53655881 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileWriteStrategyFactory.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileWriteStrategyFactory.cs @@ -13,7 +13,7 @@ internal static class FileWriteStrategyFactory .GetTypes() .FirstOrDefault(t => typeof(TFileWriteStrategy).IsAssignableFrom(t) && - string.Equals(t.Name, file.Target.Extension.TrimStart('.'), StringComparison.OrdinalIgnoreCase)); + string.Equals(t.Name, file.SourceExtension.TrimStart('.'), StringComparison.OrdinalIgnoreCase)); return strategyType is default(Type) ? default : (TFileWriteStrategy)Activator.CreateInstance(strategyType); diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs index f01fed380..0776418b2 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs @@ -5,7 +5,9 @@ namespace FilterLists.Archival.Infrastructure.Persistence { public interface IFileToArchive { - FileInfo Target { get; } + // for now, assume all source segments have same extension + string SourceExtension { get; } IEnumerable Contents { get; } + FileInfo Target { get; } } }