From 98b9ef00374bde0dec4ccc0d864ffb58421700e8 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sat, 26 Sep 2020 15:21:52 -0500 Subject: [PATCH] =?UTF-8?q?fix(archival):=20=F0=9F=90=9B=20only=20add=20no?= =?UTF-8?q?n-empty=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Commands/ArchiveList.cs | 8 +++--- .../Persistence/IFileRepository.cs | 25 +++++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs index a1e265ec1..13da161d3 100644 --- a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs +++ b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs @@ -101,9 +101,11 @@ private async IAsyncEnumerable GetSegmentsAsync( { var sourceFileName = Uri.UnescapeDataString(segment.Url.Segments.Last()); var sourceExtension = Path.GetExtension(sourceFileName); - yield return new FileSegment( - sourceExtension, - await _client.GetContentAsync(segment.Url, cancellationToken)); + var contentAsync = await _client.GetContentAsync(segment.Url, cancellationToken); + if (contentAsync != Stream.Null) + { + yield return new FileSegment(sourceExtension, contentAsync); + } } } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileRepository.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileRepository.cs index 809ad7cda..d122c78da 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileRepository.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileRepository.cs @@ -53,19 +53,22 @@ public async Task AddFileAsync(IFile file, CancellationToken cancellationToken) textStreams.Add(strategy.Convert(segment, cancellationToken)); } - _logger.LogInformation("Writing {FileName}", file.TargetFileName); - - var fileInfo = new FileInfo(Path.Combine(_options.RepositoryPath, file.TargetFileName)); - _writtenFiles.Add(fileInfo); - await using var target = fileInfo.OpenWrite(); - - // TODO: validate multi-segment lists are concatenated correctly and in order - foreach (var textStream in textStreams) + if (textStreams.Count > 0) { - await textStream.CopyToAsync(target, cancellationToken); - } + _logger.LogInformation("Writing {FileName}", file.TargetFileName); - _logger.LogInformation("Finished writing {FileName}", file.TargetFileName); + var fileInfo = new FileInfo(Path.Combine(_options.RepositoryPath, file.TargetFileName)); + _writtenFiles.Add(fileInfo); + await using var target = fileInfo.OpenWrite(); + + // TODO: validate multi-segment lists are concatenated correctly and in order + foreach (var textStream in textStreams) + { + await textStream.CopyToAsync(target, cancellationToken); + } + + _logger.LogInformation("Finished writing {FileName}", file.TargetFileName); + } } public void Commit()