prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(archival): 🐛 only add non-empty files

Browse source
This commit is contained in:
Collin M. Barrett 2020-09-26 15:21:52 -05:00
parent dc3f50a3cc
commit 98b9ef0037
2 changed files with 19 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs
View file

@ -101,9 +101,11 @@ private async IAsyncEnumerable<IFileSegment> GetSegmentsAsync(
{
var sourceFileName = Uri.UnescapeDataString(segment.Url.Segments.Last());
var sourceExtension = Path.GetExtension(sourceFileName);
yield return new FileSegment(
sourceExtension,
await _client.GetContentAsync(segment.Url, cancellationToken));
var contentAsync = await _client.GetContentAsync(segment.Url, cancellationToken);
if (contentAsync != Stream.Null)
{
yield return new FileSegment(sourceExtension, contentAsync);
}
}
}
}

25
services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileRepository.cs
View file

@ -53,19 +53,22 @@ public async Task AddFileAsync(IFile file, CancellationToken cancellationToken)
textStreams.Add(strategy.Convert(segment, cancellationToken));
}
_logger.LogInformation("Writing {FileName}", file.TargetFileName);
var fileInfo = new FileInfo(Path.Combine(_options.RepositoryPath, file.TargetFileName));
_writtenFiles.Add(fileInfo);
await using var target = fileInfo.OpenWrite();
// TODO: validate multi-segment lists are concatenated correctly and in order
foreach (var textStream in textStreams)
if (textStreams.Count > 0)
{
await textStream.CopyToAsync(target, cancellationToken);
}
_logger.LogInformation("Writing {FileName}", file.TargetFileName);
_logger.LogInformation("Finished writing {FileName}", file.TargetFileName);
var fileInfo = new FileInfo(Path.Combine(_options.RepositoryPath, file.TargetFileName));
_writtenFiles.Add(fileInfo);
await using var target = fileInfo.OpenWrite();
// TODO: validate multi-segment lists are concatenated correctly and in order
foreach (var textStream in textStreams)
{
await textStream.CopyToAsync(target, cancellationToken);
}
_logger.LogInformation("Finished writing {FileName}", file.TargetFileName);
}
}
public void Commit()