diff --git a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs index e0d9cd187..43dfcf415 100644 --- a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs +++ b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs @@ -53,12 +53,7 @@ public async Task Handle(Command request, CancellationToken cancellationTo var segments = await GetSegmentsAsync(request.ListId, cancellationToken); if (segments.Count > 0) { - var fetchTasks = segments.Select(s => FetchStreamAsync(s.Url, cancellationToken)); - var streams = await Task.WhenAll(fetchTasks); - - // TODO: add ".txt" to no extension source names - var target = new FileInfo(Uri.UnescapeDataString(segments.First().Url.Segments.Last())); - await _archiver.ArchiveFileAsync(new FileToArchive(target, streams), cancellationToken); + await DownloadSegments(segments, cancellationToken); _archiver.Commit(); _logger.LogDebug( @@ -68,7 +63,7 @@ public async Task Handle(Command request, CancellationToken cancellationTo } else { - _logger.LogInformation("List {ListId} has no view URLs to archive", request.ListId); + _logger.LogInformation("List {ListId} has no URLs to archive", request.ListId); } return Unit.Value; @@ -87,18 +82,37 @@ private async Task> GetSegmentsAsync( new List(); } - private async Task FetchStreamAsync( - Uri url, + private async Task DownloadSegments( + IReadOnlyCollection segments, CancellationToken cancellationToken) { - using var response = await _httpClient.GetAsync(url, cancellationToken); - response.EnsureSuccessStatusCode(); - var sourceStream = await response.Content.ReadAsStreamAsync(); + var responses = new List(); + try + { + var readTasks = new List>(); + foreach (var segment in segments) + { + var response = await _httpClient.GetAsync(segment.Url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + responses.Add(response); + response.EnsureSuccessStatusCode(); + readTasks.Add(response.Content.ReadAsStreamAsync()); + } - // TODO: verify efficiency. buffered to MemoryStream because Stream from HttpClient is disposed prematurely - var targetStream = new MemoryStream(); - await sourceStream.CopyToAsync(targetStream, cancellationToken); - return targetStream; + var streams = await Task.WhenAll(readTasks); + + // TODO: prefix fileName with listId + // TODO: add ".txt" sources with no extension + var fileName = Uri.UnescapeDataString(segments.First().Url.Segments.Last()); + var target = new FileInfo(fileName); + await _archiver.ArchiveFileAsync(new FileToArchive(target, streams), cancellationToken); + } + finally + { + foreach (var response in responses) + { + response.Dispose(); + } + } } } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs index 64d2a9763..fce41567f 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs @@ -48,6 +48,8 @@ public async Task ArchiveFileAsync( strategy.GetType().Name); _writtenFiles.Add(file.Target); + + // TODO: write to _options.RepositoryPath await strategy.WriteAsync(file, cancellationToken); _logger.LogDebug("Finished writing {Filename}", file.Target.Name);