From 0e1f4b791f4fdc1d33cecbc9f06e4615e13caa74 Mon Sep 17 00:00:00 2001
From: "Collin M. Barrett" <collinbarrett@users.noreply.github.com>
Date: Thu, 17 Sep 2020 17:14:47 -0500
Subject: [PATCH] =?UTF-8?q?fix(archival):=20=F0=9F=90=9B=F0=9F=93=9D=20fin?=
 =?UTF-8?q?ish=20writing=20Stream=20before=20disposing=20HttpResponseMessa?=
 =?UTF-8?q?ge,=20add=20TODOs?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../Commands/ArchiveList.cs                   | 46 ++++++++++++-------
 .../Persistence/GitFileArchiver.cs            |  2 +
 2 files changed, 32 insertions(+), 16 deletions(-)

diff --git a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs
index e0d9cd187..43dfcf415 100644
--- a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs
+++ b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs
@@ -53,12 +53,7 @@ public async Task<Unit> Handle(Command request, CancellationToken cancellationTo
                 var segments = await GetSegmentsAsync(request.ListId, cancellationToken);
                 if (segments.Count > 0)
                 {
-                    var fetchTasks = segments.Select(s => FetchStreamAsync(s.Url, cancellationToken));
-                    var streams = await Task.WhenAll(fetchTasks);
-
-                    // TODO: add ".txt" to no extension source names
-                    var target = new FileInfo(Uri.UnescapeDataString(segments.First().Url.Segments.Last()));
-                    await _archiver.ArchiveFileAsync(new FileToArchive(target, streams), cancellationToken);
+                    await DownloadSegments(segments, cancellationToken);
                     _archiver.Commit();
 
                     _logger.LogDebug(
@@ -68,7 +63,7 @@ public async Task<Unit> Handle(Command request, CancellationToken cancellationTo
                 }
                 else
                 {
-                    _logger.LogInformation("List {ListId} has no view URLs to archive", request.ListId);
+                    _logger.LogInformation("List {ListId} has no URLs to archive", request.ListId);
                 }
 
                 return Unit.Value;
@@ -87,18 +82,37 @@ private async Task<List<ListDetailsViewUrlVm>> GetSegmentsAsync(
                        new List<ListDetailsViewUrlVm>();
             }
 
-            private async Task<Stream> FetchStreamAsync(
-                Uri url,
+            private async Task DownloadSegments(
+                IReadOnlyCollection<ListDetailsViewUrlVm> segments,
                 CancellationToken cancellationToken)
             {
-                using var response = await _httpClient.GetAsync(url, cancellationToken);
-                response.EnsureSuccessStatusCode();
-                var sourceStream = await response.Content.ReadAsStreamAsync();
+                var responses = new List<HttpResponseMessage>();
+                try
+                {
+                    var readTasks = new List<Task<Stream>>();
+                    foreach (var segment in segments)
+                    {
+                        var response = await _httpClient.GetAsync(segment.Url, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+                        responses.Add(response);
+                        response.EnsureSuccessStatusCode();
+                        readTasks.Add(response.Content.ReadAsStreamAsync());
+                    }
 
-                // TODO: verify efficiency. buffered to MemoryStream because Stream from HttpClient is disposed prematurely
-                var targetStream = new MemoryStream();
-                await sourceStream.CopyToAsync(targetStream, cancellationToken);
-                return targetStream;
+                    var streams = await Task.WhenAll(readTasks);
+
+                    // TODO: prefix fileName with listId
+                    // TODO: add ".txt" sources with no extension
+                    var fileName = Uri.UnescapeDataString(segments.First().Url.Segments.Last());
+                    var target = new FileInfo(fileName);
+                    await _archiver.ArchiveFileAsync(new FileToArchive(target, streams), cancellationToken);
+                }
+                finally
+                {
+                    foreach (var response in responses)
+                    {
+                        response.Dispose();
+                    }
+                }
             }
         }
     }
diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs
index 64d2a9763..fce41567f 100644
--- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs
+++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs
@@ -48,6 +48,8 @@ public async Task ArchiveFileAsync(
                     strategy.GetType().Name);
 
                 _writtenFiles.Add(file.Target);
+
+                // TODO: write to _options.RepositoryPath
                 await strategy.WriteAsync(file, cancellationToken);
 
                 _logger.LogDebug("Finished writing {Filename}", file.Target.Name);