prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(archival): 🐛📝 finish writing Stream before disposing HttpResponseMessage, add TODOs

Browse source
This commit is contained in:
Collin M. Barrett 2020-09-17 17:14:47 -05:00
parent 66cb92eabc
commit 0e1f4b791f
2 changed files with 32 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs
View file

@ -53,12 +53,7 @@ public async Task<Unit> Handle(Command request, CancellationToken cancellationTo
var segments = await GetSegmentsAsync(request.ListId, cancellationToken);
if (segments.Count > 0)
{
var fetchTasks = segments.Select(s => FetchStreamAsync(s.Url, cancellationToken));
var streams = await Task.WhenAll(fetchTasks);
// TODO: add ".txt" to no extension source names
var target = new FileInfo(Uri.UnescapeDataString(segments.First().Url.Segments.Last()));
await _archiver.ArchiveFileAsync(new FileToArchive(target, streams), cancellationToken);
await DownloadSegments(segments, cancellationToken);
_archiver.Commit();
_logger.LogDebug(
@ -68,7 +63,7 @@ public async Task<Unit> Handle(Command request, CancellationToken cancellationTo
}
else
{
_logger.LogInformation("List {ListId} has no view URLs to archive", request.ListId);
_logger.LogInformation("List {ListId} has no URLs to archive", request.ListId);
}
return Unit.Value;
@ -87,18 +82,37 @@ private async Task<List<ListDetailsViewUrlVm>> GetSegmentsAsync(
new List<ListDetailsViewUrlVm>();
}
private async Task<Stream> FetchStreamAsync(
Uri url,
private async Task DownloadSegments(
IReadOnlyCollection<ListDetailsViewUrlVm> segments,
CancellationToken cancellationToken)
{
using var response = await _httpClient.GetAsync(url, cancellationToken);
response.EnsureSuccessStatusCode();
var sourceStream = await response.Content.ReadAsStreamAsync();
var responses = new List<HttpResponseMessage>();
try
{
var readTasks = new List<Task<Stream>>();
foreach (var segment in segments)
{
var response = await _httpClient.GetAsync(segment.Url, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
responses.Add(response);
response.EnsureSuccessStatusCode();
readTasks.Add(response.Content.ReadAsStreamAsync());
}
// TODO: verify efficiency. buffered to MemoryStream because Stream from HttpClient is disposed prematurely
var targetStream = new MemoryStream();
await sourceStream.CopyToAsync(targetStream, cancellationToken);
return targetStream;
var streams = await Task.WhenAll(readTasks);
// TODO: prefix fileName with listId
// TODO: add ".txt" sources with no extension
var fileName = Uri.UnescapeDataString(segments.First().Url.Segments.Last());
var target = new FileInfo(fileName);
await _archiver.ArchiveFileAsync(new FileToArchive(target, streams), cancellationToken);
}
finally
{
foreach (var response in responses)
{
response.Dispose();
}
}
}
}
}

2
services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs
View file

@ -48,6 +48,8 @@ public async Task ArchiveFileAsync(
strategy.GetType().Name);
_writtenFiles.Add(file.Target);
// TODO: write to _options.RepositoryPath
await strategy.WriteAsync(file, cancellationToken);
_logger.LogDebug("Finished writing {Filename}", file.Target.Name);