prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(archival): ✨ normalize target filenames

Browse source
This commit is contained in:
Collin M. Barrett 2020-09-17 20:54:37 -05:00
parent 218b9ac268
commit c4d97dd8b4
4 changed files with 32 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs
View file

@ -50,15 +50,16 @@ public async Task<Unit> Handle(Command request, CancellationToken cancellationTo
_ = request ?? throw new ArgumentNullException(nameof(request));
_logger.LogDebug("Archiving list {ListId}", request.ListId);
var segments = await GetSegmentsAsync(request.ListId, cancellationToken);
if (segments.Count > 0)
var segmentUrls = await GetSegmentUrlsAsync(request.ListId, cancellationToken);
if (segmentUrls.Count > 0)
{
await DownloadSegments(segments, cancellationToken);
var file = await GetFileAsync(request.ListId, segmentUrls, cancellationToken);
await _archiver.ArchiveFileAsync(file, cancellationToken);
_archiver.Commit();
_logger.LogDebug(
"Archived segments {@SegmentNumbers} of list {ListId}",
segments.Select(s => s.SegmentNumber),
segmentUrls.Select(s => s.SegmentNumber),
request.ListId);
}
else
@ -69,7 +70,7 @@ public async Task<Unit> Handle(Command request, CancellationToken cancellationTo
return Unit.Value;
}
private async Task<List<ListDetailsViewUrlVm>> GetSegmentsAsync(
private async Task<List<ListDetailsViewUrlVm>> GetSegmentUrlsAsync(
int listId,
CancellationToken cancellationToken)
{
@ -82,24 +83,30 @@ private async Task<List<ListDetailsViewUrlVm>> GetSegmentsAsync(
new List<ListDetailsViewUrlVm>();
}
private async Task DownloadSegments(
IReadOnlyCollection<ListDetailsViewUrlVm> segments,
private async Task<FileToArchive> GetFileAsync(
int listId,
IReadOnlyCollection<ListDetailsViewUrlVm> segmentUrls,
CancellationToken cancellationToken)
{
var contentsAsync = GetContentsAsync(segmentUrls, cancellationToken);
var sourceFileName = Uri.UnescapeDataString(segmentUrls.First().Url.Segments.Last());
var sourceExtension = Path.GetExtension(sourceFileName);
var target = new FileInfo($"{listId}.txt");
return new FileToArchive(sourceExtension, await contentsAsync, target);
}
// TODO: IAsyncEnumerable?
private async Task<IEnumerable<Stream>> GetContentsAsync(
IEnumerable<ListDetailsViewUrlVm> segmentUrls,
CancellationToken cancellationToken)
{
var downloads = new List<Task<Stream>>();
foreach (var segment in segments)
foreach (var segment in segmentUrls)
{
downloads.Add(_client.DownloadFileAsync(segment.Url, cancellationToken));
}
var streams = await Task.WhenAll(downloads);
// TODO: prefix fileName with listId
// TODO: add ".txt" for sources with no extension
var fileName = Uri.UnescapeDataString(segments.First().Url.Segments.Last());
var target = new FileInfo(fileName);
var file = new FileToArchive(target, streams);
await _archiver.ArchiveFileAsync(file, cancellationToken);
return await Task.WhenAll(downloads);
}
}
}

8
services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs
View file

@ -6,13 +6,15 @@ namespace FilterLists.Archival.Application.Models
{
internal class FileToArchive : IFileToArchive
{
public FileToArchive(FileInfo target, IEnumerable<Stream> contents)
public FileToArchive(string sourceExtension, IEnumerable<Stream> contents, FileInfo target)
{
Target = target;
SourceExtension = sourceExtension;
Contents = contents;
Target = target;
}
public FileInfo Target { get; }
public string SourceExtension { get; }
public IEnumerable<Stream> Contents { get; }
public FileInfo Target { get; }
}
}

2
services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileWriteStrategyFactory.cs
View file

@ -13,7 +13,7 @@ internal static class FileWriteStrategyFactory
.GetTypes()
.FirstOrDefault(t =>
typeof(TFileWriteStrategy).IsAssignableFrom(t) &&
string.Equals(t.Name, file.Target.Extension.TrimStart('.'), StringComparison.OrdinalIgnoreCase));
string.Equals(t.Name, file.SourceExtension.TrimStart('.'), StringComparison.OrdinalIgnoreCase));
return strategyType is default(Type)
? default
: (TFileWriteStrategy)Activator.CreateInstance(strategyType);

4
services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs
View file

@ -5,7 +5,9 @@ namespace FilterLists.Archival.Infrastructure.Persistence
{
public interface IFileToArchive
{
FileInfo Target { get; }
// for now, assume all source segments have same extension
string SourceExtension { get; }
IEnumerable<Stream> Contents { get; }
FileInfo Target { get; }
}
}