prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(archival): ✨♻ store txt files in configured dir, ren many members

Browse source
This commit is contained in:
Collin M. Barrett 2020-09-19 17:13:52 -05:00
parent 501cad0ee8
commit 2f187cd9da
13 changed files with 82 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs
View file

@ -13,6 +13,7 @@
using FilterLists.SharedKernel.Apis.Contracts.Directory;
using MediatR;
using Microsoft.Extensions.Logging;
using File = FilterLists.Archival.Application.Models.File;
namespace FilterLists.Archival.Application.Commands
{
@ -30,21 +31,21 @@ public Command(int listId)
public class Handler : IRequestHandler<Command, Unit>
{
private readonly IFileArchiver _archiver;
private readonly IFileClient _client;
private readonly IHttpContentClient _client;
private readonly IDirectoryApi _directory;
private readonly ILogger _logger;
private readonly ITxtFileRepository _repo;
public Handler(
IFileArchiver archiver,
IFileClient fileClient,
IDirectoryApi directory,
ILogger<Handler> logger)
IHttpContentClient httpContentClient,
IDirectoryApi directoryApi,
ILogger<Handler> logger,
ITxtFileRepository txtFileRepository)
{
_archiver = archiver;
_client = fileClient;
_directory = directory;
_client = httpContentClient;
_directory = directoryApi;
_logger = logger;
_repo = txtFileRepository;
}
public async Task<Unit> Handle(Command request, CancellationToken cancellationToken)
@ -52,15 +53,15 @@ public async Task<Unit> Handle(Command request, CancellationToken cancellationTo
_ = request ?? throw new ArgumentNullException(nameof(request));
_logger.LogDebug("Archiving list {ListId}", request.ListId);
var segmentUrls = await GetSegmentUrlsAsync(request.ListId, cancellationToken);
var segmentUrls = (await GetSegmentUrlsAsync(request.ListId, cancellationToken)).ToList();
if (segmentUrls.Count > 0)
{
var file = GetFileToArchive(request.ListId, segmentUrls, cancellationToken);
await _archiver.ArchiveFileAsync(file, cancellationToken);
_archiver.Commit();
await _repo.AddFileAsync(file, cancellationToken);
_repo.Commit();
_logger.LogDebug(
"Archived segments {@SegmentNumbers} of list {ListId}",
"Archived segment(s) {@SegmentNumbers} of list {ListId}",
segmentUrls.Select(s => s.SegmentNumber),
request.ListId);
}
@ -72,28 +73,27 @@ public async Task<Unit> Handle(Command request, CancellationToken cancellationTo
return Unit.Value;
}
private async Task<List<ListDetailsViewUrlVm>> GetSegmentUrlsAsync(
private async Task<IEnumerable<ListDetailsViewUrlVm>> GetSegmentUrlsAsync(
int listId,
CancellationToken cancellationToken)
{
var listDetails = await _directory.GetListDetailsAsync(listId, cancellationToken);
return listDetails.ViewUrls?
.GroupBy(u => u.SegmentNumber, (_, g) => g.OrderBy(e => e.Primariness).First())
?.ToList() ??
.GroupBy(u => u.SegmentNumber, (_, ue) => ue.OrderBy(u => u.Primariness).First()) ??
new List<ListDetailsViewUrlVm>();
}
private IFileToArchive GetFileToArchive(
private IFile GetFileToArchive(
int listId,
IEnumerable<ListDetailsViewUrlVm> segmentUrls,
CancellationToken cancellationToken)
{
var segmentsAsync = GetSegmentsAsync(segmentUrls, cancellationToken);
var target = new FileInfo($"{listId.ToString(CultureInfo.InvariantCulture).PadLeft(5, '0')}.txt");
return new FileToArchive(segmentsAsync, target);
var target = $"{listId.ToString(CultureInfo.InvariantCulture).PadLeft(5, '0')}.txt";
return new File(segmentsAsync, target);
}
private async IAsyncEnumerable<IFileToArchiveSegment> GetSegmentsAsync(
private async IAsyncEnumerable<IFileSegment> GetSegmentsAsync(
IEnumerable<ListDetailsViewUrlVm> segmentUrls,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
@ -101,9 +101,9 @@ private async IAsyncEnumerable<IFileToArchiveSegment> GetSegmentsAsync(
{
var sourceFileName = Uri.UnescapeDataString(segment.Url.Segments.Last());
var sourceExtension = Path.GetExtension(sourceFileName);
yield return new FileToArchiveSegment(
yield return new FileSegment(
sourceExtension,
await _client.DownloadFileAsync(segment.Url, cancellationToken));
await _client.GetContentAsync(segment.Url, cancellationToken));
}
}
}

14
services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs → services/Archival/FilterLists.Archival.Application/Models/File.cs
View file

@ -4,21 +4,21 @@
namespace FilterLists.Archival.Application.Models
{
internal class FileToArchive : IFileToArchive
internal class File : IFile
{
public FileToArchive(IAsyncEnumerable<IFileToArchiveSegment> segments, FileInfo target)
public File(IAsyncEnumerable<IFileSegment> segments, string targetFileName)
{
Segments = segments;
Target = target;
TargetFileName = targetFileName;
}
public IAsyncEnumerable<IFileToArchiveSegment> Segments { get; }
public FileInfo Target { get; }
public IAsyncEnumerable<IFileSegment> Segments { get; }
public string TargetFileName { get; }
}
internal class FileToArchiveSegment : IFileToArchiveSegment
internal class FileSegment : IFileSegment
{
public FileToArchiveSegment(string sourceExtension, Stream contents)
public FileSegment(string sourceExtension, Stream contents)
{
SourceExtension = sourceExtension;
Contents = contents;

2
services/Archival/FilterLists.Archival.Infrastructure/Clients/ConfigurationExtensions.cs
View file

@ -8,7 +8,7 @@ internal static class ConfigurationExtensions
{
public static void AddClients(this IServiceCollection services)
{
services.AddHttpClient<IFileClient, FileClient>()
services.AddHttpClient<IHttpContentClient, HttpContentClient>()
.AddTransientHttpErrorPolicy(b => b.WaitAndRetryAsync(new[]
{
TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10)

12
services/Archival/FilterLists.Archival.Infrastructure/Clients/IFileClient.cs
View file

@ -1,12 +0,0 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace FilterLists.Archival.Infrastructure.Clients
{
public interface IFileClient : IDisposable
{
Task<Stream> DownloadFileAsync(Uri url, CancellationToken cancellationToken);
}
}

13
services/Archival/FilterLists.Archival.Infrastructure/Clients/FileClient.cs → services/Archival/FilterLists.Archival.Infrastructure/Clients/IHttpContentClient.cs
View file

@ -7,17 +7,22 @@
namespace FilterLists.Archival.Infrastructure.Clients
{
internal sealed class FileClient : IFileClient
public interface IHttpContentClient : IDisposable
{
Task<Stream> GetContentAsync(Uri url, CancellationToken cancellationToken);
}
internal sealed class HttpContentClient : IHttpContentClient
{
private readonly HttpClient _httpClient;
private readonly ICollection<HttpResponseMessage> _httpResponseMessages = new List<HttpResponseMessage>();
public FileClient(HttpClient httpClient)
public HttpContentClient(HttpClient httpClient)
{
_httpClient = httpClient;
}
public async Task<Stream> DownloadFileAsync(Uri url, CancellationToken cancellationToken)
public async Task<Stream> GetContentAsync(Uri url, CancellationToken cancellationToken)
{
var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
_httpResponseMessages.Add(response);
@ -31,8 +36,6 @@ public void Dispose()
{
message.Dispose();
}
_httpClient.Dispose();
}
}
}

2
services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs
View file

@ -20,7 +20,7 @@ public static void AddPersistenceServices(this IServiceCollection services, ICon
return new Repository(gitOptions.RepositoryPath);
});
services.AddTransient<IFileArchiver, GitFileArchiver>();
services.AddTransient<ITxtFileRepository, GitTxtFileRepository>();
}
}
}

10
services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IFileStreamConversionStrategy.cs
View file

@ -1,10 +0,0 @@
using System.IO;
using System.Threading;
namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies
{
internal interface IFileStreamConversionStrategy
{
Stream Convert(IFileToArchiveSegment segment, CancellationToken cancellationToken);
}
}

10
services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToTxtConversionStrategy.cs Normal file
View file

@ -0,0 +1,10 @@
using System.IO;
using System.Threading;
namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies
{
internal interface IStreamToTxtConversionStrategy
{
Stream Convert(IFileSegment fileSegment, CancellationToken cancellationToken);
}
}

11
services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileStreamConversionStrategyFactory.cs → services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/StreamToTxtConversionStrategyFactory.cs
View file

@ -4,20 +4,19 @@
namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies
{
internal static class FileStreamConversionStrategyFactory
internal static class StreamToTxtConversionStrategyFactory
{
public static TFileStreamConversionStrategy? GetStrategy<TFileStreamConversionStrategy>(
this IFileToArchiveSegment segment)
where TFileStreamConversionStrategy : class, IFileStreamConversionStrategy
public static TStreamToTxtConversionStrategy? GetStrategy<TStreamToTxtConversionStrategy>(
this IFileSegment segment) where TStreamToTxtConversionStrategy : class, IStreamToTxtConversionStrategy
{
var strategyType = Assembly.GetExecutingAssembly()
.GetTypes()
.FirstOrDefault(t =>
typeof(TFileStreamConversionStrategy).IsAssignableFrom(t) &&
typeof(TStreamToTxtConversionStrategy).IsAssignableFrom(t) &&
string.Equals(t.Name, segment.SourceExtension.TrimStart('.'), StringComparison.OrdinalIgnoreCase));
return strategyType is default(Type)
? default
: (TFileStreamConversionStrategy)Activator.CreateInstance(strategyType);
: (TStreamToTxtConversionStrategy)Activator.CreateInstance(strategyType);
}
}
}

8
services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/Txt.cs
View file

@ -4,13 +4,13 @@
namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies
{
public class Txt : IFileStreamConversionStrategy
public class Txt : IStreamToTxtConversionStrategy
{
public Stream Convert(IFileToArchiveSegment segment, CancellationToken cancellationToken)
public Stream Convert(IFileSegment fileSegment, CancellationToken cancellationToken)
{
_ = segment ?? throw new ArgumentNullException(nameof(segment));
_ = fileSegment ?? throw new ArgumentNullException(nameof(fileSegment));
return segment.Contents;
return fileSegment.Contents;
}
}
}

8
services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs → services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFile.cs
View file

@ -3,13 +3,13 @@
namespace FilterLists.Archival.Infrastructure.Persistence
{
public interface IFileToArchive
public interface IFile
{
IAsyncEnumerable<IFileToArchiveSegment> Segments { get; }
FileInfo Target { get; }
IAsyncEnumerable<IFileSegment> Segments { get; }
string TargetFileName { get; }
}
public interface IFileToArchiveSegment
public interface IFileSegment
{
string SourceExtension { get; }
Stream Contents { get; }

13
services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileArchiver.cs
View file

@ -1,13 +0,0 @@
using System.Threading;
using System.Threading.Tasks;
using FilterLists.SharedKernel.SeedWork;
namespace FilterLists.Archival.Infrastructure.Persistence
{
public interface IFileArchiver : IUnitOfWork
{
Task ArchiveFileAsync(
IFileToArchive file,
CancellationToken cancellationToken);
}
}

37
services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs → services/Archival/FilterLists.Archival.Infrastructure/Persistence/ITxtFileRepository.cs
View file

@ -6,21 +6,27 @@
using System.Threading.Tasks;
using FilterLists.Archival.Infrastructure.Options;
using FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies;
using FilterLists.SharedKernel.SeedWork;
using LibGit2Sharp;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace FilterLists.Archival.Infrastructure.Persistence
{
internal sealed class GitFileArchiver : IFileArchiver
public interface ITxtFileRepository : IUnitOfWork
{
Task AddFileAsync(IFile file, CancellationToken cancellationToken);
}
internal sealed class GitTxtFileRepository : ITxtFileRepository
{
private readonly ILogger _logger;
private readonly GitOptions _options;
private readonly IRepository _repo;
private readonly ICollection<FileInfo> _writtenFiles = new HashSet<FileInfo>();
public GitFileArchiver(
ILogger<GitFileArchiver> logger,
public GitTxtFileRepository(
ILogger<GitTxtFileRepository> logger,
IOptions<GitOptions> gitOptions,
IRepository repository)
{
@ -29,32 +35,29 @@ public GitFileArchiver(
_repo = repository;
}
public async Task ArchiveFileAsync(
IFileToArchive file,
CancellationToken cancellationToken)
public async Task AddFileAsync(IFile file, CancellationToken cancellationToken)
{
var textStreams = new List<Stream>();
await foreach (var segment in file.Segments.WithCancellation(cancellationToken))
{
var strategy = segment.GetStrategy<IFileStreamConversionStrategy>();
if (strategy is default(IFileStreamConversionStrategy))
var strategy = segment.GetStrategy<IStreamToTxtConversionStrategy>();
if (strategy is default(IStreamToTxtConversionStrategy))
{
_logger.LogWarning(
"No file stream conversion strategy found for extension {Extension} for target {Target}. Skipping file",
"No stream to txt conversion strategy found for extension {Extension} for target {Target}. Skipping file",
segment.SourceExtension,
file.Target.Name);
file.TargetFileName);
return;
}
textStreams.Add(strategy.Convert(segment, cancellationToken));
}
_logger.LogDebug("Writing {FileName}", file.Target.Name);
_logger.LogDebug("Writing {FileName}", file.TargetFileName);
_writtenFiles.Add(file.Target);
// TODO: write to _options.RepositoryPath
await using var target = file.Target.OpenWrite();
var fileInfo = new FileInfo(Path.Combine(_options.RepositoryPath, file.TargetFileName));
_writtenFiles.Add(fileInfo);
await using var target = fileInfo.OpenWrite();
// TODO: validate multi-segment lists are concatenated correctly and in order
foreach (var textStream in textStreams)
@ -62,14 +65,14 @@ public async Task ArchiveFileAsync(
await textStream.CopyToAsync(target, cancellationToken);
}
_logger.LogDebug("Finished writing {FileName}", file.Target.Name);
_logger.LogDebug("Finished writing {FileName}", file.TargetFileName);
}
public void Commit()
{
var fileNames = _writtenFiles.Select(f => f.Name).ToList();
var signature = new Signature(_options.UserName, _options.UserEmail, DateTime.UtcNow);
var message = $"feat(archives): archive {fileNames.Count} files{Environment.NewLine}{string.Join(Environment.NewLine, fileNames)}";
var message = $"feat(archives): archive {fileNames.Count} file(s){Environment.NewLine}{string.Join(Environment.NewLine, fileNames)}";
Commands.Stage(_repo, fileNames);
_repo.Commit(message, signature, signature);