prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(archival): ✨ implement strategy pattern for file writing

Browse source
This commit is contained in:
Collin M. Barrett 2020-09-16 08:28:48 -05:00
parent 478b90a806
commit 8166dec254
10 changed files with 163 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs
View file

@ -1,12 +1,14 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Archival.Application.Models;
using FilterLists.Archival.Infrastructure.Persistence;
using FilterLists.SharedKernel.Apis.Clients;
using FilterLists.SharedKernel.Apis.Contracts.Directory;
using MediatR;
using Microsoft.Extensions.Logging;
@ -28,12 +30,18 @@ public class Handler : IRequestHandler<Command, Unit>
{
private readonly IFileArchiver _archiver;
private readonly IDirectoryApi _directory;
private readonly HttpClient _httpClient;
private readonly ILogger _logger;
public Handler(IFileArchiver archiver, IDirectoryApi directory, ILogger<Handler> logger)
public Handler(
IFileArchiver archiver,
IDirectoryApi directory,
IHttpClientFactory httpClientFactory,
ILogger<Handler> logger)
{
_archiver = archiver;
_directory = directory;
_httpClient = httpClientFactory.CreateClient();
_logger = logger;
}
@ -42,37 +50,49 @@ public async Task<Unit> Handle(Command request, CancellationToken cancellationTo
_ = request ?? throw new ArgumentNullException(nameof(request));
_logger.LogDebug("Archiving list {ListId}", request.ListId);
var listDetails = await _directory.GetListDetailsAsync(request.ListId, cancellationToken);
var segments = listDetails.ViewUrls?
.GroupBy(u => u.SegmentNumber)
.Select(g => g.OrderBy(u => u.Primariness))
.First()
.ToList();
if (segments != null)
var segments = await GetSegmentsAsync(request.ListId, cancellationToken);
if (segments.Count > 0)
{
var archiveTasks = new List<Task>();
foreach (var segment in segments)
{
Stream contents = null!; // TODO: fetch list file stream
var path = Path.Combine(
listDetails.Id.ToString(CultureInfo.InvariantCulture),
segment.SegmentNumber.ToString(CultureInfo.InvariantCulture));
_logger.LogDebug("Archiving segment {SegmentNumber} of list {ListId}", segment.SegmentNumber, request.ListId);
archiveTasks.Add(_archiver.ArchiveFileAsync(contents, path, cancellationToken));
}
await Task.WhenAll(archiveTasks);
var fetchTasks = segments.Select(s => FetchStreamAsync(s.Url, cancellationToken));
var streams = await Task.WhenAll(fetchTasks);
var target = new FileInfo(Uri.UnescapeDataString(segments.First().Url.Segments.Last()));
await _archiver.ArchiveFileAsync(new FileToArchive(target, streams), cancellationToken);
_archiver.Commit();
_logger.LogDebug("Archived segments {@SegmentNumbers} of list {ListId}", segments.Select(s => s.SegmentNumber), request.ListId);
_logger.LogDebug(
"Archived segments {@SegmentNumbers} of list {ListId}",
segments.Select(s => s.SegmentNumber),
request.ListId);
}
else
{
_logger.LogDebug("List {ListId} has no view URLs to archive", request.ListId);
_logger.LogInformation("List {ListId} has no view URLs to archive", request.ListId);
}
return Unit.Value;
}
private async Task<List<ListDetailsViewUrlVm>> GetSegmentsAsync(
int listId,
CancellationToken cancellationToken)
{
var listDetails = await _directory.GetListDetailsAsync(listId, cancellationToken);
return listDetails.ViewUrls?
.GroupBy(u => u.SegmentNumber)
.Select(g => g.OrderBy(u => u.Primariness))
.FirstOrDefault()
?.ToList() ??
new List<ListDetailsViewUrlVm>();
}
private async Task<Stream> FetchStreamAsync(
Uri url,
CancellationToken cancellationToken)
{
using var response = await _httpClient.GetAsync(url, cancellationToken);
response.EnsureSuccessStatusCode();
return await response.Content.ReadAsStreamAsync();
}
}
}
}

18
services/Archival/FilterLists.Archival.Application/Models/FileToArchive.cs Normal file
View file

@ -0,0 +1,18 @@
using System.Collections.Generic;
using System.IO;
using FilterLists.Archival.Infrastructure.Persistence;
namespace FilterLists.Archival.Application.Models
{
internal class FileToArchive : IFileToArchive
{
public FileToArchive(FileInfo target, IEnumerable<Stream> contents)
{
Target = target;
Contents = contents;
}
public FileInfo Target { get; }
public IEnumerable<Stream> Contents { get; }
}
}

22
services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/FileWriteStrategyFactory.cs Normal file
View file

@ -0,0 +1,22 @@
using System;
using System.Linq;
using System.Reflection;
namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies
{
internal static class FileWriteStrategyFactory
{
public static TFileWriteStrategy? GetStrategy<TFileWriteStrategy>(this IFileToArchive file)
where TFileWriteStrategy : class, IFileWriteStrategy
{
var strategyType = Assembly.GetExecutingAssembly()
.GetTypes()
.FirstOrDefault(t =>
typeof(TFileWriteStrategy).IsAssignableFrom(t) &&
string.Equals(t.Name, file.Target.Extension.TrimStart('.'), StringComparison.OrdinalIgnoreCase));
return strategyType is default(Type)
? default
: (TFileWriteStrategy)Activator.CreateInstance(strategyType);
}
}
}

10
services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IFileWriteStrategy.cs Normal file
View file

@ -0,0 +1,10 @@
using System.Threading;
using System.Threading.Tasks;
namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies
{
internal interface IFileWriteStrategy
{
Task WriteAsync(IFileToArchive file, CancellationToken cancellationToken = default);
}
}

20
services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/Txt.cs Normal file
View file

@ -0,0 +1,20 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies
{
public class Txt : IFileWriteStrategy
{
public async Task WriteAsync(IFileToArchive file, CancellationToken cancellationToken = default)
{
_ = file ?? throw new ArgumentNullException(nameof(file));
await using var target = file.Target.OpenWrite();
foreach (var source in file.Contents)
{
await source.CopyToAsync(target, cancellationToken);
}
}
}
}

52
services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitFileArchiver.cs
View file

@ -1,9 +1,11 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Archival.Infrastructure.Options;
using FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies;
using LibGit2Sharp;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@ -12,56 +14,68 @@ namespace FilterLists.Archival.Infrastructure.Persistence
{
internal sealed class GitFileArchiver : IFileArchiver
{
private readonly IList<string> _filePaths = new List<string>();
private readonly ILogger _logger;
private readonly GitOptions _options;
private readonly IRepository _repository;
private readonly IRepository _repo;
private readonly ICollection<FileInfo> _writtenFiles = new HashSet<FileInfo>();
public GitFileArchiver(
ILogger<GitFileArchiver> logger,
IOptions<GitOptions> gitOptions,
IRepository gitRepository)
IRepository repository)
{
_logger = logger;
_options = gitOptions.Value;
_repository = gitRepository;
_repo = repository;
}
public async Task ArchiveFileAsync(
Stream fileContents,
string filePath,
IFileToArchive file,
CancellationToken cancellationToken = default)
{
_logger.LogDebug("Writing {FilePath}", filePath);
var strategy = file.GetStrategy<IFileWriteStrategy>();
if (strategy is default(IFileWriteStrategy))
{
_logger.LogWarning(
"No write strategy found for {Filename}. Skipping",
file.Target.Name);
}
else
{
_logger.LogDebug(
"Writing {Filename} with strategy {FileWriteStrategy}",
file.Target.Name,
strategy.GetType().Name);
_filePaths.Add(filePath);
await using var output = File.OpenWrite(filePath);
await fileContents.CopyToAsync(output, cancellationToken);
_writtenFiles.Add(file.Target);
await strategy.WriteAsync(file, cancellationToken);
_logger.LogDebug("Finished writing {FilePath}", filePath);
_logger.LogDebug("Finished writing {Filename}", file.Target.Name);
}
}
public void Commit()
{
Commands.Stage(_repository, _filePaths);
var filenames = _writtenFiles.Select(f => f.Name).ToList();
Commands.Stage(_repo, filenames);
var signature = new Signature(_options.UserName, _options.UserEmail, DateTime.UtcNow);
var message = $"feat(archives): archive {_filePaths.Count} files{Environment.NewLine}{string.Join(Environment.NewLine, _filePaths)}";
_repository.Commit(message, signature, signature);
var message = $"feat(archives): archive {filenames.Count} files{Environment.NewLine}{string.Join(Environment.NewLine, filenames)}";
_repo.Commit(message, signature, signature);
_logger.LogDebug("Committed {@FilePaths}", _filePaths);
_logger.LogDebug("Committed {@Filenames}", filenames);
}
public void Dispose()
{
foreach (var file in _filePaths)
foreach (var file in _writtenFiles)
{
if (File.Exists(file))
if (File.Exists(file.Name))
{
File.Delete(file);
File.Delete(file.Name);
}
}
_repository.CheckoutPaths("HEAD", _filePaths);
_repo.CheckoutPaths("HEAD", _writtenFiles.Select(f => f.Name));
}
}
}

6
services/Archival/FilterLists.Archival.Infrastructure/Persistence/IArchiveFiles.cs → services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileArchiver.cs
View file

@ -1,5 +1,4 @@
using System.IO;
using System.Threading;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.SharedKernel.SeedWork;
@ -8,8 +7,7 @@ namespace FilterLists.Archival.Infrastructure.Persistence
public interface IFileArchiver : IUnitOfWork
{
Task ArchiveFileAsync(
Stream fileContents,
string filePath,
IFileToArchive file,
CancellationToken cancellationToken = default);
}
}

11
services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileToArchive.cs Normal file
View file

@ -0,0 +1,11 @@
using System.Collections.Generic;
using System.IO;
namespace FilterLists.Archival.Infrastructure.Persistence
{
public interface IFileToArchive
{
FileInfo Target { get; }
IEnumerable<Stream> Contents { get; }
}
}

3
services/Archival/FilterLists.Archival.Infrastructure/Scheduling/ConfigurationExtensions.cs
View file

@ -24,7 +24,8 @@ public static void AddSchedulingServices(this IServiceCollection services, IConf
public static void UseScheduling(this IApplicationBuilder app)
{
app.UseHangfireServer(new BackgroundJobServerOptions {WorkerCount = Environment.ProcessorCount * 1});
// TODO: re-evaluate git in parallel
app.UseHangfireServer(new BackgroundJobServerOptions {WorkerCount = 1});
}
}
}

1
services/FilterLists.sln.DotSettings
View file

@ -10,6 +10,7 @@
<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/WRAP_CHAINED_BINARY_EXPRESSIONS/@EntryValue">CHOP_IF_LONG</s:String>
<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/WRAP_CHAINED_METHOD_CALLS/@EntryValue">CHOP_IF_LONG</s:String>
<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/WRAP_PARAMETERS_STYLE/@EntryValue">CHOP_IF_LONG</s:String>
<s:Boolean x:Key="/Default/UserDictionary/Words/=filenames/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Hangfire/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Mediat/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Primariness/@EntryIndexedValue">True</s:Boolean>