From 24d8dc045f0febe9a93174b1c5a1037e921d5225 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Mon, 28 Sep 2020 07:46:02 -0500 Subject: [PATCH] =?UTF-8?q?feat(archival):=20=E2=9C=A8=E2=99=BB=20handle?= =?UTF-8?q?=20more=20source=20extensions,=20extract=20domain=20layer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../FilterLists.Archival.Api/Dockerfile | 2 + .../FilterLists.Archival.Api/dev.Dockerfile | 2 + .../Commands/ArchiveList.cs | 24 +++--- .../FilterLists.Archival.Application.csproj | 1 + .../Models/File.cs | 30 -------- .../FilterLists.Archival.Domain.csproj | 26 +++++++ .../Lists/IListArchiveRepository.cs | 11 +++ .../Lists/ListArchive.cs | 16 ++++ .../Lists/ListArchiveSegment.cs | 17 +++++ .../Lists/ListFileExtension.cs | 76 +++++++++++++++++++ .../SeedWork/IUnitOfWork.cs | 2 +- .../SeedWork/ValueObject.cs | 42 ++++++++++ ...FilterLists.Archival.Infrastructure.csproj | 1 + .../Persistence/ConfigurationExtensions.cs | 5 +- .../IStreamToPlainTextConversionStrategy.cs | 3 +- .../FileWriteStrategies/PlainText.cs | 9 ++- ...eamToPlainTextConversionStrategyFactory.cs | 17 ++--- ...ository.cs => GitListArchiveRepository.cs} | 64 ++++++++-------- .../Persistence/IFile.cs | 17 ----- services/Archival/azure-pipelines.api.yaml | 2 + services/FilterLists.sln | 7 ++ services/FilterLists.sln.DotSettings | 1 + 22 files changed, 264 insertions(+), 111 deletions(-) delete mode 100644 services/Archival/FilterLists.Archival.Application/Models/File.cs create mode 100644 services/Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj create mode 100644 services/Archival/FilterLists.Archival.Domain/Lists/IListArchiveRepository.cs create mode 100644 services/Archival/FilterLists.Archival.Domain/Lists/ListArchive.cs create mode 100644 services/Archival/FilterLists.Archival.Domain/Lists/ListArchiveSegment.cs create mode 100644 services/Archival/FilterLists.Archival.Domain/Lists/ListFileExtension.cs rename services/Archival/{FilterLists.Archival.Infrastructure => FilterLists.Archival.Domain}/SeedWork/IUnitOfWork.cs (65%) create mode 100644 services/Archival/FilterLists.Archival.Domain/SeedWork/ValueObject.cs rename services/Archival/FilterLists.Archival.Infrastructure/Persistence/{IFileRepository.cs => GitListArchiveRepository.cs} (62%) delete mode 100644 services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFile.cs diff --git a/services/Archival/FilterLists.Archival.Api/Dockerfile b/services/Archival/FilterLists.Archival.Api/Dockerfile index 7db35fa50..f7f1a2adb 100644 --- a/services/Archival/FilterLists.Archival.Api/Dockerfile +++ b/services/Archival/FilterLists.Archival.Api/Dockerfile @@ -12,6 +12,7 @@ ENV DOTNET_CLI_TELEMETRY_OPTOUT=true WORKDIR /app COPY SharedKernel/FilterLists.SharedKernel.Logging/FilterLists.SharedKernel.Logging.csproj SharedKernel/FilterLists.SharedKernel.Logging/ COPY Directory/FilterLists.Directory.Api.Contracts/FilterLists.Directory.Api.Contracts.csproj Directory/FilterLists.Directory.Api.Contracts/ +COPY Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj Archival/FilterLists.Archival.Domain/ COPY Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj Archival/FilterLists.Archival.Infrastructure/ COPY Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj Archival/FilterLists.Archival.Application/ WORKDIR /app/Archival/FilterLists.Archival.Api @@ -23,6 +24,7 @@ WORKDIR /app COPY /.editorconfig . COPY SharedKernel/FilterLists.SharedKernel.Logging/. SharedKernel/FilterLists.SharedKernel.Logging/ COPY Directory/FilterLists.Directory.Api.Contracts/. Directory/FilterLists.Directory.Api.Contracts/ +COPY Archival/FilterLists.Archival.Domain/. Archival/FilterLists.Archival.Domain/ COPY Archival/FilterLists.Archival.Infrastructure/. Archival/FilterLists.Archival.Infrastructure/ COPY Archival/FilterLists.Archival.Application/. Archival/FilterLists.Archival.Application/ WORKDIR /app/Archival/FilterLists.Archival.Api diff --git a/services/Archival/FilterLists.Archival.Api/dev.Dockerfile b/services/Archival/FilterLists.Archival.Api/dev.Dockerfile index d0ade5482..f3d00904d 100644 --- a/services/Archival/FilterLists.Archival.Api/dev.Dockerfile +++ b/services/Archival/FilterLists.Archival.Api/dev.Dockerfile @@ -11,6 +11,7 @@ ENV DOTNET_CLI_TELEMETRY_OPTOUT=true WORKDIR /app COPY SharedKernel/FilterLists.SharedKernel.Logging/FilterLists.SharedKernel.Logging.csproj SharedKernel/FilterLists.SharedKernel.Logging/ COPY Directory/FilterLists.Directory.Api.Contracts/FilterLists.Directory.Api.Contracts.csproj Directory/FilterLists.Directory.Api.Contracts/ +COPY Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj Archival/FilterLists.Archival.Domain/ COPY Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj Archival/FilterLists.Archival.Infrastructure/ COPY Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj Archival/FilterLists.Archival.Application/ WORKDIR /app/Archival/FilterLists.Archival.Api @@ -22,6 +23,7 @@ WORKDIR /app COPY /.editorconfig . COPY SharedKernel/FilterLists.SharedKernel.Logging/. SharedKernel/FilterLists.SharedKernel.Logging/ COPY Directory/FilterLists.Directory.Api.Contracts/. Directory/FilterLists.Directory.Api.Contracts/ +COPY Archival/FilterLists.Archival.Domain/. Archival/FilterLists.Archival.Domain/ COPY Archival/FilterLists.Archival.Infrastructure/. Archival/FilterLists.Archival.Infrastructure/ COPY Archival/FilterLists.Archival.Application/. Archival/FilterLists.Archival.Application/ WORKDIR /app/Archival/FilterLists.Archival.Api diff --git a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs index 13da161d3..8292a5132 100644 --- a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs +++ b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs @@ -6,14 +6,12 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; -using FilterLists.Archival.Application.Models; +using FilterLists.Archival.Domain.Lists; using FilterLists.Archival.Infrastructure.Clients; -using FilterLists.Archival.Infrastructure.Persistence; using FilterLists.Directory.Api.Contracts; using FilterLists.Directory.Api.Contracts.Models; using MediatR; using Microsoft.Extensions.Logging; -using File = FilterLists.Archival.Application.Models.File; namespace FilterLists.Archival.Application.Commands { @@ -34,18 +32,18 @@ public class Handler : IRequestHandler private readonly IHttpContentClient _client; private readonly IDirectoryApi _directory; private readonly ILogger _logger; - private readonly IFileRepository _repo; + private readonly IListArchiveRepository _repo; public Handler( IHttpContentClient httpContentClient, IDirectoryApi directoryApi, ILogger logger, - IFileRepository fileRepository) + IListArchiveRepository listArchiveRepository) { _client = httpContentClient; _directory = directoryApi; _logger = logger; - _repo = fileRepository; + _repo = listArchiveRepository; } public async Task Handle(Command request, CancellationToken cancellationToken) @@ -56,8 +54,8 @@ public async Task Handle(Command request, CancellationToken cancellationTo var segmentUrls = (await GetSegmentUrlsAsync(request.ListId, cancellationToken)).ToList(); if (segmentUrls.Count > 0) { - var file = GetFileToArchive(request.ListId, segmentUrls, cancellationToken); - await _repo.AddFileAsync(file, cancellationToken); + var list = GetList(request.ListId, segmentUrls, cancellationToken); + await _repo.AddAsync(list, cancellationToken); _repo.Commit(); _logger.LogInformation( @@ -83,28 +81,26 @@ private async Task> GetSegmentUrlsAsync( new List(); } - private IFile GetFileToArchive( + private ListArchive GetList( int listId, IEnumerable segmentUrls, CancellationToken cancellationToken) { var segmentsAsync = GetSegmentsAsync(segmentUrls, cancellationToken); var target = listId.ToString(CultureInfo.InvariantCulture).PadLeft(5, '0'); - return new File(segmentsAsync, target); + return new ListArchive(segmentsAsync, target); } - private async IAsyncEnumerable GetSegmentsAsync( + private async IAsyncEnumerable GetSegmentsAsync( IEnumerable segmentUrls, [EnumeratorCancellation] CancellationToken cancellationToken) { foreach (var segment in segmentUrls) { - var sourceFileName = Uri.UnescapeDataString(segment.Url.Segments.Last()); - var sourceExtension = Path.GetExtension(sourceFileName); var contentAsync = await _client.GetContentAsync(segment.Url, cancellationToken); if (contentAsync != Stream.Null) { - yield return new FileSegment(sourceExtension, contentAsync); + yield return new ListArchiveSegment(segment.Url, contentAsync); } } } diff --git a/services/Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj b/services/Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj index 7b1deb630..43bfd25ff 100644 --- a/services/Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj +++ b/services/Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj @@ -24,6 +24,7 @@ + diff --git a/services/Archival/FilterLists.Archival.Application/Models/File.cs b/services/Archival/FilterLists.Archival.Application/Models/File.cs deleted file mode 100644 index aec39b45d..000000000 --- a/services/Archival/FilterLists.Archival.Application/Models/File.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using FilterLists.Archival.Infrastructure.Persistence; - -namespace FilterLists.Archival.Application.Models -{ - internal class File : IFile - { - public File(IAsyncEnumerable segments, string targetFileName) - { - Segments = segments; - TargetFileName = targetFileName; - } - - public IAsyncEnumerable Segments { get; } - public string TargetFileName { get; } - } - - internal class FileSegment : IFileSegment - { - public FileSegment(string sourceExtension, Stream contents) - { - SourceExtension = sourceExtension; - Contents = contents; - } - - public string SourceExtension { get; } - public Stream Contents { get; } - } -} diff --git a/services/Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj b/services/Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj new file mode 100644 index 000000000..d2b2d9161 --- /dev/null +++ b/services/Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj @@ -0,0 +1,26 @@ + + + + netstandard2.1 + latest + enable + + + + Collin M. Barrett + FilterLists + FilterLists Archival + The independent, comprehensive directory of filter and host lists for advertisements, trackers, malware, and annoyances. + Copyright (c) 2020 Collin M. Barrett + https://github.com/collinbarrett/FilterLists + https://filterlists.com + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + \ No newline at end of file diff --git a/services/Archival/FilterLists.Archival.Domain/Lists/IListArchiveRepository.cs b/services/Archival/FilterLists.Archival.Domain/Lists/IListArchiveRepository.cs new file mode 100644 index 000000000..2eb3450a6 --- /dev/null +++ b/services/Archival/FilterLists.Archival.Domain/Lists/IListArchiveRepository.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; +using FilterLists.Archival.Domain.SeedWork; + +namespace FilterLists.Archival.Domain.Lists +{ + public interface IListArchiveRepository : IUnitOfWork + { + Task AddAsync(ListArchive listArchive, CancellationToken cancellationToken); + } +} diff --git a/services/Archival/FilterLists.Archival.Domain/Lists/ListArchive.cs b/services/Archival/FilterLists.Archival.Domain/Lists/ListArchive.cs new file mode 100644 index 000000000..2db80be78 --- /dev/null +++ b/services/Archival/FilterLists.Archival.Domain/Lists/ListArchive.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; + +namespace FilterLists.Archival.Domain.Lists +{ + public class ListArchive + { + public ListArchive(IAsyncEnumerable segments, string targetFileName) + { + Segments = segments; + TargetFileName = targetFileName; + } + + public IAsyncEnumerable Segments { get; } + public string TargetFileName { get; } + } +} diff --git a/services/Archival/FilterLists.Archival.Domain/Lists/ListArchiveSegment.cs b/services/Archival/FilterLists.Archival.Domain/Lists/ListArchiveSegment.cs new file mode 100644 index 000000000..245ff7112 --- /dev/null +++ b/services/Archival/FilterLists.Archival.Domain/Lists/ListArchiveSegment.cs @@ -0,0 +1,17 @@ +using System; +using System.IO; + +namespace FilterLists.Archival.Domain.Lists +{ + public class ListArchiveSegment + { + public ListArchiveSegment(Uri sourceUri, Stream content) + { + Extension = ListFileExtension.FromUri(sourceUri); + Content = content; + } + + public ListFileExtension Extension { get; } + public Stream Content { get; } + } +} diff --git a/services/Archival/FilterLists.Archival.Domain/Lists/ListFileExtension.cs b/services/Archival/FilterLists.Archival.Domain/Lists/ListFileExtension.cs new file mode 100644 index 000000000..b583de7a1 --- /dev/null +++ b/services/Archival/FilterLists.Archival.Domain/Lists/ListFileExtension.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using FilterLists.Archival.Domain.SeedWork; + +namespace FilterLists.Archival.Domain.Lists +{ + public sealed class ListFileExtension : ValueObject + { + private static readonly IDictionary Info = + new Dictionary + { + {string.Empty, (true, false)}, + {".txt", (true, false)}, + {".7z", (false, false)}, + {".action", (true, true)}, // Privoxy + {".all", (true, false)}, + {".aspx", (true, false)}, + {".bat", (true, true)}, + {".blacklist", (true, false)}, + {".cidr", (true, true)}, + {".conf", (true, true)}, // dnsmasq / Unbound / BIND + {".csv", (true, true)}, + {".dat", (true, true)}, // Halite for Windows + {".deny", (true, true)}, + {".gz", (false, false)}, + {".hosts", (true, true)}, + {".ips", (true, false)}, + {".ipset", (true, true)}, // Firehol + {".json", (true, true)}, + {".list", (true, false)}, + {".lsrules", (true, true)}, // Little Snitch + {".md", (true, true)}, + {".netset", (true, true)}, // Firehol + {".p2p", (true, true)}, // Peer Guardian + {".php", (true, false)}, + {".raw", (true, false)}, + {".rpz", (true, true)}, // Response Policy Zone + {".tpl", (true, true)}, // Internet Explorer + {".uBl", (true, false)}, + {".zip", (false, false)}, + {".zone", (true, false)} + }; + + private ListFileExtension(string value) + { + Value = value; + } + + public string Value { get; } + + public bool IsPlainText => Info[Value].IsPlainText; + + public bool IsMeaningfulToConsumer => Info[Value].IsMeaningfulToConsumer; + + public static ListFileExtension FromUri(Uri uri) + { + _ = uri ?? throw new ArgumentNullException(nameof(uri)); + + if (!uri.IsAbsoluteUri) + { + // TODO: implement + throw new NotImplementedException(); + } + + var extension = Path.GetExtension(Uri.UnescapeDataString(uri.Segments.Last())); + return new ListFileExtension(extension); + } + + protected override IEnumerable GetEqualityComponents() + { + return new[] {Value}; + } + } +} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/SeedWork/IUnitOfWork.cs b/services/Archival/FilterLists.Archival.Domain/SeedWork/IUnitOfWork.cs similarity index 65% rename from services/Archival/FilterLists.Archival.Infrastructure/SeedWork/IUnitOfWork.cs rename to services/Archival/FilterLists.Archival.Domain/SeedWork/IUnitOfWork.cs index adf6a2c78..1034e3630 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/SeedWork/IUnitOfWork.cs +++ b/services/Archival/FilterLists.Archival.Domain/SeedWork/IUnitOfWork.cs @@ -1,6 +1,6 @@ using System; -namespace FilterLists.Archival.Infrastructure.SeedWork +namespace FilterLists.Archival.Domain.SeedWork { public interface IUnitOfWork : IDisposable { diff --git a/services/Archival/FilterLists.Archival.Domain/SeedWork/ValueObject.cs b/services/Archival/FilterLists.Archival.Domain/SeedWork/ValueObject.cs new file mode 100644 index 000000000..6676bd00c --- /dev/null +++ b/services/Archival/FilterLists.Archival.Domain/SeedWork/ValueObject.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace FilterLists.Archival.Domain.SeedWork +{ + public abstract class ValueObject + { + protected static bool EqualOperator(ValueObject left, ValueObject right) + { + _ = left ?? throw new ArgumentNullException(nameof(left)); + + return left.Equals(right); + } + + protected static bool NotEqualOperator(ValueObject left, ValueObject right) + { + return !EqualOperator(left, right); + } + + protected abstract IEnumerable GetEqualityComponents(); + + public override bool Equals(object obj) + { + if (obj?.GetType() != GetType()) + { + return false; + } + + var other = (ValueObject)obj; + + return GetEqualityComponents().SequenceEqual(other.GetEqualityComponents()); + } + + public override int GetHashCode() + { + return GetEqualityComponents() + .Select(x => x?.GetHashCode() ?? 0) + .Aggregate((x, y) => x ^ y); + } + } +} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj b/services/Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj index b194a44e0..232dd2943 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj +++ b/services/Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj @@ -31,6 +31,7 @@ + \ No newline at end of file diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs index 05beacce5..729e0c63a 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs @@ -1,4 +1,5 @@ -using FilterLists.Archival.Infrastructure.Options; +using FilterLists.Archival.Domain.Lists; +using FilterLists.Archival.Infrastructure.Options; using LibGit2Sharp; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -20,7 +21,7 @@ public static void AddPersistenceServices(this IServiceCollection services, ICon return new Repository(gitOptions.RepositoryPath); }); - services.AddTransient(); + services.AddTransient(); } } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToPlainTextConversionStrategy.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToPlainTextConversionStrategy.cs index 3384ace74..30a811d60 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToPlainTextConversionStrategy.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToPlainTextConversionStrategy.cs @@ -1,10 +1,11 @@ using System.IO; using System.Threading; +using FilterLists.Archival.Domain.Lists; namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies { internal interface IStreamToPlainTextConversionStrategy { - Stream Convert(IFileSegment fileSegment, CancellationToken cancellationToken); + Stream Convert(ListArchiveSegment listArchiveSegment, CancellationToken cancellationToken); } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/PlainText.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/PlainText.cs index 994358a5f..5a9641581 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/PlainText.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/PlainText.cs @@ -1,16 +1,17 @@ using System; using System.IO; using System.Threading; +using FilterLists.Archival.Domain.Lists; namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies { - public class PlainText : IStreamToPlainTextConversionStrategy + internal class PlainText : IStreamToPlainTextConversionStrategy { - public Stream Convert(IFileSegment fileSegment, CancellationToken cancellationToken) + public Stream Convert(ListArchiveSegment listArchiveSegment, CancellationToken cancellationToken) { - _ = fileSegment ?? throw new ArgumentNullException(nameof(fileSegment)); + _ = listArchiveSegment ?? throw new ArgumentNullException(nameof(listArchiveSegment)); - return fileSegment.Contents; + return listArchiveSegment.Content; } } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/StreamToPlainTextConversionStrategyFactory.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/StreamToPlainTextConversionStrategyFactory.cs index 0437abdc0..4308ca3e8 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/StreamToPlainTextConversionStrategyFactory.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/StreamToPlainTextConversionStrategyFactory.cs @@ -1,22 +1,15 @@ -using System; -using System.Collections.Generic; +using FilterLists.Archival.Domain.Lists; namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies { internal static class StreamToPlainTextConversionStrategyFactory { - private static readonly IDictionary> Strategies = - new Dictionary> - { - {string.Empty, () => new PlainText()}, - {".txt", () => new PlainText()} - }; - - public static TStrategy? GetStrategy(this IFileSegment segment) + public static TStrategy? GetStrategy(this ListArchiveSegment segment) where TStrategy : class, IStreamToPlainTextConversionStrategy { - return Strategies.TryGetValue(segment.SourceExtension, out var strategy) - ? (TStrategy?)strategy() + // TODO: implement non-plain text strategies + return segment.Extension.IsPlainText + ? new PlainText() as TStrategy : default; } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileRepository.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitListArchiveRepository.cs similarity index 62% rename from services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileRepository.cs rename to services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitListArchiveRepository.cs index d122c78da..b52ce39a5 100644 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFileRepository.cs +++ b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitListArchiveRepository.cs @@ -4,29 +4,24 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using FilterLists.Archival.Domain.Lists; using FilterLists.Archival.Infrastructure.Options; using FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies; -using FilterLists.Archival.Infrastructure.SeedWork; using LibGit2Sharp; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace FilterLists.Archival.Infrastructure.Persistence { - public interface IFileRepository : IUnitOfWork - { - Task AddFileAsync(IFile file, CancellationToken cancellationToken); - } - - internal sealed class GitFileRepository : IFileRepository + internal sealed class GitListArchiveRepository : IListArchiveRepository { private readonly ILogger _logger; private readonly GitOptions _options; private readonly IRepository _repo; private readonly ICollection _writtenFiles = new HashSet(); - public GitFileRepository( - ILogger logger, + public GitListArchiveRepository( + ILogger logger, IOptions gitOptions, IRepository repository) { @@ -35,39 +30,48 @@ public GitFileRepository( _repo = repository; } - public async Task AddFileAsync(IFile file, CancellationToken cancellationToken) + public async Task AddAsync(ListArchive listArchive, CancellationToken cancellationToken) { - var textStreams = new List(); - await foreach (var segment in file.Segments.WithCancellation(cancellationToken)) + int segmentCount = 0; + await foreach (var segment in listArchive.Segments.WithCancellation(cancellationToken)) { var strategy = segment.GetStrategy(); if (strategy is default(IStreamToPlainTextConversionStrategy)) { _logger.LogWarning( - "No stream to txt conversion strategy found for extension {Extension} for target {Target}. Skipping file", - segment.SourceExtension, - file.TargetFileName); + "No stream to plain text conversion strategy found for extension {Extension} for target {Target}. Skipping list", + segment.Extension, + listArchive.TargetFileName); return; } - textStreams.Add(strategy.Convert(segment, cancellationToken)); - } - - if (textStreams.Count > 0) - { - _logger.LogInformation("Writing {FileName}", file.TargetFileName); - - var fileInfo = new FileInfo(Path.Combine(_options.RepositoryPath, file.TargetFileName)); - _writtenFiles.Add(fileInfo); - await using var target = fileInfo.OpenWrite(); - - // TODO: validate multi-segment lists are concatenated correctly and in order - foreach (var textStream in textStreams) + string targetExtension; + if (segment.Extension.IsPlainText) { - await textStream.CopyToAsync(target, cancellationToken); + targetExtension = segment.Extension.IsMeaningfulToConsumer ? segment.Extension.Value : ".txt"; + } + else + { + // TODO: implement + _logger.LogWarning( + "Writing from non-plain text extension {Extension} for target {Target} not yet supported. Skipping list", + segment.Extension, + listArchive.TargetFileName); + return; } - _logger.LogInformation("Finished writing {FileName}", file.TargetFileName); + var targetFileName = listArchive.TargetFileName + (segmentCount == 0 ? string.Empty : $"-{segmentCount}") + targetExtension; + var fileInfo = new FileInfo(Path.Combine(_options.RepositoryPath, targetFileName)); + _writtenFiles.Add(fileInfo); + + _logger.LogInformation("Writing {FileName}", fileInfo.Name); + + await using var target = fileInfo.OpenWrite(); + await segment.Content.CopyToAsync(target, cancellationToken); + + _logger.LogInformation("Finished writing {FileName}", fileInfo.Name); + + segmentCount++; } } diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFile.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFile.cs deleted file mode 100644 index 1989376c3..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/IFile.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System.Collections.Generic; -using System.IO; - -namespace FilterLists.Archival.Infrastructure.Persistence -{ - public interface IFile - { - IAsyncEnumerable Segments { get; } - string TargetFileName { get; } - } - - public interface IFileSegment - { - string SourceExtension { get; } - Stream Contents { get; } - } -} diff --git a/services/Archival/azure-pipelines.api.yaml b/services/Archival/azure-pipelines.api.yaml index e0250c3c1..4a56a8eeb 100644 --- a/services/Archival/azure-pipelines.api.yaml +++ b/services/Archival/azure-pipelines.api.yaml @@ -8,6 +8,7 @@ trigger: services/Archival/azure-pipelines.api.yaml, services/SharedKernel/FilterLists.SharedKernel.Logging/*, services/Directory/FilterLists.Directory.Api.Contracts/*, + services/Archival/FilterLists.Archival.Domain/*, services/Archival/FilterLists.Archival.Infrastructure/*, services/Archival/FilterLists.Archival.Application/*, services/Archival/FilterLists.Archival.Api/*, @@ -27,6 +28,7 @@ pr: services/Archival/azure-pipelines.api.yaml, services/SharedKernel/FilterLists.SharedKernel.Logging/*, services/Directory/FilterLists.Directory.Api.Contracts/*, + services/Archival/FilterLists.Archival.Domain/*, services/Archival/FilterLists.Archival.Infrastructure/*, services/Archival/FilterLists.Archival.Application/*, services/Archival/FilterLists.Archival.Api/*, diff --git a/services/FilterLists.sln b/services/FilterLists.sln index b84859506..6e0e7d9d5 100644 --- a/services/FilterLists.sln +++ b/services/FilterLists.sln @@ -66,6 +66,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.SharedKernel.Lo EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.Directory.Api.Contracts", "Directory\FilterLists.Directory.Api.Contracts\FilterLists.Directory.Api.Contracts.csproj", "{4D3D2508-CCC6-4C0B-B5E4-734DD5EC588F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FilterLists.Archival.Domain", "Archival\FilterLists.Archival.Domain\FilterLists.Archival.Domain.csproj", "{6402E81A-75C2-422F-933B-AAC1ED8C4128}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -116,6 +118,10 @@ Global {4D3D2508-CCC6-4C0B-B5E4-734DD5EC588F}.Debug|Any CPU.Build.0 = Debug|Any CPU {4D3D2508-CCC6-4C0B-B5E4-734DD5EC588F}.Release|Any CPU.ActiveCfg = Release|Any CPU {4D3D2508-CCC6-4C0B-B5E4-734DD5EC588F}.Release|Any CPU.Build.0 = Release|Any CPU + {6402E81A-75C2-422F-933B-AAC1ED8C4128}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6402E81A-75C2-422F-933B-AAC1ED8C4128}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6402E81A-75C2-422F-933B-AAC1ED8C4128}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6402E81A-75C2-422F-933B-AAC1ED8C4128}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -132,6 +138,7 @@ Global {A0B77015-E8C4-41D3-B63D-BF818026298C} = {5E52FEF9-9605-4ACB-8A42-A20E06331599} {5485F55E-DFED-4DA0-8A00-220B081FAC96} = {59197818-6C72-4A3C-A595-49F3D7D2EB18} {4D3D2508-CCC6-4C0B-B5E4-734DD5EC588F} = {AE5D6471-1B6E-4B06-A313-34EF81F35342} + {6402E81A-75C2-422F-933B-AAC1ED8C4128} = {5E52FEF9-9605-4ACB-8A42-A20E06331599} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {758B57EF-7505-4BE2-90A2-E2DE2EC32909} diff --git a/services/FilterLists.sln.DotSettings b/services/FilterLists.sln.DotSettings index 992684d36..148f60e7a 100644 --- a/services/FilterLists.sln.DotSettings +++ b/services/FilterLists.sln.DotSettings @@ -14,4 +14,5 @@ True True True + True True \ No newline at end of file