mirror of
https://github.com/collinbarrett/FilterLists.git
synced 2026-03-11 09:04:27 +00:00
feat(archival): ✨♻ handle more source extensions, extract domain layer
This commit is contained in:
parent
09afeb3373
commit
24d8dc045f
22 changed files with 264 additions and 111 deletions
|
|
@ -12,6 +12,7 @@ ENV DOTNET_CLI_TELEMETRY_OPTOUT=true
|
|||
WORKDIR /app
|
||||
COPY SharedKernel/FilterLists.SharedKernel.Logging/FilterLists.SharedKernel.Logging.csproj SharedKernel/FilterLists.SharedKernel.Logging/
|
||||
COPY Directory/FilterLists.Directory.Api.Contracts/FilterLists.Directory.Api.Contracts.csproj Directory/FilterLists.Directory.Api.Contracts/
|
||||
COPY Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj Archival/FilterLists.Archival.Domain/
|
||||
COPY Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj Archival/FilterLists.Archival.Infrastructure/
|
||||
COPY Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj Archival/FilterLists.Archival.Application/
|
||||
WORKDIR /app/Archival/FilterLists.Archival.Api
|
||||
|
|
@ -23,6 +24,7 @@ WORKDIR /app
|
|||
COPY /.editorconfig .
|
||||
COPY SharedKernel/FilterLists.SharedKernel.Logging/. SharedKernel/FilterLists.SharedKernel.Logging/
|
||||
COPY Directory/FilterLists.Directory.Api.Contracts/. Directory/FilterLists.Directory.Api.Contracts/
|
||||
COPY Archival/FilterLists.Archival.Domain/. Archival/FilterLists.Archival.Domain/
|
||||
COPY Archival/FilterLists.Archival.Infrastructure/. Archival/FilterLists.Archival.Infrastructure/
|
||||
COPY Archival/FilterLists.Archival.Application/. Archival/FilterLists.Archival.Application/
|
||||
WORKDIR /app/Archival/FilterLists.Archival.Api
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ ENV DOTNET_CLI_TELEMETRY_OPTOUT=true
|
|||
WORKDIR /app
|
||||
COPY SharedKernel/FilterLists.SharedKernel.Logging/FilterLists.SharedKernel.Logging.csproj SharedKernel/FilterLists.SharedKernel.Logging/
|
||||
COPY Directory/FilterLists.Directory.Api.Contracts/FilterLists.Directory.Api.Contracts.csproj Directory/FilterLists.Directory.Api.Contracts/
|
||||
COPY Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj Archival/FilterLists.Archival.Domain/
|
||||
COPY Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj Archival/FilterLists.Archival.Infrastructure/
|
||||
COPY Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj Archival/FilterLists.Archival.Application/
|
||||
WORKDIR /app/Archival/FilterLists.Archival.Api
|
||||
|
|
@ -22,6 +23,7 @@ WORKDIR /app
|
|||
COPY /.editorconfig .
|
||||
COPY SharedKernel/FilterLists.SharedKernel.Logging/. SharedKernel/FilterLists.SharedKernel.Logging/
|
||||
COPY Directory/FilterLists.Directory.Api.Contracts/. Directory/FilterLists.Directory.Api.Contracts/
|
||||
COPY Archival/FilterLists.Archival.Domain/. Archival/FilterLists.Archival.Domain/
|
||||
COPY Archival/FilterLists.Archival.Infrastructure/. Archival/FilterLists.Archival.Infrastructure/
|
||||
COPY Archival/FilterLists.Archival.Application/. Archival/FilterLists.Archival.Application/
|
||||
WORKDIR /app/Archival/FilterLists.Archival.Api
|
||||
|
|
|
|||
|
|
@ -6,14 +6,12 @@
|
|||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FilterLists.Archival.Application.Models;
|
||||
using FilterLists.Archival.Domain.Lists;
|
||||
using FilterLists.Archival.Infrastructure.Clients;
|
||||
using FilterLists.Archival.Infrastructure.Persistence;
|
||||
using FilterLists.Directory.Api.Contracts;
|
||||
using FilterLists.Directory.Api.Contracts.Models;
|
||||
using MediatR;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using File = FilterLists.Archival.Application.Models.File;
|
||||
|
||||
namespace FilterLists.Archival.Application.Commands
|
||||
{
|
||||
|
|
@ -34,18 +32,18 @@ public class Handler : IRequestHandler<Command, Unit>
|
|||
private readonly IHttpContentClient _client;
|
||||
private readonly IDirectoryApi _directory;
|
||||
private readonly ILogger _logger;
|
||||
private readonly IFileRepository _repo;
|
||||
private readonly IListArchiveRepository _repo;
|
||||
|
||||
public Handler(
|
||||
IHttpContentClient httpContentClient,
|
||||
IDirectoryApi directoryApi,
|
||||
ILogger<Handler> logger,
|
||||
IFileRepository fileRepository)
|
||||
IListArchiveRepository listArchiveRepository)
|
||||
{
|
||||
_client = httpContentClient;
|
||||
_directory = directoryApi;
|
||||
_logger = logger;
|
||||
_repo = fileRepository;
|
||||
_repo = listArchiveRepository;
|
||||
}
|
||||
|
||||
public async Task<Unit> Handle(Command request, CancellationToken cancellationToken)
|
||||
|
|
@ -56,8 +54,8 @@ public async Task<Unit> Handle(Command request, CancellationToken cancellationTo
|
|||
var segmentUrls = (await GetSegmentUrlsAsync(request.ListId, cancellationToken)).ToList();
|
||||
if (segmentUrls.Count > 0)
|
||||
{
|
||||
var file = GetFileToArchive(request.ListId, segmentUrls, cancellationToken);
|
||||
await _repo.AddFileAsync(file, cancellationToken);
|
||||
var list = GetList(request.ListId, segmentUrls, cancellationToken);
|
||||
await _repo.AddAsync(list, cancellationToken);
|
||||
_repo.Commit();
|
||||
|
||||
_logger.LogInformation(
|
||||
|
|
@ -83,28 +81,26 @@ private async Task<IEnumerable<ListDetailsViewUrlVm>> GetSegmentUrlsAsync(
|
|||
new List<ListDetailsViewUrlVm>();
|
||||
}
|
||||
|
||||
private IFile GetFileToArchive(
|
||||
private ListArchive GetList(
|
||||
int listId,
|
||||
IEnumerable<ListDetailsViewUrlVm> segmentUrls,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var segmentsAsync = GetSegmentsAsync(segmentUrls, cancellationToken);
|
||||
var target = listId.ToString(CultureInfo.InvariantCulture).PadLeft(5, '0');
|
||||
return new File(segmentsAsync, target);
|
||||
return new ListArchive(segmentsAsync, target);
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<IFileSegment> GetSegmentsAsync(
|
||||
private async IAsyncEnumerable<ListArchiveSegment> GetSegmentsAsync(
|
||||
IEnumerable<ListDetailsViewUrlVm> segmentUrls,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var segment in segmentUrls)
|
||||
{
|
||||
var sourceFileName = Uri.UnescapeDataString(segment.Url.Segments.Last());
|
||||
var sourceExtension = Path.GetExtension(sourceFileName);
|
||||
var contentAsync = await _client.GetContentAsync(segment.Url, cancellationToken);
|
||||
if (contentAsync != Stream.Null)
|
||||
{
|
||||
yield return new FileSegment(sourceExtension, contentAsync);
|
||||
yield return new ListArchiveSegment(segment.Url, contentAsync);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@
|
|||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\FilterLists.Archival.Domain\FilterLists.Archival.Domain.csproj" />
|
||||
<ProjectReference Include="..\FilterLists.Archival.Infrastructure\FilterLists.Archival.Infrastructure.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using FilterLists.Archival.Infrastructure.Persistence;
|
||||
|
||||
namespace FilterLists.Archival.Application.Models
|
||||
{
|
||||
internal class File : IFile
|
||||
{
|
||||
public File(IAsyncEnumerable<IFileSegment> segments, string targetFileName)
|
||||
{
|
||||
Segments = segments;
|
||||
TargetFileName = targetFileName;
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<IFileSegment> Segments { get; }
|
||||
public string TargetFileName { get; }
|
||||
}
|
||||
|
||||
internal class FileSegment : IFileSegment
|
||||
{
|
||||
public FileSegment(string sourceExtension, Stream contents)
|
||||
{
|
||||
SourceExtension = sourceExtension;
|
||||
Contents = contents;
|
||||
}
|
||||
|
||||
public string SourceExtension { get; }
|
||||
public Stream Contents { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.1</TargetFramework>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<Authors>Collin M. Barrett</Authors>
|
||||
<Company>FilterLists</Company>
|
||||
<Product>FilterLists Archival</Product>
|
||||
<Description>The independent, comprehensive directory of filter and host lists for advertisements, trackers, malware, and annoyances.</Description>
|
||||
<Copyright>Copyright (c) 2020 Collin M. Barrett</Copyright>
|
||||
<RepositoryUrl>https://github.com/collinbarrett/FilterLists</RepositoryUrl>
|
||||
<PackageProjectUrl>https://filterlists.com</PackageProjectUrl>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="3.3.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FilterLists.Archival.Domain.SeedWork;
|
||||
|
||||
namespace FilterLists.Archival.Domain.Lists
|
||||
{
|
||||
public interface IListArchiveRepository : IUnitOfWork
|
||||
{
|
||||
Task AddAsync(ListArchive listArchive, CancellationToken cancellationToken);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
using System.Collections.Generic;
|
||||
|
||||
namespace FilterLists.Archival.Domain.Lists
|
||||
{
|
||||
public class ListArchive
|
||||
{
|
||||
public ListArchive(IAsyncEnumerable<ListArchiveSegment> segments, string targetFileName)
|
||||
{
|
||||
Segments = segments;
|
||||
TargetFileName = targetFileName;
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<ListArchiveSegment> Segments { get; }
|
||||
public string TargetFileName { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace FilterLists.Archival.Domain.Lists
|
||||
{
|
||||
public class ListArchiveSegment
|
||||
{
|
||||
public ListArchiveSegment(Uri sourceUri, Stream content)
|
||||
{
|
||||
Extension = ListFileExtension.FromUri(sourceUri);
|
||||
Content = content;
|
||||
}
|
||||
|
||||
public ListFileExtension Extension { get; }
|
||||
public Stream Content { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using FilterLists.Archival.Domain.SeedWork;
|
||||
|
||||
namespace FilterLists.Archival.Domain.Lists
|
||||
{
|
||||
public sealed class ListFileExtension : ValueObject
|
||||
{
|
||||
private static readonly IDictionary<string, (bool IsPlainText, bool IsMeaningfulToConsumer)> Info =
|
||||
new Dictionary<string, (bool, bool)>
|
||||
{
|
||||
{string.Empty, (true, false)},
|
||||
{".txt", (true, false)},
|
||||
{".7z", (false, false)},
|
||||
{".action", (true, true)}, // Privoxy
|
||||
{".all", (true, false)},
|
||||
{".aspx", (true, false)},
|
||||
{".bat", (true, true)},
|
||||
{".blacklist", (true, false)},
|
||||
{".cidr", (true, true)},
|
||||
{".conf", (true, true)}, // dnsmasq / Unbound / BIND
|
||||
{".csv", (true, true)},
|
||||
{".dat", (true, true)}, // Halite for Windows
|
||||
{".deny", (true, true)},
|
||||
{".gz", (false, false)},
|
||||
{".hosts", (true, true)},
|
||||
{".ips", (true, false)},
|
||||
{".ipset", (true, true)}, // Firehol
|
||||
{".json", (true, true)},
|
||||
{".list", (true, false)},
|
||||
{".lsrules", (true, true)}, // Little Snitch
|
||||
{".md", (true, true)},
|
||||
{".netset", (true, true)}, // Firehol
|
||||
{".p2p", (true, true)}, // Peer Guardian
|
||||
{".php", (true, false)},
|
||||
{".raw", (true, false)},
|
||||
{".rpz", (true, true)}, // Response Policy Zone
|
||||
{".tpl", (true, true)}, // Internet Explorer
|
||||
{".uBl", (true, false)},
|
||||
{".zip", (false, false)},
|
||||
{".zone", (true, false)}
|
||||
};
|
||||
|
||||
private ListFileExtension(string value)
|
||||
{
|
||||
Value = value;
|
||||
}
|
||||
|
||||
public string Value { get; }
|
||||
|
||||
public bool IsPlainText => Info[Value].IsPlainText;
|
||||
|
||||
public bool IsMeaningfulToConsumer => Info[Value].IsMeaningfulToConsumer;
|
||||
|
||||
public static ListFileExtension FromUri(Uri uri)
|
||||
{
|
||||
_ = uri ?? throw new ArgumentNullException(nameof(uri));
|
||||
|
||||
if (!uri.IsAbsoluteUri)
|
||||
{
|
||||
// TODO: implement
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
var extension = Path.GetExtension(Uri.UnescapeDataString(uri.Segments.Last()));
|
||||
return new ListFileExtension(extension);
|
||||
}
|
||||
|
||||
protected override IEnumerable<object> GetEqualityComponents()
|
||||
{
|
||||
return new[] {Value};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
using System;
|
||||
|
||||
namespace FilterLists.Archival.Infrastructure.SeedWork
|
||||
namespace FilterLists.Archival.Domain.SeedWork
|
||||
{
|
||||
public interface IUnitOfWork : IDisposable
|
||||
{
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace FilterLists.Archival.Domain.SeedWork
|
||||
{
|
||||
public abstract class ValueObject
|
||||
{
|
||||
protected static bool EqualOperator(ValueObject left, ValueObject right)
|
||||
{
|
||||
_ = left ?? throw new ArgumentNullException(nameof(left));
|
||||
|
||||
return left.Equals(right);
|
||||
}
|
||||
|
||||
protected static bool NotEqualOperator(ValueObject left, ValueObject right)
|
||||
{
|
||||
return !EqualOperator(left, right);
|
||||
}
|
||||
|
||||
protected abstract IEnumerable<object> GetEqualityComponents();
|
||||
|
||||
public override bool Equals(object obj)
|
||||
{
|
||||
if (obj?.GetType() != GetType())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var other = (ValueObject)obj;
|
||||
|
||||
return GetEqualityComponents().SequenceEqual(other.GetEqualityComponents());
|
||||
}
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return GetEqualityComponents()
|
||||
.Select(x => x?.GetHashCode() ?? 0)
|
||||
.Aggregate((x, y) => x ^ y);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -31,6 +31,7 @@
|
|||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\Directory\FilterLists.Directory.Api.Contracts\FilterLists.Directory.Api.Contracts.csproj" />
|
||||
<ProjectReference Include="..\..\SharedKernel\FilterLists.SharedKernel.Logging\FilterLists.SharedKernel.Logging.csproj" />
|
||||
<ProjectReference Include="..\FilterLists.Archival.Domain\FilterLists.Archival.Domain.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
using FilterLists.Archival.Infrastructure.Options;
|
||||
using FilterLists.Archival.Domain.Lists;
|
||||
using FilterLists.Archival.Infrastructure.Options;
|
||||
using LibGit2Sharp;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
|
@ -20,7 +21,7 @@ public static void AddPersistenceServices(this IServiceCollection services, ICon
|
|||
|
||||
return new Repository(gitOptions.RepositoryPath);
|
||||
});
|
||||
services.AddTransient<IFileRepository, GitFileRepository>();
|
||||
services.AddTransient<IListArchiveRepository, GitListArchiveRepository>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
using System.IO;
|
||||
using System.Threading;
|
||||
using FilterLists.Archival.Domain.Lists;
|
||||
|
||||
namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies
|
||||
{
|
||||
internal interface IStreamToPlainTextConversionStrategy
|
||||
{
|
||||
Stream Convert(IFileSegment fileSegment, CancellationToken cancellationToken);
|
||||
Stream Convert(ListArchiveSegment listArchiveSegment, CancellationToken cancellationToken);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +1,17 @@
|
|||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using FilterLists.Archival.Domain.Lists;
|
||||
|
||||
namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies
|
||||
{
|
||||
public class PlainText : IStreamToPlainTextConversionStrategy
|
||||
internal class PlainText : IStreamToPlainTextConversionStrategy
|
||||
{
|
||||
public Stream Convert(IFileSegment fileSegment, CancellationToken cancellationToken)
|
||||
public Stream Convert(ListArchiveSegment listArchiveSegment, CancellationToken cancellationToken)
|
||||
{
|
||||
_ = fileSegment ?? throw new ArgumentNullException(nameof(fileSegment));
|
||||
_ = listArchiveSegment ?? throw new ArgumentNullException(nameof(listArchiveSegment));
|
||||
|
||||
return fileSegment.Contents;
|
||||
return listArchiveSegment.Content;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,22 +1,15 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using FilterLists.Archival.Domain.Lists;
|
||||
|
||||
namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies
|
||||
{
|
||||
internal static class StreamToPlainTextConversionStrategyFactory
|
||||
{
|
||||
private static readonly IDictionary<string, Func<IStreamToPlainTextConversionStrategy>> Strategies =
|
||||
new Dictionary<string, Func<IStreamToPlainTextConversionStrategy>>
|
||||
{
|
||||
{string.Empty, () => new PlainText()},
|
||||
{".txt", () => new PlainText()}
|
||||
};
|
||||
|
||||
public static TStrategy? GetStrategy<TStrategy>(this IFileSegment segment)
|
||||
public static TStrategy? GetStrategy<TStrategy>(this ListArchiveSegment segment)
|
||||
where TStrategy : class, IStreamToPlainTextConversionStrategy
|
||||
{
|
||||
return Strategies.TryGetValue(segment.SourceExtension, out var strategy)
|
||||
? (TStrategy?)strategy()
|
||||
// TODO: implement non-plain text strategies
|
||||
return segment.Extension.IsPlainText
|
||||
? new PlainText() as TStrategy
|
||||
: default;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,29 +4,24 @@
|
|||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FilterLists.Archival.Domain.Lists;
|
||||
using FilterLists.Archival.Infrastructure.Options;
|
||||
using FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies;
|
||||
using FilterLists.Archival.Infrastructure.SeedWork;
|
||||
using LibGit2Sharp;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace FilterLists.Archival.Infrastructure.Persistence
|
||||
{
|
||||
public interface IFileRepository : IUnitOfWork
|
||||
{
|
||||
Task AddFileAsync(IFile file, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class GitFileRepository : IFileRepository
|
||||
internal sealed class GitListArchiveRepository : IListArchiveRepository
|
||||
{
|
||||
private readonly ILogger _logger;
|
||||
private readonly GitOptions _options;
|
||||
private readonly IRepository _repo;
|
||||
private readonly ICollection<FileInfo> _writtenFiles = new HashSet<FileInfo>();
|
||||
|
||||
public GitFileRepository(
|
||||
ILogger<GitFileRepository> logger,
|
||||
public GitListArchiveRepository(
|
||||
ILogger<GitListArchiveRepository> logger,
|
||||
IOptions<GitOptions> gitOptions,
|
||||
IRepository repository)
|
||||
{
|
||||
|
|
@ -35,39 +30,48 @@ public GitFileRepository(
|
|||
_repo = repository;
|
||||
}
|
||||
|
||||
public async Task AddFileAsync(IFile file, CancellationToken cancellationToken)
|
||||
public async Task AddAsync(ListArchive listArchive, CancellationToken cancellationToken)
|
||||
{
|
||||
var textStreams = new List<Stream>();
|
||||
await foreach (var segment in file.Segments.WithCancellation(cancellationToken))
|
||||
int segmentCount = 0;
|
||||
await foreach (var segment in listArchive.Segments.WithCancellation(cancellationToken))
|
||||
{
|
||||
var strategy = segment.GetStrategy<IStreamToPlainTextConversionStrategy>();
|
||||
if (strategy is default(IStreamToPlainTextConversionStrategy))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"No stream to txt conversion strategy found for extension {Extension} for target {Target}. Skipping file",
|
||||
segment.SourceExtension,
|
||||
file.TargetFileName);
|
||||
"No stream to plain text conversion strategy found for extension {Extension} for target {Target}. Skipping list",
|
||||
segment.Extension,
|
||||
listArchive.TargetFileName);
|
||||
return;
|
||||
}
|
||||
|
||||
textStreams.Add(strategy.Convert(segment, cancellationToken));
|
||||
}
|
||||
|
||||
if (textStreams.Count > 0)
|
||||
{
|
||||
_logger.LogInformation("Writing {FileName}", file.TargetFileName);
|
||||
|
||||
var fileInfo = new FileInfo(Path.Combine(_options.RepositoryPath, file.TargetFileName));
|
||||
_writtenFiles.Add(fileInfo);
|
||||
await using var target = fileInfo.OpenWrite();
|
||||
|
||||
// TODO: validate multi-segment lists are concatenated correctly and in order
|
||||
foreach (var textStream in textStreams)
|
||||
string targetExtension;
|
||||
if (segment.Extension.IsPlainText)
|
||||
{
|
||||
await textStream.CopyToAsync(target, cancellationToken);
|
||||
targetExtension = segment.Extension.IsMeaningfulToConsumer ? segment.Extension.Value : ".txt";
|
||||
}
|
||||
else
|
||||
{
|
||||
// TODO: implement
|
||||
_logger.LogWarning(
|
||||
"Writing from non-plain text extension {Extension} for target {Target} not yet supported. Skipping list",
|
||||
segment.Extension,
|
||||
listArchive.TargetFileName);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Finished writing {FileName}", file.TargetFileName);
|
||||
var targetFileName = listArchive.TargetFileName + (segmentCount == 0 ? string.Empty : $"-{segmentCount}") + targetExtension;
|
||||
var fileInfo = new FileInfo(Path.Combine(_options.RepositoryPath, targetFileName));
|
||||
_writtenFiles.Add(fileInfo);
|
||||
|
||||
_logger.LogInformation("Writing {FileName}", fileInfo.Name);
|
||||
|
||||
await using var target = fileInfo.OpenWrite();
|
||||
await segment.Content.CopyToAsync(target, cancellationToken);
|
||||
|
||||
_logger.LogInformation("Finished writing {FileName}", fileInfo.Name);
|
||||
|
||||
segmentCount++;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
||||
namespace FilterLists.Archival.Infrastructure.Persistence
|
||||
{
|
||||
public interface IFile
|
||||
{
|
||||
IAsyncEnumerable<IFileSegment> Segments { get; }
|
||||
string TargetFileName { get; }
|
||||
}
|
||||
|
||||
public interface IFileSegment
|
||||
{
|
||||
string SourceExtension { get; }
|
||||
Stream Contents { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@ trigger:
|
|||
services/Archival/azure-pipelines.api.yaml,
|
||||
services/SharedKernel/FilterLists.SharedKernel.Logging/*,
|
||||
services/Directory/FilterLists.Directory.Api.Contracts/*,
|
||||
services/Archival/FilterLists.Archival.Domain/*,
|
||||
services/Archival/FilterLists.Archival.Infrastructure/*,
|
||||
services/Archival/FilterLists.Archival.Application/*,
|
||||
services/Archival/FilterLists.Archival.Api/*,
|
||||
|
|
@ -27,6 +28,7 @@ pr:
|
|||
services/Archival/azure-pipelines.api.yaml,
|
||||
services/SharedKernel/FilterLists.SharedKernel.Logging/*,
|
||||
services/Directory/FilterLists.Directory.Api.Contracts/*,
|
||||
services/Archival/FilterLists.Archival.Domain/*,
|
||||
services/Archival/FilterLists.Archival.Infrastructure/*,
|
||||
services/Archival/FilterLists.Archival.Application/*,
|
||||
services/Archival/FilterLists.Archival.Api/*,
|
||||
|
|
|
|||
|
|
@ -66,6 +66,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.SharedKernel.Lo
|
|||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.Directory.Api.Contracts", "Directory\FilterLists.Directory.Api.Contracts\FilterLists.Directory.Api.Contracts.csproj", "{4D3D2508-CCC6-4C0B-B5E4-734DD5EC588F}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FilterLists.Archival.Domain", "Archival\FilterLists.Archival.Domain\FilterLists.Archival.Domain.csproj", "{6402E81A-75C2-422F-933B-AAC1ED8C4128}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
|
|
@ -116,6 +118,10 @@ Global
|
|||
{4D3D2508-CCC6-4C0B-B5E4-734DD5EC588F}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{4D3D2508-CCC6-4C0B-B5E4-734DD5EC588F}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{4D3D2508-CCC6-4C0B-B5E4-734DD5EC588F}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{6402E81A-75C2-422F-933B-AAC1ED8C4128}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{6402E81A-75C2-422F-933B-AAC1ED8C4128}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{6402E81A-75C2-422F-933B-AAC1ED8C4128}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{6402E81A-75C2-422F-933B-AAC1ED8C4128}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
@ -132,6 +138,7 @@ Global
|
|||
{A0B77015-E8C4-41D3-B63D-BF818026298C} = {5E52FEF9-9605-4ACB-8A42-A20E06331599}
|
||||
{5485F55E-DFED-4DA0-8A00-220B081FAC96} = {59197818-6C72-4A3C-A595-49F3D7D2EB18}
|
||||
{4D3D2508-CCC6-4C0B-B5E4-734DD5EC588F} = {AE5D6471-1B6E-4B06-A313-34EF81F35342}
|
||||
{6402E81A-75C2-422F-933B-AAC1ED8C4128} = {5E52FEF9-9605-4ACB-8A42-A20E06331599}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {758B57EF-7505-4BE2-90A2-E2DE2EC32909}
|
||||
|
|
|
|||
|
|
@ -14,4 +14,5 @@
|
|||
<s:Boolean x:Key="/Default/UserDictionary/Words/=Hangfire/@EntryIndexedValue">True</s:Boolean>
|
||||
<s:Boolean x:Key="/Default/UserDictionary/Words/=Mediat/@EntryIndexedValue">True</s:Boolean>
|
||||
<s:Boolean x:Key="/Default/UserDictionary/Words/=Primariness/@EntryIndexedValue">True</s:Boolean>
|
||||
<s:Boolean x:Key="/Default/UserDictionary/Words/=Privoxy/@EntryIndexedValue">True</s:Boolean>
|
||||
<s:Boolean x:Key="/Default/UserDictionary/Words/=Syntaxes/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
|
||||
Loading…
Reference in a new issue