diff --git a/src/FilterLists.Agent/Core/List/IListRepository.cs b/src/FilterLists.Agent/Core/List/IListRepository.cs index 3850b34a5..09cc98236 100644 --- a/src/FilterLists.Agent/Core/List/IListRepository.cs +++ b/src/FilterLists.Agent/Core/List/IListRepository.cs @@ -1,10 +1,12 @@ -using System.Net.Http; +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; namespace FilterLists.Agent.Core.List { public interface IListRepository { - //TODO: expose GetList() method rather than HttpClient - HttpClient HttpClient { get; } + Task GetListStreamAsync(Uri url, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/FilterLists.Agent/Features/Lists/DownloadRawText.cs b/src/FilterLists.Agent/Features/Lists/DownloadRawText.cs index 47ab203f9..335d7454a 100644 --- a/src/FilterLists.Agent/Features/Lists/DownloadRawText.cs +++ b/src/FilterLists.Agent/Features/Lists/DownloadRawText.cs @@ -1,12 +1,10 @@ using System.IO; using System.Linq; -using System.Net.Http; using System.Threading; using System.Threading.Tasks; using FilterLists.Agent.Core.List; using FilterLists.Agent.Extensions; using MediatR; -using Microsoft.Extensions.Logging; namespace FilterLists.Agent.Features.Lists { @@ -26,32 +24,27 @@ public class Handler : AsyncRequestHandler { private const string RepoDirectory = "archives"; private readonly string[] _extensionsToRewrite = {"", ".aspx", ".p2p", ".php"}; - private readonly HttpClient _httpClient; - private readonly ILogger _logger; + private readonly IListRepository _repo; - public Handler(IListRepository listRepository, ILogger logger) + public Handler(IListRepository listRepository) { - _httpClient = listRepository.HttpClient; - _logger = logger; + _repo = listRepository; } protected override async Task Handle(Command request, CancellationToken cancellationToken) + { + var destinationPath = BuildDestinationPath(request); + using var input = await _repo.GetListStreamAsync(request.ListUrl.ViewUrl, cancellationToken); + using var output = File.OpenWrite(destinationPath); + await input.CopyToAsync(output, cancellationToken); + } + + private string BuildDestinationPath(Command request) { var sourceExtension = request.ListUrl.ViewUrl.GetExtension(); var destinationExtension = _extensionsToRewrite.Contains(sourceExtension) ? ".txt" : sourceExtension; var destinationPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}{destinationExtension}"); - using (var response = await _httpClient.GetAsync(request.ListUrl.ViewUrl, cancellationToken)) - { - if (response.IsSuccessStatusCode) - using (var input = await response.Content.ReadAsStreamAsync()) - using (var output = File.OpenWrite(destinationPath)) - { - await input.CopyToAsync(output, cancellationToken); - } - else - _logger.LogError( - $"Error downloading list {request.ListUrl.Id} from {request.ListUrl.ViewUrl}. {response.StatusCode}"); - } + return destinationPath; } } } diff --git a/src/FilterLists.Agent/Features/Lists/DownloadSevenZip.cs b/src/FilterLists.Agent/Features/Lists/DownloadSevenZip.cs index c3539e61d..adbe92a32 100644 --- a/src/FilterLists.Agent/Features/Lists/DownloadSevenZip.cs +++ b/src/FilterLists.Agent/Features/Lists/DownloadSevenZip.cs @@ -1,10 +1,8 @@ using System.IO; -using System.Net.Http; using System.Threading; using System.Threading.Tasks; using FilterLists.Agent.Core.List; using MediatR; -using Microsoft.Extensions.Logging; using SharpCompress.Archives; using SharpCompress.Archives.SevenZip; using SharpCompress.Common; @@ -26,33 +24,22 @@ public Command(ListUrl listUrl) public class Handler : AsyncRequestHandler { private const string RepoDirectory = "archives"; - private readonly HttpClient _httpClient; - private readonly ILogger _logger; + private readonly IListRepository _repo; - public Handler(IListRepository listRepository, ILogger logger) + public Handler(IListRepository listRepository) { - _httpClient = listRepository.HttpClient; - _logger = logger; + _repo = listRepository; } protected override async Task Handle(Command request, CancellationToken cancellationToken) { var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}"); - using (var response = await _httpClient.GetAsync(request.ListUrl.ViewUrl, cancellationToken)) - { - if (response.IsSuccessStatusCode) - using (var input = await response.Content.ReadAsStreamAsync()) - using (var archive = SevenZipArchive.Open(input)) - { - foreach (var entry in archive.Entries) - if (!entry.IsDirectory) - entry.WriteToDirectory(destinationDirectoryPath, - new ExtractionOptions {ExtractFullPath = true, Overwrite = true}); - } - else - _logger.LogError( - $"Error downloading list {request.ListUrl.Id} from {request.ListUrl.ViewUrl}. {response.StatusCode}"); - } + using var input = await _repo.GetListStreamAsync(request.ListUrl.ViewUrl, cancellationToken); + using var archive = SevenZipArchive.Open(input); + foreach (var entry in archive.Entries) + if (!entry.IsDirectory) + entry.WriteToDirectory(destinationDirectoryPath, + new ExtractionOptions {ExtractFullPath = true, Overwrite = true}); } } } diff --git a/src/FilterLists.Agent/Features/Lists/DownloadZip.cs b/src/FilterLists.Agent/Features/Lists/DownloadZip.cs index dd4df995a..8c8d2cfbc 100644 --- a/src/FilterLists.Agent/Features/Lists/DownloadZip.cs +++ b/src/FilterLists.Agent/Features/Lists/DownloadZip.cs @@ -1,10 +1,8 @@ using System.IO; -using System.Net.Http; using System.Threading; using System.Threading.Tasks; using FilterLists.Agent.Core.List; using MediatR; -using Microsoft.Extensions.Logging; using SharpCompress.Common; using SharpCompress.Readers; @@ -25,33 +23,22 @@ public Command(ListUrl listUrl) public class Handler : AsyncRequestHandler { private const string RepoDirectory = "archives"; - private readonly HttpClient _httpClient; - private readonly ILogger _logger; + private readonly IListRepository _repo; - public Handler(IListRepository listRepository, ILogger logger) + public Handler(IListRepository listRepository) { - _httpClient = listRepository.HttpClient; - _logger = logger; + _repo = listRepository; } protected override async Task Handle(Command request, CancellationToken cancellationToken) { var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}"); - using (var response = await _httpClient.GetAsync(request.ListUrl.ViewUrl, cancellationToken)) - { - if (response.IsSuccessStatusCode) - using (var input = await response.Content.ReadAsStreamAsync()) - using (var reader = ReaderFactory.Open(input)) - { - while (reader.MoveToNextEntry()) - if (!reader.Entry.IsDirectory) - reader.WriteEntryToDirectory(destinationDirectoryPath, - new ExtractionOptions {ExtractFullPath = true, Overwrite = true}); - } - else - _logger.LogError( - $"Error downloading list {request.ListUrl.Id} from {request.ListUrl.ViewUrl}. {response.StatusCode}"); - } + using var input = await _repo.GetListStreamAsync(request.ListUrl.ViewUrl, cancellationToken); + using var reader = ReaderFactory.Open(input); + while (reader.MoveToNextEntry()) + if (!reader.Entry.IsDirectory) + reader.WriteEntryToDirectory(destinationDirectoryPath, + new ExtractionOptions {ExtractFullPath = true, Overwrite = true}); } } } diff --git a/src/FilterLists.Agent/Infrastructure/ListRepository.cs b/src/FilterLists.Agent/Infrastructure/ListRepository.cs index 2e4ed8f1e..20484db2f 100644 --- a/src/FilterLists.Agent/Infrastructure/ListRepository.cs +++ b/src/FilterLists.Agent/Infrastructure/ListRepository.cs @@ -1,25 +1,39 @@ using System; +using System.IO; using System.Net.Http; using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; using FilterLists.Agent.Core.List; using JetBrains.Annotations; +using Microsoft.Extensions.Logging; namespace FilterLists.Agent.Infrastructure { [UsedImplicitly] public class ListRepository : IListRepository { - public ListRepository(HttpClient client) - { - client.Timeout = TimeSpan.FromSeconds(90); + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + public ListRepository(HttpClient httpClient, ILogger logger) + { + httpClient.Timeout = TimeSpan.FromSeconds(90); var header = new ProductHeaderValue("FilterLists.Agent"); var userAgent = new ProductInfoHeaderValue(header); - client.DefaultRequestHeaders.UserAgent.Add(userAgent); + httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent); + _httpClient = httpClient; - HttpClient = client; + _logger = logger; } - public HttpClient HttpClient { get; } + public async Task GetListStreamAsync(Uri url, CancellationToken cancellationToken) + { + using var response = await _httpClient.GetAsync(url, cancellationToken); + if (response.IsSuccessStatusCode) + return await response.Content.ReadAsStreamAsync(); + _logger.LogError($"Error downloading list from {url}. {response.StatusCode}"); + return Stream.Null; + } } } \ No newline at end of file