prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

encapsulate HttpClient behind IListRepository method

Browse source
This commit is contained in:
Collin M. Barrett 2019-07-09 17:22:51 -05:00
parent 75e19da3fa
commit 2df35847bc
5 changed files with 55 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/FilterLists.Agent/Core/List/IListRepository.cs
View file

@ -1,10 +1,12 @@
using System.Net.Http;
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace FilterLists.Agent.Core.List
{
public interface IListRepository
{
//TODO: expose GetList() method rather than HttpClient
HttpClient HttpClient { get; }
Task<Stream> GetListStreamAsync(Uri url, CancellationToken cancellationToken);
}
}

31
src/FilterLists.Agent/Features/Lists/DownloadRawText.cs
View file

@ -1,12 +1,10 @@
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Core.List;
using FilterLists.Agent.Extensions;
using MediatR;
using Microsoft.Extensions.Logging;
namespace FilterLists.Agent.Features.Lists
{
@ -26,32 +24,27 @@ public class Handler : AsyncRequestHandler<Command>
{
private const string RepoDirectory = "archives";
private readonly string[] _extensionsToRewrite = {"", ".aspx", ".p2p", ".php"};
private readonly HttpClient _httpClient;
private readonly ILogger<Handler> _logger;
private readonly IListRepository _repo;
public Handler(IListRepository listRepository, ILogger<Handler> logger)
public Handler(IListRepository listRepository)
{
_httpClient = listRepository.HttpClient;
_logger = logger;
_repo = listRepository;
}
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
var destinationPath = BuildDestinationPath(request);
using var input = await _repo.GetListStreamAsync(request.ListUrl.ViewUrl, cancellationToken);
using var output = File.OpenWrite(destinationPath);
await input.CopyToAsync(output, cancellationToken);
}
private string BuildDestinationPath(Command request)
{
var sourceExtension = request.ListUrl.ViewUrl.GetExtension();
var destinationExtension = _extensionsToRewrite.Contains(sourceExtension) ? ".txt" : sourceExtension;
var destinationPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}{destinationExtension}");
using (var response = await _httpClient.GetAsync(request.ListUrl.ViewUrl, cancellationToken))
{
if (response.IsSuccessStatusCode)
using (var input = await response.Content.ReadAsStreamAsync())
using (var output = File.OpenWrite(destinationPath))
{
await input.CopyToAsync(output, cancellationToken);
}
else
_logger.LogError(
$"Error downloading list {request.ListUrl.Id} from {request.ListUrl.ViewUrl}. {response.StatusCode}");
}
return destinationPath;
}
}
}

31
src/FilterLists.Agent/Features/Lists/DownloadSevenZip.cs
View file

@ -1,10 +1,8 @@
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Core.List;
using MediatR;
using Microsoft.Extensions.Logging;
using SharpCompress.Archives;
using SharpCompress.Archives.SevenZip;
using SharpCompress.Common;
@ -26,33 +24,22 @@ public Command(ListUrl listUrl)
public class Handler : AsyncRequestHandler<Command>
{
private const string RepoDirectory = "archives";
private readonly HttpClient _httpClient;
private readonly ILogger<Handler> _logger;
private readonly IListRepository _repo;
public Handler(IListRepository listRepository, ILogger<Handler> logger)
public Handler(IListRepository listRepository)
{
_httpClient = listRepository.HttpClient;
_logger = logger;
_repo = listRepository;
}
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}");
using (var response = await _httpClient.GetAsync(request.ListUrl.ViewUrl, cancellationToken))
{
if (response.IsSuccessStatusCode)
using (var input = await response.Content.ReadAsStreamAsync())
using (var archive = SevenZipArchive.Open(input))
{
foreach (var entry in archive.Entries)
if (!entry.IsDirectory)
entry.WriteToDirectory(destinationDirectoryPath,
new ExtractionOptions {ExtractFullPath = true, Overwrite = true});
}
else
_logger.LogError(
$"Error downloading list {request.ListUrl.Id} from {request.ListUrl.ViewUrl}. {response.StatusCode}");
}
using var input = await _repo.GetListStreamAsync(request.ListUrl.ViewUrl, cancellationToken);
using var archive = SevenZipArchive.Open(input);
foreach (var entry in archive.Entries)
if (!entry.IsDirectory)
entry.WriteToDirectory(destinationDirectoryPath,
new ExtractionOptions {ExtractFullPath = true, Overwrite = true});
}
}
}

31
src/FilterLists.Agent/Features/Lists/DownloadZip.cs
View file

@ -1,10 +1,8 @@
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Core.List;
using MediatR;
using Microsoft.Extensions.Logging;
using SharpCompress.Common;
using SharpCompress.Readers;
@ -25,33 +23,22 @@ public Command(ListUrl listUrl)
public class Handler : AsyncRequestHandler<Command>
{
private const string RepoDirectory = "archives";
private readonly HttpClient _httpClient;
private readonly ILogger<Handler> _logger;
private readonly IListRepository _repo;
public Handler(IListRepository listRepository, ILogger<Handler> logger)
public Handler(IListRepository listRepository)
{
_httpClient = listRepository.HttpClient;
_logger = logger;
_repo = listRepository;
}
protected override async Task Handle(Command request, CancellationToken cancellationToken)
{
var destinationDirectoryPath = Path.Combine(RepoDirectory, $"{request.ListUrl.Id}");
using (var response = await _httpClient.GetAsync(request.ListUrl.ViewUrl, cancellationToken))
{
if (response.IsSuccessStatusCode)
using (var input = await response.Content.ReadAsStreamAsync())
using (var reader = ReaderFactory.Open(input))
{
while (reader.MoveToNextEntry())
if (!reader.Entry.IsDirectory)
reader.WriteEntryToDirectory(destinationDirectoryPath,
new ExtractionOptions {ExtractFullPath = true, Overwrite = true});
}
else
_logger.LogError(
$"Error downloading list {request.ListUrl.Id} from {request.ListUrl.ViewUrl}. {response.StatusCode}");
}
using var input = await _repo.GetListStreamAsync(request.ListUrl.ViewUrl, cancellationToken);
using var reader = ReaderFactory.Open(input);
while (reader.MoveToNextEntry())
if (!reader.Entry.IsDirectory)
reader.WriteEntryToDirectory(destinationDirectoryPath,
new ExtractionOptions {ExtractFullPath = true, Overwrite = true});
}
}
}

26
src/FilterLists.Agent/Infrastructure/ListRepository.cs
View file

@ -1,25 +1,39 @@
using System;
using System.IO;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Agent.Core.List;
using JetBrains.Annotations;
using Microsoft.Extensions.Logging;
namespace FilterLists.Agent.Infrastructure
{
[UsedImplicitly]
public class ListRepository : IListRepository
{
public ListRepository(HttpClient client)
{
client.Timeout = TimeSpan.FromSeconds(90);
private readonly HttpClient _httpClient;
private readonly ILogger<ListRepository> _logger;
public ListRepository(HttpClient httpClient, ILogger<ListRepository> logger)
{
httpClient.Timeout = TimeSpan.FromSeconds(90);
var header = new ProductHeaderValue("FilterLists.Agent");
var userAgent = new ProductInfoHeaderValue(header);
client.DefaultRequestHeaders.UserAgent.Add(userAgent);
httpClient.DefaultRequestHeaders.UserAgent.Add(userAgent);
_httpClient = httpClient;
HttpClient = client;
_logger = logger;
}
public HttpClient HttpClient { get; }
public async Task<Stream> GetListStreamAsync(Uri url, CancellationToken cancellationToken)
{
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (response.IsSuccessStatusCode)
return await response.Content.ReadAsStreamAsync();
_logger.LogError($"Error downloading list from {url}. {response.StatusCode}");
return Stream.Null;
}
}
}