prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(archival): ✨ add retry to new typed FileClient

Browse source
This commit is contained in:
Collin M. Barrett 2020-09-17 19:26:35 -05:00
parent 1f916ccb58
commit 218b9ac268
6 changed files with 87 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs
View file

@ -2,10 +2,10 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FilterLists.Archival.Application.Models;
using FilterLists.Archival.Infrastructure.Clients;
using FilterLists.Archival.Infrastructure.Persistence;
using FilterLists.SharedKernel.Apis.Clients;
using FilterLists.SharedKernel.Apis.Contracts.Directory;
@ -29,19 +29,19 @@ public Command(int listId)
public class Handler : IRequestHandler<Command, Unit>
{
private readonly IFileArchiver _archiver;
private readonly IFileClient _client;
private readonly IDirectoryApi _directory;
private readonly HttpClient _httpClient;
private readonly ILogger _logger;
public Handler(
IFileArchiver archiver,
IFileClient fileClient,
IDirectoryApi directory,
IHttpClientFactory httpClientFactory,
ILogger<Handler> logger)
{
_archiver = archiver;
_client = fileClient;
_directory = directory;
_httpClient = httpClientFactory.CreateClient();
_logger = logger;
}
@ -86,34 +86,20 @@ private async Task DownloadSegments(
IReadOnlyCollection<ListDetailsViewUrlVm> segments,
CancellationToken cancellationToken)
{
var responses = new List<HttpResponseMessage>();
try
var downloads = new List<Task<Stream>>();
foreach (var segment in segments)
{
var readTasks = new List<Task<Stream>>();
foreach (var segment in segments)
{
// TODO: add Polly for resiliency
var response = await _httpClient.GetAsync(segment.Url, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
responses.Add(response);
response.EnsureSuccessStatusCode();
readTasks.Add(response.Content.ReadAsStreamAsync());
}
var streams = await Task.WhenAll(readTasks);
// TODO: prefix fileName with listId
// TODO: add ".txt" for sources with no extension
var fileName = Uri.UnescapeDataString(segments.First().Url.Segments.Last());
var target = new FileInfo(fileName);
await _archiver.ArchiveFileAsync(new FileToArchive(target, streams), cancellationToken);
}
finally
{
foreach (var response in responses)
{
response.Dispose();
}
downloads.Add(_client.DownloadFileAsync(segment.Url, cancellationToken));
}
var streams = await Task.WhenAll(downloads);
// TODO: prefix fileName with listId
// TODO: add ".txt" for sources with no extension
var fileName = Uri.UnescapeDataString(segments.First().Url.Segments.Last());
var target = new FileInfo(fileName);
var file = new FileToArchive(target, streams);
await _archiver.ArchiveFileAsync(file, cancellationToken);
}
}
}

18
services/Archival/FilterLists.Archival.Infrastructure/Clients/ConfigurationExtensions.cs Normal file
View file

@ -0,0 +1,18 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Polly;
namespace FilterLists.Archival.Infrastructure.Clients
{
internal static class ConfigurationExtensions
{
public static void AddClients(this IServiceCollection services)
{
services.AddHttpClient<IFileClient, FileClient>()
.AddTransientHttpErrorPolicy(b => b.WaitAndRetryAsync(new[]
{
TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10)
}));
}
}
}

38
services/Archival/FilterLists.Archival.Infrastructure/Clients/FileClient.cs Normal file
View file

@ -0,0 +1,38 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
namespace FilterLists.Archival.Infrastructure.Clients
{
internal sealed class FileClient : IFileClient
{
private readonly HttpClient _httpClient;
private readonly ICollection<HttpResponseMessage> _httpResponseMessages = new List<HttpResponseMessage>();
public FileClient(HttpClient httpClient)
{
_httpClient = httpClient;
}
public async Task<Stream> DownloadFileAsync(Uri url, CancellationToken cancellationToken)
{
var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
_httpResponseMessages.Add(response);
response.EnsureSuccessStatusCode();
return await response.Content.ReadAsStreamAsync();
}
public void Dispose()
{
foreach (var message in _httpResponseMessages)
{
message.Dispose();
}
_httpClient.Dispose();
}
}
}

12
services/Archival/FilterLists.Archival.Infrastructure/Clients/IFileClient.cs Normal file
View file

@ -0,0 +1,12 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace FilterLists.Archival.Infrastructure.Clients
{
public interface IFileClient : IDisposable
{
Task<Stream> DownloadFileAsync(Uri url, CancellationToken cancellationToken);
}
}

2
services/Archival/FilterLists.Archival.Infrastructure/ConfigurationExtensions.cs
View file

@ -1,4 +1,5 @@
using System;
using FilterLists.Archival.Infrastructure.Clients;
using FilterLists.Archival.Infrastructure.Persistence;
using FilterLists.Archival.Infrastructure.Scheduling;
using FilterLists.SharedKernel.Apis.Clients;
@ -24,6 +25,7 @@ public static void AddInfrastructureServices(this IServiceCollection services, I
services.AddSharedKernelLogging(configuration);
services.AddSchedulingServices(configuration);
services.AddApiClients(configuration);
services.AddClients();
services.AddPersistenceServices(configuration);
}

1
services/Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj
View file

@ -25,6 +25,7 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="3.1.8" />
</ItemGroup>
<ItemGroup>