diff --git a/.github/README.md b/.github/README.md index 3247f60b1..2d58e45e3 100644 --- a/.github/README.md +++ b/.github/README.md @@ -33,7 +33,6 @@ Typical adblockers run as an extension in popular web browsers. As we browse the | ------------- | ----------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Website | A React & TypeScript UI built with Ant Design. | [![Website](https://img.shields.io/website-up-down-green-red/http/shields.io.svg?label=Website)](https://filterlists.com/) [![Website Azure DevOps builds](https://dev.azure.com/collinbarrett/FilterLists/_apis/build/status/Web?branchName=main)](https://dev.azure.com/collinbarrett/FilterLists/_build/latest?definitionId=18) [![Website Azure DevOps releases](https://vsrm.dev.azure.com/collinbarrett/_apis/public/Release/badge/b06a3d5c-459e-4789-9735-0f5969006fe8/4/5)](https://dev.azure.com/collinbarrett/FilterLists/_release?definitionId=4) [![Website Docker Image](https://img.shields.io/badge/docker%20image-web-blue?label=Docker%20Image)](https://github.com/users/collinbarrett/packages/container/package/filterlists-web) [![Website Security Headers](https://img.shields.io/security-headers?url=https%3A%2F%2Ffilterlists.com)](https://securityheaders.com/?q=https%3A%2F%2Ffilterlists.com) | | Directory API | An ASP.NET Core API serving the core FilterList information. | [![Directory API Swagger UI](https://img.shields.io/website-up-down-green-red/http/shields.io.svg?label=API%20Docs)](https://filterlists.com/api/?urls.primaryName=Directory) [![Directory API OpenAPI Specification](https://img.shields.io/swagger/valid/3.0?specUrl=https%3A%2F%2Ffilterlists.com%2Fapi%2Fdirectory%2Fv1%2Fswagger.json)](https://filterlists.com/api/directory/v1/swagger.json) [![Directory API Azure DevOps builds](https://dev.azure.com/collinbarrett/FilterLists/_apis/build/status/Directory%20API?branchName=main)](https://dev.azure.com/collinbarrett/FilterLists/_build/latest?definitionId=27) [![Directory API Azure DevOps releases](https://vsrm.dev.azure.com/collinbarrett/_apis/public/Release/badge/b06a3d5c-459e-4789-9735-0f5969006fe8/3/4)](https://dev.azure.com/collinbarrett/FilterLists/_release?definitionId=3) [![Directory API Docker Image](https://img.shields.io/badge/docker%20image-directory--api-blue?label=Docker%20Image)](https://github.com/users/collinbarrett/packages/container/package/filterlists-directory-api) [![Directory API Security Headers](https://img.shields.io/security-headers?url=https%3A%2F%2Ffilterlists.com%2Fapi%2Fdirectory%2Fv1%2Fswagger.json)](https://securityheaders.com/?q=https%3A%2F%2Ffilterlists.com%2Fapi%2Fdirectory%2Fv1%2Fswagger.json) | -| Archival API | An ASP.NET Core API archiving and serving copies of FilterLists for mirrors and analysis. | [![Archival API Swagger UI](https://img.shields.io/website-up-down-green-red/http/shields.io.svg?label=API%20Docs)](https://filterlists.com/api/?urls.primaryName=Archival) [![Archival API OpenAPI Specification](https://img.shields.io/swagger/valid/3.0?specUrl=https%3A%2F%2Ffilterlists.com%2Fapi%2Farchival%2Fv1%2Fswagger.json)](https://filterlists.com/api/archival/v1/swagger.json) [![Archival API Azure DevOps builds](https://dev.azure.com/collinbarrett/FilterLists/_apis/build/status/Archival%20API?branchName=main)](https://dev.azure.com/collinbarrett/FilterLists/_build/latest?definitionId=29) [![Archival API Azure DevOps releases](https://vsrm.dev.azure.com/collinbarrett/_apis/public/Release/badge/b06a3d5c-459e-4789-9735-0f5969006fe8/7/8)](https://dev.azure.com/collinbarrett/FilterLists/_release?definitionId=7) [![Archival API Docker Image](https://img.shields.io/badge/docker%20image-archival--api-blue?label=Docker%20Image)](https://github.com/users/collinbarrett/packages/container/package/filterlists-archival-api) [![Archival API Security Headers](https://img.shields.io/security-headers?url=https%3A%2F%2Ffilterlists.com%2Fapi%2Farchival%2Fv1%2Fswagger.json)](https://securityheaders.com/?q=https%3A%2F%2Ffilterlists.com%2Fapi%2Farchival%2Fv1%2Fswagger.json) | | Reverse Proxy | An NGINX instance forwarding requests to the respective services above. | [![Reverse Proxy Azure DevOps builds](https://dev.azure.com/collinbarrett/FilterLists/_apis/build/status/Reverse%20Proxy?branchName=main)](https://dev.azure.com/collinbarrett/FilterLists/_build/latest?definitionId=21) [![Reverse Proxy Azure DevOps releases](https://vsrm.dev.azure.com/collinbarrett/_apis/public/Release/badge/b06a3d5c-459e-4789-9735-0f5969006fe8/5/6)](https://dev.azure.com/collinbarrett/FilterLists/_release?definitionId=5) [![Reverse Proxy Mozilla HTTP Observatory Grade](https://img.shields.io/mozilla-observatory/grade/filterlists.com?publish)](https://observatory.mozilla.org/analyze/filterlists.com) [![Reverse Proxy Chromium HSTS preload](https://img.shields.io/hsts/preload/filterlists.com)](https://hstspreload.org/?domain=filterlists.com) | # Contributing diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0b5a15e5e..e731a4fdf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,13 +1,5 @@ version: 2 updates: - - package-ecosystem: docker - directory: /services/Archival/FilterLists.Archival.Api - schedule: - interval: daily - - package-ecosystem: nuget - directory: /services/Archival/FilterLists.Archival.Api - schedule: - interval: daily - package-ecosystem: docker directory: /services/Directory/FilterLists.Directory.Api schedule: diff --git a/docker-compose/.env b/docker-compose/.env index 4d483c680..c3b8c7957 100644 --- a/docker-compose/.env +++ b/docker-compose/.env @@ -1,5 +1,2 @@ -ARCHIVAL_APPLICATION_INSIGHTS_SERVER_TELEMETRY_CHANNEL_STORAGE_PATH=application-insights -ARCHIVAL_GIT_REPOSITORY_PATH=archives - DIRECTORY_HOST=directory-api DIRECTORY_APPLICATION_INSIGHTS_SERVER_TELEMETRY_CHANNEL_STORAGE_PATH=application-insights \ No newline at end of file diff --git a/docker-compose/api-docs.env b/docker-compose/api-docs.env index ac7417e4a..a32cf71d3 100644 --- a/docker-compose/api-docs.env +++ b/docker-compose/api-docs.env @@ -1,2 +1,2 @@ -URLS=[{url:'http://localhost:8080/api/directory/v1/swagger.json',name:'Directory'},{url:'http://localhost:8080/api/archival/v1/swagger.json',name:'Archival'}] +URLS=[{url:'http://localhost:8080/api/directory/v1/swagger.json',name:'Directory'}] URLS_PRIMARY_NAME=Directory \ No newline at end of file diff --git a/docker-compose/archival-api.env b/docker-compose/archival-api.env deleted file mode 100644 index a2321aeb0..000000000 --- a/docker-compose/archival-api.env +++ /dev/null @@ -1,4 +0,0 @@ -ApplicationInsights__InstrumentationKey= -ConnectionStrings__SchedulingConnection=archival-scheduling-db:6379 -Git__UserName=FilterLists Archival -Git__UserEmail=noreply@filterlists.com \ No newline at end of file diff --git a/docker-compose/docker-compose.dcproj b/docker-compose/docker-compose.dcproj index ad3c98f5a..b9d730b4d 100644 --- a/docker-compose/docker-compose.dcproj +++ b/docker-compose/docker-compose.dcproj @@ -12,7 +12,6 @@ - diff --git a/docker-compose/docker-compose.override.yml b/docker-compose/docker-compose.override.yml index 412c98f09..855532c1d 100644 --- a/docker-compose/docker-compose.override.yml +++ b/docker-compose/docker-compose.override.yml @@ -12,13 +12,6 @@ services: - ../reverse-proxy/conf.d.dev:/etc/nginx/conf.d:ro command: ["nginx-debug", "-g", "daemon off;"] - archival-api: - build: - context: ../services - dockerfile: Archival/FilterLists.Archival.Api/dev.Dockerfile - environment: - - ASPNETCORE_ENVIRONMENT=Development - directory-api: build: context: ../services diff --git a/docker-compose/docker-compose.prod.build.yml b/docker-compose/docker-compose.prod.build.yml index e407ca655..a9a1be570 100644 --- a/docker-compose/docker-compose.prod.build.yml +++ b/docker-compose/docker-compose.prod.build.yml @@ -4,11 +4,6 @@ version: "3.7" services: - archival-api: - build: - context: ../services - dockerfile: Archival/FilterLists.Archival.Api/Dockerfile - directory-api: build: context: ../services diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 8282de9fe..d579e11b2 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -8,7 +8,6 @@ services: - reverse-proxy depends_on: - api-docs - - archival-api - directory-api api-docs: @@ -17,38 +16,10 @@ services: networks: - reverse-proxy depends_on: - - archival-api - directory-api env_file: - api-docs.env - archival-api: - image: ghcr.io/collinbarrett/filterlists-archival-api - restart: always - networks: - - reverse-proxy - - archival - depends_on: - - archival-scheduling-db - volumes: - - archival-application-insights:/app/${ARCHIVAL_APPLICATION_INSIGHTS_SERVER_TELEMETRY_CHANNEL_STORAGE_PATH} - - archival-archives:/app/${ARCHIVAL_GIT_REPOSITORY_PATH} - env_file: - - archival-api.env - environment: - ApplicationInsights__ServerTelemetryChannelStoragePath: ${ARCHIVAL_APPLICATION_INSIGHTS_SERVER_TELEMETRY_CHANNEL_STORAGE_PATH} - Api__DirectoryHost: ${DIRECTORY_HOST} - Git__RepositoryPath: ${ARCHIVAL_GIT_REPOSITORY_PATH} - - archival-scheduling-db: - image: redis:alpine - restart: always - networks: - - archival - volumes: - - archival-scheduling-db:/data - entrypoint: ["redis-server", "--appendonly", "yes"] - directory-api: image: ghcr.io/collinbarrett/filterlists-directory-api restart: always @@ -78,12 +49,8 @@ services: networks: reverse-proxy: - archival: directory: volumes: - archival-application-insights: - archival-scheduling-db: - archival-archives: directory-application-insights: directory-db: diff --git a/reverse-proxy/conf.d.dev/default.conf b/reverse-proxy/conf.d.dev/default.conf index fd464a2f4..d666fb2c7 100644 --- a/reverse-proxy/conf.d.dev/default.conf +++ b/reverse-proxy/conf.d.dev/default.conf @@ -2,15 +2,6 @@ server { listen 80 default_server; include server-includes/*; - location ^~ /api/archival/ { - resolver 127.0.0.11 valid=30s; - set $archival_api "archival-api"; - rewrite ^/api/archival/(.*)$ /$1 break; - proxy_pass http://$archival_api:80; - include location-includes/*; - add_header Content-Security-Policy "upgrade-insecure-requests; default-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'none';" always; - } - location ^~ /api/directory/ { resolver 127.0.0.11 valid=30s; set $directory_api "directory-api"; diff --git a/reverse-proxy/conf.d/default.conf b/reverse-proxy/conf.d/default.conf index 044addb86..5b50fe354 100644 --- a/reverse-proxy/conf.d/default.conf +++ b/reverse-proxy/conf.d/default.conf @@ -5,15 +5,6 @@ server { ssl_certificate /etc/nginx/cert/filterlists_cert.pem; ssl_certificate_key /etc/nginx/cert/filterlists_key.pem; - location ^~ /api/archival/ { - resolver 127.0.0.11 valid=30s; - set $archival_api "archival-api"; - rewrite ^/api/archival/(.*)$ /$1 break; - proxy_pass http://$archival_api:80; - include location-includes/*; - add_header Content-Security-Policy "upgrade-insecure-requests; default-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'none';" always; - } - location ^~ /api/directory/ { resolver 127.0.0.11 valid=30s; set $directory_api "directory-api"; diff --git a/services/Archival/FilterLists.Archival.Api/Controllers/ErrorController.cs b/services/Archival/FilterLists.Archival.Api/Controllers/ErrorController.cs deleted file mode 100644 index d3a148b7d..000000000 --- a/services/Archival/FilterLists.Archival.Api/Controllers/ErrorController.cs +++ /dev/null @@ -1,28 +0,0 @@ -using Microsoft.AspNetCore.Diagnostics; -using Microsoft.AspNetCore.Mvc; - -namespace FilterLists.Archival.Api.Controllers; - -[ApiController] -[ApiExplorerSettings(IgnoreApi = true)] -// TODO: de-duplicate into SharedKernel -public class ErrorController : ControllerBase -{ - [Route("/error-local-development")] - public IActionResult ErrorLocalDevelopment([FromServices] IWebHostEnvironment webHostEnvironment) - { - if (webHostEnvironment.EnvironmentName != "Development") - { - throw new InvalidOperationException("This shouldn't be invoked in non-development environments."); - } - - var context = HttpContext.Features.Get(); - return Problem(context?.Error.StackTrace, title: context?.Error.Message); - } - - [Route("/error")] - public IActionResult Error() - { - return Problem(); - } -} diff --git a/services/Archival/FilterLists.Archival.Api/Controllers/PingController.cs b/services/Archival/FilterLists.Archival.Api/Controllers/PingController.cs deleted file mode 100644 index a4160e2e6..000000000 --- a/services/Archival/FilterLists.Archival.Api/Controllers/PingController.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Microsoft.AspNetCore.Mvc; - -namespace FilterLists.Archival.Api.Controllers; - -[ApiController] -[Route("[controller]")] -[Produces("application/json")] -public class PingController : ControllerBase -{ - /// - /// A sample endpoint. - /// - /// Pong. - [HttpGet] - [ProducesResponseType(typeof(string), StatusCodes.Status200OK)] - public ActionResult Ping() - { - return "pong"; - } -} diff --git a/services/Archival/FilterLists.Archival.Api/Dockerfile b/services/Archival/FilterLists.Archival.Api/Dockerfile deleted file mode 100644 index d0f8aa68c..000000000 --- a/services/Archival/FilterLists.Archival.Api/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# init base -FROM mcr.microsoft.com/dotnet/runtime-deps:7.0-alpine AS base -LABEL org.opencontainers.image.description="filterlists.com | github.com/collinbarrett/filterlists | An ASP.NET Core API archiving and serving copies of FilterLists for mirrors and analysis." -WORKDIR /app -EXPOSE 80 - -# init build -FROM mcr.microsoft.com/dotnet/sdk:7.0-alpine AS build -ENV DOTNET_CLI_TELEMETRY_OPTOUT=true - -# restore API -WORKDIR /app -COPY SharedKernel/FilterLists.SharedKernel.Domain.SeedWork/FilterLists.SharedKernel.Domain.SeedWork.csproj SharedKernel/FilterLists.SharedKernel.Domain.SeedWork/ -COPY SharedKernel/FilterLists.SharedKernel.Logging/FilterLists.SharedKernel.Logging.csproj SharedKernel/FilterLists.SharedKernel.Logging/ -COPY Directory/FilterLists.Directory.Api.Contracts/FilterLists.Directory.Api.Contracts.csproj Directory/FilterLists.Directory.Api.Contracts/ -COPY Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj Archival/FilterLists.Archival.Domain/ -COPY Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj Archival/FilterLists.Archival.Infrastructure/ -COPY Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj Archival/FilterLists.Archival.Application/ -WORKDIR /app/Archival/FilterLists.Archival.Api -COPY Archival/FilterLists.Archival.Api/FilterLists.Archival.Api.csproj . -RUN dotnet restore - -# build API -WORKDIR /app -COPY /.editorconfig . -COPY SharedKernel/FilterLists.SharedKernel.Domain.SeedWork/. SharedKernel/FilterLists.SharedKernel.Domain.SeedWork/ -COPY SharedKernel/FilterLists.SharedKernel.Logging/. SharedKernel/FilterLists.SharedKernel.Logging/ -COPY Directory/FilterLists.Directory.Api.Contracts/. Directory/FilterLists.Directory.Api.Contracts/ -COPY Archival/FilterLists.Archival.Domain/. Archival/FilterLists.Archival.Domain/ -COPY Archival/FilterLists.Archival.Infrastructure/. Archival/FilterLists.Archival.Infrastructure/ -COPY Archival/FilterLists.Archival.Application/. Archival/FilterLists.Archival.Application/ -WORKDIR /app/Archival/FilterLists.Archival.Api -COPY Archival/FilterLists.Archival.Api/. . - -# TODO: enable PublishTrimmed in csproj file -# TODO: enable PublishReadyToRun -RUN dotnet publish -c Release -p:DebugType=None -p:DebugSymbols=false --no-restore -o /app/publish -p:TreatWarningsAsErrors=true -r linux-musl-x64 --self-contained true -p:PublishSingleFile=true - -# package final -FROM base AS final -COPY --from=build /app/publish . -ENTRYPOINT ./FilterLists.Archival.Api \ No newline at end of file diff --git a/services/Archival/FilterLists.Archival.Api/FilterLists.Archival.Api.csproj b/services/Archival/FilterLists.Archival.Api/FilterLists.Archival.Api.csproj deleted file mode 100644 index e5f8f2a4e..000000000 --- a/services/Archival/FilterLists.Archival.Api/FilterLists.Archival.Api.csproj +++ /dev/null @@ -1,43 +0,0 @@ - - - - Exe - net7.0 - enable - enable - true - true - linux-musl-x64;win10-x64 - - - - Collin M. Barrett - FilterLists - FilterLists Archival - The independent, comprehensive directory of filter and host lists for advertisements, trackers, malware, and annoyances. - Copyright (c) 2021 Collin M. Barrett - https://github.com/collinbarrett/FilterLists - https://filterlists.com - Linux - ..\.. - ..\..\..\docker-compose\docker-compose.dcproj - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/services/Archival/FilterLists.Archival.Api/Program.cs b/services/Archival/FilterLists.Archival.Api/Program.cs deleted file mode 100644 index 439e00cd9..000000000 --- a/services/Archival/FilterLists.Archival.Api/Program.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Text.Json.Serialization; -using FilterLists.Archival.Api; -using FilterLists.Archival.Application; -using FilterLists.SharedKernel.Logging; - -var builder = WebApplication.CreateBuilder(args); - -builder.Host.ConfigureApplication(); -builder.Services.Configure(options => options.LowercaseUrls = true); -builder.Services.AddControllers() - .AddJsonOptions(o => o.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull); -builder.Services.AddEndpointsApiExplorer(); -builder.Services.AddSwaggerGen(); -builder.Services.AddApplication(builder.Configuration); - -var app = builder.Build(); - -app.UseApplication(); -app.UseExceptionHandler(app.Environment.IsDevelopment() ? "/error-local-development" : "/error"); -app.UseSwagger(); -app.MapControllers(); -app.Use(async (context, next) => -{ - context.Request.PathBase = "/api/archival"; - await next.Invoke(); -}); - -await app.TryRunWithLoggingAsync(); diff --git a/services/Archival/FilterLists.Archival.Api/Properties/launchSettings.json b/services/Archival/FilterLists.Archival.Api/Properties/launchSettings.json deleted file mode 100644 index e5afbd083..000000000 --- a/services/Archival/FilterLists.Archival.Api/Properties/launchSettings.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "$schema": "http://json.schemastore.org/launchsettings.json", - "iisSettings": { - "windowsAuthentication": false, - "anonymousAuthentication": true, - "iisExpress": { - "applicationUrl": "http://localhost:60531/", - "sslPort": 0 - } - }, - "profiles": { - "IIS Express": { - "commandName": "IISExpress", - "launchBrowser": true, - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "FilterLists.Archival.Api": { - "commandName": "Project", - "launchBrowser": true, - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "applicationUrl": "http://localhost:5000", - "dotnetRunMessages": "true" - }, - "Docker": { - "commandName": "Docker", - "launchBrowser": true, - "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}/swagger", - "publishAllPorts": true - } - } -} \ No newline at end of file diff --git a/services/Archival/FilterLists.Archival.Api/SwaggerConfigurationExtensions.cs b/services/Archival/FilterLists.Archival.Api/SwaggerConfigurationExtensions.cs deleted file mode 100644 index f53df6a91..000000000 --- a/services/Archival/FilterLists.Archival.Api/SwaggerConfigurationExtensions.cs +++ /dev/null @@ -1,52 +0,0 @@ -using System.Reflection; -using Microsoft.OpenApi.Models; - -namespace FilterLists.Archival.Api; - -internal static class SwaggerConfigurationExtensions -{ - public static void AddSwaggerGen(this IServiceCollection services) - { - services.AddSwaggerGen(o => - { - o.SupportNonNullableReferenceTypes(); - - o.SwaggerDoc( - "v1", - new OpenApiInfo - { - Title = "FilterLists Archival API", - Description = "An ASP.NET Core API archiving and serving copies of FilterLists for mirrors and analysis.", - Version = "v1", - //TermsOfService = "", - Contact = new OpenApiContact { Name = "FilterLists", Url = new Uri("https://filterlists.com") }, - License = new OpenApiLicense - { - Name = "MIT License", - Url = new Uri("https://github.com/collinbarrett/FilterLists/blob/main/LICENSE") - } - }); - - // Swagger UI struggles with lookups of nested types represented by concatenated '+' - o.CustomSchemaIds(t => t.FullName?.Replace("+", ".")); - - var apiXmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml"; - o.IncludeXmlComments(Path.Combine(AppContext.BaseDirectory, apiXmlFile)); - - var applicationXmlFile = $"{typeof(Application.ConfigurationExtensions).Assembly.GetName().Name}.xml"; - o.IncludeXmlComments(Path.Combine(AppContext.BaseDirectory, applicationXmlFile)); - }); - } - - public static void UseSwagger(this IApplicationBuilder app) - { - app.UseSwagger(o => - { - o.RouteTemplate = "{documentName}/swagger.json"; - o.PreSerializeFilters.Add((swaggerDoc, httpReq) => swaggerDoc.Servers = new List - { - new() { Url = $"{httpReq.Scheme}://{httpReq.Host.Value}/api/archival" } - }); - }); - } -} diff --git a/services/Archival/FilterLists.Archival.Api/dev.Dockerfile b/services/Archival/FilterLists.Archival.Api/dev.Dockerfile deleted file mode 100644 index 6fa514fd7..000000000 --- a/services/Archival/FilterLists.Archival.Api/dev.Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# init base for Visual Studio debugging -FROM mcr.microsoft.com/dotnet/aspnet:7.0-alpine AS base -WORKDIR /app - -# init build -FROM mcr.microsoft.com/dotnet/sdk:7.0-alpine AS build -ENV DOTNET_CLI_TELEMETRY_OPTOUT=true - -# restore API -WORKDIR /app -COPY SharedKernel/FilterLists.SharedKernel.Domain.SeedWork/FilterLists.SharedKernel.Domain.SeedWork.csproj SharedKernel/FilterLists.SharedKernel.Domain.SeedWork/ -COPY SharedKernel/FilterLists.SharedKernel.Logging/FilterLists.SharedKernel.Logging.csproj SharedKernel/FilterLists.SharedKernel.Logging/ -COPY Directory/FilterLists.Directory.Api.Contracts/FilterLists.Directory.Api.Contracts.csproj Directory/FilterLists.Directory.Api.Contracts/ -COPY Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj Archival/FilterLists.Archival.Domain/ -COPY Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj Archival/FilterLists.Archival.Infrastructure/ -COPY Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj Archival/FilterLists.Archival.Application/ -WORKDIR /app/Archival/FilterLists.Archival.Api -COPY Archival/FilterLists.Archival.Api/FilterLists.Archival.Api.csproj . -RUN dotnet restore - -# build API -WORKDIR /app -COPY /.editorconfig . -COPY SharedKernel/FilterLists.SharedKernel.Domain.SeedWork/. SharedKernel/FilterLists.SharedKernel.Domain.SeedWork/ -COPY SharedKernel/FilterLists.SharedKernel.Logging/. SharedKernel/FilterLists.SharedKernel.Logging/ -COPY Directory/FilterLists.Directory.Api.Contracts/. Directory/FilterLists.Directory.Api.Contracts/ -COPY Archival/FilterLists.Archival.Domain/. Archival/FilterLists.Archival.Domain/ -COPY Archival/FilterLists.Archival.Infrastructure/. Archival/FilterLists.Archival.Infrastructure/ -COPY Archival/FilterLists.Archival.Application/. Archival/FilterLists.Archival.Application/ -WORKDIR /app/Archival/FilterLists.Archival.Api -COPY Archival/FilterLists.Archival.Api/. . -RUN dotnet publish --no-restore -o /app/publish -r linux-musl-x64 - -# package final -FROM base AS final -COPY --from=build /app/publish . -ENTRYPOINT ./FilterLists.Archival.Api \ No newline at end of file diff --git a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs b/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs deleted file mode 100644 index 842a38618..000000000 --- a/services/Archival/FilterLists.Archival.Application/Commands/ArchiveList.cs +++ /dev/null @@ -1,91 +0,0 @@ -using System.Runtime.CompilerServices; -using FilterLists.Archival.Domain.ListArchives; -using FilterLists.Archival.Infrastructure.Clients; -using FilterLists.Directory.Api.Contracts; -using FilterLists.Directory.Api.Contracts.Models; -using MediatR; -using Microsoft.Extensions.Logging; - -namespace FilterLists.Archival.Application.Commands; - -public static class ArchiveList -{ - public record Command(long ListId) : IRequest; - - internal class Handler : IRequestHandler - { - private readonly IHttpContentClient _client; - private readonly IDirectoryApi _directory; - private readonly ILogger _logger; - private readonly IListArchiveRepository _repo; - - public Handler( - IHttpContentClient httpContentClient, - IDirectoryApi directoryApi, - ILogger logger, - IListArchiveRepository listArchiveRepository) - { - _client = httpContentClient; - _directory = directoryApi; - _logger = logger; - _repo = listArchiveRepository; - } - - public async Task Handle(Command request, CancellationToken cancellationToken) - { - _logger.LogInformation("Archiving list {ListId}", request.ListId); - - var segmentUrls = (await GetSegmentUrlsAsync(request.ListId, cancellationToken)).ToList(); - if (segmentUrls.Count > 0) - { - var list = GetList(request.ListId, segmentUrls, cancellationToken); - await _repo.AddAsync(list, cancellationToken); - _repo.Commit(); - - _logger.LogInformation( - "Archived segment(s) {@SegmentNumbers} of list {ListId}", - segmentUrls.Select(s => s.SegmentNumber), - request.ListId); - } - else - { - _logger.LogWarning("List {ListId} has no URLs to archive", request.ListId); - } - - return Unit.Value; - } - - private async Task> GetSegmentUrlsAsync( - long listId, - CancellationToken cancellationToken) - { - var listDetails = await _directory.GetListDetailsAsync(listId, cancellationToken); - return listDetails.ViewUrls? - .GroupBy(u => u.SegmentNumber, (_, ue) => ue.OrderBy(u => u.Primariness).First()) ?? - new List(); - } - - private ListArchive GetList( - long listId, - IEnumerable segmentUrls, - CancellationToken cancellationToken) - { - var segmentsAsync = GetSegmentsAsync(segmentUrls, cancellationToken); - return new ListArchive(listId, segmentsAsync); - } - - private async IAsyncEnumerable GetSegmentsAsync( - IEnumerable segmentUrls, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - foreach (var segment in segmentUrls) - { - var content = await _client.GetContentAsync(segment.Url, cancellationToken); - if (content != Stream.Null) - { - yield return new ListArchiveSegment(segment.Url, content); - } - } - } - } -} diff --git a/services/Archival/FilterLists.Archival.Application/Commands/EnqueueArchiveAllLists.cs b/services/Archival/FilterLists.Archival.Application/Commands/EnqueueArchiveAllLists.cs deleted file mode 100644 index 85ac2ee11..000000000 --- a/services/Archival/FilterLists.Archival.Application/Commands/EnqueueArchiveAllLists.cs +++ /dev/null @@ -1,50 +0,0 @@ -using FilterLists.Archival.Infrastructure.Scheduling; -using FilterLists.Directory.Api.Contracts; -using MediatR; -using Microsoft.Extensions.Logging; - -namespace FilterLists.Archival.Application.Commands; - -public static class EnqueueArchiveAllLists -{ - public record Command : IRequest; - - internal class Handler : IRequestHandler - { - private readonly IDirectoryApi _directory; - private readonly ILogger _logger; - - public Handler(IDirectoryApi directory, ILogger logger) - { - _directory = directory; - _logger = logger; - } - - public async Task Handle(Command request, CancellationToken cancellationToken) - { - var r = new Random(); - var lists = (await _directory.GetListsAsync(cancellationToken)).OrderBy(_ => r.Next()).ToList(); - - int archiveCount; - TimeSpan spacing; -#if DEBUG - archiveCount = 0; - spacing = TimeSpan.FromSeconds(5); -#else - archiveCount = 0; - //archiveCount = lists.Count; - spacing = TimeSpan.FromSeconds((double)86400 / lists.Count); -#endif - - _logger.LogInformation("Enqueuing archival of {ArchiveCount} lists spaced {Spacing} seconds apart.", - archiveCount, spacing.Seconds); - - for (var i = 0; i < archiveCount; i++) - { - new ArchiveList.Command(lists[i].Id).ScheduleBackgroundJob(i * spacing); - } - - return Unit.Value; - } - } -} diff --git a/services/Archival/FilterLists.Archival.Application/ConfigurationExtensions.cs b/services/Archival/FilterLists.Archival.Application/ConfigurationExtensions.cs deleted file mode 100644 index c09da3880..000000000 --- a/services/Archival/FilterLists.Archival.Application/ConfigurationExtensions.cs +++ /dev/null @@ -1,42 +0,0 @@ -using FilterLists.Archival.Application.Commands; -using FilterLists.Archival.Infrastructure; -using FilterLists.Archival.Infrastructure.Scheduling; -using Hangfire; -using MediatR; -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; - -namespace FilterLists.Archival.Application; - -public static class ConfigurationExtensions -{ - public static IHostBuilder ConfigureApplication(this IHostBuilder hostBuilder) - { - return hostBuilder.ConfigureInfrastructure(); - } - - public static void AddApplication(this IServiceCollection services, IConfiguration configuration) - { - services.AddMediatR(typeof(ConfigurationExtensions).Assembly); - services.AddInfrastructure(configuration); - } - - public static void UseApplication(this IApplicationBuilder app) - { - app.UseInfrastructure(); - ScheduleArchival(); - } - - private static void ScheduleArchival() - { - // TODO: fix InvalidOperationException and re-enable -#if DEBUG - //JobStorage.Current?.GetMonitoringApi()?.PurgeJobs(); - //new EnqueueArchiveAllLists.Command().EnqueueBackgroundJob(); -#else - //new EnqueueArchiveAllLists.Command().AddOrUpdateRecurringJob(Cron.Daily); -#endif - } -} diff --git a/services/Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj b/services/Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj deleted file mode 100644 index 9c1c6fd9e..000000000 --- a/services/Archival/FilterLists.Archival.Application/FilterLists.Archival.Application.csproj +++ /dev/null @@ -1,25 +0,0 @@ - - - - net7.0 - enable - enable - true - - - - Collin M. Barrett - FilterLists - FilterLists Archival - The independent, comprehensive directory of filter and host lists for advertisements, trackers, malware, and annoyances. - Copyright (c) 2021 Collin M. Barrett - https://github.com/collinbarrett/FilterLists - https://filterlists.com - - - - - - - - \ No newline at end of file diff --git a/services/Archival/FilterLists.Archival.Domain/Extensions/UriExtension.cs b/services/Archival/FilterLists.Archival.Domain/Extensions/UriExtension.cs deleted file mode 100644 index 2a512b00a..000000000 --- a/services/Archival/FilterLists.Archival.Domain/Extensions/UriExtension.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace FilterLists.Archival.Domain.Extensions; - -internal static class UriExtension -{ - private static readonly Uri DummyBaseUri = new("http://localhost"); - - public static string GetFileExtension(this Uri uri) - { - if (!uri.IsAbsoluteUri) - { - uri = new Uri(DummyBaseUri, uri); - } - - return Path.GetExtension(uri.LocalPath); - } -} diff --git a/services/Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj b/services/Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj deleted file mode 100644 index 578784d36..000000000 --- a/services/Archival/FilterLists.Archival.Domain/FilterLists.Archival.Domain.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - - net7.0 - enable - enable - - - - Collin M. Barrett - FilterLists - FilterLists Archival - The independent, comprehensive directory of filter and host lists for advertisements, trackers, malware, and annoyances. - Copyright (c) 2021 Collin M. Barrett - https://github.com/collinbarrett/FilterLists - https://filterlists.com - - - - - - - \ No newline at end of file diff --git a/services/Archival/FilterLists.Archival.Domain/ListArchives/IListArchiveRepository.cs b/services/Archival/FilterLists.Archival.Domain/ListArchives/IListArchiveRepository.cs deleted file mode 100644 index 26aa1a5fe..000000000 --- a/services/Archival/FilterLists.Archival.Domain/ListArchives/IListArchiveRepository.cs +++ /dev/null @@ -1,8 +0,0 @@ -using FilterLists.SharedKernel.Domain.SeedWork; - -namespace FilterLists.Archival.Domain.ListArchives; - -public interface IListArchiveRepository : IUnitOfWork -{ - Task AddAsync(ListArchive listArchive, CancellationToken cancellationToken); -} diff --git a/services/Archival/FilterLists.Archival.Domain/ListArchives/ListArchive.cs b/services/Archival/FilterLists.Archival.Domain/ListArchives/ListArchive.cs deleted file mode 100644 index 027673ce0..000000000 --- a/services/Archival/FilterLists.Archival.Domain/ListArchives/ListArchive.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace FilterLists.Archival.Domain.ListArchives; - -public record ListArchive(long Id, IAsyncEnumerable Segments); diff --git a/services/Archival/FilterLists.Archival.Domain/ListArchives/ListArchiveSegment.cs b/services/Archival/FilterLists.Archival.Domain/ListArchives/ListArchiveSegment.cs deleted file mode 100644 index da3b12059..000000000 --- a/services/Archival/FilterLists.Archival.Domain/ListArchives/ListArchiveSegment.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace FilterLists.Archival.Domain.ListArchives; - -public record ListArchiveSegment -{ - public ListArchiveSegment(Uri sourceUri, Stream content) - { - Extension = ListFileExtension.FromUri(sourceUri); - Content = content; - } - - public ListFileExtension Extension { get; } - - public Stream Content { get; } -} diff --git a/services/Archival/FilterLists.Archival.Domain/ListArchives/ListFileExtension.cs b/services/Archival/FilterLists.Archival.Domain/ListArchives/ListFileExtension.cs deleted file mode 100644 index 7a5a2d01e..000000000 --- a/services/Archival/FilterLists.Archival.Domain/ListArchives/ListFileExtension.cs +++ /dev/null @@ -1,63 +0,0 @@ -using FilterLists.Archival.Domain.Extensions; -using FilterLists.SharedKernel.Domain.SeedWork; - -namespace FilterLists.Archival.Domain.ListArchives; - -public sealed class ListFileExtension : ValueObject -{ - private static readonly IDictionary Info = - new Dictionary - { - { string.Empty, (true, false) }, - { ".txt", (true, false) }, - { ".7z", (false, false) }, - { ".action", (true, true) }, // Privoxy - { ".all", (true, false) }, - { ".aspx", (true, false) }, - { ".bat", (true, true) }, - { ".blacklist", (true, false) }, - { ".cidr", (true, true) }, - { ".conf", (true, true) }, // dnsmasq / Unbound / BIND - { ".csv", (true, true) }, - { ".dat", (true, true) }, // Halite for Windows - { ".deny", (true, true) }, - { ".gz", (false, false) }, - { ".hosts", (true, true) }, - { ".ips", (true, false) }, - { ".ipset", (true, true) }, // Firehol - { ".json", (true, true) }, - { ".list", (true, false) }, - { ".lsrules", (true, true) }, // Little Snitch - { ".md", (true, true) }, - { ".netset", (true, true) }, // Firehol - { ".p2p", (true, true) }, // Peer Guardian - { ".php", (true, false) }, - { ".raw", (true, false) }, - { ".rpz", (true, true) }, // Response Policy Zone - { ".tpl", (true, true) }, // Internet Explorer - { ".uBl", (true, false) }, - { ".zip", (false, false) }, - { ".zone", (true, false) } - }; - - private ListFileExtension(string value) - { - Value = value; - } - - public string Value { get; } - - public bool IsPlainText => Info[Value].IsPlainText; - - public bool IsMeaningfulToConsumer => Info[Value].IsMeaningfulToConsumer; - - public static ListFileExtension FromUri(Uri uri) - { - return new ListFileExtension(uri.GetFileExtension()); - } - - protected override IEnumerable GetEqualityComponents() - { - return new[] { Value }; - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Clients/ConfigurationExtensions.cs b/services/Archival/FilterLists.Archival.Infrastructure/Clients/ConfigurationExtensions.cs deleted file mode 100644 index 086493783..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Clients/ConfigurationExtensions.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Polly; - -namespace FilterLists.Archival.Infrastructure.Clients; - -internal static class ConfigurationExtensions -{ - public static void AddClients(this IServiceCollection services) - { - services.AddHttpClient() - .AddTransientHttpErrorPolicy(b => b.WaitAndRetryAsync(new[] - { - TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10) - })); - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Clients/IHttpContentClient.cs b/services/Archival/FilterLists.Archival.Infrastructure/Clients/IHttpContentClient.cs deleted file mode 100644 index 9340ea7dd..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Clients/IHttpContentClient.cs +++ /dev/null @@ -1,48 +0,0 @@ -using Microsoft.Extensions.Logging; - -namespace FilterLists.Archival.Infrastructure.Clients; - -public interface IHttpContentClient : IDisposable -{ - Task GetContentAsync(Uri url, CancellationToken cancellationToken); -} - -internal sealed class HttpContentClient : IHttpContentClient -{ - private readonly HttpClient _httpClient; - private readonly ICollection _httpResponseMessages = new HashSet(); - private readonly ILogger _logger; - - public HttpContentClient(HttpClient httpClient, ILogger logger) - { - _httpClient = httpClient; - _httpClient.DefaultRequestHeaders.Add( - "User-Agent", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36"); - _logger = logger; - } - - public async Task GetContentAsync(Uri url, CancellationToken cancellationToken) - { - var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); - _httpResponseMessages.Add(response); - try - { - response.EnsureSuccessStatusCode(); - return await response.Content.ReadAsStreamAsync(cancellationToken); - } - catch (HttpRequestException ex) - { - _logger.LogError(ex, "Failed to get content from {Url}", url); - return Stream.Null; - } - } - - public void Dispose() - { - foreach (var message in _httpResponseMessages) - { - message.Dispose(); - } - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/ConfigurationExtensions.cs b/services/Archival/FilterLists.Archival.Infrastructure/ConfigurationExtensions.cs deleted file mode 100644 index 77aee76ae..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/ConfigurationExtensions.cs +++ /dev/null @@ -1,33 +0,0 @@ -using FilterLists.Archival.Infrastructure.Clients; -using FilterLists.Archival.Infrastructure.Persistence; -using FilterLists.Archival.Infrastructure.Scheduling; -using FilterLists.Directory.Api.Contracts; -using FilterLists.SharedKernel.Logging; -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; - -namespace FilterLists.Archival.Infrastructure; - -public static class ConfigurationExtensions -{ - public static IHostBuilder ConfigureInfrastructure(this IHostBuilder hostBuilder) - { - return hostBuilder.ConfigureLogging(); - } - - public static void AddInfrastructure(this IServiceCollection services, IConfiguration configuration) - { - services.AddSharedKernelLogging(configuration); - services.AddScheduling(configuration); - services.AddDirectoryApiClient(configuration); - services.AddClients(); - services.AddPersistence(configuration); - } - - public static void UseInfrastructure(this IApplicationBuilder app) - { - app.UseLogging(); - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj b/services/Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj deleted file mode 100644 index b4b1f4406..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/FilterLists.Archival.Infrastructure.csproj +++ /dev/null @@ -1,33 +0,0 @@ - - - - net7.0 - enable - enable - - - - Collin M. Barrett - FilterLists - FilterLists Archival - The independent, comprehensive directory of filter and host lists for advertisements, trackers, malware, and annoyances. - Copyright (c) 2021 Collin M. Barrett - https://github.com/collinbarrett/FilterLists - https://filterlists.com - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Options/GitOptions.cs b/services/Archival/FilterLists.Archival.Infrastructure/Options/GitOptions.cs deleted file mode 100644 index 0aac6da98..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Options/GitOptions.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace FilterLists.Archival.Infrastructure.Options; - -internal record GitOptions -{ - public const string Key = "Git"; - - public string RepositoryPath { get; init; } = default!; - public string UserName { get; init; } = default!; - public string UserEmail { get; init; } = default!; -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs deleted file mode 100644 index ce29a1578..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/ConfigurationExtensions.cs +++ /dev/null @@ -1,28 +0,0 @@ -using FilterLists.Archival.Domain.ListArchives; -using FilterLists.Archival.Infrastructure.Options; -using LibGit2Sharp; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; - -namespace FilterLists.Archival.Infrastructure.Persistence; - -internal static class ConfigurationExtensions -{ - public static void AddPersistence(this IServiceCollection services, IConfiguration configuration) - { - services.Configure(configuration.GetSection(GitOptions.Key)); - services.AddTransient( - _ => - { - var gitOptions = configuration.GetSection(GitOptions.Key) - .Get(); - if (!Repository.IsValid(gitOptions?.RepositoryPath)) - { - Repository.Init(gitOptions?.RepositoryPath); - } - - return new Repository(gitOptions?.RepositoryPath); - }); - services.AddTransient(); - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToPlainTextConversionStrategy.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToPlainTextConversionStrategy.cs deleted file mode 100644 index fa3626390..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/IStreamToPlainTextConversionStrategy.cs +++ /dev/null @@ -1,8 +0,0 @@ -using FilterLists.Archival.Domain.ListArchives; - -namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies; - -internal interface IStreamToPlainTextConversionStrategy -{ - Stream Convert(ListArchiveSegment listArchiveSegment, CancellationToken cancellationToken); -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/PlainText.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/PlainText.cs deleted file mode 100644 index a59af1b45..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/PlainText.cs +++ /dev/null @@ -1,11 +0,0 @@ -using FilterLists.Archival.Domain.ListArchives; - -namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies; - -internal class PlainText : IStreamToPlainTextConversionStrategy -{ - public Stream Convert(ListArchiveSegment listArchiveSegment, CancellationToken cancellationToken) - { - return listArchiveSegment.Content; - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/StreamToPlainTextConversionStrategyFactory.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/StreamToPlainTextConversionStrategyFactory.cs deleted file mode 100644 index 138ad2121..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/FileWriteStrategies/StreamToPlainTextConversionStrategyFactory.cs +++ /dev/null @@ -1,15 +0,0 @@ -using FilterLists.Archival.Domain.ListArchives; - -namespace FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies; - -internal static class StreamToPlainTextConversionStrategyFactory -{ - public static TStrategy? GetStrategy(this ListArchiveSegment segment) - where TStrategy : class, IStreamToPlainTextConversionStrategy - { - // TODO: implement non-plain text strategies - return segment.Extension.IsPlainText - ? new PlainText() as TStrategy - : default; - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitListArchiveRepository.cs b/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitListArchiveRepository.cs deleted file mode 100644 index 07f332558..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Persistence/GitListArchiveRepository.cs +++ /dev/null @@ -1,128 +0,0 @@ -using System.Globalization; -using FilterLists.Archival.Domain.ListArchives; -using FilterLists.Archival.Infrastructure.Options; -using FilterLists.Archival.Infrastructure.Persistence.FileWriteStrategies; -using LibGit2Sharp; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace FilterLists.Archival.Infrastructure.Persistence; - -internal sealed class GitListArchiveRepository : IListArchiveRepository -{ - private readonly ILogger _logger; - private readonly GitOptions _options; - private readonly IRepository _repo; - private readonly ICollection _writtenFiles = new HashSet(); - - public GitListArchiveRepository( - ILogger logger, - IOptions gitOptions, - IRepository repository) - { - _logger = logger; - _options = gitOptions.Value; - _repo = repository; - } - - public async Task AddAsync(ListArchive listArchive, CancellationToken cancellationToken) - { - var segmentNumber = 1; - await foreach (var segment in listArchive.Segments.WithCancellation(cancellationToken)) - { - var strategy = segment.GetStrategy(); - if (strategy is default(IStreamToPlainTextConversionStrategy)) - { - _logger.LogWarning( - "No stream to plain text conversion strategy found for extension {Extension} for list {ListId}. Skipping list", - segment.Extension, - listArchive.Id); - return; - } - - var fileInfo = GetTargetFile(listArchive.Id, segmentNumber, segment.Extension); - if (fileInfo is default(FileInfo)) - { - _logger.LogWarning( - "Writing from non-plain text extension {Extension} for list {ListId} not yet supported. Skipping list", - segment.Extension, - listArchive.Id); - return; - } - - _writtenFiles.Add(fileInfo); - - _logger.LogInformation("Writing {FileName}", fileInfo.Name); - - await using var target = fileInfo.OpenWrite(); - await strategy.Convert(segment, cancellationToken).CopyToAsync(target, cancellationToken); - - _logger.LogInformation("Finished writing {FileName}", fileInfo.Name); - - segmentNumber++; - } - } - - public void Commit() - { - if (_writtenFiles.Count > 0) - { - var fileNames = _writtenFiles.Select(f => f.Name).ToList(); - var signature = new Signature(_options.UserName, _options.UserEmail, DateTime.UtcNow); - var message = - $"feat(archives): archive {fileNames.Count} file(s){Environment.NewLine}{string.Join(Environment.NewLine, fileNames)}"; - Commands.Stage(_repo, fileNames); - try - { - _repo.Commit(message, signature, signature); - } - catch (EmptyCommitException ex) - { - _logger.LogInformation(ex, "No changes to commit for {@FileNames}", fileNames); - } - - _logger.LogInformation("Committed {@FileNames}", fileNames); - } - else - { - _logger.LogInformation("No written files to commit"); - } - } - - public void Dispose() - { - foreach (var file in _writtenFiles) - { - if (File.Exists(file.Name)) - { - File.Delete(file.Name); - } - } - - _repo.CheckoutPaths("HEAD", _writtenFiles.Select(f => f.Name)); - } - - private FileInfo? GetTargetFile(long listId, int segmentNumber, ListFileExtension extension) - { - string targetExtension; - if (extension.IsPlainText) - { - targetExtension = extension.IsMeaningfulToConsumer ? extension.Value : ".txt"; - } - else - { - // TODO: implement - return default; - } - - var targetFileName = GetTargetFileNamePrefix(listId) + - (segmentNumber == 1 ? string.Empty : $"-{segmentNumber}") + - targetExtension; - return new FileInfo(Path.Combine(_options.RepositoryPath, targetFileName)); - } - - private static string GetTargetFileNamePrefix(long listId) - { - return listId.ToString(CultureInfo.InvariantCulture).PadLeft(5, '0'); - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/ConfigurationExtensions.cs b/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/ConfigurationExtensions.cs deleted file mode 100644 index c62854785..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/ConfigurationExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Hangfire; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StackExchange.Redis; - -namespace FilterLists.Archival.Infrastructure.Scheduling; - -internal static class ConfigurationExtensions -{ - private static ConnectionMultiplexer _redis = default!; - - public static void AddScheduling(this IServiceCollection services, IConfiguration configuration) - { - _redis = ConnectionMultiplexer.Connect( - configuration.GetConnectionString("SchedulingConnection") ?? string.Empty); - services.AddHangfire( - (_, globalConfiguration) => globalConfiguration.UseRedisStorage(_redis) - .UseMediatR()); - services.AddHangfireServer(o => o.WorkerCount = Environment.ProcessorCount); - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/HangfireExtensions.cs b/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/HangfireExtensions.cs deleted file mode 100644 index 34c270327..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/HangfireExtensions.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Hangfire; -using Hangfire.Storage; - -namespace FilterLists.Archival.Infrastructure.Scheduling; - -public static class HangfireExtensions -{ - // https://github.com/HangfireIO/Hangfire/issues/394#issuecomment-179924221 - public static void PurgeJobs(this IMonitoringApi monitoringApi) - { - var toDelete = new List(); - foreach (var queue in monitoringApi.Queues()) - { - for (var i = 0; i < Math.Ceiling(queue.Length / 1000d); i++) - { - toDelete.AddRange(monitoringApi.EnqueuedJobs(queue.Name, 1000 * i, 1000).Select(x => x.Key)); - } - } - - foreach (var jobId in toDelete) - { - BackgroundJob.Delete(jobId); - } - } -} diff --git a/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/ScheduleMediator.cs b/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/ScheduleMediator.cs deleted file mode 100644 index f39245201..000000000 --- a/services/Archival/FilterLists.Archival.Infrastructure/Scheduling/ScheduleMediator.cs +++ /dev/null @@ -1,36 +0,0 @@ -using Hangfire; -using MediatR; -using Newtonsoft.Json; - -namespace FilterLists.Archival.Infrastructure.Scheduling; - -public static class RequestExtensions -{ - public static void EnqueueBackgroundJob(this IRequest request) - { - // Hangfire replaces CancellationToken at runtime with its own. We just need any in the signature. - // https://docs.hangfire.io/en/latest/background-methods/using-cancellation-tokens.html#cancellationtoken - BackgroundJob.Enqueue(m => m.Send(request, CancellationToken.None)); - } - - public static void ScheduleBackgroundJob(this IRequest request, TimeSpan delay) - { - BackgroundJob.Schedule(m => m.Send(request, CancellationToken.None), delay); - } - - public static void AddOrUpdateRecurringJob(this IRequest request, Func cronExpression) - { - RecurringJob.AddOrUpdate(m => m.Send(request, CancellationToken.None), cronExpression); - } -} - -internal static class HangfireExtension -{ - public static IGlobalConfiguration UseMediatR(this IGlobalConfiguration globalConfiguration) - { - /// https://codeopinion.com/background-commands-mediatr-hangfire/ - GlobalConfiguration.Configuration.UseSerializerSettings( - new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Objects }); - return globalConfiguration; - } -} diff --git a/services/Archival/azure-pipelines.api.yaml b/services/Archival/azure-pipelines.api.yaml deleted file mode 100644 index 0b5feb79a..000000000 --- a/services/Archival/azure-pipelines.api.yaml +++ /dev/null @@ -1,97 +0,0 @@ -trigger: - batch: false - branches: - include: [main] - paths: - include: - [ - services/Archival/azure-pipelines.api.yaml, - services/SharedKernel/FilterLists.SharedKernel.Logging/*, - services/Directory/FilterLists.Directory.Api.Contracts/*, - services/Archival/FilterLists.Archival.Domain/*, - services/Archival/FilterLists.Archival.Infrastructure/*, - services/Archival/FilterLists.Archival.Application/*, - services/Archival/FilterLists.Archival.Api/*, - services/.editorignore, - services/.dockerignore, - docker-compose/docker-compose.yml, - docker-compose/docker-compose.prod.yml, - docker-compose/docker-compose.prod.build.yml, - docker-compose/.env, - docker-compose/archival-api.env, - ] - -pr: - autoCancel: false - branches: - include: [main] - paths: - include: - [ - services/Archival/azure-pipelines.api.yaml, - services/SharedKernel/FilterLists.SharedKernel.Logging/*, - services/Directory/FilterLists.Directory.Api.Contracts/*, - services/Archival/FilterLists.Archival.Domain/*, - services/Archival/FilterLists.Archival.Infrastructure/*, - services/Archival/FilterLists.Archival.Application/*, - services/Archival/FilterLists.Archival.Api/*, - services/.editorignore, - services/.dockerignore, - docker-compose/docker-compose.yml, - docker-compose/docker-compose.prod.yml, - docker-compose/docker-compose.prod.build.yml, - docker-compose/.env, - docker-compose/archival-api.env, - ] - -pool: - vmImage: ubuntu-latest - -steps: - - checkout: self - fetchDepth: 5 - - - task: Docker@2 - displayName: build - inputs: - containerRegistry: github-container-registry - repository: collinbarrett/filterlists-archival-api - command: build - Dockerfile: services/Archival/FilterLists.Archival.Api/Dockerfile - buildContext: ./services - tags: | - latest - $(Build.BuildId) - env: - DOCKER_BUILDKIT: 1 - - - task: Docker@2 - displayName: push - inputs: - containerRegistry: github-container-registry - repository: collinbarrett/filterlists-archival-api - command: push - tags: | - latest - $(Build.BuildId) - env: - DOCKER_BUILDKIT: 1 - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) - - - task: CopyFiles@2 - displayName: stage docker-compose files - inputs: - SourceFolder: $(Build.SourcesDirectory)/docker-compose - Contents: | - docker-compose.yml - docker-compose.prod.yml - TargetFolder: $(Build.ArtifactStagingDirectory) - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) - - - task: PublishBuildArtifacts@1 - displayName: publish docker-compose files - inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory) - ArtifactName: drop - publishLocation: Container - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) diff --git a/services/FilterLists.sln b/services/FilterLists.sln index 1157c1a61..6782b7009 100644 --- a/services/FilterLists.sln +++ b/services/FilterLists.sln @@ -50,25 +50,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Data", "Data", "{8F419098-3 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.Directory.Infrastructure.Migrations.Tests", "Directory\FilterLists.Directory.Infrastructure.Migrations.Tests\FilterLists.Directory.Infrastructure.Migrations.Tests.csproj", "{8B07EF04-5516-40BB-B21B-CEC6F90EC631}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Archival", "Archival", "{5E52FEF9-9605-4ACB-8A42-A20E06331599}" - ProjectSection(SolutionItems) = preProject - Archival\azure-pipelines.api.yaml = Archival\azure-pipelines.api.yaml - EndProjectSection -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.Archival.Api", "Archival\FilterLists.Archival.Api\FilterLists.Archival.Api.csproj", "{F625C241-0501-4AEB-8B82-609C93EE6B7D}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.Archival.Application", "Archival\FilterLists.Archival.Application\FilterLists.Archival.Application.csproj", "{11669839-E666-477B-A1B6-276041AD3768}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.Archival.Infrastructure", "Archival\FilterLists.Archival.Infrastructure\FilterLists.Archival.Infrastructure.csproj", "{A0B77015-E8C4-41D3-B63D-BF818026298C}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SharedKernel", "SharedKernel", "{59197818-6C72-4A3C-A595-49F3D7D2EB18}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.SharedKernel.Logging", "SharedKernel\FilterLists.SharedKernel.Logging\FilterLists.SharedKernel.Logging.csproj", "{5485F55E-DFED-4DA0-8A00-220B081FAC96}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.Directory.Api.Contracts", "Directory\FilterLists.Directory.Api.Contracts\FilterLists.Directory.Api.Contracts.csproj", "{4D3D2508-CCC6-4C0B-B5E4-734DD5EC588F}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.Archival.Domain", "Archival\FilterLists.Archival.Domain\FilterLists.Archival.Domain.csproj", "{6402E81A-75C2-422F-933B-AAC1ED8C4128}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.Directory.Domain", "Directory\FilterLists.Directory.Domain\FilterLists.Directory.Domain.csproj", "{EE844873-71D7-4B9D-93BF-DF65B9E4770D}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterLists.SharedKernel.Domain.SeedWork", "SharedKernel\FilterLists.SharedKernel.Domain.SeedWork\FilterLists.SharedKernel.Domain.SeedWork.csproj", "{D30024F5-B73C-452E-8EE3-C3A6D238BF23}"