From f0dbe3cc238bbe9364bc39c0eb41cfb3b2507a34 Mon Sep 17 00:00:00 2001 From: "Collin M. Barrett" Date: Sat, 6 Jul 2019 19:17:02 -0500 Subject: [PATCH] update README/CONTRIBUTING docker guides, add guide for Agent, and module summaries to README --- .github/CONTRIBUTING.md | 150 ++++++++++-------- .github/README.md | 47 ++++-- .../SeedFilterListsDbContextTests.cs | 5 + 3 files changed, 121 insertions(+), 81 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index f73d73f69..8ca5b25cb 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,67 +1,83 @@ -## Community - -Discourse posts - -Check out the FilterLists Hub, a place for discussing how to write rules, maintain FilterLists, and chat about the state of the adblocking community. - -## Adding or Updating Lists - -To submit a new list or update data about an existing list, please submit a pull request to [data](https://github.com/collinbarrett/FilterLists/tree/master/data) in conjunction with the data model described [here](https://github.com/collinbarrett/FilterLists/wiki/Data-Model_sidebar). Alternatively, you can [open a new issue](https://github.com/collinbarrett/FilterLists/issues/new) providing information for all of the fields described in the [data model](https://github.com/collinbarrett/FilterLists/wiki/Data-Model_sidebar). - -## Adding or Updating Rules - -FilterLists does not maintain any of these lists. It serves only as a discovery tool to direct users to lists that they may want to use. If you want to request addition, modification, or removal of a rule from a list, you will need to contact the maintainers of that list directly. FilterLists provides a variety of ways you can get in contact with the maintainers to do so. - -## Building and Running Locally - -We have containerized FilterLists to make it as easy as possible for contributers to get the project up and running locally. - -### Up and Running - -1. Install Docker CE for your computer's operating system. [Docs](https://docs.docker.com/install/) -2. Clone the FilterLists git repository to your computer. [Docs](https://help.github.com/en/articles/cloning-a-repository) -3. Navigate to the root directory of your locally cloned FilterLists git repository in a command-line interface. -4. Execute `docker-compose up`. (Optionally, include the `-d` flag to launch in [detached mode](https://docs.docker.com/compose/reference/up/).) -5. After a minute or so, visit the locally running version of FilterLists in a web browser at `http://localhost/`. - -### Testing changes to the data (.json files) - -#### Automated - -Execute: - -`docker volume create test-data-results && docker-compose -f docker-compose.data.tests.yml down -v && docker-compose -f docker-compose.data.tests.yml build api && docker-compose -f docker-compose.data.tests.yml run api` - -#### Manual - -1. Execute `docker container ls` to find the `CONTAINER ID` of the `filterlists.api` container. -2. Execute `docker-compose up -d --build [CONTAINER ID]` replacing `[CONTAINER ID]` with the hash from step 1. -3. Verify your changes are properly reflected at `http://localhost/`. - -### Testing changes to the `Api`, `Services`, or `Data` projects - -#### Automated - -- To run `FilterLists.Services.Tests`: - - `docker volume create test-services-results && docker-compose -f docker-compose.services.tests.yml build api && docker-compose -f docker-compose.services.tests.yml run --rm api` - -- To run `FilterLists.Data.Tests`: - - `docker volume create test-data-results && docker-compose -f docker-compose.data.tests.yml down -v && docker-compose -f docker-compose.data.tests.yml build api && docker-compose -f docker-compose.data.tests.yml run api` - -#### Manual - -1. Execute `docker container ls` to find the `CONTAINER ID` of the `filterlists.api` container. -2. Execute `docker-compose up -d --build [CONTAINER ID]` replacing `[CONTAINER ID]` with the hash from step 1. -3. Verify your changes are properly reflected at `http://localhost/api`. - -### Testing changes to the `Web` project - -1. Execute `docker container ls` to find the `CONTAINER ID` of the `filterlists.web` container. -2. Execute `docker-compose up -d --build [CONTAINER ID]` replacing `[CONTAINER ID]` with the hash from step 1. - -### Debugging - -1. Execute `docker container ls -a` to find the `CONTAINER ID` of the container in question. -2. Execute `docker logs [CONTAINER ID]` replacing `[CONTAINER ID]` with the hash from step 1 to view the logs from that container. +## Community + +Discourse posts + +Check out the FilterLists Hub, a place for discussing how to write rules, maintain FilterLists, and chat about the state of the adblocking community. + +## Adding or Updating Lists + +To submit a new list or update data about an existing list, please submit a pull request to [data](https://github.com/collinbarrett/FilterLists/tree/master/data) in conjunction with the data model described [here](https://github.com/collinbarrett/FilterLists/wiki/Data-Model_sidebar). Alternatively, you can [open a new issue](https://github.com/collinbarrett/FilterLists/issues/new) providing information for all of the fields described in the [data model](https://github.com/collinbarrett/FilterLists/wiki/Data-Model_sidebar). + +## Adding or Updating Rules + +FilterLists does not maintain any of these lists. It serves only as a discovery tool to direct users to lists that they may want to use. If you want to request addition, modification, or removal of a rule from a list, you will need to contact the maintainers of that list directly. FilterLists provides a variety of ways you can get in contact with the maintainers to do so. + +## Building and Running Locally + +We have containerized FilterLists to make it as easy as possible for contributers to get the project up and running locally. + +### Up and Running + +1. Install Docker CE for your computer's operating system. [Docs](https://docs.docker.com/install/) +2. Clone the FilterLists git repository to your computer. [Docs](https://help.github.com/en/articles/cloning-a-repository) +3. Navigate to the root directory of your locally cloned FilterLists git repository in a command-line interface. +4. Execute `docker-compose up`. (Optionally, include the `-d` flag to launch in [detached mode](https://docs.docker.com/compose/reference/up/).) +5. After a minute or so, visit the locally running version of FilterLists in a web browser at `http://localhost/`. + +### Testing changes to the data (.json files) + +#### Automated + +Execute: + +`docker volume create test-data-results && docker-compose -p test-data -f docker-compose.data.tests.yml down -v && docker-compose -f docker-compose.data.tests.yml build api && docker-compose -p test-data -f docker-compose.data.tests.yml run --rm api` + +#### Manual + +1. Execute `docker-compose up -d --build api`. +2. Verify your changes are properly reflected at `http://localhost/`. + +### Testing changes to the `Api`, `Services`, or `Data` projects + +#### Automated + +- To run `FilterLists.Services.Tests`: + + `docker volume create test-services-results && docker-compose -f docker-compose.services.tests.yml build api && docker-compose -p test-services -f docker-compose.services.tests.yml run --rm api` + +- To run `FilterLists.Data.Tests`: + + `docker volume create test-data-results && docker-compose -p test-data -f docker-compose.data.tests.yml down -v && docker-compose -f docker-compose.data.tests.yml build api && docker-compose -p test-data -f docker-compose.data.tests.yml run --rm api` + +#### Manual + +1. Execute `docker-compose up -d --build api`. +2. Verify your changes are properly reflected at `http://localhost/api`. + +### Testing changes to the `Web` project + +1. Execute `docker-compose up -d --build web`. +2. Verify your changes are properly reflected at `http://localhost/`. + +### Testing changes to the `Agent` project + +The Agent takes the following command line arguments: + +- `-a` Archive copies of all lists in a git repository. +- `-c` Validate all URLs in the FilterLists database. + +#### Automated + +- To run `FilterLists.Agent.Tests`: + + `docker volume create test-agent-results && docker-compose -f docker-compose.agent.tests.yml build agent && docker-compose -p test-agent -f docker-compose.agent.tests.yml run --rm agent` + +#### Manual + +1. Execute `docker-compose build agent && docker-compose run agent [-c] [-v]` (don't include the square brackets, they indicate optional command line arguments). +2. Verify your changes are properly reflected in the console logger. + +### Debugging + +1. Execute `docker container ls -a` to find the `CONTAINER ID` of the container in question. +2. Execute `docker logs [CONTAINER ID]` replacing `[CONTAINER ID]` with the hash from step 1 to view the logs from that container. \ No newline at end of file diff --git a/.github/README.md b/.github/README.md index 92101ffb7..c16e68889 100644 --- a/.github/README.md +++ b/.github/README.md @@ -12,18 +12,21 @@ Azure DevOps builds Azure DevOps tests Azure DevOps releases -Docker Pulls

+Docker Pulls +
An ASP.NET Core API serving data from a MariaDB instance.

Website: Website Azure DevOps builds Azure DevOps releases -Docker Pulls

+Docker Pulls +
A simple React & TypeScript UI primarily featuring a [react-table](https://www.npmjs.com/package/react-table) instance.

Agent: Azure DevOps builds Azure DevOps tests Azure DevOps releases -Docker Pulls

+Docker Pulls +
A .NET Core console application performing crawling, archiving, and URL validation.

# Background @@ -67,13 +70,12 @@ We have containerized FilterLists to make it as easy as possible for contributer Execute: -`docker volume create test-data-results && docker-compose -f docker-compose.data.tests.yml down -v && docker-compose -f docker-compose.data.tests.yml build api && docker-compose -f docker-compose.data.tests.yml run api` +`docker volume create test-data-results && docker-compose -p test-data -f docker-compose.data.tests.yml down -v && docker-compose -f docker-compose.data.tests.yml build api && docker-compose -p test-data -f docker-compose.data.tests.yml run --rm api` #### Manual -1. Execute `docker container ls` to find the `CONTAINER ID` of the `filterlists.api` container. -2. Execute `docker-compose up -d --build [CONTAINER ID]` replacing `[CONTAINER ID]` with the hash from step 1. -3. Verify your changes are properly reflected at `http://localhost/`. +1. Execute `docker-compose up -d --build api`. +2. Verify your changes are properly reflected at `http://localhost/`. ### Testing changes to the `Api`, `Services`, or `Data` projects @@ -81,22 +83,39 @@ Execute: - To run `FilterLists.Services.Tests`: - `docker volume create test-services-results && docker-compose -f docker-compose.services.tests.yml build api && docker-compose -f docker-compose.services.tests.yml run --rm api` + `docker volume create test-services-results && docker-compose -f docker-compose.services.tests.yml build api && docker-compose -p test-services -f docker-compose.services.tests.yml run --rm api` - To run `FilterLists.Data.Tests`: - `docker volume create test-data-results && docker-compose -f docker-compose.data.tests.yml down -v && docker-compose -f docker-compose.data.tests.yml build api && docker-compose -f docker-compose.data.tests.yml run api` + `docker volume create test-data-results && docker-compose -p test-data -f docker-compose.data.tests.yml down -v && docker-compose -f docker-compose.data.tests.yml build api && docker-compose -p test-data -f docker-compose.data.tests.yml run --rm api` #### Manual -1. Execute `docker container ls` to find the `CONTAINER ID` of the `filterlists.api` container. -2. Execute `docker-compose up -d --build [CONTAINER ID]` replacing `[CONTAINER ID]` with the hash from step 1. -3. Verify your changes are properly reflected at `http://localhost/api`. +1. Execute `docker-compose up -d --build api`. +2. Verify your changes are properly reflected at `http://localhost/api`. ### Testing changes to the `Web` project -1. Execute `docker container ls` to find the `CONTAINER ID` of the `filterlists.web` container. -2. Execute `docker-compose up -d --build [CONTAINER ID]` replacing `[CONTAINER ID]` with the hash from step 1. +1. Execute `docker-compose up -d --build web`. +2. Verify your changes are properly reflected at `http://localhost/`. + +### Testing changes to the `Agent` project + +The Agent takes the following command line arguments: + +- `-a` Archive copies of all lists in a git repository. +- `-c` Validate all URLs in the FilterLists database. + +#### Automated + +- To run `FilterLists.Agent.Tests`: + + `docker volume create test-agent-results && docker-compose -f docker-compose.agent.tests.yml build agent && docker-compose -p test-agent -f docker-compose.agent.tests.yml run --rm agent` + +#### Manual + +1. Execute `docker-compose build agent && docker-compose run agent [-c] [-v]` (don't include the square brackets, they indicate optional command line arguments). +2. Verify your changes are properly reflected in the console logger. ### Debugging diff --git a/tests/FilterLists.Data.Tests/SeedFilterListsDbContextTests.cs b/tests/FilterLists.Data.Tests/SeedFilterListsDbContextTests.cs index 244b3b17c..a8ff2ea21 100644 --- a/tests/FilterLists.Data.Tests/SeedFilterListsDbContextTests.cs +++ b/tests/FilterLists.Data.Tests/SeedFilterListsDbContextTests.cs @@ -1,3 +1,4 @@ +using System.Threading; using FilterLists.Data.Seed.Extensions; using Microsoft.EntityFrameworkCore; using Xunit; @@ -9,6 +10,10 @@ public class SeedFilterListsDbContextTests [Fact] public async void SeedOrUpdateAsync_DoesNotThrowException() { + //TODO: replace with Polly or similar to optimize time waste + // allow time for db to init + Thread.Sleep(10000); + const string connString = "Server=mariadb;Database=filterlists;Uid=filterlists;Pwd=filterlists;"; var options = new DbContextOptionsBuilder() .UseMySql(connString, m => m.MigrationsAssembly("FilterLists.Api"))