diff --git a/ops/FilterLists.postman_collection.json b/ops/FilterLists.postman_collection.json new file mode 100644 index 000000000..5223a4d1f --- /dev/null +++ b/ops/FilterLists.postman_collection.json @@ -0,0 +1,656 @@ +{ + "info": { + "_postman_id": "918a11ab-a25e-3c3c-dfb2-295e487ee4d6", + "name": "FilterLists", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "item": [ + { + "name": "dependents", + "item": [ + { + "name": "dependents/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/dependents/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "dependents", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "forks", + "item": [ + { + "name": "forks/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/forks/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "forks", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "languages", + "item": [ + { + "name": "languages", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/languages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "languages" + ] + } + }, + "response": [] + }, + { + "name": "languages/{id}", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/languages/122", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "languages", + "122" + ] + } + }, + "response": [] + }, + { + "name": "languages/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/languages/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "languages", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "licenses", + "item": [ + { + "name": "licenses", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/licenses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "licenses" + ] + } + }, + "response": [] + }, + { + "name": "licenses/{id}", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/licenses/1", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "licenses", + "1" + ] + } + }, + "response": [] + }, + { + "name": "licenses/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/licenses/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "licenses", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "lists", + "item": [ + { + "name": "lists", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/lists", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "lists" + ] + } + }, + "response": [] + }, + { + "name": "lists/{id}", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/lists/1", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "lists", + "1" + ] + } + }, + "response": [] + }, + { + "name": "lists/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/lists/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "lists", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "listslanguages", + "item": [ + { + "name": "listslanguages/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/listslanguages/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "listslanguages", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "listsmaintainers", + "item": [ + { + "name": "listsmaintainers/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/listsmaintainers/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "listsmaintainers", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "liststags", + "item": [ + { + "name": "liststags/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/liststags/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "liststags", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "maintainers", + "item": [ + { + "name": "maintainers", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/maintainers", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "maintainers" + ] + } + }, + "response": [] + }, + { + "name": "maintainers/{id}", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/maintainers/2", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "maintainers", + "2" + ] + } + }, + "response": [] + }, + { + "name": "maintainers/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/maintainers/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "maintainers", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "merges", + "item": [ + { + "name": "merges/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/merges/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "merges", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "rules", + "item": [ + { + "name": "rules", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/rules", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "rules" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "software", + "item": [ + { + "name": "software", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/software", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "software" + ] + } + }, + "response": [] + }, + { + "name": "software/{id}", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/software/1", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "software", + "1" + ] + } + }, + "response": [] + }, + { + "name": "software/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/software/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "software", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "softwaresyntaxes", + "item": [ + { + "name": "softwaresyntaxes/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/softwaresyntaxes/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "softwaresyntaxes", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "syntaxes", + "item": [ + { + "name": "syntaxes", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/syntaxes", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "syntaxes" + ] + } + }, + "response": [] + }, + { + "name": "syntaxes/{id}", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/syntaxes/1", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "syntaxes", + "1" + ] + } + }, + "response": [] + }, + { + "name": "syntaxes/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/syntaxes/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "syntaxes", + "seed" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "tags", + "item": [ + { + "name": "tags", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/tags", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "tags" + ] + } + }, + "response": [] + }, + { + "name": "tags/{id}", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/tags/1", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "tags", + "1" + ] + } + }, + "response": [] + }, + { + "name": "tags/seed", + "request": { + "method": "GET", + "header": [], + "body": {}, + "url": { + "raw": "{{baseUrl}}/{{version}}/tags/seed", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "{{version}}", + "tags", + "seed" + ] + } + }, + "response": [] + } + ] + } + ] +} \ No newline at end of file diff --git a/ops/nginx/conf.d/filterlists.conf b/ops/nginx/conf.d/filterlists.conf new file mode 100644 index 000000000..9bc5af745 --- /dev/null +++ b/ops/nginx/conf.d/filterlists.conf @@ -0,0 +1,46 @@ +# https://github.com/collinbarrett/FilterLists +server { + listen 80 default_server; + listen 443 default_server ssl http2; + server_name filterlists.com; + + include svr-includes/*; + + ssl_certificate /etc/nginx/cert/filterlists_cert.pem; + ssl_certificate_key /etc/nginx/cert/filterlists_key.pem; + + location ^~ /api { + rewrite ^/api/(.*)$ /$1 break; + proxy_pass http://api:5000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection keep-alive; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location / { + proxy_pass http://web:5000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection keep-alive; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +# Redirect www to non-www +server { + listen 80; + listen 443 ssl http2; + server_name www.filterlists.com; + + ssl_certificate /etc/nginx/cert/filterlists_cert.pem; + ssl_certificate_key /etc/nginx/cert/filterlists_key.pem; + + return 301 https://filterlists.com$request_uri; +} diff --git a/ops/nginx/svr-includes/security.conf b/ops/nginx/svr-includes/security.conf new file mode 100644 index 000000000..a53c73a47 --- /dev/null +++ b/ops/nginx/svr-includes/security.conf @@ -0,0 +1,6 @@ +server_tokens off; + +add_header X-Content-Type-Options "nosniff" always; +add_header X-Frame-Options "SAMEORIGIN" always; +add_header X-XSS-Protection "1; mode=block" always; +add_header Referrer-Policy "no-referrer" always; diff --git a/ops/nginx/svr-includes/ssl.conf b/ops/nginx/svr-includes/ssl.conf new file mode 100644 index 000000000..4cfb00994 --- /dev/null +++ b/ops/nginx/svr-includes/ssl.conf @@ -0,0 +1,18 @@ +# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=nginx-1.15.9&openssl=1.0.1e&hsts=yes&profile=modern + +ssl_session_timeout 1d; +ssl_session_cache shared:SSL:50m; +ssl_session_tickets off; + +ssl_protocols TLSv1.2; +ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; +ssl_prefer_server_ciphers on; + +ssl_dhparam /etc/nginx/cert/dhparam.pem; + +add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"; + +# https://support.cloudflare.com/hc/en-us/articles/204899617-Authenticated-Origin-Pulls + +ssl_client_certificate /etc/nginx/cert/cloudflare.crt; +ssl_verify_client on; diff --git a/ops/scripts/cron b/ops/scripts/cron new file mode 100644 index 000000000..24c3f09c6 --- /dev/null +++ b/ops/scripts/cron @@ -0,0 +1,2 @@ +# FilterLists Capture Snapshots +*/1 * * * * nice -n 19 /usr/bin/flock -n /tmp/filterlistsagent.lockfile /var/www/filterlists/FilterLists.Agent > /dev/null 2>&1 \ No newline at end of file diff --git a/ops/scripts/sql/createDb.sql b/ops/scripts/sql/createDb.sql new file mode 100644 index 000000000..9c8ea6f8f --- /dev/null +++ b/ops/scripts/sql/createDb.sql @@ -0,0 +1,4 @@ +CREATE USER 'filterlists'@'localhost' IDENTIFIED BY 'password'; +CREATE DATABASE filterlistsdata CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci; +GRANT ALL PRIVILEGES ON filterlistsdata.* TO 'filterlists'@'localhost'; +FLUSH PRIVILEGES; \ No newline at end of file diff --git a/ops/scripts/sql/duplicateRules.sql b/ops/scripts/sql/duplicateRules.sql new file mode 100644 index 000000000..0d58615c5 --- /dev/null +++ b/ops/scripts/sql/duplicateRules.sql @@ -0,0 +1,4 @@ +SELECT Raw, COUNT(*) 'Count' +FROM rules +GROUP BY Raw COLLATE utf8mb4_bin +HAVING COUNT > 1; \ No newline at end of file diff --git a/ops/scripts/sql/orphanedSnapshotsRules.sql b/ops/scripts/sql/orphanedSnapshotsRules.sql new file mode 100644 index 000000000..75cef7238 --- /dev/null +++ b/ops/scripts/sql/orphanedSnapshotsRules.sql @@ -0,0 +1,5 @@ +SELECT SnapshotId, COUNT(*) +FROM snapshots_rules +JOIN snapshots ON snapshots_rules.SnapshotId = snapshots.Id +WHERE snapshots.WasSuccessful = FALSE +GROUP BY snapshots.Id; \ No newline at end of file diff --git a/ops/scripts/sql/resetSnapshotsRules.sql b/ops/scripts/sql/resetSnapshotsRules.sql new file mode 100644 index 000000000..42156539f --- /dev/null +++ b/ops/scripts/sql/resetSnapshotsRules.sql @@ -0,0 +1,7 @@ +SET FOREIGN_KEY_CHECKS = 0; +TRUNCATE snapshots_rules; +TRUNCATE snapshots; +TRUNCATE rules; +SET FOREIGN_KEY_CHECKS = 1; +ALTER TABLE snapshots AUTO_INCREMENT = 1; +ALTER TABLE rules AUTO_INCREMENT = 1; \ No newline at end of file diff --git a/ops/scripts/sql/snapshotMetrics.sql b/ops/scripts/sql/snapshotMetrics.sql new file mode 100644 index 000000000..4f0e14e86 --- /dev/null +++ b/ops/scripts/sql/snapshotMetrics.sql @@ -0,0 +1,6 @@ +SELECT snapshots.CreatedDateUtc, FilterlistId, HttpStatusCode, CAST(WasSuccessful AS UNSIGNED), WaybackTimestamp, COUNT(snapshots_rules.RuleId) AS 'Count' +FROM snapshots +JOIN snapshots_rules ON snapshots.Id = snapshots_rules.SnapshotId +GROUP BY snapshots.Id +ORDER BY snapshots.CreatedDateUtc DESC +LIMIT 5; \ No newline at end of file diff --git a/ops/scripts/sql/snapshotTimeAnalysis.sql b/ops/scripts/sql/snapshotTimeAnalysis.sql new file mode 100644 index 000000000..1491adb3d --- /dev/null +++ b/ops/scripts/sql/snapshotTimeAnalysis.sql @@ -0,0 +1,9 @@ +SELECT + FilterListId, + ModifiedDateUtc - CreatedDateUtc AS ProcessingTimeSecs, + CreatedDateUtc, + ModifiedDateUtc, + WasSuccessful +FROM snapshots +WHERE ModifiedDateUtc - CreatedDateUtc > 30 OR WasSuccessful = 0 +ORDER BY ProcessingTimeSecs DESC; \ No newline at end of file