prehistoric-systems/FilterLists
1
0
Fork 0
mirror of https://github.com/collinbarrett/FilterLists.git synced 2026-03-11 09:04:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

revert last

Browse source
This commit is contained in:
Collin M. Barrett 2019-05-15 18:46:33 -05:00
parent 06d7f11a05
commit 22c0fb354d
11 changed files with 763 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

656
ops/FilterLists.postman_collection.json Normal file
View file

@ -0,0 +1,656 @@
{
"info": {
"_postman_id": "918a11ab-a25e-3c3c-dfb2-295e487ee4d6",
"name": "FilterLists",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
},
"item": [
{
"name": "dependents",
"item": [
{
"name": "dependents/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/dependents/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"dependents",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "forks",
"item": [
{
"name": "forks/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/forks/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"forks",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "languages",
"item": [
{
"name": "languages",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/languages",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"languages"
]
}
},
"response": []
},
{
"name": "languages/{id}",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/languages/122",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"languages",
"122"
]
}
},
"response": []
},
{
"name": "languages/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/languages/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"languages",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "licenses",
"item": [
{
"name": "licenses",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/licenses",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"licenses"
]
}
},
"response": []
},
{
"name": "licenses/{id}",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/licenses/1",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"licenses",
"1"
]
}
},
"response": []
},
{
"name": "licenses/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/licenses/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"licenses",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "lists",
"item": [
{
"name": "lists",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/lists",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"lists"
]
}
},
"response": []
},
{
"name": "lists/{id}",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/lists/1",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"lists",
"1"
]
}
},
"response": []
},
{
"name": "lists/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/lists/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"lists",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "listslanguages",
"item": [
{
"name": "listslanguages/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/listslanguages/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"listslanguages",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "listsmaintainers",
"item": [
{
"name": "listsmaintainers/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/listsmaintainers/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"listsmaintainers",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "liststags",
"item": [
{
"name": "liststags/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/liststags/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"liststags",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "maintainers",
"item": [
{
"name": "maintainers",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/maintainers",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"maintainers"
]
}
},
"response": []
},
{
"name": "maintainers/{id}",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/maintainers/2",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"maintainers",
"2"
]
}
},
"response": []
},
{
"name": "maintainers/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/maintainers/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"maintainers",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "merges",
"item": [
{
"name": "merges/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/merges/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"merges",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "rules",
"item": [
{
"name": "rules",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/rules",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"rules"
]
}
},
"response": []
}
]
},
{
"name": "software",
"item": [
{
"name": "software",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/software",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"software"
]
}
},
"response": []
},
{
"name": "software/{id}",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/software/1",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"software",
"1"
]
}
},
"response": []
},
{
"name": "software/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/software/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"software",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "softwaresyntaxes",
"item": [
{
"name": "softwaresyntaxes/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/softwaresyntaxes/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"softwaresyntaxes",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "syntaxes",
"item": [
{
"name": "syntaxes",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/syntaxes",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"syntaxes"
]
}
},
"response": []
},
{
"name": "syntaxes/{id}",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/syntaxes/1",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"syntaxes",
"1"
]
}
},
"response": []
},
{
"name": "syntaxes/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/syntaxes/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"syntaxes",
"seed"
]
}
},
"response": []
}
]
},
{
"name": "tags",
"item": [
{
"name": "tags",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/tags",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"tags"
]
}
},
"response": []
},
{
"name": "tags/{id}",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/tags/1",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"tags",
"1"
]
}
},
"response": []
},
{
"name": "tags/seed",
"request": {
"method": "GET",
"header": [],
"body": {},
"url": {
"raw": "{{baseUrl}}/{{version}}/tags/seed",
"host": [
"{{baseUrl}}"
],
"path": [
"{{version}}",
"tags",
"seed"
]
}
},
"response": []
}
]
}
]
}

46
ops/nginx/conf.d/filterlists.conf Normal file
View file

@ -0,0 +1,46 @@
# https://github.com/collinbarrett/FilterLists
server {
listen 80 default_server;
listen 443 default_server ssl http2;
server_name filterlists.com;
include svr-includes/*;
ssl_certificate /etc/nginx/cert/filterlists_cert.pem;
ssl_certificate_key /etc/nginx/cert/filterlists_key.pem;
location ^~ /api {
rewrite ^/api/(.*)$ /$1 break;
proxy_pass http://api:5000;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection keep-alive;
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
location / {
proxy_pass http://web:5000;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection keep-alive;
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
# Redirect www to non-www
server {
listen 80;
listen 443 ssl http2;
server_name www.filterlists.com;
ssl_certificate /etc/nginx/cert/filterlists_cert.pem;
ssl_certificate_key /etc/nginx/cert/filterlists_key.pem;
return 301 https://filterlists.com$request_uri;
}

6
ops/nginx/svr-includes/security.conf Normal file
View file

@ -0,0 +1,6 @@
server_tokens off;
add_header X-Content-Type-Options "nosniff" always;
add_header X-Frame-Options "SAMEORIGIN" always;
add_header X-XSS-Protection "1; mode=block" always;
add_header Referrer-Policy "no-referrer" always;

18
ops/nginx/svr-includes/ssl.conf Normal file
View file

@ -0,0 +1,18 @@
# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=nginx-1.15.9&openssl=1.0.1e&hsts=yes&profile=modern
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_session_tickets off;
ssl_protocols TLSv1.2;
ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/nginx/cert/dhparam.pem;
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload";
# https://support.cloudflare.com/hc/en-us/articles/204899617-Authenticated-Origin-Pulls
ssl_client_certificate /etc/nginx/cert/cloudflare.crt;
ssl_verify_client on;

2
ops/scripts/cron Normal file
View file

@ -0,0 +1,2 @@
# FilterLists Capture Snapshots
*/1 * * * * nice -n 19 /usr/bin/flock -n /tmp/filterlistsagent.lockfile /var/www/filterlists/FilterLists.Agent > /dev/null 2>&1

4
ops/scripts/sql/createDb.sql Normal file
View file

@ -0,0 +1,4 @@
CREATE USER 'filterlists'@'localhost' IDENTIFIED BY 'password';
CREATE DATABASE filterlistsdata CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci;
GRANT ALL PRIVILEGES ON filterlistsdata.* TO 'filterlists'@'localhost';
FLUSH PRIVILEGES;

4
ops/scripts/sql/duplicateRules.sql Normal file
View file

@ -0,0 +1,4 @@
SELECT Raw, COUNT(*) 'Count'
FROM rules
GROUP BY Raw COLLATE utf8mb4_bin
HAVING COUNT > 1;

5
ops/scripts/sql/orphanedSnapshotsRules.sql Normal file
View file

@ -0,0 +1,5 @@
SELECT SnapshotId, COUNT(*)
FROM snapshots_rules
JOIN snapshots ON snapshots_rules.SnapshotId = snapshots.Id
WHERE snapshots.WasSuccessful = FALSE
GROUP BY snapshots.Id;

7
ops/scripts/sql/resetSnapshotsRules.sql Normal file
View file

@ -0,0 +1,7 @@
SET FOREIGN_KEY_CHECKS = 0;
TRUNCATE snapshots_rules;
TRUNCATE snapshots;
TRUNCATE rules;
SET FOREIGN_KEY_CHECKS = 1;
ALTER TABLE snapshots AUTO_INCREMENT = 1;
ALTER TABLE rules AUTO_INCREMENT = 1;

6
ops/scripts/sql/snapshotMetrics.sql Normal file
View file

@ -0,0 +1,6 @@
SELECT snapshots.CreatedDateUtc, FilterlistId, HttpStatusCode, CAST(WasSuccessful AS UNSIGNED), WaybackTimestamp, COUNT(snapshots_rules.RuleId) AS 'Count'
FROM snapshots
JOIN snapshots_rules ON snapshots.Id = snapshots_rules.SnapshotId
GROUP BY snapshots.Id
ORDER BY snapshots.CreatedDateUtc DESC
LIMIT 5;

9
ops/scripts/sql/snapshotTimeAnalysis.sql Normal file
View file

@ -0,0 +1,9 @@
SELECT
FilterListId,
ModifiedDateUtc - CreatedDateUtc AS ProcessingTimeSecs,
CreatedDateUtc,
ModifiedDateUtc,
WasSuccessful
FROM snapshots
WHERE ModifiedDateUtc - CreatedDateUtc > 30 OR WasSuccessful = 0
ORDER BY ProcessingTimeSecs DESC;