From 313cec11d51a4ab3991d308ef82b46f92765b837 Mon Sep 17 00:00:00 2001 From: ZhymabekRoman Date: Tue, 29 Oct 2024 14:45:06 +0500 Subject: [PATCH] feat(analytics) - plausible, refactor docker network --- README.md | 156 ++++++++++++--- caddy/{CaddyfileDev => Caddyfile} | 39 +++- caddy/CaddyfileDevTemplate | 14 -- caddy/CaddyfileProd | 211 -------------------- caddy/CaddyfileProdTemplate | 15 -- caddy/CaddyfileTemplate | 37 ++++ caddy/generate_caddy_file.py | 8 +- docker-compose/docker-compose.main.yml | 28 ++- docker-compose/docker-compose.plausible.yml | 110 ++++++++++ docker-compose/docker-compose.proxy.yml | 2 - docker-compose/docker-compose.yml | 1 + plausible/clickhouse/docker.xml | 3 + plausible/clickhouse/ipv4-only.xml | 3 + plausible/clickhouse/logs.xml | 36 ++++ 14 files changed, 374 insertions(+), 289 deletions(-) rename caddy/{CaddyfileDev => Caddyfile} (88%) delete mode 100644 caddy/CaddyfileDevTemplate delete mode 100644 caddy/CaddyfileProd delete mode 100644 caddy/CaddyfileProdTemplate create mode 100644 caddy/CaddyfileTemplate mode change 100644 => 100755 caddy/generate_caddy_file.py create mode 100644 docker-compose/docker-compose.plausible.yml create mode 100644 plausible/clickhouse/docker.xml create mode 100644 plausible/clickhouse/ipv4-only.xml create mode 100644 plausible/clickhouse/logs.xml diff --git a/README.md b/README.md index 9ffddc0..d061689 100644 --- a/README.md +++ b/README.md @@ -6,25 +6,7 @@ Become a Patron -## FAQ - -### Why did we create Freedium? - -In mid-June to mid-July 2023, Medium changed their paywall method, and all old paywall bypass methods we had stopped working. So I became obsessed with the idea of creating a service to bypass Medium's paywalled posts. Honestly I am not a big fan of Medium, but I sometimes read articles to improve my knowledge. - -### How does Freedium work? - -In the first version of Freedium, we reverse-engineered Medium.com's GraphQL endpoints and built our own parser and toolkits to show you unpaywalled Medium posts. Unfortunately, Medium closed this loophole and nowadays we just pay subscriptions and share access through Freedium. Sometimes we got a bugs because of the self-written parser, but we are working to make Freedium bug-free. - -### Wow! I would like to contribute to Freedium. How can I do that? - -We need volunteers who have Medium subscriptions because we might get banned by Medium. And if you developer you can start from the this (https://codeberg.org/Freedium-cfd/web) repository. - -### Plans, future? - -Speed up Freedium, add support for more services than just Medium and (probably) create open source Medium frontend (in next life) - -## Technologies: +## Stack: - Backend: Python 3.9+, Unicorn, FastAPI, Jinja2, Sentry - Frontend: Tailwinds CSS v3 @@ -33,6 +15,13 @@ Speed up Freedium, add support for more services than just Medium and (probably) ## Local run: +There is two profiles: + +- `min` - without 2 Cluster of Cloudflare WARP proxy, HAProxy proxy balancer, Plausible, Grafana. +- `prod` - with all services for production. + +For local development, we recommend to use `min` profile. + Requirements: - Docker @@ -44,8 +33,8 @@ To configure your Freedium instance, follow these steps: 1. Clone the repository: ``` - git clone https://codeberg.org/Freedium-cfd/web/ ./web --depth 1 - cd ./web + git clone https://codeberg.org/Freedium-cfd/web/ ./freedium-web --depth 1 + cd ./freedium-web ``` 2. Create and configure the environment file: @@ -56,24 +45,135 @@ To configure your Freedium instance, follow these steps: Open the `.env` file and adjust the values as needed for your setup. -3. Set up the Docker network: +3. (Optional) Set up the Docker network: ``` - sudo docker network create caddy_net + sudo docker network create caddy_freedium_net ``` -4. Start the Freedium services: +4. Change your hosts file: + ``` - sudo docker compose -f ./docker-compose/docker-compose.yml up + sudo nano /etc/hosts ``` -And now you can access local instance of Freedium by opening browser and type `http://localhost:6752`. + Add the following line: -These steps will set up and run your local Freedium instance. + ``` + 127.0.0.1 freedium.local + ``` + +5. Start the Freedium services (`min` profile): + + ``` + sudo docker compose --profile min -f ./docker-compose/docker-compose.yml up + ``` + + Stopping the services: + + ``` + sudo docker compose --profile min -f ./docker-compose/docker-compose.yml down + ``` + +6. (Optional) Configure your reverse proxy (Caddy, Nginx, etc.) to use `freedium.local` as a host. + +If you use Dockerized reverse proxy, you can specify network `caddy_freedium_net` with `external: true` option in networks section of your reverse proxy container. Specify `caddy_freedium` hostname with port `80` (or `443`) in your reverse proxy configuration. + +And now you can access local instance of Freedium by opening browser and type `https://freedium.local`. There is would be a warning about insecure connection, because we use self-signed TLS certificate. Ignore it. + +## Architecture: + +```mermaid +graph TB + subgraph "Main Application" + fw[freedium_web] + cdy[caddy_freedium] + end + + subgraph "Database Layer" + rds[redis_service
DragonFlyDB] + pg[(postgres_freedium
PostgreSQL)] + pgadm[pgadmin4_freedium] + end + + subgraph "Proxy Layer" + hpb[haproxy-proxy-balancer] + + subgraph "WGCF Cluster 1" + wg1[wgcf1] + d1[dante_1] + wh1[wgcf1_healthcare_service] + end + + subgraph "WGCF Cluster 2" + wg2[wgcf2] + d2[dante_2] + wh2[wgcf2_healthcare_service] + end + end + + subgraph "Analytics" + pls[freedium_plausible] + pldb[(plausible_db)] + pledb[(plausible_events_db
ClickHouse)] + end + + subgraph "Utility" + ah[autoheal] + end + + %% Dependencies + fw -->|depends_on| hpb + d1 -->|depends_on| wg1 + d2 -->|depends_on| wg2 + wg2 -->|depends_on| wg1 + hpb -->|depends_on| wg1 + hpb -->|depends_on| wg2 + pls -->|depends_on| pldb + pls -->|depends_on| pledb + + %% Network Connections + subgraph "Networks" + fn[freedium_net] + cn[caddy_net] + pn[plausible_net] + end + + fw ---|freedium_net| fn + cdy ---|freedium_net & caddy_net| fn + rds ---|freedium_net| fn + pg ---|freedium_net| fn + pgadm ---|freedium_net| fn + hpb ---|freedium_net| fn + wg1 ---|freedium_net| fn + wg2 ---|freedium_net| fn + wh1 ---|freedium_net| fn + wh2 ---|freedium_net| fn + pls ---|all networks| fn + + %% Port Exposures + cdy -->|":6752"| ext1[External Access] + fw -->|":7080"| ext2[External Access] + pgadm -->|":5433"| ext3[External Access] + + classDef service fill:#2ecc71,stroke:#27ae60,color:white + classDef network fill:#3498db,stroke:#2980b9,color:white + classDef database fill:#e74c3c,stroke:#c0392b,color:white + classDef proxy fill:#f1c40f,stroke:#f39c12,color:black + classDef utility fill:#9b59b6,stroke:#8e44ad,color:white + classDef external fill:#95a5a6,stroke:#7f8c8d,color:white + + class fw,cdy,pls service + class rds,pg,pldb,pledb database + class hpb,wg1,wg2,d1,d2 proxy + class ah,wh1,wh2 utility + class fn,cn,pn network + class ext1,ext2,ext3 external +``` ## TODO: -- Integrate library notifiers - https://github.com/liiight/notifiers +- ~~Integrate library notifiers - https://github.com/liiight/notifiers~~ Use Graphana and Loki instead - Do not use 'shturman/dante' image, because it is does not have updates for a long time. (Probably) Use https://hub.docker.com/r/vimagick/dante/ ## Roadmap diff --git a/caddy/CaddyfileDev b/caddy/Caddyfile similarity index 88% rename from caddy/CaddyfileDev rename to caddy/Caddyfile index afaee57..6a5a437 100644 --- a/caddy/CaddyfileDev +++ b/caddy/Caddyfile @@ -1,9 +1,9 @@ -:6752 { - # header Server "nginx" +(common) { + tls internal encode gzip header -Server - + handle_path /android-chrome-192x192.png { root * /static/android-chrome-192x192.png file_server @@ -200,11 +200,34 @@ respond "Access denied" 403 } +} - route /* { - reverse_proxy freedium_web:7080 { - lb_try_duration 30s - lb_try_interval 1s - } +(header_up) { + header_up Host {host} + header_up X-Real-IP {remote_host} + header_up X-Forwarded-For {remote_host} + header_up X-Forwarded-Proto {scheme} +} + +(lb_try) { + lb_try_duration 30s + lb_try_interval 1s +} + +plausible.freedium.local { + import common + + reverse_proxy freedium_plausible:8000 { + import header_up + import lb_try + } +} + +freedium.local { + import common + + reverse_proxy freedium_web:7080 { + import header_up + import lb_try } } \ No newline at end of file diff --git a/caddy/CaddyfileDevTemplate b/caddy/CaddyfileDevTemplate deleted file mode 100644 index 4f78960..0000000 --- a/caddy/CaddyfileDevTemplate +++ /dev/null @@ -1,14 +0,0 @@ -:6752 { - # header Server "nginx" - encode gzip - header -Server - -{{ template }} - - route /* { - reverse_proxy freedium_web:7080 { - lb_try_duration 30s - lb_try_interval 1s - } - } -} diff --git a/caddy/CaddyfileProd b/caddy/CaddyfileProd deleted file mode 100644 index e0aa1b0..0000000 --- a/caddy/CaddyfileProd +++ /dev/null @@ -1,211 +0,0 @@ -# https://futurestud.io/tutorials/caddy-reverse-proxy-a-node-js-app -freedium.cfd { - # header Server "nginx" - encode gzip - header -Server - - - handle_path /android-chrome-192x192.png { - root * /static/android-chrome-192x192.png - file_server - } - - - handle_path /android-chrome-512x512.png { - root * /static/android-chrome-512x512.png - file_server - } - - - handle_path /apple-touch-icon.png { - root * /static/apple-touch-icon.png - file_server - } - - - handle_path /browserconfig.xml { - root * /static/browserconfig.xml - file_server - } - - - handle_path /favicon-16x16.png { - root * /static/favicon-16x16.png - file_server - } - - - handle_path /favicon-32x32.png { - root * /static/favicon-32x32.png - file_server - } - - - handle_path /favicon.ico { - root * /static/favicon.ico - file_server - } - - - handle_path /humans.txt { - root * /static/humans.txt - file_server - } - - - handle_path /mstile-144x144.png { - root * /static/mstile-144x144.png - file_server - } - - - handle_path /mstile-150x150.png { - root * /static/mstile-150x150.png - file_server - } - - - handle_path /mstile-310x150.png { - root * /static/mstile-310x150.png - file_server - } - - - handle_path /mstile-310x310.png { - root * /static/mstile-310x310.png - file_server - } - - - handle_path /mstile-70x70.png { - root * /static/mstile-70x70.png - file_server - } - - - handle_path /robots.txt { - root * /static/robots.txt - file_server - } - - - handle_path /safari-pinned-tab.svg { - root * /static/safari-pinned-tab.svg - file_server - } - - - handle_path /security.txt { - root * /static/security.txt - file_server - } - - - handle_path /site.webmanifest { - root * /static/site.webmanifest - file_server - } - - - handle_path /websocket { - respond "Access denied" 403 - } - - - handle_path /meta.json { - respond "Access denied" 403 - } - - - handle_path /cdn-cgi/challenge-platform/scripts/jsd/main.js { - respond "Access denied" 403 - } - - - handle_path /cdn-cgi/rum { - respond "Access denied" 403 - } - - - handle_path /graphql/websocket { - respond "Access denied" 403 - } - - - handle_path /onboarding/* { - respond "Access denied" 403 - } - - - handle_path /wp-* { - respond "Access denied" 403 - } - - - handle_path /.env { - respond "Access denied" 403 - } - - - handle_path /api* { - respond "Access denied" 403 - } - - - handle_path /apple-touch-icon-precomposed.png { - respond "Access denied" 403 - } - - - handle_path /rss.xml { - respond "Access denied" 403 - } - - - handle_path /.git/* { - respond "Access denied" 403 - } - - - handle_path /apple-touch-icon-120x120.png { - respond "Access denied" 403 - } - - - handle_path /apple-touch-icon-120x120-precomposed.png { - respond "Access denied" 403 - } - - - handle_path /apple-touch-icon-152x152.png { - respond "Access denied" 403 - } - - - handle_path /apple-touch-icon-152x152-precomposed.png { - respond "Access denied" 403 - } - - - handle_path /.well-known/* { - respond "Access denied" 403 - } - - - handle_path /cdn-cgi/challenge-platform/h/b/orchestrate/chl_page/v1 { - respond "Access denied" 403 - } - - - handle_path /cdn-cgi/challenge-platform/h/g/orchestrate/chl_page/v1 { - respond "Access denied" 403 - } - - - route /* { - reverse_proxy freedium_web:7080 { - lb_try_duration 30s - lb_try_interval 1s - } - } -} \ No newline at end of file diff --git a/caddy/CaddyfileProdTemplate b/caddy/CaddyfileProdTemplate deleted file mode 100644 index 681ff9a..0000000 --- a/caddy/CaddyfileProdTemplate +++ /dev/null @@ -1,15 +0,0 @@ -# https://futurestud.io/tutorials/caddy-reverse-proxy-a-node-js-app -freedium.cfd { - # header Server "nginx" - encode gzip - header -Server - - {{ template }} - - route /* { - reverse_proxy freedium_web:7080 { - lb_try_duration 30s - lb_try_interval 1s - } - } -} diff --git a/caddy/CaddyfileTemplate b/caddy/CaddyfileTemplate new file mode 100644 index 0000000..592b14b --- /dev/null +++ b/caddy/CaddyfileTemplate @@ -0,0 +1,37 @@ +(common) { + tls internal + encode gzip + header -Server + + {{ template }} +} + +(header_up) { + header_up Host {host} + header_up X-Real-IP {remote_host} + header_up X-Forwarded-For {remote_host} + header_up X-Forwarded-Proto {scheme} +} + +(lb_try) { + lb_try_duration 30s + lb_try_interval 1s +} + +plausible.freedium.local { + import common + + reverse_proxy freedium_plausible:8000 { + import header_up + import lb_try + } +} + +freedium.local { + import common + + reverse_proxy freedium_web:7080 { + import header_up + import lb_try + } +} diff --git a/caddy/generate_caddy_file.py b/caddy/generate_caddy_file.py old mode 100644 new mode 100755 index 5788720..91cfe9c --- a/caddy/generate_caddy_file.py +++ b/caddy/generate_caddy_file.py @@ -1,6 +1,9 @@ +#!/bin/env python3 + from os import listdir from os.path import isfile, join -from typing import List, Dict +from typing import Dict, List + from jinja2 import Template # Constants @@ -29,8 +32,7 @@ ACCESS_DENIED_PATHS: List[str] = [ ] CADDY_FILE_TEMPLATES: Dict[str, str] = { - "CaddyfileDevTemplate": "CaddyfileDev", - "CaddyfileProdTemplate": "CaddyfileProd", + "CaddyfileTemplate": "Caddyfile", } diff --git a/docker-compose/docker-compose.main.yml b/docker-compose/docker-compose.main.yml index c5cad55..077020e 100644 --- a/docker-compose/docker-compose.main.yml +++ b/docker-compose/docker-compose.main.yml @@ -8,19 +8,23 @@ services: dockerfile: Dockerfile cap_add: - NET_ADMIN + networks: + freedium_local_net: + ipv4_address: 177.28.0.5 + freedium_net: + caddy_freedium_net: # caddy_net: ports: - - "6752:6752" + - "80:6752" + # - "80:80" + # - "443:443" volumes: - - ../caddy/CaddyfileDev:/etc/caddy/Caddyfile + - ../caddy/Caddyfile:/etc/caddy/Caddyfile - freedium_caddy_data:/data - freedium_caddy_config:/config - ../caddy/static:/static - networks: - - freedium_net - - caddy_net restart: always healthcheck: - test: [ "CMD-SHELL", "curl -f http://localhost:6752/ --max-time 80 --header 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3 Safari/605.1.15'" ] + test: [ "CMD-SHELL", "curl -f http://localhost:80/ --max-time 80 --header 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3 Safari/605.1.15'" ] interval: 30s start_period: 20s timeout: 80s @@ -62,6 +66,14 @@ volumes: networks: - caddy_net: - external: true + # caddy_net: + # external: true freedium_net: + caddy_freedium_net: + freedium_local_net: + driver: bridge + ipam: + driver: default + config: + - subnet: 177.28.0.0/16 + gateway: 177.28.0.1 diff --git a/docker-compose/docker-compose.plausible.yml b/docker-compose/docker-compose.plausible.yml new file mode 100644 index 0000000..5c02283 --- /dev/null +++ b/docker-compose/docker-compose.plausible.yml @@ -0,0 +1,110 @@ +version: '3.7' + +# Based on: https://github.com/plausible/community-edition/blob/v2.1.4/compose.yml +services: + plausible_db: + image: postgres:16.3-alpine3.20 + profiles: [ prod ] + restart: always + networks: + - plausible_net + volumes: + - plausible-db-data:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=postgres + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U postgres" ] + start_period: 1m + + plausible_events_db: + image: clickhouse/clickhouse-server:24.3.3.102-alpine + restart: always + profiles: [ prod ] + networks: + - plausible_net + volumes: + - plausible-event-data:/var/lib/clickhouse + - plausible-event-logs:/var/log/clickhouse-server + - ../plausible/clickhouse/logs.xml:/etc/clickhouse-server/config.d/logs.xml:ro + - ../plausible/clickhouse/docker.xml:/etc/clickhouse-server/config.d/docker.xml:ro + # This makes ClickHouse bind to IPv4 only, since Docker doesn't enable IPv6 in bridge networks by default. + # Fixes "Listen [::]:9000 failed: Address family for hostname not supported" warnings. + - ../plausible/clickhouse/ipv4-only.xml:/etc/clickhouse-server/config.d/ipv4-only.xml:ro + ulimits: + nofile: + soft: 262144 + hard: 262144 + healthcheck: + test: [ "CMD-SHELL", "wget --no-verbose --tries=1 -O - http://127.0.0.1:8123/ping || exit 1" ] + start_period: 1m + + freedium_plausible: + image: ghcr.io/plausible/community-edition:v2.1.4 + restart: always + profiles: [ prod ] + command: sh -c "/entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run" + networks: + - plausible_net + - caddy_freedium_net + - freedium_net + depends_on: + plausible_db: + condition: service_healthy + plausible_events_db: + condition: service_healthy + volumes: + - plausible-data:/var/lib/plausible + ulimits: + nofile: + soft: 65535 + hard: 65535 + environment: + - TMPDIR=/var/lib/plausible/tmp + # required: https://github.com/plausible/community-edition/wiki/configuration#required + - BASE_URL=${PLAUSIBLE_BASE_URL:-https://plausible.freedium.local} + - SECRET_KEY_BASE=${PLAUSIBLE_SECRET_KEY_BASE} + - DISABLE_REGISTRATION=${PLAUSIBLE_DISABLE_REGISTRATION} + # - DISABLE_REGISTRATION + # optional: https://github.com/plausible/community-edition/wiki/configuration#optional + # registration: https://github.com/plausible/community-edition/wiki/configuration#registration + # - TOTP_VAULT_KEY=${PLAUSIBLE_TOTP_VAULT_KEY} + # - ENABLE_EMAIL_VERIFICATION=${PLAUSIBLE_ENABLE_EMAIL_VERIFICATION} + # web: https://github.com/plausible/community-edition/wiki/configuration#web + - HTTP_PORT=${PLAUSIBLE_HTTP_PORT:-8000} # Default: 8000 + # - HTTPS_PORT + # databases: https://github.com/plausible/community-edition/wiki/configuration#database + - DATABASE_URL=postgres://postgres:postgres@plausible_db:5432/plausible_db + - CLICKHOUSE_DATABASE_URL=http://plausible_events_db:8123/plausible_events_db + # Google: https://github.com/plausible/community-edition/wiki/configuration#google + # - GOOGLE_CLIENT_ID + # - GOOGLE_CLIENT_SECRET + # geolocation: https://github.com/plausible/community-edition/wiki/configuration#ip-geolocation + # - IP_GEOLOCATION_DB + # - GEONAMES_SOURCE_FILE + # - MAXMIND_LICENSE_KEY + # - MAXMIND_EDITION + # email: https://github.com/plausible/community-edition/wiki/configuration#email + # - MAILER_ADAPTER + # - MAILER_EMAIL + # - MAILER_NAME + # - SMTP_HOST_ADDR + # - SMTP_HOST_PORT + # - SMTP_USER_NAME + # - SMTP_USER_PWD + # - SMTP_HOST_SSL_ENABLED + # - POSTMARK_API_KEY + # - MAILGUN_API_KEY + # - MAILGUN_DOMAIN + # - MAILGUN_BASE_URI + # - MANDRILL_API_KEY + # - SENDGRID_API_KEY + +volumes: + plausible-db-data: + plausible-event-data: + plausible-event-logs: + plausible-data: + + +networks: + plausible_net: diff --git a/docker-compose/docker-compose.proxy.yml b/docker-compose/docker-compose.proxy.yml index 9657e78..3d3f020 100644 --- a/docker-compose/docker-compose.proxy.yml +++ b/docker-compose/docker-compose.proxy.yml @@ -7,8 +7,6 @@ services: dockerfile: Dockerfile container_name: haproxy-proxy-balancer hostname: haproxy-pb - # ports: - # - "1080:1080" volumes: - ../proxy-balancer/haproxy:/usr/local/etc/haproxy/ restart: always diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 3b64906..f44825f 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -6,3 +6,4 @@ include: - docker-compose.db.yml - docker-compose.utility.yml - docker-compose.proxy.yml + - docker-compose.plausible.yml diff --git a/plausible/clickhouse/docker.xml b/plausible/clickhouse/docker.xml new file mode 100644 index 0000000..a9f3181 --- /dev/null +++ b/plausible/clickhouse/docker.xml @@ -0,0 +1,3 @@ + + 0 + \ No newline at end of file diff --git a/plausible/clickhouse/ipv4-only.xml b/plausible/clickhouse/ipv4-only.xml new file mode 100644 index 0000000..e4f4b22 --- /dev/null +++ b/plausible/clickhouse/ipv4-only.xml @@ -0,0 +1,3 @@ + + 0.0.0.0 + \ No newline at end of file diff --git a/plausible/clickhouse/logs.xml b/plausible/clickhouse/logs.xml new file mode 100644 index 0000000..d1c0a62 --- /dev/null +++ b/plausible/clickhouse/logs.xml @@ -0,0 +1,36 @@ + + + warning + true + + + + system + query_log
+ 7500 + + ENGINE = MergeTree + PARTITION BY event_date + ORDER BY (event_time) + TTL event_date + interval 30 day + SETTINGS ttl_only_drop_parts=1 + + + + + event_date + event_date + INTERVAL 30 DAY DELETE + + + + event_date + event_date + INTERVAL 30 DAY DELETE + + + + + + + + + \ No newline at end of file