diff --git a/CaddyfileDev b/CaddyfileDev index 5036d03..d0f87d7 100644 --- a/CaddyfileDev +++ b/CaddyfileDev @@ -155,6 +155,6 @@ } route /* { - reverse_proxy localhost:7080 + reverse_proxy web:7080 } } \ No newline at end of file diff --git a/CaddyfileDevTemplate b/CaddyfileDevTemplate index 480a7cd..f02d5df 100644 --- a/CaddyfileDevTemplate +++ b/CaddyfileDevTemplate @@ -6,6 +6,6 @@ {{ template }} route /* { - reverse_proxy localhost:7080 + reverse_proxy web:7080 } } diff --git a/CaddyfileProd b/CaddyfileProd index 0ecbe65..793ebd9 100644 --- a/CaddyfileProd +++ b/CaddyfileProd @@ -156,6 +156,6 @@ freedium.cfd { } route /* { - reverse_proxy localhost:7080 + reverse_proxy web:7080 } } \ No newline at end of file diff --git a/CaddyfileProdTemplate b/CaddyfileProdTemplate index 4bfa411..8b1abb6 100644 --- a/CaddyfileProdTemplate +++ b/CaddyfileProdTemplate @@ -7,6 +7,6 @@ freedium.cfd { {{ template }} route /* { - reverse_proxy localhost:7080 + reverse_proxy web:7080 } } diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..23dbd84 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM debian:bullseye-slim + +WORKDIR /app + +RUN apt-get update && \ + apt-get install -y python3 python3-pip && \ + apt-get clean + +COPY ./requirements.txt ./ +COPY ./requirements-fast.txt ./ + +COPY ./core ./core +COPY ./rl_string_helper ./rl_string_helper + +COPY ./other/sqlite_zstd-0.1.dev1+g5aaeb60-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl ./ + +RUN pip install --no-cache-dir wheel + +RUN pip3 install --no-cache-dir ./rl_string_helper +RUN pip3 install --no-cache-dir ./core + +RUN pip3 install --no-cache-dir -r requirements.txt +RUN pip3 install --no-cache-dir -r requirements-fast.txt + +RUN pip3 install --no-cache-dir sqlite_zstd-0.1.dev1+g5aaeb60-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + +EXPOSE 7080 + +CMD ["python3", "-m", "server", "server"] \ No newline at end of file diff --git a/bin/x86_64/dragonfly b/bin/x86_64/dragonfly deleted file mode 100755 index a167ea1..0000000 Binary files a/bin/x86_64/dragonfly and /dev/null differ diff --git a/core/requirements.txt b/core/requirements.txt index 06f48a7..0962c0f 100644 --- a/core/requirements.txt +++ b/core/requirements.txt @@ -1,3 +1,5 @@ +rl_string_helper==0.1.0 + loguru==0.6.0 aiohttp==3.8.5 aiohttp-retry==2.8.3 diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 0000000..ffeef53 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,65 @@ +version: '3.8' + +services: + caddy: + image: caddy:2-alpine + ports: + - "6752:6752" + volumes: + - ./CaddyfileDev:/etc/caddy/Caddyfile + - caddy_data:/data + - caddy_config:/config + networks: + - web_network + depends_on: + - web + healthcheck: + test: ["CMD", "curl", "-f", "http://caddy:6752"] + interval: 30s + timeout: 10s + retries: 3 + restart: on-failure + + web: + build: . + command: python3 -m server server + volumes: + - .:/app + expose: + - 7080 + networks: + - web_network + depends_on: + - dragonfly + healthcheck: + test: ["CMD", "curl", "-f", "http://web:7080"] + interval: 30s + timeout: 10s + retries: 3 + restart: on-failure + + dragonfly: + image: 'docker.dragonflydb.io/dragonflydb/dragonfly' + ulimits: + memlock: -1 + expose: + - 6379 + networks: + - web_network + volumes: + - dragonflydata:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 30s + timeout: 10s + retries: 3 + restart: on-failure + +volumes: + caddy_data: + caddy_config: + dragonflydata: + +networks: + web_network: + driver: bridge diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml new file mode 100644 index 0000000..b2d57dc --- /dev/null +++ b/docker-compose-prod.yml @@ -0,0 +1,66 @@ +version: '3.8' + +services: + caddy: + image: caddy:2-alpine + ports: + - "80:80" + - "443:443" + volumes: + - ./CaddyfileProd:/etc/caddy/Caddyfile + - caddy_data:/data + - caddy_config:/config + networks: + - web_network + depends_on: + - web + healthcheck: + test: ["CMD", "curl", "-f", "http://caddy:6752"] + interval: 30s + timeout: 10s + retries: 3 + restart: on-failure + + web: + build: . + command: python3 -m server server + volumes: + - .:/app + expose: + - 7080 + networks: + - web_network + depends_on: + - dragonfly + healthcheck: + test: ["CMD", "curl", "-f", "http://web:7080"] + interval: 30s + timeout: 10s + retries: 3 + restart: on-failure + + dragonfly: + image: 'docker.dragonflydb.io/dragonflydb/dragonfly' + ulimits: + memlock: -1 + expose: + - 6379 + networks: + - web_network + volumes: + - dragonflydata:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 30s + timeout: 10s + retries: 3 + restart: on-failure + +volumes: + caddy_data: + caddy_config: + dragonflydata: + +networks: + web_network: + driver: bridge diff --git a/other/sqlite_zstd-0.1.dev1+g5aaeb60-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/other/sqlite_zstd-0.1.dev1+g5aaeb60-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 0000000..da162b2 Binary files /dev/null and b/other/sqlite_zstd-0.1.dev1+g5aaeb60-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/scripts/start_dev.sh b/scripts/start_dev.sh deleted file mode 100755 index 3d7f33a..0000000 --- a/scripts/start_dev.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -# Same script as start_prod, but adopted to dev environment - -check_env_var() { - if [[ -z "${!1}" ]]; then - echo "$1 var is blank" - else - echo "$1 var is set to '${!1}'" - fi -} - -check_env_var "TELEGRAM_ADMIN_ID" -check_env_var "TELEGRAM_BOT_TOKEN" - -arch=$(lscpu | grep Architecture | awk {'print $2'}) -echo $arch - -redis-cli flushall -./bin/$arch/caddy run --config CaddyfileDev & -CADDY_PID=$! -PYTHONASYNCIODEBUG=1 python3 -m server server & -SERVER_PID=$! - -onexit() { - echo "onexit" - kill $CADDY_PID - kill $SERVER_PID -} - -trap onexit EXIT - -sendMessageTelegram(){ - echo ${1} - local message=${1} - - curl -X POST \ - -H 'Content-Type: application/json' \ - -d "{\"chat_id\":\"$TELEGRAM_ADMIN_ID\",\"text\":\"$message\"}" \ - "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" -} - -while true -do - sleep 15 - - CHECK_CADDY_PID=$(ps -A| grep $CADDY_PID |wc -l) - if [[ $CHECK_CADDY_PID -eq 0 ]]; then - # sendMessageTelegram "Restarting caddy, since it's down" - ./bin/$arch/caddy start --config CaddyfileDev & - CADDY_PID=$! - fi - - CHECK_SERVER_PID=$(ps -A| grep $SERVER_PID |wc -l) - if [[ $CHECK_SERVER_PID -eq 0 ]]; then - # sendMessageTelegram "Restarting server, since it's down" - PYTHONASYNCIODEBUG=1 python3 -m server server & - SERVER_PID=%! - fi - - sleep 35 - - backend_service_url="http://localhost:7080" - backend_status_code=$(curl -m 10 -s -o /dev/null -w "%{http_code}" "$backend_service_url") - - if [ "$backend_status_code" -lt 200 ]; then - sendMessageTelegram "Restarting backend, since it's down" - kill $SERVER_PID - PYTHONASYNCIODEBUG=1 python3 -m server server & - SERVER_PID=$! - fi - - reverse_service_url="http://localhost:6752" - reverse_status_code=$(curl -m 10 -s -o /dev/null -w "%{http_code}" "$reverse_service_url") - - if [ "$reverse_status_code" -lt 200 ]; then - sendMessageTelegram "Restarting reverse, since it's down" - kill $CADDY_PID - ./bin/$arch/caddy start --config CaddyfileDev & - CADDY_PID=$! - fi - - sleep 65 -done - diff --git a/scripts/start_prod.sh b/scripts/start_prod.sh deleted file mode 100755 index 1200b4b..0000000 --- a/scripts/start_prod.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -if [ -z "$TELEGRAM_ADMIN_ID" ]; then echo "TELEGRAM_ADMIN_ID var is blank"; else echo "TELEGRAM_ADMIN_ID var is set to '$TELEGRAM_ADMIN_ID'"; fi -if [ -z "$TELEGRAM_BOT_TOKEN" ]; then echo "TELEGRAM_BOT_TOKEN var is blank"; else echo "TELEGRAM_BOT_TOKEN var is set to '$TELEGRAM_BOT_TOKEN'"; fi - -arch=$(lscpu | grep Architecture | awk {'print $2'}) - -redis-cli flushall -./bin/$arch/caddy run --config CaddyfileProd & -CADDY_PID=$! -python3 -m server server & -SERVER_PID=$! - -function onexit() { - echo "onexit" - sleep 25 - kill $CADDY_PID - kill $SERVER_PID -} -# trap onexit EXIT - -sendMessageTelegram(){ - echo ${1} - local message=${1} - - curl -X POST \ - -H 'Content-Type: application/json' \ - -d "{\"chat_id\":\"$TELEGRAM_ADMIN_ID\",\"text\":\"$message\"}" \ - "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" -} - -while true -do - sleep 5 - - CHECK_CADDY_PID=$(ps -A| grep $CADDY_PID |wc -l) - if [[ $CHECK_CADDY_PID -eq 0 ]]; then - sendMessageTelegram "Restarting caddy, since it's down" - ./bin/$arch/caddy run --config CaddyfileProd & - CADDY_PID=$! - fi - - CHECK_SERVER_PID=$(ps -A| grep $SERVER_PID |wc -l) - if [[ $CHECK_SERVER_PID -eq 0 ]]; then - sendMessageTelegram "Restarting server, since it's down" - python3 -m server server & - SERVER_PID=$! - fi - - sleep 5 - - backend_service_url="http://localhost:7080" - backend_status_code=$(curl -m 10 -s -o /dev/null -w "%{http_code}" "$backend_service_url") - - if [ "$backend_status_code" -lt 200 ]; then - sendMessageTelegram "Restarting backend, since it's down" - kill $SERVER_PID - python3 -m server server & - SERVER_PID=$! - fi - - reverse_service_url="http://localhost" - reverse_status_code=$(curl -m 10 -s -o /dev/null -w "%{http_code}" "$reverse_service_url") - - if [ "$reverse_status_code" -lt 308 ]; then - sendMessageTelegram "Restarting reverse, since it's down" - kill $CADDY_PID - ./bin/$arch/caddy run --config CaddyfileProd & - CADDY_PID=$! - fi - - sleep 65 -done - diff --git a/server/__init__.py b/server/__init__.py index 8f9de21..963b770 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -8,10 +8,9 @@ from jinja2 import Environment, DebugUndefined, FileSystemLoader import redis.asyncio as redis from xkcdpass import xkcd_password as xp -from server import config from server.utils.loguru_handler import InterceptHandler -redis_storage = redis.Redis(host="localhost", port=6379, db=0) +redis_storage = redis.Redis(host="dragonfly", port=6379, db=0) jinja_env = Environment(enable_async=True) jinja_safe_env = Environment(undefined=DebugUndefined) @@ -33,6 +32,7 @@ error_template = jinja_env.from_string(error_template_raw_rendered) logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True) url_correlation: ContextVar[Optional[str]] = ContextVar("url_correlation", default="UNKNOWN_URL") +home_page_process: ContextVar[Optional[list]] = ContextVar("home_page_process", default=[]) transponder_code_correlation: ContextVar[Optional[str]] = ContextVar("transponder_code_correlation", default="unknown transponder location... Beep!") ban_db = pickledb.load('ban_post_list.db', True) diff --git a/server/handlers/main.py b/server/handlers/main.py index 3975914..d67fdeb 100644 --- a/server/handlers/main.py +++ b/server/handlers/main.py @@ -1,10 +1,10 @@ from html5lib.html5parser import parse from html5lib import serialize -from fastapi.responses import HTMLResponse from server import base_template, main_template, config from fastapi import Request +from fastapi.responses import HTMLResponse from server.handlers.post import render_medium_post_link, render_postleter from server.handlers.reverse_proxy import miro_proxy, iframe_proxy @@ -22,13 +22,15 @@ async def route_processing(path: str, request: Request): path = request.url.path path = path.removeprefix("/") + """ if path.startswith("render-no-cache/"): path = path.removeprefix("render-no-cache/") if path.startswith("/no-redis/"): path = path.removeprefix("/no-redis/") return await render_medium_post_link(path, True, False) return await render_medium_post_link(path, False) - elif path.startswith("@miro/"): + """ + if path.startswith("@miro/"): miro_data = path.removeprefix("@miro/") return await miro_proxy(miro_data) elif path.startswith("render_iframe/"): diff --git a/server/handlers/post.py b/server/handlers/post.py index 5fdb84f..d0cad05 100644 --- a/server/handlers/post.py +++ b/server/handlers/post.py @@ -5,12 +5,12 @@ from html5lib.html5parser import parse from html5lib import serialize from loguru import logger +from server import base_template, config, url_correlation, redis_storage, postleter_template, home_page_process from server.utils.error import generate_error from server.utils.logger_trace import trace from server.utils.notify import send_message from server.utils.cache import aio_redis_cache from server.utils.utils import correct_url, safe_check_redis_connection -from server import base_template, config, url_correlation, redis_storage, postleter_template from medium_parser import medium_parser_exceptions from medium_parser import cache as medium_cache @@ -21,6 +21,7 @@ from medium_parser.utils import is_valid_medium_post_id_hexadecimal @aio_redis_cache(10 * 60) async def render_postleter(limit: int = 60, as_html: bool = False): random_post_id_list = [i[0] for i in medium_cache.random(limit)] + home_page_process.set(random_post_id_list) outlenget_posts_list = [] for post_id in random_post_id_list: diff --git a/server/middlewares/logger.py b/server/middlewares/logger.py index f90921d..3c922cd 100644 --- a/server/middlewares/logger.py +++ b/server/middlewares/logger.py @@ -8,7 +8,7 @@ from starlette.requests import Request from starlette.responses import Response, StreamingResponse from starlette.types import Message -from server import transponder_code_correlation, url_correlation, xkcd_passwd, xp, config +from server import transponder_code_correlation, url_correlation, xkcd_passwd, xp, config, home_page_process from server.utils.notify import send_message from server.utils.error import generate_error from server.utils.utils import string_to_number_ascii @@ -48,7 +48,7 @@ class LoggerMiddleware(BaseHTTPMiddleware): logger.debug("< Headers:") for name, value in request.headers.items(): - value = self._sanitize_header_value(name, value) + value = self._sanitize_header(name, value) logger.debug(f"\t< {name}: {value}") if hasattr(request, "cookies") and request.cookies: @@ -62,8 +62,9 @@ class LoggerMiddleware(BaseHTTPMiddleware): try: response = await asyncio.wait_for(call_next(request), timeout=config.REQUEST_TIMEOUT) except Exception as ex: + exception_class = type(ex) logger.exception(ex) - await send_message(f"Error while processing url: {url_correlation.get()}, transponder_code: {transponder_code_correlation.get()}, error: {ex}") + await send_message(f"Error while processing url: {url_correlation.get()}, transponder_code: {transponder_code_correlation.get()}, error: {ex}. exception: {exception_class.__name__}. {home_page_process.get()}") response = await generate_error() logger.trace(response.__dict__) @@ -75,7 +76,7 @@ class LoggerMiddleware(BaseHTTPMiddleware): logger.debug("> Headers:") for name, value in response.headers.items(): - value = self._sanitize_header_value(name, value) + value = self._sanitize_header(name, value) logger.debug(f"\t> {name}: {value}") if hasattr(response, "cookies"): @@ -88,7 +89,7 @@ class LoggerMiddleware(BaseHTTPMiddleware): return response - def _sanitize_header_value(self, name, value): + def _sanitize_header(self, name, value): if name.lower() == "authorization": value = f"{value[:25]}******" return value diff --git a/server/templates/base.html b/server/templates/base.html index bb5c0f1..a899b72 100644 --- a/server/templates/base.html +++ b/server/templates/base.html @@ -109,10 +109,8 @@