Migrate to Docker & logger enhances

2026-03-11 09:04:37 +00:00 · 2024-02-07 17:17:07 +06:00 · 2024-02-07 17:17:07 +06:00 · 11b6de1c53
commit 11b6de1c53
parent 86a7ec2033
17 changed files with 190 additions and 180 deletions
--- a/2
+++ b/2
@ -155,6 +155,6 @@
    }

    route /* {
-		reverse_proxy localhost:7080
+		reverse_proxy web:7080
 	}
 }
--- a/2
+++ b/2
@ -6,6 +6,6 @@
 {{ template }}

    route /* {
-		reverse_proxy localhost:7080
+		reverse_proxy web:7080
 	}
 }
--- a/2
+++ b/2
@ -156,6 +156,6 @@ freedium.cfd {
    }

    route /* {
-		reverse_proxy localhost:7080
+		reverse_proxy web:7080
 	}
 }
--- a/2
+++ b/2
@ -7,6 +7,6 @@ freedium.cfd {
    {{ template }}

    route /* {
-		reverse_proxy localhost:7080
+		reverse_proxy web:7080
 	}
 }
--- a/29
+++ b/29
@ -0,0 +1,29 @@
+FROM debian:bullseye-slim
+
+WORKDIR /app
+
+RUN apt-get update && \
+    apt-get install -y python3 python3-pip && \
+    apt-get clean
+
+COPY ./requirements.txt ./
+COPY ./requirements-fast.txt ./
+
+COPY ./core ./core
+COPY ./rl_string_helper ./rl_string_helper
+
+COPY ./other/sqlite_zstd-0.1.dev1+g5aaeb60-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl ./
+
+RUN pip install --no-cache-dir wheel
+
+RUN pip3 install --no-cache-dir ./rl_string_helper
+RUN pip3 install --no-cache-dir ./core
+
+RUN pip3 install --no-cache-dir -r requirements.txt
+RUN pip3 install --no-cache-dir -r requirements-fast.txt
+
+RUN pip3 install --no-cache-dir sqlite_zstd-0.1.dev1+g5aaeb60-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
+
+EXPOSE 7080
+
+CMD ["python3", "-m", "server", "server"]
--- a/bin/x86_64/dragonfly
+++ b/bin/x86_64/dragonfly
--- a/core/requirements.txt
+++ b/core/requirements.txt
@ -1,3 +1,5 @@
+rl_string_helper==0.1.0
+
 loguru==0.6.0
 aiohttp==3.8.5
 aiohttp-retry==2.8.3
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@ -0,0 +1,65 @@
+version: '3.8'
+
+services:
+  caddy:
+    image: caddy:2-alpine
+    ports:
+      - "6752:6752"
+    volumes:
+      - ./CaddyfileDev:/etc/caddy/Caddyfile
+      - caddy_data:/data
+      - caddy_config:/config
+    networks:
+      - web_network
+    depends_on:
+      - web
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://caddy:6752"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    restart: on-failure
+
+  web:
+    build: .
+    command: python3 -m server server
+    volumes:
+      - .:/app
+    expose:
+      - 7080
+    networks:
+      - web_network
+    depends_on:
+      - dragonfly
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://web:7080"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    restart: on-failure
+
+  dragonfly:
+    image: 'docker.dragonflydb.io/dragonflydb/dragonfly'
+    ulimits:
+      memlock: -1
+    expose:
+      - 6379
+    networks:
+      - web_network
+    volumes:
+      - dragonflydata:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    restart: on-failure
+
+volumes:
+  caddy_data:
+  caddy_config:
+  dragonflydata:
+
+networks:
+  web_network:
+    driver: bridge
--- a/docker-compose-prod.yml
+++ b/docker-compose-prod.yml
@ -0,0 +1,66 @@
+version: '3.8'
+
+services:
+  caddy:
+    image: caddy:2-alpine
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./CaddyfileProd:/etc/caddy/Caddyfile
+      - caddy_data:/data
+      - caddy_config:/config
+    networks:
+      - web_network
+    depends_on:
+      - web
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://caddy:6752"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    restart: on-failure
+
+  web:
+    build: .
+    command: python3 -m server server
+    volumes:
+      - .:/app
+    expose:
+      - 7080
+    networks:
+      - web_network
+    depends_on:
+      - dragonfly
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://web:7080"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    restart: on-failure
+
+  dragonfly:
+    image: 'docker.dragonflydb.io/dragonflydb/dragonfly'
+    ulimits:
+      memlock: -1
+    expose:
+      - 6379
+    networks:
+      - web_network
+    volumes:
+      - dragonflydata:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    restart: on-failure
+
+volumes:
+  caddy_data:
+  caddy_config:
+  dragonflydata:
+
+networks:
+  web_network:
+    driver: bridge
--- a/other/sqlite_zstd-0.1.dev1+g5aaeb60-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
+++ b/other/sqlite_zstd-0.1.dev1+g5aaeb60-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
--- a/scripts/start_dev.sh
+++ b/scripts/start_dev.sh
@ -1,85 +0,0 @@
-#!/bin/bash
-
-# Same script as start_prod, but adopted to dev environment
-
-check_env_var() {
-    if [[ -z "${!1}" ]]; then
-        echo "$1 var is blank"
-    else
-        echo "$1 var is set to '${!1}'"
-    fi
-}
-
-check_env_var "TELEGRAM_ADMIN_ID"
-check_env_var "TELEGRAM_BOT_TOKEN"
-
-arch=$(lscpu | grep Architecture | awk {'print $2'})
-echo $arch
-
-redis-cli flushall
-./bin/$arch/caddy run --config CaddyfileDev &
-CADDY_PID=$!
-PYTHONASYNCIODEBUG=1 python3 -m server server &
-SERVER_PID=$!
-
-onexit() {
-        echo "onexit"
-        kill $CADDY_PID
-        kill $SERVER_PID
-}
-
-trap onexit EXIT
-
-sendMessageTelegram(){
-    echo ${1}
-    local message=${1}
-
-    curl -X POST \
-         -H 'Content-Type: application/json' \
-         -d "{\"chat_id\":\"$TELEGRAM_ADMIN_ID\",\"text\":\"$message\"}" \
-         "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage"
-}
-
-while true
-do
-  sleep 15
-
-  CHECK_CADDY_PID=$(ps -A| grep $CADDY_PID |wc -l)
-  if [[ $CHECK_CADDY_PID -eq 0 ]]; then
-          # sendMessageTelegram "Restarting caddy, since it's down"
-          ./bin/$arch/caddy start --config CaddyfileDev &
-          CADDY_PID=$!
-  fi
-
-  CHECK_SERVER_PID=$(ps -A| grep $SERVER_PID |wc -l)
-  if [[ $CHECK_SERVER_PID -eq 0 ]]; then
-        # sendMessageTelegram "Restarting server, since it's down"
-        PYTHONASYNCIODEBUG=1 python3 -m server server &
-        SERVER_PID=%!
-  fi
-
-  sleep 35
-
-  backend_service_url="http://localhost:7080"
-  backend_status_code=$(curl -m 10 -s -o /dev/null -w "%{http_code}" "$backend_service_url")
-
-  if [ "$backend_status_code" -lt 200 ]; then
-    sendMessageTelegram "Restarting backend, since it's down"
-    kill $SERVER_PID
-    PYTHONASYNCIODEBUG=1 python3 -m server server &
-    SERVER_PID=$!
-  fi
-
-  reverse_service_url="http://localhost:6752"
-  reverse_status_code=$(curl -m 10 -s -o /dev/null -w "%{http_code}" "$reverse_service_url")
-
-  if [ "$reverse_status_code" -lt 200 ]; then
-    sendMessageTelegram "Restarting reverse, since it's down"
-    kill $CADDY_PID
-    ./bin/$arch/caddy start --config CaddyfileDev &
-    CADDY_PID=$!
-  fi
-
-  sleep 65
-done
-
--- a/scripts/start_prod.sh
+++ b/scripts/start_prod.sh
@ -1,74 +0,0 @@
-#!/bin/bash
-
-if [ -z "$TELEGRAM_ADMIN_ID" ]; then echo "TELEGRAM_ADMIN_ID var is blank"; else echo "TELEGRAM_ADMIN_ID var is set to '$TELEGRAM_ADMIN_ID'"; fi
-if [ -z "$TELEGRAM_BOT_TOKEN" ]; then echo "TELEGRAM_BOT_TOKEN var is blank"; else echo "TELEGRAM_BOT_TOKEN var is set to '$TELEGRAM_BOT_TOKEN'"; fi
-
-arch=$(lscpu | grep Architecture | awk {'print $2'})
-
-redis-cli flushall
-./bin/$arch/caddy run --config CaddyfileProd &
-CADDY_PID=$!
-python3 -m server server &
-SERVER_PID=$!
-
-function onexit() {
-        echo "onexit"
-        sleep 25
-        kill $CADDY_PID
-        kill $SERVER_PID
-}
-# trap onexit EXIT
-
-sendMessageTelegram(){
-    echo ${1}
-    local message=${1}
-
-    curl -X POST \
-         -H 'Content-Type: application/json' \
-         -d "{\"chat_id\":\"$TELEGRAM_ADMIN_ID\",\"text\":\"$message\"}" \
-         "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage"
-}
-
-while true
-do
-  sleep 5
-
-  CHECK_CADDY_PID=$(ps -A| grep $CADDY_PID |wc -l)
-  if [[ $CHECK_CADDY_PID -eq 0 ]]; then
-          sendMessageTelegram "Restarting caddy, since it's down"
-          ./bin/$arch/caddy run --config CaddyfileProd &
-          CADDY_PID=$!
-  fi
-
-  CHECK_SERVER_PID=$(ps -A| grep $SERVER_PID |wc -l)
-  if [[ $CHECK_SERVER_PID -eq 0 ]]; then
-        sendMessageTelegram "Restarting server, since it's down"
-        python3 -m server server &
-        SERVER_PID=$!
-  fi
-
-  sleep 5
-
-  backend_service_url="http://localhost:7080"
-  backend_status_code=$(curl -m 10 -s -o /dev/null -w "%{http_code}" "$backend_service_url")
-
-  if [ "$backend_status_code" -lt 200 ]; then
-    sendMessageTelegram "Restarting backend, since it's down"
-    kill $SERVER_PID
-    python3 -m server server &
-    SERVER_PID=$!
-  fi
-
-  reverse_service_url="http://localhost"
-  reverse_status_code=$(curl -m 10 -s -o /dev/null -w "%{http_code}" "$reverse_service_url")
-
-  if [ "$reverse_status_code" -lt 308 ]; then
-    sendMessageTelegram "Restarting reverse, since it's down"
-    kill $CADDY_PID
-    ./bin/$arch/caddy run --config CaddyfileProd &
-    CADDY_PID=$!
-  fi
-
-  sleep 65
-done
-
--- a/server/init.py
+++ b/server/init.py
@ -8,10 +8,9 @@ from jinja2 import Environment, DebugUndefined, FileSystemLoader
 import redis.asyncio as redis
 from xkcdpass import xkcd_password as xp

-from server import config
 from server.utils.loguru_handler import InterceptHandler

-redis_storage = redis.Redis(host="localhost", port=6379, db=0)
+redis_storage = redis.Redis(host="dragonfly", port=6379, db=0)

 jinja_env = Environment(enable_async=True)
 jinja_safe_env = Environment(undefined=DebugUndefined)
@ -33,6 +32,7 @@ error_template = jinja_env.from_string(error_template_raw_rendered)
 logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True)

 url_correlation: ContextVar[Optional[str]] = ContextVar("url_correlation", default="UNKNOWN_URL")
+home_page_process: ContextVar[Optional[list]] = ContextVar("home_page_process", default=[])
 transponder_code_correlation: ContextVar[Optional[str]] = ContextVar("transponder_code_correlation", default="unknown transponder location... Beep!")

 ban_db = pickledb.load('ban_post_list.db', True)
--- a/server/handlers/main.py
+++ b/server/handlers/main.py
@ -1,10 +1,10 @@
 from html5lib.html5parser import parse
 from html5lib import serialize
-from fastapi.responses import HTMLResponse

 from server import base_template, main_template, config

 from fastapi import Request
+from fastapi.responses import HTMLResponse

 from server.handlers.post import render_medium_post_link, render_postleter
 from server.handlers.reverse_proxy import miro_proxy, iframe_proxy
@ -22,13 +22,15 @@ async def route_processing(path: str, request: Request):
        path = request.url.path
    path = path.removeprefix("/")

+    """
    if path.startswith("render-no-cache/"):
        path = path.removeprefix("render-no-cache/")
        if path.startswith("/no-redis/"):
            path = path.removeprefix("/no-redis/")
            return await render_medium_post_link(path, True, False)
        return await render_medium_post_link(path, False)
-    elif path.startswith("@miro/"):
+    """
+    if path.startswith("@miro/"):
        miro_data = path.removeprefix("@miro/")
        return await miro_proxy(miro_data)
    elif path.startswith("render_iframe/"):
--- a/server/handlers/post.py
+++ b/server/handlers/post.py
@ -5,12 +5,12 @@ from html5lib.html5parser import parse
 from html5lib import serialize
 from loguru import logger

+from server import base_template, config, url_correlation, redis_storage, postleter_template, home_page_process
 from server.utils.error import generate_error
 from server.utils.logger_trace import trace
 from server.utils.notify import send_message
 from server.utils.cache import aio_redis_cache
 from server.utils.utils import correct_url, safe_check_redis_connection
-from server import base_template, config, url_correlation, redis_storage, postleter_template

 from medium_parser import medium_parser_exceptions
 from medium_parser import cache as medium_cache
@ -21,6 +21,7 @@ from medium_parser.utils import is_valid_medium_post_id_hexadecimal
@aio_redis_cache(10 * 60)
 async def render_postleter(limit: int = 60, as_html: bool = False):
    random_post_id_list = [i[0] for i in medium_cache.random(limit)]
+    home_page_process.set(random_post_id_list)

    outlenget_posts_list = []
    for post_id in random_post_id_list:
--- a/server/middlewares/logger.py
+++ b/server/middlewares/logger.py
@ -8,7 +8,7 @@ from starlette.requests import Request
 from starlette.responses import Response, StreamingResponse
 from starlette.types import Message

-from server import transponder_code_correlation, url_correlation, xkcd_passwd, xp, config
+from server import transponder_code_correlation, url_correlation, xkcd_passwd, xp, config, home_page_process
 from server.utils.notify import send_message
 from server.utils.error import generate_error
 from server.utils.utils import string_to_number_ascii
@ -48,7 +48,7 @@ class LoggerMiddleware(BaseHTTPMiddleware):

            logger.debug("< Headers:")
            for name, value in request.headers.items():
-                value = self._sanitize_header_value(name, value)
+                value = self._sanitize_header(name, value)
                logger.debug(f"\t< {name}: {value}")

            if hasattr(request, "cookies") and request.cookies:
@ -62,8 +62,9 @@ class LoggerMiddleware(BaseHTTPMiddleware):
            try:
                response = await asyncio.wait_for(call_next(request), timeout=config.REQUEST_TIMEOUT)
            except Exception as ex:
+                exception_class = type(ex)
                logger.exception(ex)
-                await send_message(f"Error while processing url: <code>{url_correlation.get()}</code>, transponder_code: <code>{transponder_code_correlation.get()}</code>, error: <code>{ex}</code>")
+                await send_message(f"Error while processing url: <code>{url_correlation.get()}</code>, transponder_code: <code>{transponder_code_correlation.get()}</code>, error: <code>{ex}</code>. exception: <code>{exception_class.__name__}</code>. {home_page_process.get()}")
                response = await generate_error()

            logger.trace(response.__dict__)
@ -75,7 +76,7 @@ class LoggerMiddleware(BaseHTTPMiddleware):

            logger.debug("> Headers:")
            for name, value in response.headers.items():
-                value = self._sanitize_header_value(name, value)
+                value = self._sanitize_header(name, value)
                logger.debug(f"\t> {name}: {value}")

            if hasattr(response, "cookies"):
@ -88,7 +89,7 @@ class LoggerMiddleware(BaseHTTPMiddleware):

            return response

-    def _sanitize_header_value(self, name, value):
+    def _sanitize_header(self, name, value):
        if name.lower() == "authorization":
            value = f"{value[:25]}******"
        return value
--- a/server/templates/base.html
+++ b/server/templates/base.html
@ -109,10 +109,8 @@
    <nav id="header" class="fixed w-full z-9 top-0 dark:bg-gray-800 dark:text-white bg-white shadow">
        <div class="notification-container">
    <div class="notification-card dark:bg-gray-800 bg-white">
-      <p class="text-2xl pb-5 text-black dark:text-white">Good news !!!</p>
-      <p class="pb-3 text-black dark:text-white">We are thrilled to share that we have migrated to Codeberg. We are eager to carry on our journey in this fresh setting. We appreciate the backing we received on Github and Reddit, and we aspire to see our community thrive further on Codeberg.</br></br>
-        
-        <strong>UPDATE 4.02.2024: In response to potential blocks from Medium, we have also established mirrors to ensure uninterrupted access to our content. This proactive measure aims to safeguard our community's ability to reach our resources without hindrance.</strong></bold>
+      <p class="text-2xl pb-5 text-black dark:text-white">Great news !!!</p>
+      <p class="pb-3 text-black dark:text-white">We are finally unblocked on GitHub! Check it out!</br></br></p>
      <a href="https://patreon.com/Freedium" target="_blank" title="Patreon">
      <button class="bg-red-400 mx-1 text-white hover:bg-red-500 font-semibold py-1 px-2 rounded mt-2">
        Patreon
@ -128,10 +126,15 @@
        Close
      </button>
      <a href="https://codeberg.org/Freedium-cfd/web" target="_blank" title="Codeberg">
-      <button class="bg-gray-700 hover:bg-gray-600 mx-1 text-white font-semibold py-1 px-2 rounded mt-2">
+      <button class="bg-blue-800 hover:bg-blue-900 mx-1 text-white font-semibold py-1 px-2 rounded mt-2">
        Source code - Codeberg
      </button>
      </a>
+      <a href="https://github.com/Freedium-cfd/web" target="_blank" title="GitHub"></a>
+      <button class="bg-gray-700 hover:bg-gray-600 mx-1 text-white font-semibold py-1 px-2 rounded mt-2">
+        Source code - GitHub
+      </button>
+      </a>
    </div>
  </div>

@ -363,14 +366,14 @@ document.addEventListener('scroll', function () {
        const closeButton = document.querySelector('.close-button');
  
        function showNotification() {
-          if (!localStorage.getItem('showNotification-update146-block')) {
+          if (!localStorage.getItem('showNotification-update46-block')) {
                  notificationContainer.style.display = 'block';
          }
        }
  
        // Hide the notification
        function hideNotification() {
-          localStorage.setItem('showNotification-update146-block', 'false');
+          localStorage.setItem('showNotification-update46-block', 'false');
          notificationContainer.style.display = 'none';
        }