Minor fixes

docker-compose-dev.yml

@@ -14,7 +14,7 @@ services:
     depends_on:
       - web
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://caddy:6752"]
+      test: ["CMD", "curl", "http://caddy:6752"]
       interval: 30s
       timeout: 10s
       retries: 3
@@ -32,7 +32,7 @@ services:
     depends_on:
       - dragonfly
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://web:7080"]
+      test: ["CMD", "curl", "http://web:7080"]
       interval: 30s
       timeout: 10s
       retries: 3

docker-compose-prod.yml

@@ -15,7 +15,7 @@ services:
     depends_on:
       - web
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://caddy:6752"]
+      test: ["CMD", "curl", "http://caddy:6752"]
       interval: 30s
       timeout: 10s
       retries: 3
@@ -33,7 +33,7 @@ services:
     depends_on:
       - dragonfly
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://web:7080"]
+      test: ["CMD", "curl", "http://web:7080"]
       interval: 30s
       timeout: 10s
       retries: 3

server/handlers/misc.py

@@ -2,6 +2,8 @@ from loguru import logger
 from pydantic import BaseModel
 from fastapi.responses import JSONResponse
 
+from medium_parser.core import MediumParser
+
 from server import config, ban_db
 from server.utils.notify import send_message
 from server.utils.logger_trace import trace
@@ -18,7 +20,7 @@ class DeleteFromCache(BaseModel):
 
 @trace
 async def report_problem(problem: ReportProblem):
-    await send_message(f"New problem report: \n{problem.description}\n\n{problem.page}")
+    send_message(f"New problem report: \n{problem.description}\n\n{problem.page}")
     return JSONResponse({"message": "OK"}, status_code=200)
 
 

server/handlers/post.py

@@ -32,7 +32,7 @@ async def render_postleter(limit: int = 30, as_html: bool = False):
             outlet_posts_list.append(post_metadata)
         except Exception as ex:
             logger.error(f"Couldn't render post_id for postleter: {post_id}, ex: {ex}")
-            # await send_message(f"Couldn't render post_id for postleter: {post_id}, ex: {ex}")
+            # send_message(f"Couldn't render post_id for postleter: {post_id}, ex: {ex}")
 
     postleter_template_rendered = await postleter_template.render_async(post_list=outlet_posts_list)
     if as_html:
@@ -97,9 +97,9 @@ async def render_medium_post_link(path: str, use_cache: bool = True, use_redis:
 
         if not redis_result:
             if not redis_available:
-                await send_message("ERROR: Redis is not available. Please check your configuration.")
+                send_message("ERROR: Redis is not available. Please check your configuration.")
             elif use_redis:
                 await redis_storage.setex(medium_post_id, config.CACHE_LIFE_TIME, pickle.dumps(rendered_medium_post))
-            await send_message(f"✅ Successfully rendered post: {url_correlation.get()}", True, "GOOD")
+            send_message(f"✅ Successfully rendered post: {url_correlation.get()}", True, "GOOD")
 
         return HTMLResponse(serialized_rendered_post)

server/middlewares/logger.py

@@ -64,7 +64,7 @@ class LoggerMiddleware(BaseHTTPMiddleware):
             except Exception as ex:
                 exception_class = type(ex)
                 logger.exception(ex)
-                await send_message(f"Error while processing url: <code>{url_correlation.get()}</code>, transponder_code: <code>{transponder_code_correlation.get()}</code>, error: <code>{ex}</code>. exception: <code>{exception_class.__name__}</code>. {home_page_process.get(transponder_code_correlation.get(), '')}")
+                send_message(f"Error while processing url: <code>{url_correlation.get()}</code>, transponder_code: <code>{transponder_code_correlation.get()}</code>, error: <code>{ex}</code>. exception: <code>{exception_class.__name__}</code>. {home_page_process.get(transponder_code_correlation.get(), '')}")
                 response = await generate_error()
 
             logger.trace(response.__dict__)

server/utils/error.py

@@ -37,12 +37,10 @@ async def generate_error(error_msg: str = None, title: str = "Error", status_cod
     if not error_msg:
         error_msg = random.choice(ERROR_MSG_LIST)
 
-    """
     if not quiet:
-        await send_message(
+        send_message(
             f"📛 Error while processing url: <code>{url_correlation.get()}</code>, transponder_code: <code>{transponder_code_correlation.get()}</code>, error: <code>{error_msg}</code>"
         )
-    """
 
     error_template_rendered = await error_template.render_async(error_msg=error_msg, transponder_code=transponder_code_correlation.get())
     base_context = {

server/utils/notify.py

@@ -12,7 +12,7 @@ class MessageStatus(Enum):
     GOOD = "GOOD"
 
 
-async def send_message(text: str, silent: bool = False, status: MessageStatus = "ERROR") -> None:
+def send_message(text: str, silent: bool = False, status: MessageStatus = "ERROR") -> None:
     asyncio.create_task(task_send_message(text, silent, status))
 
 

server/worker.py

@@ -1,13 +1,16 @@
+import io
 import atexit
 import multiprocessing
 from multiprocessing.util import _exit_function
 
+import traceback
 import gunicorn.app.base
 import uvicorn
 from loguru import logger
 
 from server import config, ban_db
 from server.main import app
+from server.utils.notify import send_message
 from server.utils.logger import GunicornLogger
 from server.utils.logger_trace import trace
 
@@ -26,6 +29,25 @@ def on_exit():
     ban_db.dump()
 
 
+def worker_exit(server, worker):
+    """Called just after a worker has been exited, in the worker process."""
+    logger.warning("Shutting down gunicorn worker.")
+
+    send_message("Shutting down gunicorn worker.", status="ERROR")
+
+
+def worker_abort(worker):
+    """Log the stack trace when a worker timeout occurs"""
+    buffer = io.StringIO()
+    traceback.print_stack(file=buffer)
+    data = buffer.getvalue()
+    buffer.close()
+
+    logger.error(f"Killing worker {worker.pid}\n{data}")
+
+    send_message(f"Killing worker {worker.pid}\n{data}", status="ERROR")
+
+
 @trace
 def number_of_workers():
     cores = multiprocessing.cpu_count()
@@ -62,6 +84,8 @@ def execute_server_worker(host: str, port: int):
         "preload_app": True,
         "post_worker_init": post_worker_init,
         "timeout": config.WORKER_TIMEOUT,
+        "worker_exit": worker_exit,
+        "worker_abort": worker_abort,
         # "on_exit": on_exit,
     }
     GunicornStandaloneApplication(app, options).run()