diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index ffeef53..d6d7c17 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -14,7 +14,7 @@ services: depends_on: - web healthcheck: - test: ["CMD", "curl", "-f", "http://caddy:6752"] + test: ["CMD", "curl", "http://caddy:6752"] interval: 30s timeout: 10s retries: 3 @@ -32,7 +32,7 @@ services: depends_on: - dragonfly healthcheck: - test: ["CMD", "curl", "-f", "http://web:7080"] + test: ["CMD", "curl", "http://web:7080"] interval: 30s timeout: 10s retries: 3 diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index b2d57dc..aa7df8c 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -15,7 +15,7 @@ services: depends_on: - web healthcheck: - test: ["CMD", "curl", "-f", "http://caddy:6752"] + test: ["CMD", "curl", "http://caddy:6752"] interval: 30s timeout: 10s retries: 3 @@ -33,7 +33,7 @@ services: depends_on: - dragonfly healthcheck: - test: ["CMD", "curl", "-f", "http://web:7080"] + test: ["CMD", "curl", "http://web:7080"] interval: 30s timeout: 10s retries: 3 diff --git a/server/handlers/misc.py b/server/handlers/misc.py index b6e1770..d9e00b9 100644 --- a/server/handlers/misc.py +++ b/server/handlers/misc.py @@ -2,6 +2,8 @@ from loguru import logger from pydantic import BaseModel from fastapi.responses import JSONResponse +from medium_parser.core import MediumParser + from server import config, ban_db from server.utils.notify import send_message from server.utils.logger_trace import trace @@ -18,7 +20,7 @@ class DeleteFromCache(BaseModel): @trace async def report_problem(problem: ReportProblem): - await send_message(f"New problem report: \n{problem.description}\n\n{problem.page}") + send_message(f"New problem report: \n{problem.description}\n\n{problem.page}") return JSONResponse({"message": "OK"}, status_code=200) diff --git a/server/handlers/post.py b/server/handlers/post.py index 21cd6ea..cf139ef 100644 --- a/server/handlers/post.py +++ b/server/handlers/post.py @@ -32,7 +32,7 @@ async def render_postleter(limit: int = 30, as_html: bool = False): outlet_posts_list.append(post_metadata) except Exception as ex: logger.error(f"Couldn't render post_id for postleter: {post_id}, ex: {ex}") - # await send_message(f"Couldn't render post_id for postleter: {post_id}, ex: {ex}") + # send_message(f"Couldn't render post_id for postleter: {post_id}, ex: {ex}") postleter_template_rendered = await postleter_template.render_async(post_list=outlet_posts_list) if as_html: @@ -97,9 +97,9 @@ async def render_medium_post_link(path: str, use_cache: bool = True, use_redis: if not redis_result: if not redis_available: - await send_message("ERROR: Redis is not available. Please check your configuration.") + send_message("ERROR: Redis is not available. Please check your configuration.") elif use_redis: await redis_storage.setex(medium_post_id, config.CACHE_LIFE_TIME, pickle.dumps(rendered_medium_post)) - await send_message(f"✅ Successfully rendered post: {url_correlation.get()}", True, "GOOD") + send_message(f"✅ Successfully rendered post: {url_correlation.get()}", True, "GOOD") return HTMLResponse(serialized_rendered_post) diff --git a/server/middlewares/logger.py b/server/middlewares/logger.py index a647b00..88fbc70 100644 --- a/server/middlewares/logger.py +++ b/server/middlewares/logger.py @@ -64,7 +64,7 @@ class LoggerMiddleware(BaseHTTPMiddleware): except Exception as ex: exception_class = type(ex) logger.exception(ex) - await send_message(f"Error while processing url: {url_correlation.get()}, transponder_code: {transponder_code_correlation.get()}, error: {ex}. exception: {exception_class.__name__}. {home_page_process.get(transponder_code_correlation.get(), '')}") + send_message(f"Error while processing url: {url_correlation.get()}, transponder_code: {transponder_code_correlation.get()}, error: {ex}. exception: {exception_class.__name__}. {home_page_process.get(transponder_code_correlation.get(), '')}") response = await generate_error() logger.trace(response.__dict__) diff --git a/server/utils/error.py b/server/utils/error.py index e6dfda5..a6b0d8e 100644 --- a/server/utils/error.py +++ b/server/utils/error.py @@ -37,12 +37,10 @@ async def generate_error(error_msg: str = None, title: str = "Error", status_cod if not error_msg: error_msg = random.choice(ERROR_MSG_LIST) - """ if not quiet: - await send_message( + send_message( f"📛 Error while processing url: {url_correlation.get()}, transponder_code: {transponder_code_correlation.get()}, error: {error_msg}" ) - """ error_template_rendered = await error_template.render_async(error_msg=error_msg, transponder_code=transponder_code_correlation.get()) base_context = { diff --git a/server/utils/notify.py b/server/utils/notify.py index 985d7d3..6f07a81 100644 --- a/server/utils/notify.py +++ b/server/utils/notify.py @@ -12,7 +12,7 @@ class MessageStatus(Enum): GOOD = "GOOD" -async def send_message(text: str, silent: bool = False, status: MessageStatus = "ERROR") -> None: +def send_message(text: str, silent: bool = False, status: MessageStatus = "ERROR") -> None: asyncio.create_task(task_send_message(text, silent, status)) diff --git a/server/worker.py b/server/worker.py index 455721c..3a3547a 100644 --- a/server/worker.py +++ b/server/worker.py @@ -1,13 +1,16 @@ +import io import atexit import multiprocessing from multiprocessing.util import _exit_function +import traceback import gunicorn.app.base import uvicorn from loguru import logger from server import config, ban_db from server.main import app +from server.utils.notify import send_message from server.utils.logger import GunicornLogger from server.utils.logger_trace import trace @@ -26,6 +29,25 @@ def on_exit(): ban_db.dump() +def worker_exit(server, worker): + """Called just after a worker has been exited, in the worker process.""" + logger.warning("Shutting down gunicorn worker.") + + send_message("Shutting down gunicorn worker.", status="ERROR") + + +def worker_abort(worker): + """Log the stack trace when a worker timeout occurs""" + buffer = io.StringIO() + traceback.print_stack(file=buffer) + data = buffer.getvalue() + buffer.close() + + logger.error(f"Killing worker {worker.pid}\n{data}") + + send_message(f"Killing worker {worker.pid}\n{data}", status="ERROR") + + @trace def number_of_workers(): cores = multiprocessing.cpu_count() @@ -62,6 +84,8 @@ def execute_server_worker(host: str, port: int): "preload_app": True, "post_worker_init": post_worker_init, "timeout": config.WORKER_TIMEOUT, + "worker_exit": worker_exit, + "worker_abort": worker_abort, # "on_exit": on_exit, } GunicornStandaloneApplication(app, options).run()