From 23bdb735249a4689e5ed236aeae9a4f5c23d0b94 Mon Sep 17 00:00:00 2001 From: ZhymabekRoman Date: Wed, 27 Mar 2024 21:49:52 +0500 Subject: [PATCH] Major Refactor --- .env_template | 8 +++++++- README.md | 24 ++++++++++-------------- server/__init__.py | 22 +--------------------- server/__main__.py | 2 +- server/handlers/main.py | 4 ++-- server/handlers/post.py | 3 ++- server/services/__init__.py | 1 + server/{ => services}/cli.py | 6 +++--- server/services/jinja.py | 18 ++++++++++++++++++ server/services/scheduler.py | 3 +++ server/{ => services}/worker.py | 0 server/utils/error.py | 16 ++++------------ server/utils/maintenance_scheduler.py | 16 ++++++++++++---- 13 files changed, 64 insertions(+), 59 deletions(-) create mode 100644 server/services/__init__.py rename server/{ => services}/cli.py (79%) create mode 100644 server/services/jinja.py create mode 100644 server/services/scheduler.py rename server/{ => services}/worker.py (100%) diff --git a/.env_template b/.env_template index 0074fd2..feb2857 100644 --- a/.env_template +++ b/.env_template @@ -1,4 +1,10 @@ +# This is the address of the server that will be used to send the requests to HOST_ADDRESS = "http://localhost:6752" + +# This is the timeout in seconds for the requests TIMEOUT=3 + +# This is the secret key that will be used to get access for some internal API endpoints ADMIN_SECRET_KEY="test" -# MEDIUM_AUTH_COOKIES="Get your premium subscription account coockies here, uid and sid properties is required" + +# MEDIUM_AUTH_COOKIES="Get your Medium subscription account coockies here ONLY IF YOU HAVE A PAID SUBSCRIPTION, uid and sid properties is required" diff --git a/README.md b/README.md index 970a467..b2753c4 100644 --- a/README.md +++ b/README.md @@ -44,21 +44,17 @@ Speed up Freedium, and probably create open source Medium frontend in next life Requirements: -- Python 3.9+ -- pip -- git +- Docker +- Linux. Officially, we can't guarantee that Freedium will work on other OS. -And probably any `Linux` based distro (like Arch btw). - -```bash -git clone https://github.com/Freedium-cfd/web ./web -cd ./web -pip install -r requirements.txt -# for linux also do: pip install -r requirements-fast.txt -pip install ./database-lib -pip install ./core -pip install ./rl_string_helper +We need configure our Freedium instance. Copy `.env_template` to `.env` configuration file and set values, required for you. +``` +git clone https://github.com/Freedium-cfd/web ./web +cd ./web +cp .env_template .env +# do some changes in .env, if you want +sudo docker-compose -f docker-compose-dev.yml up ``` -Now we need configure our Freedium instance. Copy `.env_template` to `.env` configuration file and set values, required for you. +And now you can access local instance of Freedium by opening browser and type `http://localhost:6752`. \ No newline at end of file diff --git a/server/__init__.py b/server/__init__.py index 11c5b23..707e1fb 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -1,12 +1,10 @@ import datetime as dt import pickledb -from multiprocessing import Value, Manager +from multiprocessing import Value import logging from contextvars import ContextVar from typing import Optional -from apscheduler.schedulers.asyncio import AsyncIOScheduler -from jinja2 import Environment, DebugUndefined, FileSystemLoader import redis.asyncio as redis from xkcdpass import xkcd_password as xp @@ -14,24 +12,6 @@ from server.utils.loguru_handler import InterceptHandler redis_storage = redis.Redis(host="dragonfly", port=6379, db=0) -scheduler = AsyncIOScheduler() - -jinja_env = Environment(enable_async=True) -jinja_safe_env = Environment(undefined=DebugUndefined) -template_env = Environment(loader=FileSystemLoader("./server/templates"), enable_async=True) -template_safe_env = Environment(loader=FileSystemLoader("./server/templates"), undefined=DebugUndefined) - -base_template = template_env.get_template("base.html") -url_line_template = template_env.get_template("url_line.html").render() -main_template_raw = template_safe_env.get_template("main.html") -postleter_template = template_env.get_template("postleter.html") -error_template_raw = template_safe_env.get_template("error.html") - -main_template_raw_rendered = main_template_raw.render(url_line=url_line_template) -main_template = jinja_env.from_string(main_template_raw_rendered) - -error_template_raw_rendered = error_template_raw.render(url_line=url_line_template) -error_template = jinja_env.from_string(error_template_raw_rendered) logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True) diff --git a/server/__main__.py b/server/__main__.py index 67c3821..871b17d 100644 --- a/server/__main__.py +++ b/server/__main__.py @@ -1,3 +1,3 @@ -from server.cli import cli +from server.services.cli import cli cli() diff --git a/server/handlers/main.py b/server/handlers/main.py index 1bcc5c9..69739fc 100644 --- a/server/handlers/main.py +++ b/server/handlers/main.py @@ -1,11 +1,11 @@ from html5lib.html5parser import parse from html5lib import serialize -from server import base_template, main_template, config - from fastapi import Request from fastapi.responses import HTMLResponse, JSONResponse +from server import config +from server.services.jinja import base_template, main_template from server.handlers.post import render_medium_post_link, render_postleter from server.handlers.reverse_proxy import miro_proxy, iframe_proxy from server.handlers.misc import report_problem, delete_from_cache diff --git a/server/handlers/post.py b/server/handlers/post.py index d197285..f639998 100644 --- a/server/handlers/post.py +++ b/server/handlers/post.py @@ -6,7 +6,8 @@ from html5lib.html5parser import parse from html5lib import serialize from loguru import logger -from server import base_template, config, url_correlation, redis_storage, postleter_template, home_page_process, transponder_code_correlation +from server import config, url_correlation, redis_storage, home_page_process, transponder_code_correlation +from server.services.jinja import base_template, postleter_template from server.utils.error import generate_error from server.utils.logger_trace import trace from server.utils.notify import send_message diff --git a/server/services/__init__.py b/server/services/__init__.py new file mode 100644 index 0000000..9137fa6 --- /dev/null +++ b/server/services/__init__.py @@ -0,0 +1 @@ +from .scheduler import scheduler \ No newline at end of file diff --git a/server/cli.py b/server/services/cli.py similarity index 79% rename from server/cli.py rename to server/services/cli.py index 21652c7..2fb34b3 100644 --- a/server/cli.py +++ b/server/services/cli.py @@ -24,8 +24,8 @@ def server_cmd(cmd, opts): if is_port_in_use(opts.port): cmd.error(f"Port {opts.port} is in use or permission denied") - from server.worker import execute_server_worker - from server.utils.maintenance_scheduler import enable_maintenance_mode + from server.services.worker import execute_server_worker + from server.utils.maintenance_scheduler import do_maintenance - threading.Thread(target=enable_maintenance_mode, daemon=True).start() + threading.Thread(target=do_maintenance, daemon=True).start() execute_server_worker(host="0.0.0.0", port=opts.port) diff --git a/server/services/jinja.py b/server/services/jinja.py new file mode 100644 index 0000000..74b2a77 --- /dev/null +++ b/server/services/jinja.py @@ -0,0 +1,18 @@ +from jinja2 import Environment, DebugUndefined, FileSystemLoader + +jinja_env = Environment(enable_async=True) +jinja_safe_env = Environment(undefined=DebugUndefined) +template_env = Environment(loader=FileSystemLoader("./server/templates"), enable_async=True) +template_safe_env = Environment(loader=FileSystemLoader("./server/templates"), undefined=DebugUndefined) + +base_template = template_env.get_template("base.html") +url_line_template = template_env.get_template("url_line.html").render() +main_template_raw = template_safe_env.get_template("main.html") +postleter_template = template_env.get_template("postleter.html") +error_template_raw = template_safe_env.get_template("error.html") + +main_template_raw_rendered = main_template_raw.render(url_line=url_line_template) +main_template = jinja_env.from_string(main_template_raw_rendered) + +error_template_raw_rendered = error_template_raw.render(url_line=url_line_template) +error_template = jinja_env.from_string(error_template_raw_rendered) \ No newline at end of file diff --git a/server/services/scheduler.py b/server/services/scheduler.py new file mode 100644 index 0000000..0ec4214 --- /dev/null +++ b/server/services/scheduler.py @@ -0,0 +1,3 @@ +from apscheduler.schedulers.asyncio import AsyncIOScheduler + +scheduler = AsyncIOScheduler() \ No newline at end of file diff --git a/server/worker.py b/server/services/worker.py similarity index 100% rename from server/worker.py rename to server/services/worker.py diff --git a/server/utils/error.py b/server/utils/error.py index a6b0d8e..03f1d1d 100644 --- a/server/utils/error.py +++ b/server/utils/error.py @@ -2,13 +2,8 @@ import random from fastapi.responses import HTMLResponse -from server import ( - base_template, - config, - error_template, - transponder_code_correlation, - url_correlation -) +from server import config, transponder_code_correlation, url_correlation +from server.services.jinja import base_template, error_template from server.utils.logger_trace import trace from server.utils.notify import send_message @@ -28,7 +23,7 @@ ERROR_MSG_LIST = [ "Oops! We've encountered a problem, but don't worry, it's not as disastrous as my attempt at karaoke! 🎀", "Sorry to hear that, but we've got a problem that's more stubborn than a toddler refusing to eat their veggies! πŸ‘ΆπŸ₯¦", "Apologies, but we've run into a problem - it's not as amusing as my dog trying to catch its tail, but it's a problem! 🐢", - "Sorry to hear that, but we've got a problem that's more elusive than the end of a rainbow! 🌈" + "Sorry to hear that, but we've got a problem that's more elusive than the end of a rainbow! 🌈", ] @@ -38,9 +33,7 @@ async def generate_error(error_msg: str = None, title: str = "Error", status_cod error_msg = random.choice(ERROR_MSG_LIST) if not quiet: - send_message( - f"πŸ“› Error while processing url: {url_correlation.get()}, transponder_code: {transponder_code_correlation.get()}, error: {error_msg}" - ) + send_message(f"πŸ“› Error while processing url: {url_correlation.get()}, transponder_code: {transponder_code_correlation.get()}, error: {error_msg}") error_template_rendered = await error_template.render_async(error_msg=error_msg, transponder_code=transponder_code_correlation.get()) base_context = { @@ -50,4 +43,3 @@ async def generate_error(error_msg: str = None, title: str = "Error", status_cod } base_template_rendered = await base_template.render_async(base_context, HOST_ADDRESS=config.HOST_ADDRESS) return HTMLResponse(base_template_rendered, status_code=status_code) - diff --git a/server/utils/maintenance_scheduler.py b/server/utils/maintenance_scheduler.py index c45c6f7..b7c127e 100644 --- a/server/utils/maintenance_scheduler.py +++ b/server/utils/maintenance_scheduler.py @@ -1,18 +1,19 @@ from contextlib import suppress from datetime import datetime -from server import maintenance_mode, scheduler +from server import maintenance_mode from server.utils.notify import send_message from medium_parser import cache as medium_cache +from time import sleep from loguru import logger def enable_maintenance_mode(): global maintenance_mode maintenance_mode.value = True - logger.debug("Maintenance mode enabled") + logger.debug("Maintenance mode enabled") send_message("Maintenance mode enabled") now = datetime.now() @@ -26,6 +27,13 @@ def enable_maintenance_mode(): logger.debug("Maintenance mode disabled") send_message("Maintenance mode disabled") -scheduler.add_job(enable_maintenance_mode, 'cron', hour='*/2') -scheduler.start() \ No newline at end of file +def do_maintenance(sleep_time: int = 60 * 60): + while True: + try: + enable_maintenance_mode() + except Exception as e: + logger.error(f"Error enabling maintenance mode: {e}") + send_message(f"Error enabling maintenance mode: {e}") + finally: + sleep(sleep_time)