diff --git a/web/server/__init__.py b/web/server/__init__.py index 4b371a4..2e5ad85 100644 --- a/web/server/__init__.py +++ b/web/server/__init__.py @@ -1,25 +1,30 @@ import sentry_sdk + from server import config if config.SENTRY_SDK_DSN: - sentry_sdk.init(dsn=config.SENTRY_SDK_DSN, traces_sample_rate=config.SENTRY_TRACES_SAMPLE_RATE, profiles_sample_rate=config.SENTRY_PROFILES_SAMPLE_RATE) + sentry_sdk.init( + dsn=config.SENTRY_SDK_DSN, + traces_sample_rate=config.SENTRY_TRACES_SAMPLE_RATE, + profiles_sample_rate=config.SENTRY_PROFILES_SAMPLE_RATE, + ) +import time from contextvars import ContextVar from multiprocessing import Value from typing import Optional import pickledb import redis.asyncio as redis -import time -from psycopg2 import OperationalError, connect -from database_lib import PostgreSQLCacheBackend, migrate_to_postgres, execute_migrate_to_postgres_in_thread +from database_lib import PostgreSQLCacheBackend, execute_migrate_to_postgres_in_thread, migrate_to_postgres from loguru import logger +from medium_parser.api import MediumApi +from medium_parser.core import MediumParser +from psycopg2 import OperationalError, connect from xkcdpass import xkcd_password as xp from server.utils.logger import configure_logger -from medium_parser.core import MediumParser -from medium_parser.api import MediumApi def wait_for_postgres(max_retries=5, retry_interval=5): @@ -30,8 +35,10 @@ def wait_for_postgres(max_retries=5, retry_interval=5): conn.close() logger.info("Successfully connected to PostgreSQL") return - except OperationalError as e: - logger.warning(f"PostgreSQL is not ready. Retrying in {retry_interval} seconds... (Attempt {retries + 1}/{max_retries})") + except OperationalError: + logger.warning( + f"PostgreSQL is not ready. Retrying in {retry_interval} seconds... (Attempt {retries + 1}/{max_retries})" + ) time.sleep(retry_interval) retries += 1 @@ -48,8 +55,16 @@ medium_cache.init_db() logger.debug(f"Database length: {medium_cache.all_length()}") -medium_api = MediumApi(auth_cookies=config.MEDIUM_AUTH_COOKIES, timeout=3, proxy_list=config.PROXY_LIST) -medium_parser = MediumParser(cache=medium_cache, medium_api=medium_api, timeout=3, host_address=config.HOST_ADDRESS, template_folder="server/templates") +medium_api = MediumApi( + auth_cookies=config.MEDIUM_AUTH_COOKIES, timeout=config.REQUEST_TIMEOUT, proxy_list=config.PROXY_LIST +) +medium_parser = MediumParser( + cache=medium_cache, + medium_api=medium_api, + timeout=config.REQUEST_TIMEOUT, + host_address=config.HOST_ADDRESS, + template_folder="server/templates", +) redis_storage = redis.Redis( host=config.REDIS_HOST, @@ -61,7 +76,9 @@ redis_storage = redis.Redis( ) url_correlation: ContextVar[Optional[str]] = ContextVar("url_correlation", default="UNKNOWN_URL") -transponder_code_correlation: ContextVar[Optional[str]] = ContextVar("transponder_code_correlation", default="unknown transponder location... Beep!") +transponder_code_correlation: ContextVar[Optional[str]] = ContextVar( + "transponder_code_correlation", default="unknown transponder location... Beep!" +) ban_db = pickledb.load("ban_post_list.db", True) diff --git a/web/server/config.py b/web/server/config.py index f79b71a..e2201ac 100644 --- a/web/server/config.py +++ b/web/server/config.py @@ -16,9 +16,9 @@ MORE_LOGS: bool = config("MORE_LOGS", cast=bool, default=False) DISABLE_EXTERNAL_DOCS: bool = config("DISABLE_EXTERNAL_DOCS", cast=bool, default=True) -TIMEOUT: int = config("TIMEOUT", cast=int, default=8) -REQUEST_TIMEOUT: int = config("REQUEST_TIMEOUT", cast=int, default=40) -WORKER_TIMEOUT: int = config("WORKER_TIMEOUT", cast=int, default=120) +TIMEOUT: int = config("TIMEOUT", cast=int, default=38) +REQUEST_TIMEOUT: int = config("REQUEST_TIMEOUT", cast=int, default=12) +WORKER_TIMEOUT: int = config("WORKER_TIMEOUT", cast=int, default=85) CACHE_LIFE_TIME: int = config("CACHE_LIFE_TIME", cast=int, default=60 * 60 * 5) diff --git a/web/server/handlers/iframe.py b/web/server/handlers/iframe.py index 3218298..1e56447 100644 --- a/web/server/handlers/iframe.py +++ b/web/server/handlers/iframe.py @@ -1,14 +1,14 @@ +import random + import aiohttp -from fastapi import Response from aiohttp_retry import RetryClient from aiohttp_socks import ProxyConnector -import random +from bs4 import BeautifulSoup +from fastapi import Response +from medium_parser import retry_options from server import config from server.utils.logger_trace import trace -from medium_parser import retry_options - -from bs4 import BeautifulSoup IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"} @@ -18,11 +18,15 @@ async def iframe_proxy(iframe_id: str): connector = ProxyConnector.from_url(random.choice(config.PROXY_LIST)) if config.PROXY_LIST else None async with aiohttp.ClientSession(connector=connector) as session: - async with RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) as retry_client: + async with RetryClient( + client_session=session, raise_for_status=False, retry_options=retry_options + ) as retry_client: async with retry_client.get( f"https://medium.com/media/{iframe_id}", - timeout=config.TIMEOUT, - headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}, + timeout=config.REQUEST_TIMEOUT, + headers={ + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36" + }, ) as request: request_content = await request.text() @@ -31,10 +35,14 @@ async def iframe_proxy(iframe_id: str): def patch_iframe_content(content: str) -> str: - content = content.replace("document.domain = document.domain", 'console.log("[FREEDIUM] iframe workaround started")') + content = content.replace( + "document.domain = document.domain", 'console.log("[FREEDIUM] iframe workaround started")' + ) soup = BeautifulSoup(content, "html.parser") - iframe_resizer_script = BeautifulSoup('', "html.parser").script + iframe_resizer_script = BeautifulSoup( + '', "html.parser" + ).script soup.head.append(iframe_resizer_script) return soup.prettify() diff --git a/web/server/handlers/miro.py b/web/server/handlers/miro.py index 2e489c2..44fcc37 100644 --- a/web/server/handlers/miro.py +++ b/web/server/handlers/miro.py @@ -1,18 +1,21 @@ -import aiohttp -from fastapi import Response -from aiohttp_retry import RetryClient -from aiohttp_socks import ProxyConnector import random -from server import config +import aiohttp +from aiohttp_retry import RetryClient +from aiohttp_socks import ProxyConnector +from fastapi import Response from medium_parser import retry_options +from server import config + IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"} async def miro_proxy(miro_data: str, use_proxy: bool = False): url = f"https://miro.medium.com/{miro_data}" - headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"} + headers = { + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36" + } if use_proxy and config.PROXY_LIST: proxy = random.choice(config.PROXY_LIST) @@ -21,9 +24,9 @@ async def miro_proxy(miro_data: str, use_proxy: bool = False): connector = None async with aiohttp.ClientSession(connector=connector) as session: - client = session if not use_proxy else RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) + client = RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) - async with client.get(url, timeout=config.TIMEOUT, headers=headers) as request: + async with client.get(url, timeout=config.REQUEST_TIMEOUT, headers=headers) as request: request_content = await request.read() content_type = request.headers["Content-Type"] diff --git a/web/server/middlewares/logger.py b/web/server/middlewares/logger.py index 3705598..86f01bb 100644 --- a/web/server/middlewares/logger.py +++ b/web/server/middlewares/logger.py @@ -45,7 +45,7 @@ class LoggerMiddleware(BaseHTTPMiddleware): logger.debug(f"\t< {name}: {value}") try: - response = await asyncio.wait_for(call_next(request), timeout=config.REQUEST_TIMEOUT) + response = await asyncio.wait_for(call_next(request), timeout=config.TIMEOUT) except Exception as ex: exception_class = type(ex) logger.exception(ex)