prod: fixing timeout values

web/server/__init__.py

@@ -1,25 +1,30 @@
 import sentry_sdk
+
 from server import config
 
 if config.SENTRY_SDK_DSN:
-    sentry_sdk.init(dsn=config.SENTRY_SDK_DSN, traces_sample_rate=config.SENTRY_TRACES_SAMPLE_RATE, profiles_sample_rate=config.SENTRY_PROFILES_SAMPLE_RATE)
+    sentry_sdk.init(
+        dsn=config.SENTRY_SDK_DSN,
+        traces_sample_rate=config.SENTRY_TRACES_SAMPLE_RATE,
+        profiles_sample_rate=config.SENTRY_PROFILES_SAMPLE_RATE,
+    )
 
 
+import time
 from contextvars import ContextVar
 from multiprocessing import Value
 from typing import Optional
 
 import pickledb
 import redis.asyncio as redis
-import time
-from psycopg2 import OperationalError, connect
-from database_lib import PostgreSQLCacheBackend, migrate_to_postgres, execute_migrate_to_postgres_in_thread
+from database_lib import PostgreSQLCacheBackend, execute_migrate_to_postgres_in_thread, migrate_to_postgres
 from loguru import logger
+from medium_parser.api import MediumApi
+from medium_parser.core import MediumParser
+from psycopg2 import OperationalError, connect
 from xkcdpass import xkcd_password as xp
 
 from server.utils.logger import configure_logger
-from medium_parser.core import MediumParser
-from medium_parser.api import MediumApi
 
 
 def wait_for_postgres(max_retries=5, retry_interval=5):
@@ -30,8 +35,10 @@ def wait_for_postgres(max_retries=5, retry_interval=5):
             conn.close()
             logger.info("Successfully connected to PostgreSQL")
             return
-        except OperationalError as e:
-            logger.warning(f"PostgreSQL is not ready. Retrying in {retry_interval} seconds... (Attempt {retries + 1}/{max_retries})")
+        except OperationalError:
+            logger.warning(
+                f"PostgreSQL is not ready. Retrying in {retry_interval} seconds... (Attempt {retries + 1}/{max_retries})"
+            )
             time.sleep(retry_interval)
             retries += 1
 
@@ -48,8 +55,16 @@ medium_cache.init_db()
 
 logger.debug(f"Database length: {medium_cache.all_length()}")
 
-medium_api = MediumApi(auth_cookies=config.MEDIUM_AUTH_COOKIES, timeout=3, proxy_list=config.PROXY_LIST)
-medium_parser = MediumParser(cache=medium_cache, medium_api=medium_api, timeout=3, host_address=config.HOST_ADDRESS, template_folder="server/templates")
+medium_api = MediumApi(
+    auth_cookies=config.MEDIUM_AUTH_COOKIES, timeout=config.REQUEST_TIMEOUT, proxy_list=config.PROXY_LIST
+)
+medium_parser = MediumParser(
+    cache=medium_cache,
+    medium_api=medium_api,
+    timeout=config.REQUEST_TIMEOUT,
+    host_address=config.HOST_ADDRESS,
+    template_folder="server/templates",
+)
 
 redis_storage = redis.Redis(
     host=config.REDIS_HOST,
@@ -61,7 +76,9 @@ redis_storage = redis.Redis(
 )
 
 url_correlation: ContextVar[Optional[str]] = ContextVar("url_correlation", default="UNKNOWN_URL")
-transponder_code_correlation: ContextVar[Optional[str]] = ContextVar("transponder_code_correlation", default="unknown transponder location... Beep!")
+transponder_code_correlation: ContextVar[Optional[str]] = ContextVar(
+    "transponder_code_correlation", default="unknown transponder location... Beep!"
+)
 
 ban_db = pickledb.load("ban_post_list.db", True)
 

web/server/config.py

@@ -16,9 +16,9 @@ MORE_LOGS: bool = config("MORE_LOGS", cast=bool, default=False)
 
 DISABLE_EXTERNAL_DOCS: bool = config("DISABLE_EXTERNAL_DOCS", cast=bool, default=True)
 
-TIMEOUT: int = config("TIMEOUT", cast=int, default=8)
-REQUEST_TIMEOUT: int = config("REQUEST_TIMEOUT", cast=int, default=40)
-WORKER_TIMEOUT: int = config("WORKER_TIMEOUT", cast=int, default=120)
+TIMEOUT: int = config("TIMEOUT", cast=int, default=38)
+REQUEST_TIMEOUT: int = config("REQUEST_TIMEOUT", cast=int, default=12)
+WORKER_TIMEOUT: int = config("WORKER_TIMEOUT", cast=int, default=85)
 
 CACHE_LIFE_TIME: int = config("CACHE_LIFE_TIME", cast=int, default=60 * 60 * 5)
 

web/server/handlers/iframe.py

@@ -1,14 +1,14 @@
+import random
+
 import aiohttp
-from fastapi import Response
 from aiohttp_retry import RetryClient
 from aiohttp_socks import ProxyConnector
-import random
+from bs4 import BeautifulSoup
+from fastapi import Response
+from medium_parser import retry_options
 
 from server import config
 from server.utils.logger_trace import trace
-from medium_parser import retry_options
-
-from bs4 import BeautifulSoup
 
 IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"}
 
@@ -18,11 +18,15 @@ async def iframe_proxy(iframe_id: str):
     connector = ProxyConnector.from_url(random.choice(config.PROXY_LIST)) if config.PROXY_LIST else None
 
     async with aiohttp.ClientSession(connector=connector) as session:
-        async with RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) as retry_client:
+        async with RetryClient(
+            client_session=session, raise_for_status=False, retry_options=retry_options
+        ) as retry_client:
             async with retry_client.get(
                 f"https://medium.com/media/{iframe_id}",
-                timeout=config.TIMEOUT,
-                headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"},
+                timeout=config.REQUEST_TIMEOUT,
+                headers={
+                    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
+                },
             ) as request:
                 request_content = await request.text()
 
@@ -31,10 +35,14 @@ async def iframe_proxy(iframe_id: str):
 
 
 def patch_iframe_content(content: str) -> str:
-    content = content.replace("document.domain = document.domain", 'console.log("[FREEDIUM] iframe workaround started")')
+    content = content.replace(
+        "document.domain = document.domain", 'console.log("[FREEDIUM] iframe workaround started")'
+    )
 
     soup = BeautifulSoup(content, "html.parser")
-    iframe_resizer_script = BeautifulSoup('<script src="https://cdn.jsdelivr.net/npm/@iframe-resizer/child"></script>', "html.parser").script
+    iframe_resizer_script = BeautifulSoup(
+        '<script src="https://cdn.jsdelivr.net/npm/@iframe-resizer/child"></script>', "html.parser"
+    ).script
     soup.head.append(iframe_resizer_script)
 
     return soup.prettify()

web/server/handlers/miro.py

@@ -1,18 +1,21 @@
-import aiohttp
-from fastapi import Response
-from aiohttp_retry import RetryClient
-from aiohttp_socks import ProxyConnector
 import random
 
-from server import config
+import aiohttp
+from aiohttp_retry import RetryClient
+from aiohttp_socks import ProxyConnector
+from fastapi import Response
 from medium_parser import retry_options
 
+from server import config
+
 IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"}
 
 
 async def miro_proxy(miro_data: str, use_proxy: bool = False):
     url = f"https://miro.medium.com/{miro_data}"
-    headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}
+    headers = {
+        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
+    }
 
     if use_proxy and config.PROXY_LIST:
         proxy = random.choice(config.PROXY_LIST)
@@ -21,9 +24,9 @@ async def miro_proxy(miro_data: str, use_proxy: bool = False):
         connector = None
 
     async with aiohttp.ClientSession(connector=connector) as session:
-        client = session if not use_proxy else RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options)
+        client = RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options)
 
-        async with client.get(url, timeout=config.TIMEOUT, headers=headers) as request:
+        async with client.get(url, timeout=config.REQUEST_TIMEOUT, headers=headers) as request:
             request_content = await request.read()
             content_type = request.headers["Content-Type"]
 

web/server/middlewares/logger.py

@@ -45,7 +45,7 @@ class LoggerMiddleware(BaseHTTPMiddleware):
                     logger.debug(f"\t< {name}: {value}")
 
             try:
-                response = await asyncio.wait_for(call_next(request), timeout=config.REQUEST_TIMEOUT)
+                response = await asyncio.wait_for(call_next(request), timeout=config.TIMEOUT)
             except Exception as ex:
                 exception_class = type(ex)
                 logger.exception(ex)