chore: refactor & optimize code

core/medium_parser/__init__.py

@@ -1,10 +1,8 @@
+import jinja2
 from aiohttp_retry import ExponentialRetry
 
-import jinja2
+from medium_parser import exceptions as exceptions
+from medium_parser import exceptions as medium_parser_exceptions
 
 retry_options = ExponentialRetry(attempts=3)
-
-from . import exceptions as exceptions
-from . import exceptions as medium_parser_exceptions
-
-jinja_env = jinja2.Environment()
+jinja_env = jinja2.Environment()

core/medium_parser/core.py

@@ -14,25 +14,25 @@ from rl_string_helper import RLStringHelper, parse_markups, split_overlapping_ra
 
 from . import jinja_env
 from .exceptions import InvalidMediumPostID, InvalidMediumPostURL, InvalidURL, MediumParserException, MediumPostQueryError
-from .medium_api import query_post_by_id
+from .medium_api import MediumApi
 from .models.html_result import HtmlResult
 from .time import convert_datetime_to_human_readable
 from .utils import correct_url, extract_hex_string, getting_percontage_of_match, is_has_valid_medium_post_id, is_valid_medium_url, is_valid_url, resolve_medium_url
 
 if typing.TYPE_CHECKING:
-    from database_lib import SQLiteCacheBackend
+    from database_lib import AbstractCacheBackend
 
 
 class MediumParser:
-    __slots__ = ("cache", "host_address", "jinja_template", "post_template", "timeout", "auth_cookies")
+    __slots__ = ("cache", "host_address", "jinja_template", "post_template", "timeout")
 
-    def __init__(self, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: Optional[str] = None, template_folder: str = "./templates"):
-        self.timeout = timeout
-        self.cache = cache
-        self.host_address = host_address
-        self.auth_cookies = auth_cookies
-        self.jinja_template = jinja2.Environment(loader=jinja2.FileSystemLoader(template_folder))
-        self.post_template = self.jinja_template.get_template("post.html")
+    def __init__(self, cache: "AbstractCacheBackend", medium_api: MediumApi, timeout: int, host_address: str, template_folder: str = "./templates"):
+        self.timeout: int = timeout
+        self.cache: AbstractCacheBackend = cache
+        self.host_address: str = host_address
+        self.jinja_template: jinja2.Environment = jinja2.Environment(loader=jinja2.FileSystemLoader(template_folder))
+        self.post_template: jinja2.Template = self.jinja_template.get_template("post.html")
+        self.medium_api: MediumApi = medium_api
 
     async def resolve(self, unknown: str) -> str:
         logger.debug(f"We got some unknown data: {unknown=}. Trying resolve them...///")
@@ -83,7 +83,7 @@ class MediumParser:
         async def _get_from_api():
             logger.debug("Using API to gather post data")
             try:
-                return await query_post_by_id(post_id, self.timeout, self.auth_cookies)
+                return await self.medium_api.query_post_by_id(post_id)
             except Exception as ex:
                 logger.debug("Error while querying post data from Medium API")
                 logger.exception(ex)
@@ -218,20 +218,20 @@ class MediumParser:
             else:
                 text_formater = None
 
-                for highlight in highlights:
-                    for highlight_paragraph in highlight["paragraphs"]:
-                        if highlight_paragraph["name"] == paragraph["name"]:
-                            logger.trace("Apply highlight to this paragraph")
-                            if highlight_paragraph["text"] != text_formater.get_text():
-                                logger.warning("Highlighted text and paragraph text are not the same! Skip...")
-                                break
-                            quote_markup_template = '<mark class="bg-emerald-300">{{ text }}</mark>'
-                            text_formater.set_template(
-                                highlight["startOffset"],
-                                highlight["endOffset"],
-                                quote_markup_template,
-                            )
+            for highlight in highlights:
+                for highlight_paragraph in highlight["paragraphs"]:
+                    if highlight_paragraph["name"] == paragraph["name"]:
+                        logger.trace("Apply highlight to this paragraph")
+                        if highlight_paragraph["text"] != text_formater.get_text():
+                            logger.warning("Highlighted text and paragraph text are not the same! Skip...")
                             break
+                        quote_markup_template = '<mark class="bg-emerald-300">{{ text }}</mark>'
+                        text_formater.set_template(
+                            highlight["startOffset"],
+                            highlight["endOffset"],
+                            quote_markup_template,
+                        )
+                        break
 
             if paragraph["type"] == "H2":
                 css_class = []

core/medium_parser/exceptions.py

@@ -24,3 +24,15 @@ class InvalidMediumPostID(MediumParserException):
 
 class MediumPostQueryError(MediumParserException):
     pass
+
+
+class MediumPostNotFound(MediumPostQueryError):
+    pass
+
+
+class MediumPostUnavailable(MediumPostQueryError):
+    pass
+
+
+class MediumPostDeleted(MediumPostQueryError):
+    pass

core/medium_parser/utils.py

@@ -15,7 +15,7 @@ from async_lru import alru_cache
 from bs4 import BeautifulSoup
 from loguru import logger
 
-from . import exceptions, retry_options
+from medium_parser import exceptions, retry_options
 
 DEFAULT_URL_PROTOCOL = "https://"
 
@@ -157,6 +157,7 @@ def unquerify_url(url: str) -> str:
 
 @lru_cache(maxsize=500)
 def un_wwwify(url: str):
+    # TODO: enhanced type checks
     if url.startswith("www."):
         return url.removeprefix("www.")
     return url

database-lib/database_lib/__init__.py

@@ -1 +1,9 @@
-from .cache_db import SQLiteCacheBackend, PostgreSQLCacheBackend, migrate_to_postgres, execute_migrate_to_postgres_in_thread
+from database_lib.main import AbstractCacheBackend, SQLiteCacheBackend, PostgreSQLCacheBackend, migrate_to_postgres, execute_migrate_to_postgres_in_thread
+
+__all__ = [
+    "AbstractCacheBackend",
+    "SQLiteCacheBackend",
+    "PostgreSQLCacheBackend",
+    "migrate_to_postgres",
+    "execute_migrate_to_postgres_in_thread",
+]

database-lib/database_lib/main.py

@@ -4,6 +4,7 @@ import sqlite3
 import threading
 from itertools import islice
 from typing import Union, Optional
+from abc import ABC, abstractmethod
 
 import psycopg2
 import sqlite_zstd
@@ -12,7 +13,7 @@ from loguru import logger
 from psycopg2.extras import execute_batch
 
 
-class CacheResponse:
+class CacheData:
     __slots__ = ("data",)
 
     def __init__(self, data: str):
@@ -28,7 +29,52 @@ class CacheResponse:
         return self.data
 
 
-class SQLiteCacheBackend:
+class CacheResponse:
+    __slots__ = ("key", "data")
+
+    def __init__(self, key: str, data: Union[CacheData, str]):
+        self.key: str = key
+        self.data: CacheData = CacheData(data) if isinstance(data, str) else data
+
+    def json(self):
+        return self.data.json()
+
+
+class AbstractCacheBackend(ABC):
+    @abstractmethod
+    def init_db(self):
+        pass
+
+    @abstractmethod
+    def all(self):
+        pass
+
+    @abstractmethod
+    def all_length(self) -> int:
+        pass
+
+    @abstractmethod
+    def random(self, size: int) -> list[CacheResponse]:
+        pass
+
+    @abstractmethod
+    def pull(self, key: str) -> Union[CacheResponse, None]:
+        pass
+
+    @abstractmethod
+    def push(self, key: str, value: Union[str, dict]) -> None:
+        pass
+
+    @abstractmethod
+    def delete(self, key: str) -> None:
+        pass
+
+    @abstractmethod
+    def close(self):
+        pass
+
+
+class SQLiteCacheBackend(AbstractCacheBackend):
     __slots__ = ("connection", "cursor", "database", "lock")
 
     def __init__(self, database: str, zstd_enabled: bool = False):
@@ -53,9 +99,10 @@ class SQLiteCacheBackend:
         with self.connection:
             return self.cursor.execute("SELECT COUNT(*) FROM cache").fetchone()[0]
 
-    def random(self, size: int):
+    def random(self, size: int) -> list[CacheResponse]:
         with self.connection:
-            return self.cursor.execute("SELECT * FROM cache ORDER BY RANDOM() LIMIT ?", (size,)).fetchall()
+            self.cursor.execute("SELECT key, value FROM cache ORDER BY RANDOM() LIMIT ?", (size,))
+            return [CacheResponse(key, value) for key, value in self.cursor]
 
     def enable_zstd(self):
         if not self.zstd_enabled:
@@ -65,7 +112,9 @@ class SQLiteCacheBackend:
             try:
                 self.cursor.execute('SELECT zstd_enable_transparent(\'{"table": "cache", "column": "value", "compression_level": 9, "dict_chooser": "\'\'a\'\'"}\')')
             except Exception as error:
-                print(error)
+                logger.error(f"Error enabling ZSTD compression: {error}")
+                logger.exception(error)
+
             self.connection.execute("PRAGMA auto_vacuum=full")
 
     def init_db(self):
@@ -78,7 +127,7 @@ class SQLiteCacheBackend:
             cache = self.cursor.execute("SELECT value FROM cache WHERE key = :0", {"0": key}).fetchone()
             if cache:
                 logger.debug("Value found in DB, returning it")
-                return CacheResponse(cache[0])
+                return CacheResponse(key, cache[0])
             else:
                 logger.debug(f"No value found for key: {key}")
                 return None
@@ -161,7 +210,7 @@ class SQLiteCacheBackend:
         self.connection.close()
 
 
-class PostgreSQLCacheBackend:
+class PostgreSQLCacheBackend(AbstractCacheBackend):
     def __init__(self, connection_string: str):
         self.connection = psycopg2.connect(connection_string)
         self.cursor = self.connection.cursor()
@@ -187,10 +236,10 @@ class PostgreSQLCacheBackend:
             self.cursor.execute("SELECT COUNT(*) FROM cache")
             return self.cursor.fetchone()[0]
 
-    def random(self, size: int):
+    def random(self, size: int) -> list[CacheResponse]:
         with self.connection:
-            self.cursor.execute("SELECT * FROM cache ORDER BY RANDOM() LIMIT %s", (size,))
-            return self.cursor.fetchall()
+            self.cursor.execute("SELECT key, value FROM cache ORDER BY RANDOM() LIMIT %s", (size,))
+            return [CacheResponse(key, value) for key, value in self.cursor]
 
     def pull(self, key: str) -> Union[CacheResponse, None]:
         with self.connection:
@@ -198,7 +247,7 @@ class PostgreSQLCacheBackend:
             cache = self.cursor.fetchone()
             if cache:
                 logger.debug("Value found in DB, returning it")
-                return CacheResponse(cache[0])
+                return CacheResponse(key, cache[0])
             else:
                 logger.debug(f"No value found for key: {key}")
                 return None
@@ -258,7 +307,6 @@ def migrate_to_postgres(sqlite_db_path: str, pg_conn_string: str, chunk_size: in
             elapsed_time = time.time() - start_time
             rows_per_second = processed_rows / elapsed_time
             logger.info(f"Processed {processed_rows}/{total_rows} rows. Speed: {rows_per_second:.2f} rows/second")
-
     except Exception as e:
         logger.error(f"An error occurred during migration: {e}")
         pg_db.connection.rollback()

requirements-fast.txt

@@ -1,2 +1 @@
 orjson==3.10.6
-uvloop==0.19.0

requirements.txt

@@ -2,7 +2,7 @@ pickledb==0.9.2
 html5lib==1.1
 sentry-sdk[fastapi]==1.29.2
 loguru==0.6.0 # due to: https://github.com/Delgan/loguru/issues/916
-uvicorn==0.27.1
+uvicorn[standard]==0.30.3
 anyio<=4.0.0 # Workaround to: https://github.getafreenode.com/tiangolo/fastapi/discussions/11652
 Jinja2==3.1.2
 fastapi==0.108.0

server/config.py

@@ -17,7 +17,7 @@ MORE_LOGS = config("MORE_LOGS", cast=bool, default=False)
 DISABLE_EXTERNAL_DOCS = config("DISABLE_EXTERNAL_DOCS", cast=bool, default=True)
 DISABLE_RATE_LIMITER = config("DISABLE_RATE_LIMITER", cast=bool, default=True)
 
-TIMEOUT = config("TIMEOUT", cast=int, default=25)
+TIMEOUT = config("TIMEOUT", cast=int, default=8)
 REQUEST_TIMEOUT = config("REQUEST_TIMEOUT", cast=int, default=40)
 WORKER_TIMEOUT = config("WORKER_TIMEOUT", cast=int, default=120)
 

server/handlers/main.py

@@ -53,7 +53,7 @@ async def route_processing(path: str, request: Request):
 async def main_page():
     homepage_template = await render_homepage(as_html=True)
     main_template_rendered = main_template.render(postleter=homepage_template)
-    base_template_rendered = base_template.render(body_template=main_template_rendered, HOST_ADDRESS=config.HOST_ADDRESS)
+    base_template_rendered = base_template.render(body_template=main_template_rendered, host_address=config.HOST_ADDRESS)
     parsed_template = parse(base_template_rendered)
     serialized_template = serialize(parsed_template, encoding='utf-8')
     return HTMLResponse(serialized_template)

server/handlers/post.py

@@ -88,12 +88,13 @@ async def render_medium_post_link(path: str, use_cache: bool = True, use_redis:
         return await handle_exception(ex, status_code=500)
     else:
         base_context = {
+            "host_address": config.HOST_ADDRESS,
             "enable_ads_header": config.ENABLE_ADS_BANNER,
             "body_template": rendered_medium_post.data,
             "title": rendered_medium_post.title,
             "description": rendered_medium_post.description,
         }
-        rendered_post = base_template.render(base_context, HOST_ADDRESS=config.HOST_ADDRESS)
+        rendered_post = base_template.render(base_context)
         parsed_rendered_post = parse(rendered_post)
         serialized_rendered_post = serialize(parsed_rendered_post, encoding="utf-8")
 

server/handlers/reverse_proxy.py

@@ -30,13 +30,14 @@ async def iframe_proxy(iframe_id: str):
     return Response(content=request_content_soup.prettify(), media_type="text/html", headers=IFRAME_HEADERS)
 
 
-async def miro_proxy(miro_data: str):
-    async with aiohttp.ClientSession() as client:
-        request = await client.get(
-           f"https://miro.medium.com/{miro_data}",
-            timeout=config.TIMEOUT,
-            headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"},
-        )
-        request_content = await request.read()
-        content_type = request.headers["Content-Type"]
-    return Response(content=request_content, media_type=content_type)
+# async def miro_proxy(miro_data: str):
+#     async with aiohttp.ClientSession() as client:
+#         request = await client.get(
+#            f"https://miro.medium.com/{miro_data}",
+#             timeout=config.TIMEOUT,
+#             headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"},
+#         )
+#         request_content = await request.read()
+#         content_type = request.headers["Content-Type"]
+#     return Response(content=request_content, media_type=content_type)
+#

server/middlewares/logger.py

@@ -25,6 +25,7 @@ class LoggerMiddleware(BaseHTTPMiddleware):
         transponder_code = string_to_number_ascii(generated_id)
         transponder_code_correlation.set(transponder_code)
         url_correlation.set(request.url)
+
         with logger.contextualize(id=generated_id):
             logger.debug(f"Current ID '{generated_id}' transponder code is '{transponder_code}'")
             logger.trace(request.__dict__)

server/templates/base.html

@@ -347,7 +347,7 @@ document.addEventListener('scroll', function () {
                     if (element.tagName === "IMG" && element.hasAttribute("data-src")) {
                         const srcAttribute = element.attributes["data-src"].value;
                         if (srcAttribute.startsWith("https://miro.medium.com/v2/")) {
-                            element.setAttribute("src", srcAttribute.replace("https://miro.medium.com/v2/", "{{HOST_ADDRESS}}/@miro/v2/"));
+                            element.setAttribute("src", srcAttribute.replace("https://miro.medium.com/v2/", "{{host_address}}/@miro/v2/"));
                         }
                     }
                 }

server/utils/error.py

@@ -1,3 +1,4 @@
+from typing import Optional
 import random
 
 from fastapi.responses import HTMLResponse
@@ -28,18 +29,22 @@ ERROR_MSG_LIST = [
 
 
 @trace
-async def generate_error(error_msg: str = None, title: str = "Error", status_code: int = 500, quiet: bool = False):
+async def generate_error(error_msg: Optional[str] = None, title: Optional[str] = None, status_code: int = 500, quiet: bool = False):
     if not error_msg:
         error_msg = random.choice(ERROR_MSG_LIST)
 
+    if not title:
+        title = "Opppps.."
+
     if not quiet:
         send_message(f"📛 Error while processing url: <code>{url_correlation.get()}</code>, transponder_code: <code>{transponder_code_correlation.get()}</code>, error: <code>{error_msg}</code>")
 
     error_template_rendered = error_template.render(error_msg=error_msg, transponder_code=transponder_code_correlation.get())
     base_context = {
+        "host_address": config.HOST_ADDRESS,
         "enable_ads_header": config.ENABLE_ADS_BANNER,
         "body_template": error_template_rendered,
         "title": title,
     }
-    base_template_rendered = base_template.render(base_context, HOST_ADDRESS=config.HOST_ADDRESS)
+    base_template_rendered = base_template.render(base_context)
     return HTMLResponse(base_template_rendered, status_code=status_code)