CORE: improve post id parser with regex

core/medium_parser/core.py

@@ -19,8 +19,8 @@ from .medium_api import query_post_by_id
 from .models.html_result import HtmlResult
 from .time import convert_datetime_to_human_readable
 from .utils import (correct_url, getting_percontage_of_match,
-                    is_valid_medium_post_id_hexadecimal, is_valid_medium_url,
-                    is_valid_url, resolve_medium_url)
+                    is_has_valid_medium_post_id, is_valid_medium_url,
+                    is_valid_url, resolve_medium_url, extract_hex_string)
 
 if typing.TYPE_CHECKING:
     from database_lib import SQLiteCacheBackend
@@ -40,13 +40,14 @@ class MediumParser:
     async def from_unknown(cls, unknown: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser":
         logger.debug(f"We got some unknown data: {unknown=}, with {cache=}, {timeout=}, {host_address=}, {auth_cookies=}. Trying resolve them...///")
 
-        if is_valid_medium_post_id_hexadecimal(unknown):
+        if is_has_valid_medium_post_id(unknown):
             logger.debug("Seems like it's valid post_id")
             return cls(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies)
 
         logger.debug("...maybe it's URL. Let's checkout...")
         return await cls.from_url(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies)
 
+
     @classmethod
     async def from_url(cls, url: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser":
         sanitized_url = correct_url(url)
@@ -65,10 +66,10 @@ class MediumParser:
 
     @post_id.setter
     def post_id(self, value):
-        if not is_valid_medium_post_id_hexadecimal(value):
+        if not is_has_valid_medium_post_id(value):
             raise InvalidMediumPostID(f"Invalid medium post ID: {value}")
 
-        self.__post_id = value
+        self.__post_id = extract_hex_string(value)
 
     @post_id.getter
     def post_id(self):
@@ -137,13 +138,13 @@ class MediumParser:
                 if not post_data:
                     reason = "No post data returned"
                 elif not isinstance(post_data, dict):
-                    reason = "Post data is not a dictionary"
+                    reason = f"Post data is not a dictionary: {post_data=}"
                 elif post_data.get("error"):
-                    reason = "Post data contains an error"
+                    reason = f"Post data contains an error: {post_data=}"
                 elif not post_data.get("data"):
-                    reason = "Post data missing 'data' key"
+                    reason = f"Post data missing 'data' key: {post_data=}"
                 elif not post_data.get("data").get("post"):
-                    reason = "Post data missing 'data.post' key"
+                    reason = f"Post data missing 'data.post' key: {post_data=}"
 
                 if reason is None:
                     logger.debug("Post data was successfully queried")

core/medium_parser/utils.py

@@ -81,6 +81,7 @@ NOT_MEDIUM_DOMAINS = (
     "bloomberg.com",
     "www.lesechos.fr",
     "www.otz.de",
+    "www.businessinsider.com",
     "buff.ly",
     "www.delish.com",
     "www.economist.com",
@@ -191,7 +192,12 @@ def unplaginate_url(url):
 
 
 @lru_cache(maxsize=100)
-def is_valid_medium_post_id_hexadecimal(hex_string: str) -> bool:
+def is_has_valid_medium_post_id(hex_string: str) -> bool:
+    return extract_hex_string(hex_string) is not None
+
+
+@lru_cache(maxsize=100)
+def basic_hex_check(hex_string: str) -> bool:
     # Check if the string is a valid hexadecimal string
     for char in hex_string:
         if char not in VALID_ID_CHARS:
@@ -212,6 +218,16 @@ def is_valid_medium_post_id_hexadecimal(hex_string: str) -> bool:
     return True
 
 
+@lru_cache(maxsize=100)
+def extract_hex_string(input_string: str) -> str:
+    # First try to find a hexadecimal string preceded by a '-'
+    match = re.search(r'-(\b[a-fA-F0-9]{8,12}\b)', input_string)
+    if not match:
+        # If no match, try to find a hexadecimal string without the '-'
+        match = re.search(r'(\b[a-fA-F0-9]{8,12}\b)', input_string)
+    return match.group(1) if match else None
+
+
 async def resolve_medium_short_link(short_url_id: str, timeout: int = 5) -> str:
     async with aiohttp.ClientSession() as session:
         retry_client = RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options)
@@ -307,7 +323,7 @@ async def resolve_medium_url(url: str, timeout: int = 5) -> str:
         post_url = parsed_url.path.split("/")[-1]
         post_id = post_url.split("-")[-1]
 
-    if not is_valid_medium_post_id_hexadecimal(post_id):
+    if not is_has_valid_medium_post_id(post_id):
         logger.warning(f"...but hoops, that's invalid post_id: {post_id}")
         return False
 

server/handlers/post.py

@@ -4,6 +4,7 @@ import pickle
 from fastapi.responses import HTMLResponse
 from html5lib import serialize
 from html5lib.html5parser import parse
+from async_lru import alru_cache
 from loguru import logger
 from medium_parser import medium_parser_exceptions
 from medium_parser.core import MediumParser
@@ -46,7 +47,7 @@ async def render_postleter(limit: int = 30, as_html: bool = False):
     return HTMLResponse(postleter_template_rendered)
 
 
-@trace
+@alru_cache(maxsize=20)
 async def render_medium_post_link(path: str, use_cache: bool = True, use_redis: bool = True):
     redis_available = await safe_check_redis_connection(redis_storage)
     logger.debug("Redis available: {}", redis_available)