From 133dc3cf963c3746c9463deb80b9dc1d30bc6c37 Mon Sep 17 00:00:00 2001 From: ZhymabekRoman Date: Fri, 26 Apr 2024 16:22:52 +0500 Subject: [PATCH] CORE: improve post id parser with regex --- core/medium_parser/core.py | 19 ++++++++++--------- core/medium_parser/utils.py | 20 ++++++++++++++++++-- server/handlers/post.py | 3 ++- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/core/medium_parser/core.py b/core/medium_parser/core.py index a093a14..a99766f 100644 --- a/core/medium_parser/core.py +++ b/core/medium_parser/core.py @@ -19,8 +19,8 @@ from .medium_api import query_post_by_id from .models.html_result import HtmlResult from .time import convert_datetime_to_human_readable from .utils import (correct_url, getting_percontage_of_match, - is_valid_medium_post_id_hexadecimal, is_valid_medium_url, - is_valid_url, resolve_medium_url) + is_has_valid_medium_post_id, is_valid_medium_url, + is_valid_url, resolve_medium_url, extract_hex_string) if typing.TYPE_CHECKING: from database_lib import SQLiteCacheBackend @@ -40,13 +40,14 @@ class MediumParser: async def from_unknown(cls, unknown: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser": logger.debug(f"We got some unknown data: {unknown=}, with {cache=}, {timeout=}, {host_address=}, {auth_cookies=}. Trying resolve them...///") - if is_valid_medium_post_id_hexadecimal(unknown): + if is_has_valid_medium_post_id(unknown): logger.debug("Seems like it's valid post_id") return cls(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies) logger.debug("...maybe it's URL. Let's checkout...") return await cls.from_url(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies) + @classmethod async def from_url(cls, url: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser": sanitized_url = correct_url(url) @@ -65,10 +66,10 @@ class MediumParser: @post_id.setter def post_id(self, value): - if not is_valid_medium_post_id_hexadecimal(value): + if not is_has_valid_medium_post_id(value): raise InvalidMediumPostID(f"Invalid medium post ID: {value}") - self.__post_id = value + self.__post_id = extract_hex_string(value) @post_id.getter def post_id(self): @@ -137,13 +138,13 @@ class MediumParser: if not post_data: reason = "No post data returned" elif not isinstance(post_data, dict): - reason = "Post data is not a dictionary" + reason = f"Post data is not a dictionary: {post_data=}" elif post_data.get("error"): - reason = "Post data contains an error" + reason = f"Post data contains an error: {post_data=}" elif not post_data.get("data"): - reason = "Post data missing 'data' key" + reason = f"Post data missing 'data' key: {post_data=}" elif not post_data.get("data").get("post"): - reason = "Post data missing 'data.post' key" + reason = f"Post data missing 'data.post' key: {post_data=}" if reason is None: logger.debug("Post data was successfully queried") diff --git a/core/medium_parser/utils.py b/core/medium_parser/utils.py index cdd9a94..f5757c3 100644 --- a/core/medium_parser/utils.py +++ b/core/medium_parser/utils.py @@ -81,6 +81,7 @@ NOT_MEDIUM_DOMAINS = ( "bloomberg.com", "www.lesechos.fr", "www.otz.de", + "www.businessinsider.com", "buff.ly", "www.delish.com", "www.economist.com", @@ -191,7 +192,12 @@ def unplaginate_url(url): @lru_cache(maxsize=100) -def is_valid_medium_post_id_hexadecimal(hex_string: str) -> bool: +def is_has_valid_medium_post_id(hex_string: str) -> bool: + return extract_hex_string(hex_string) is not None + + +@lru_cache(maxsize=100) +def basic_hex_check(hex_string: str) -> bool: # Check if the string is a valid hexadecimal string for char in hex_string: if char not in VALID_ID_CHARS: @@ -212,6 +218,16 @@ def is_valid_medium_post_id_hexadecimal(hex_string: str) -> bool: return True +@lru_cache(maxsize=100) +def extract_hex_string(input_string: str) -> str: + # First try to find a hexadecimal string preceded by a '-' + match = re.search(r'-(\b[a-fA-F0-9]{8,12}\b)', input_string) + if not match: + # If no match, try to find a hexadecimal string without the '-' + match = re.search(r'(\b[a-fA-F0-9]{8,12}\b)', input_string) + return match.group(1) if match else None + + async def resolve_medium_short_link(short_url_id: str, timeout: int = 5) -> str: async with aiohttp.ClientSession() as session: retry_client = RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) @@ -307,7 +323,7 @@ async def resolve_medium_url(url: str, timeout: int = 5) -> str: post_url = parsed_url.path.split("/")[-1] post_id = post_url.split("-")[-1] - if not is_valid_medium_post_id_hexadecimal(post_id): + if not is_has_valid_medium_post_id(post_id): logger.warning(f"...but hoops, that's invalid post_id: {post_id}") return False diff --git a/server/handlers/post.py b/server/handlers/post.py index aac77dc..7919b1b 100644 --- a/server/handlers/post.py +++ b/server/handlers/post.py @@ -4,6 +4,7 @@ import pickle from fastapi.responses import HTMLResponse from html5lib import serialize from html5lib.html5parser import parse +from async_lru import alru_cache from loguru import logger from medium_parser import medium_parser_exceptions from medium_parser.core import MediumParser @@ -46,7 +47,7 @@ async def render_postleter(limit: int = 30, as_html: bool = False): return HTMLResponse(postleter_template_rendered) -@trace +@alru_cache(maxsize=20) async def render_medium_post_link(path: str, use_cache: bool = True, use_redis: bool = True): redis_available = await safe_check_redis_connection(redis_storage) logger.debug("Redis available: {}", redis_available)