diff --git a/Dockerfile b/Dockerfile index a3c1610..d164d44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get update && \ COPY ./requirements.txt ./ COPY ./requirements-fast.txt ./ +COPY ./server ./server COPY ./core ./core COPY ./rl_string_helper ./rl_string_helper COPY ./database-lib ./database-lib diff --git a/core/medium_parser/__init__.py b/core/medium_parser/__init__.py index f659066..3041515 100644 --- a/core/medium_parser/__init__.py +++ b/core/medium_parser/__init__.py @@ -1,15 +1,7 @@ from aiohttp_retry import ExponentialRetry -from loguru import logger import jinja2 -from database_lib import SQLiteCacheBackend - -cache = SQLiteCacheBackend('medium_db_cache.sqlite') -cache.init_db() -cache.enable_zstd() - -logger.debug(f"Database length: {cache.all_length()}") retry_options = ExponentialRetry(attempts=3) diff --git a/core/medium_parser/core.py b/core/medium_parser/core.py index b599ba1..a093a14 100644 --- a/core/medium_parser/core.py +++ b/core/medium_parser/core.py @@ -1,44 +1,63 @@ import asyncio import math import textwrap +import typing import urllib.parse +from contextlib import suppress import jinja2 import tld from loguru import logger -from contextlib import suppress -from rl_string_helper import RLStringHelper, parse_markups, split_overlapping_ranges +from rl_string_helper import (RLStringHelper, parse_markups, + split_overlapping_ranges) -from . import cache, jinja_env -from .exceptions import InvalidMediumPostID, InvalidMediumPostURL, InvalidURL, MediumParserException, MediumPostQueryError +from . import jinja_env +from .exceptions import (InvalidMediumPostID, InvalidMediumPostURL, InvalidURL, + MediumParserException, MediumPostQueryError) from .medium_api import query_post_by_id from .models.html_result import HtmlResult from .time import convert_datetime_to_human_readable -from .utils import resolve_medium_url, getting_percontage_of_match, is_valid_medium_post_id_hexadecimal, is_valid_medium_url, is_valid_url, sanitize_url +from .utils import (correct_url, getting_percontage_of_match, + is_valid_medium_post_id_hexadecimal, is_valid_medium_url, + is_valid_url, resolve_medium_url) +if typing.TYPE_CHECKING: + from database_lib import SQLiteCacheBackend class MediumParser: - __slots__ = ("__post_id", "post_data", "jinja", "timeout", "host_address", "auth_cookies") + __slots__ = ("__post_id", "auth_cookies", "cache", "host_address", "jinja", "post_data", "timeout") - def __init__(self, post_id: str, timeout: int, host_address: str, auth_cookies: str = None): + def __init__(self, post_id: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None): self.timeout = timeout + self.cache = cache self.host_address = host_address self.post_id = post_id self.post_data = None self.auth_cookies = auth_cookies @classmethod - async def from_url(cls, url: str, timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser": - sanitized_url = sanitize_url(url) - if is_valid_url(url) and not await is_valid_medium_url(sanitized_url, timeout): + async def from_unknown(cls, unknown: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser": + logger.debug(f"We got some unknown data: {unknown=}, with {cache=}, {timeout=}, {host_address=}, {auth_cookies=}. Trying resolve them...///") + + if is_valid_medium_post_id_hexadecimal(unknown): + logger.debug("Seems like it's valid post_id") + return cls(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies) + + logger.debug("...maybe it's URL. Let's checkout...") + return await cls.from_url(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies) + + @classmethod + async def from_url(cls, url: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser": + sanitized_url = correct_url(url) + if not is_valid_url(url) or not await is_valid_medium_url(sanitized_url): raise InvalidURL(f"Invalid Medium URL: {sanitized_url}") post_id = await resolve_medium_url(sanitized_url, timeout) if not post_id: raise InvalidMediumPostURL(f"Could not find Medium post ID for URL: {sanitized_url}") - return cls(post_id, timeout, host_address, auth_cookies) + return cls(post_id, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies) @property def post_id(self): @@ -59,14 +78,14 @@ class MediumParser: if not post_id: post_id = self.post_id - cache.delete(post_id) + self.cache.delete(post_id) return True async def get_post_data_from_cache(self): async def _get_from_cache(): logger.debug("Using cache backend") - post_data = cache.pull(self.post_id) + post_data = self.cache.pull(self.post_id) if post_data: logger.debug("post query was found on cache") return post_data.json() @@ -80,11 +99,11 @@ class MediumParser: async def get_post_data_from_api(self): async def _get_from_api(): - logger.debug("Using API backend") + logger.debug("Using API to gather post data") try: return await query_post_by_id(self.post_id, self.timeout, self.auth_cookies) except Exception as ex: - logger.debug("Error while querying post by Medium API") + logger.debug("Error while querying post data from Medium API") logger.exception(ex) return None @@ -127,11 +146,11 @@ class MediumParser: reason = "Post data missing 'data.post' key" if reason is None: + logger.debug("Post data was successfully queried") break except Exception as e: logger.error(f"Attempt {attempt + 1} failed with exception: {e}") - finally: - logger.info(f"Retrying in {2 ** attempt} seconds...") + logger.debug(f"Retrying in {2 ** attempt} seconds...") await asyncio.sleep(2**attempt) attempt += 1 else: @@ -141,9 +160,11 @@ class MediumParser: raise MediumPostQueryError(f"Could not query post by ID from API: {self.post_id}. Reason: {reason}") if not is_cache_used: - cache.push(self.post_id, post_data) + logger.debug("Pushing post data to cache") + self.cache.push(self.post_id, post_data) self.post_data = post_data + logger.trace(f"Query: done") return post_data async def _parse_and_render_content_html_post(self, content: dict, title: str, subtitle: str, preview_image_id: str, highlights: list, tags: list) -> tuple[list, str, str]: @@ -179,7 +200,11 @@ class MediumParser: if current_pos in range(4): if paragraph["type"] in ["H3", "H4", "H2"]: if getting_percontage_of_match(paragraph["text"], title) > 80: - logger.trace("Title was detected, ignore...") + if title.endswith("…"): + logger.trace("Title was detected, replace...") + title = paragraph["text"] + else: + logger.trace("Title was detected, ignore...") current_pos += 1 continue if paragraph["type"] in ["H4"]: @@ -366,7 +391,7 @@ class MediumParser: logger.trace(pq_template_rendered) out_paragraphs.append(pq_template_rendered) elif paragraph["type"] == "MIXTAPE_EMBED": - # TODO: redirect all Medium embeding artickles to Fredium + # TODO: redirect all Medium embeding articles to Fredium embed_template = jinja_env.from_string( """

{{ embed_title }}

{{ embed_description }}

{{ embed_site }}

@@ -389,8 +414,15 @@ class MediumParser: title_range = paragraph["markups"][1] description_range = paragraph["markups"][2] + logger.trace(f"{title_range=}") + logger.trace(f"{description_range=}") + embed_title = text_raw[title_range["start"] : title_range["end"]] embed_description = text_raw[description_range["start"] : description_range["end"]] + + logger.trace(f"{embed_title=}") + logger.trace(f"{embed_description=}") + try: embed_site = tld.get_fld(url) except Exception as ex: @@ -398,6 +430,8 @@ class MediumParser: parsed_url = urllib.parse.urlparse(url) embed_site = parsed_url.hostname + logger.trace(f"{embed_site=}") + embed_template_rendered = await embed_template.render_async(paragraph=paragraph, url=url, embed_title=embed_title, embed_description=embed_description, embed_site=embed_site) out_paragraphs.append(embed_template_rendered) elif paragraph["type"] == "IFRAME": diff --git a/core/medium_parser/medium_api.py b/core/medium_parser/medium_api.py index 84acaf1..d7268d2 100644 --- a/core/medium_parser/medium_api.py +++ b/core/medium_parser/medium_api.py @@ -9,6 +9,8 @@ from .utils import generate_random_sha256_hash # https://gist.github.com/vladar/a4e3afd608cfe8b13e5844d75447f0a4 async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: str = ""): + auth_cookies = "" if not auth_cookies else auth_cookies + headers = { "X-APOLLO-OPERATION-ID": generate_random_sha256_hash(), "X-APOLLO-OPERATION-NAME": "FullPostQuery", @@ -21,7 +23,7 @@ async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: str = " "Cache-Control": "public, max-age=-1", "Content-Type": "application/json", "Connection": "Keep-Alive", - # "Cookie": auth_cookies, + "Cookie": auth_cookies, } json_data = { diff --git a/core/medium_parser/utils.py b/core/medium_parser/utils.py index a8c0c01..cdd9a94 100644 --- a/core/medium_parser/utils.py +++ b/core/medium_parser/utils.py @@ -1,28 +1,104 @@ -import hashlib -import secrets import difflib -import urllib.parse -from aiohttp_retry import RetryClient -from datetime import datetime -from loguru import logger -from functools import lru_cache -from urllib.parse import urlparse, parse_qs -import aiohttp +import hashlib +import re +import secrets import string +import urllib.parse +from datetime import datetime +from functools import lru_cache +from urllib.parse import parse_qs, urlparse -from . import retry_options, exceptions - +import aiohttp import tld +from aiohttp_retry import RetryClient +from async_lru import alru_cache from bs4 import BeautifulSoup +from loguru import logger + +from . import exceptions, retry_options + +DEFAULT_URL_PROTOCOL = "https://" VALID_ID_CHARS = set(string.ascii_letters + string.digits) -KNOWN_MEDIUM_NETLOC = ("javascript.plainenglish.io", "python.plainenglish.io", "blog.stackademic.com", "ai.gopubby.com", "blog.devops.dev", "levelup.gitconnected.com", "betterhumans.coach.me", "ai.plainenglish.io") -KNOWN_MEDIUM_DOMAINS = ("medium.com", "uxplanet.org", "towardsdatascience.com", "thetaoist.online", "devopsquare.com", "bettermarketing.pub", "itnext.io", "eand.co", "betterprogramming.pub", "curiouse.co", "betterhumans.pub", "uxdesign.cc", "thebolditalic.com", "arcdigital.media", "codeburst.io", "psiloveyou.xyz", "writingcooperative.com", "entrepreneurshandbook.co", "prototypr.io", "theascent.pub") -NOT_MEDIUM_DOMAINS = ("github.com", "yandex.ru", "yandex.kz", "youtube.com", "nytimes.com", "wsj.com", "reddit.com", "elpais.com", "forbes.com", "bloomberg.com") +KNOWN_MEDIUM_CUSTOM_DOMAINS = ( + "javascript.plainenglish.io", + "blog.llamaindex.ai", + "code.likeagirl.io", + "medium.datadriveninvestor.com", + "blog.det.life", + "python.plainenglish.io", + "blog.stackademic.com", + "ai.gopubby.com", + "blog.devops.dev", + "levelup.gitconnected.com", + "betterhumans.coach.me", + "ai.plainenglish.io", +) +KNOWN_MEDIUM_DOMAINS = ( + "medium.com", + "uxplanet.org", + "osintteam.blog", + "ahmedelfakharany.com", + "drlee.io", + "artificialcorner.com", + "generativeai.pub", + "productcoalition.com", + "towardsdev.com", + "infosecwriteups.com", + "towardsdatascience.com", + "thetaoist.online", + "devopsquare.com", + "www.laceydearie.com", + "bettermarketing.pub", + "itnext.io", + "eand.co", + "betterprogramming.pub", + "curiouse.co", + "betterhumans.pub", + "uxdesign.cc", + "thebolditalic.com", + "arcdigital.media", + "codeburst.io", + "psiloveyou.xyz", + "writingcooperative.com", + "entrepreneurshandbook.co", + "prototypr.io", + "theascent.pub", + "storiusmag.com" +) +NOT_MEDIUM_DOMAINS = ( + "github.com", + "yandex.ru", + "yandex.kz", + "youtube.com", + "nytimes.com", + "wsj.com", + "www.wsj.com", + "reddit.com", + "elpais.com", + "forbes.com", + "bloomberg.com", + "www.lesechos.fr", + "www.otz.de", + "buff.ly", + "www.delish.com", + "www.economist.com", + "www.wired.com", + "www.rollingstone.com", +) def is_valid_url(url): + """ + Check if the given URL is valid by verifying if it has a valid scheme and netloc. + + Parameters: + url (str): The URL to be validated. + + Returns: + bool: True if the URL is valid, False otherwise. + """ fld = get_fld(url) if not fld: return False @@ -62,46 +138,81 @@ def get_unix_ms() -> int: def unquerify_url(url): """ Sanitizes a URL by removing all query parameters. - + Args: - url: The URL to sanitize. - + url: The URL to sanitize. + Returns: - A sanitized URL. + A sanitized URL. """ - + parsed_url = urllib.parse.urlparse(url) query = parsed_url.query if query: - parsed_url = parsed_url._replace(query='') + parsed_url = parsed_url._replace(query="") sanitized_url = urllib.parse.urlunparse(parsed_url) return sanitized_url.removesuffix("/") -def sanitize_url(url): +def correct_url(url: str) -> str: + # Workaround for Safari bug. We don't known by what condition this happens, but sometimes we get + # some broken URL, for example like "", and all of them based on user-agent comes from Safari browser engine, + # from some kinda different platforms like Windows, and that's strange bcz does Windows has Safari browser? lmao + + # TODO: fix + + # unsafari_url = re.sub(r"https?://", DEFAULT_URL_PROTOCOL, url) + # logger.debug(f"Is URL broken by Safari bug: {unsafari_url != url}") + + unsafari_url = url + + unquerified_url = unquerify_url(unsafari_url) + logger.debug(f"Is URL has query data: {unquerified_url != unsafari_url}") + + unplaginated_url = unplaginate_url(unquerified_url) + logger.debug(f"Is URL has plagination: {unplaginated_url != unquerified_url}") + + # parsed_url = urlparse(url) + # if not bool(parsed_url.netloc and parsed_url.scheme): + # return DEFAULT_PROTOCOL + url + + # if not re.match(r'http[s]?://', url): + # url = DEFAULT_PROTOCOL + url + + return url + + +def unplaginate_url(url): + """ + Removes page plaginations from URL + """ sanitized_url = url.removesuffix("/page/2") return sanitized_url.removesuffix("/") +@lru_cache(maxsize=100) def is_valid_medium_post_id_hexadecimal(hex_string: str) -> bool: # Check if the string is a valid hexadecimal string - # isalnum() for char in hex_string: if char not in VALID_ID_CHARS: return False + # Unfortunately, this logic doesn't works correctly sometimes, because + # there is some unique URLs that are has only digits, like this: + # https://valeman.medium.com/python-vs-r-for-time-series-forecasting-395390432598 + # Check if the string contains only lowercase hexadecimal characters # if not hex_string.islower(): # return False # Check if the length of the string is correct for a hexadecimal string (e.g., 10, 11 or 12 characters) - if len(hex_string) not in range(8, 13): + if len(hex_string) not in range(8, 12 + 1): return False return True -async def resolve_medium_short_link_v1(short_url_id: str, timeout: int = 5) -> str: +async def resolve_medium_short_link(short_url_id: str, timeout: int = 5) -> str: async with aiohttp.ClientSession() as session: retry_client = RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) request = await retry_client.get( @@ -111,54 +222,93 @@ async def resolve_medium_short_link_v1(short_url_id: str, timeout: int = 5) -> s allow_redirects=False, ) post_url = request.headers["Location"] - return await resolve_medium_url(post_url) + return post_url +@alru_cache(maxsize=500) async def resolve_medium_url(url: str, timeout: int = 5) -> str: + logger.debug(f"Trying resolve {url=}, with {timeout=}") parsed_url = urlparse(url) + if parsed_url.path.startswith("/p/"): + logger.debug("URL is Medium 'mobile' link") post_id = parsed_url.path.rsplit("/p/")[1] + elif parsed_url.netloc == "l.facebook.com" and parsed_url.path.startswith("/l.php"): + logger.debug("URL seems like is Facebook redirect (tracking) link") + parsed_query = parse_qs(parsed_url.query) if parsed_query.get("u") and len(parsed_query["u"]) == 1: post_url = parsed_query["u"][0] return await resolve_medium_url(post_url) + + logger.debug("...but we get fucked up...") return False + elif parsed_url.netloc == "webcache.googleusercontent.com" and parsed_url.path.startswith("/search"): + logger.debug("URL seems like is Google Web Archive page link") + parsed_query = parse_qs(parsed_url.query) if parsed_query.get("q") and len(parsed_query["q"]) == 1: post_url = parsed_query["q"][0].removeprefix("cache:") return await resolve_medium_url(post_url) + + logger.debug("...but we get fucked up...") return False + elif parsed_url.netloc == "www.google.com" and parsed_url.path.startswith("/url"): + logger.debug("URL seems like is Google redirect (tracking) link") + parsed_query = parse_qs(parsed_url.query) if parsed_query.get("url") and len(parsed_query["url"]) == 1: + logger.debug("..and we got 'url' passed param. Make resolve them....") post_url = parsed_query["url"][0] return await resolve_medium_url(post_url) elif parsed_query.get("q") and len(parsed_query["q"]) == 1: + logger.debug("..and we got 'q' passed param. Make resolve them....") post_url = parsed_query["q"][0] return await resolve_medium_url(post_url) + + logger.debug("...but we get fucked up...") return False + elif parsed_url.netloc == "12ft.io": + logger.debug("URL seems like is from our partner named 12ft.io") + parsed_query = parse_qs(parsed_url.query) if parsed_query.get("q") and len(parsed_query["q"]) == 1: + logger.debug("..and we got 'q' passed param. Make resolve them....") post_url = parsed_query["q"][0] return await resolve_medium_url(post_url) + + logger.debug("...but we get fucked up...") return False + elif parsed_url.path.startswith("/m/global-identity-2"): + logger.debug("URL seems like is Medium redirect (tracking) link. Possibly from email subscription") + parsed_query = parse_qs(parsed_url.query) if parsed_query.get("redirectUrl") and len(parsed_query["redirectUrl"]) == 1: + logger.debug("..and we got 'redirectUrl' passed param. Make resolve them....") post_url = parsed_query["redirectUrl"][0] return await resolve_medium_url(post_url) + + logger.debug("...but we get fucked up...") return False + elif parsed_url.netloc == "link.medium.com": + logger.debug("URL seems like is Medium short (SHORT) redirect (tracking) link. Make resolve them...") short_url_id = parsed_url.path.removeprefix("/") - return await resolve_medium_short_link_v1(short_url_id, timeout) + post_url = await resolve_medium_short_link(short_url_id, timeout) + return await resolve_medium_url(post_url) + else: + logger.debug("We can't determine the URL type. Let's just try to extract the post_id...") post_url = parsed_url.path.split("/")[-1] post_id = post_url.split("-")[-1] if not is_valid_medium_post_id_hexadecimal(post_id): + logger.warning(f"...but hoops, that's invalid post_id: {post_id}") return False return post_id @@ -182,40 +332,7 @@ async def resolve_medium_url_old(url: str, timeout: int = 5) -> str: return parsed_value -@lru_cache(maxsize=200) -def get_fld(url: str): - try: - fld = tld.get_fld(url) - except Exception as ex: - logger.trace(ex) - return None - else: - return fld - - -async def is_valid_medium_url(url: str, timeout: int = 5) -> bool: - """ - Check if the url is a valid medium.com url - - First stage of url validation is checking if the domain is in the known medium.com url list. If the domain is in the list, then the url is valid - Second stage is checking if the url is valid Medium site by performing a GET request to the url and checking the site name meta tag. If the site name meta tag is Medium, then the url is valid - """ - # First stage - domain = get_fld(url) - parsed_url = urlparse(url) - - if domain in ["12ft.io", "google.com", "facebook.com", "googleusercontent.com"]: - return True - - if domain in NOT_MEDIUM_DOMAINS: - raise exceptions.NotValidMediumURL("100% not valid Medium URL") - - if domain in KNOWN_MEDIUM_DOMAINS or parsed_url.netloc in KNOWN_MEDIUM_NETLOC: - return True - else: - logger.warning(f"url '{url}' wasn't detected in known medium domains") - - # Second stage +async def is_valid_medium_url_old(url: str, timeout: int = 5): async with aiohttp.ClientSession() as session: retry_client = RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) @@ -237,3 +354,47 @@ async def is_valid_medium_url(url: str, timeout: int = 5) -> bool: return False return True + + +@lru_cache(maxsize=500) +def get_fld(url: str): + try: + fld = tld.get_fld(url) + except Exception as ex: + logger.trace(ex) + return None + else: + return fld + + +@alru_cache(maxsize=100) +async def is_valid_medium_url(url: str) -> bool: + """ + Check if the url is a valid Medium article page + + Check if the domain is in the known Medium domains and subdomains list. If the doman/subdomain is in the list, then the url is valid + """ + domain = get_fld(url) + parsed_url = urlparse(url) + + # TODO: http://freedium.cfd/https://www.google.com.vn/url?sa=i&url=https%3A%2F%2Fmedium.com%2F%40dugguRK%2Fabout-android-hardware-abstraction-layer-hal-5d191dafeb2c&psig=AOvVaw17KP0U_haPMmhAByeMTxSg&ust=1711354113283000&source=images&cd=vfe&opi=89978449&ved=0CBQQjhxqFwoTCMCM_oG5jIUDFQAAAAAdAAAAABAa + + if domain in ["12ft.io", "google.com", "facebook.com", "googleusercontent.com"]: + return True + + if domain in NOT_MEDIUM_DOMAINS or parsed_url.netloc in NOT_MEDIUM_DOMAINS: + raise exceptions.NotValidMediumURL("100% not valid Medium URL") + + if domain in KNOWN_MEDIUM_DOMAINS or parsed_url.netloc in KNOWN_MEDIUM_CUSTOM_DOMAINS: + return True + + logger.warning(f"url '{url}' wasn't detected in known Medium domains") + + # XXX: Unfourtunately, for now we don't know ALL Medium's domains, so we need resolve links + resolve_result = bool(await resolve_medium_url(url)) + + # send_message(f"We found that {domain=}, {parsed_url.netloc=} is not listed in out known Medium database.\nURL: {url}") + + return resolve_result + + # return False diff --git a/core/requirements.txt b/core/requirements.txt index 5dce889..bad65f0 100644 --- a/core/requirements.txt +++ b/core/requirements.txt @@ -2,9 +2,10 @@ rl_string_helper==0.1.0 database_lib==0.1.0 loguru==0.6.0 -aiohttp==3.8.5 +aiohttp==3.9.4 aiohttp-retry==2.8.3 tld==0.13 bs4==0.0.1 Jinja2==3.1.2 beautifulsoup4==4.12.2 +async-lru==2.0.4 \ No newline at end of file diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 0e3727c..7041d9e 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -31,7 +31,27 @@ services: dockerfile: ./Dockerfile command: python3 -m server server volumes: + # - ./.env:/app/.env + # - ./server/user_data/logs:/app/server/user_data/logs - .:/app + develop: + watch: + - action: rebuild + path: ./server + target: /app/server + - action: rebuild + path: ./core + target: /app/core + - action: rebuild + path: ./rl_string_helper + target: /app/rl_string_helper + - action: rebuild + path: ./database-lib + target: /app/database-lib + - action: rebuild + path: "**/requirements.txt" + - action: rebuild + path: "**/requirements-fast.txt" expose: - 7080 networks: diff --git a/server/__init__.py b/server/__init__.py index 707e1fb..9c36af0 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -1,4 +1,5 @@ import datetime as dt +from loguru import logger import pickledb from multiprocessing import Value import logging @@ -9,10 +10,15 @@ import redis.asyncio as redis from xkcdpass import xkcd_password as xp from server.utils.loguru_handler import InterceptHandler +from database_lib import SQLiteCacheBackend + +medium_cache = SQLiteCacheBackend('medium_db_cache.sqlite') +medium_cache.init_db() +medium_cache.enable_zstd() +logger.debug(f"Database length: {medium_cache.all_length()}") redis_storage = redis.Redis(host="dragonfly", port=6379, db=0) - logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True) url_correlation: ContextVar[Optional[str]] = ContextVar("url_correlation", default="UNKNOWN_URL") diff --git a/server/config.py b/server/config.py index 3089bfb..c9a147a 100644 --- a/server/config.py +++ b/server/config.py @@ -16,4 +16,4 @@ REQUEST_TIMEOUT = config("REQUEST_TIMEOUT", cast=int, default=40) WORKER_TIMEOUT = config("WORKER_TIMEOUT", cast=int, default=120) SENTRY_SDK_DSN = config("SENTRY_SDK_DSN", default=None) ENABLE_ADS_BANNER = config("ENABLE_ADS_BANNER", cast=bool, default=False) -CACHE_LIFE_TIME = config("CACHE_LIFE_TIME", cast=int, default=60 * 60 * 24) \ No newline at end of file +CACHE_LIFE_TIME = config("CACHE_LIFE_TIME", cast=int, default=60 * 60 * 5) \ No newline at end of file diff --git a/server/handlers/main.py b/server/handlers/main.py index 65298b4..c0b7116 100644 --- a/server/handlers/main.py +++ b/server/handlers/main.py @@ -1,14 +1,14 @@ -from html5lib.html5parser import parse -from html5lib import serialize - from fastapi import Request from fastapi.responses import HTMLResponse, JSONResponse +from html5lib import serialize +from html5lib.html5parser import parse +from loguru import logger from server import config -from server.services.jinja import base_template, main_template +from server.handlers.misc import delete_from_cache, report_problem from server.handlers.post import render_medium_post_link, render_postleter -from server.handlers.reverse_proxy import miro_proxy, iframe_proxy -from server.handlers.misc import report_problem, delete_from_cache +from server.handlers.reverse_proxy import iframe_proxy, miro_proxy +from server.services.jinja import base_template, main_template from server.utils.logger_trace import trace @@ -16,31 +16,29 @@ from server.utils.logger_trace import trace async def route_processing(path: str, request: Request): if not path: return await main_page() - if request.scope.get("query_string"): - path = request.url.path + "?" + request.scope["query_string"].decode() - else: - path = request.url.path - path = path.removeprefix("/") - if path.startswith("render-no-cache/"): + query_params = request.query_params + redis = not "no-redis" in query_params + db_cache = not "no-db-cache" in query_params + + logger.trace(f"no_cache: {db_cache}, no_redis: {redis}") + + path = request.url.path.removeprefix("/") + + if not db_cache or not redis: key_data = request.headers.get("ADMIN_SECRET_KEY") if key_data != config.ADMIN_SECRET_KEY: return JSONResponse({"message": f"Wrong secret key: {key_data}"}, status_code=403) - path = path.removeprefix("render-no-cache/") - if path.startswith("/no-redis/"): - path = path.removeprefix("/no-redis/") - return await render_medium_post_link(path, True, False) - return await render_medium_post_link(path, False) - elif path.startswith("@miro/"): + if path.startswith("@miro/"): miro_data = path.removeprefix("@miro/") return await miro_proxy(miro_data) elif path.startswith("render_iframe/"): iframe_id = path.removeprefix("render_iframe/") return await iframe_proxy(iframe_id) - return await render_medium_post_link(path) + return await render_medium_post_link(path, db_cache, redis) @trace diff --git a/server/handlers/post.py b/server/handlers/post.py index 298a08d..aac77dc 100644 --- a/server/handlers/post.py +++ b/server/handlers/post.py @@ -1,23 +1,21 @@ -from fastapi.responses import HTMLResponse - +import asyncio import pickle -from html5lib.html5parser import parse -from html5lib import serialize -from loguru import logger -from server import config, url_correlation, redis_storage, home_page_process, transponder_code_correlation +from fastapi.responses import HTMLResponse +from html5lib import serialize +from html5lib.html5parser import parse +from loguru import logger +from medium_parser import medium_parser_exceptions +from medium_parser.core import MediumParser + +from server import config, home_page_process, medium_cache, redis_storage, transponder_code_correlation from server.services.jinja import base_template, postleter_template -from server.utils.error import generate_error +from server.utils.cache import aio_redis_cache +from server.utils.exceptions import handle_exception from server.utils.logger_trace import trace from server.utils.notify import send_message -from server.utils.cache import aio_redis_cache -from server.utils.utils import correct_url, safe_check_redis_connection -from server.utils.exceptions import handle_exception +from server.utils.utils import safe_check_redis_connection -from medium_parser import medium_parser_exceptions -from medium_parser import cache as medium_cache -from medium_parser.core import MediumParser -from medium_parser.utils import is_valid_medium_post_id_hexadecimal @trace @aio_redis_cache(10 * 60) @@ -26,14 +24,21 @@ async def render_postleter(limit: int = 30, as_html: bool = False): home_page_process[transponder_code_correlation.get()] = random_post_id_list outlet_posts_list = [] + tasks = [] for post_id in random_post_id_list: - try: - post = MediumParser(post_id, timeout=3, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES) - await post.query(force_cache=True, retry=1) - post_metadata = await post.generate_metadata(as_dict=True) - outlet_posts_list.append(post_metadata) - except Exception as ex: - await handle_exception(ex, message=f"Couldn't render post_id for postleter: {post_id}") + async def fetch_post_metadata(post_id): + try: + post = MediumParser(post_id, cache=medium_cache, timeout=3, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES) + await post.query(force_cache=True, retry=1) + post_metadata = await post.generate_metadata(as_dict=True) + outlet_posts_list.append(post_metadata) + except Exception as ex: + await handle_exception(ex, message=f"Couldn't render post_id for postleter: {post_id}") + + task = fetch_post_metadata(post_id) + tasks.append(task) + + await asyncio.gather(*tasks) postleter_template_rendered = await postleter_template.render_async(post_list=outlet_posts_list) if as_html: @@ -44,25 +49,31 @@ async def render_postleter(limit: int = 30, as_html: bool = False): @trace async def render_medium_post_link(path: str, use_cache: bool = True, use_redis: bool = True): redis_available = await safe_check_redis_connection(redis_storage) + logger.debug("Redis available: {}", redis_available) try: - if is_valid_medium_post_id_hexadecimal(path): - medium_parser = MediumParser(path, timeout=config.TIMEOUT, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES) - else: - url = correct_url(path) - medium_parser = await MediumParser.from_url(url, timeout=config.TIMEOUT, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES) - medium_post_id = medium_parser.post_id + medium_parser = await MediumParser.from_unknown(path, cache=medium_cache, timeout=config.TIMEOUT, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES) + logger.debug("MediumParser initialized for path: {}", path) + redis_result = None if redis_available and use_cache and use_redis: - redis_result = await redis_storage.get(medium_post_id) - else: - redis_result = None + redis_result = await redis_storage.get(medium_parser.post_id) + logger.debug("Redis cache hit for post_id: {}", medium_parser.post_id) + if not redis_result: + logger.debug("No cache found, querying MediumParser") await medium_parser.query(use_cache=use_cache) rendered_medium_post = await medium_parser.render_as_html("server/templates") + logger.debug("Rendered Medium post from HTML template") + if redis_available and use_redis: + await redis_storage.setex(medium_parser.post_id, config.CACHE_LIFE_TIME, pickle.dumps(rendered_medium_post)) + logger.debug("Stored rendered post in Redis cache") else: rendered_medium_post = pickle.loads(redis_result) + logger.debug("Loaded rendered post from Redis cache") + except medium_parser_exceptions.InvalidURL as ex: - return await handle_exception(ex, + return await handle_exception( + ex, "Unable to identify the Medium article URL.", status_code=404, ) @@ -87,13 +98,7 @@ async def render_medium_post_link(path: str, use_cache: bool = True, use_redis: } rendered_post = await base_template.render_async(base_context, HOST_ADDRESS=config.HOST_ADDRESS) parsed_rendered_post = parse(rendered_post) - serialized_rendered_post = serialize(parsed_rendered_post, encoding='utf-8') - - if not redis_result: - if not redis_available: - send_message("ERROR: Redis is not available. Please check your configuration.") - elif use_redis: - await redis_storage.setex(medium_post_id, config.CACHE_LIFE_TIME, pickle.dumps(rendered_medium_post)) - send_message(f"✅ Successfully rendered post: {url_correlation.get()}", True, "GOOD") + serialized_rendered_post = serialize(parsed_rendered_post, encoding="utf-8") + send_message(f"✅ Successfully rendered post: {path}", True, "GOOD") return HTMLResponse(serialized_rendered_post) diff --git a/server/utils/maintenance_scheduler.py b/server/utils/maintenance_scheduler.py index d8d5de9..f3e7543 100644 --- a/server/utils/maintenance_scheduler.py +++ b/server/utils/maintenance_scheduler.py @@ -3,7 +3,7 @@ from datetime import datetime from server import maintenance_mode from server.utils.notify import send_message -from medium_parser import cache as medium_cache +from server import medium_cache from time import sleep from loguru import logger diff --git a/server/utils/utils.py b/server/utils/utils.py index 836b0a6..35adcbc 100644 --- a/server/utils/utils.py +++ b/server/utils/utils.py @@ -1,26 +1,8 @@ import random -import re import socket from server.utils.logger_trace import trace -DEFAULT_PROTOCOL = "https://" - - -@trace -def correct_url(url: str) -> str: - # Workaround for Safari bug - url = re.sub(r"https?://?", DEFAULT_PROTOCOL, url) - - # parsed_url = urlparse(url) - # if not bool(parsed_url.netloc and parsed_url.scheme): - # return DEFAULT_PROTOCOL + url - - # if not re.match(r'http[s]?://', url): - # url = DEFAULT_PROTOCOL + url - - return url - def string_to_number_ascii(input_str: str, key_number: int = None): if not key_number: diff --git a/test.py b/test.py index 7da53e2..2ca9821 100644 --- a/test.py +++ b/test.py @@ -1,6 +1,6 @@ import requests -server_url = input("Enter your Freedium instance server URL: ") +server_url = input("Enter your Freedium instance server URL (for example: http://localhost:6752/ ): ") # List of some problematic Medium posts url_for_test = { @@ -13,7 +13,7 @@ url_for_test = { "https://valeman.medium.com/python-vs-r-for-time-series-forecasting-395390432598", "https://medium.com/@aleb/how-to-generate-random-user-agents-with-an-api-22aad3d232cb", "https://medium.com/angular-in-depth/the-best-way-to-unsubscribe-rxjs-observable-in-the-angular-applications-d8f9aa42f6a0", - "515dd5a43948", + "515dd5a43948", # full address: https://medium.com/macoclock/12-macos-apps-so-good-you-will-wonder-how-they-are-free-515dd5a43948 "https://anudeep-vysyaraju.medium.com/how-any-gitamite-can-get-free-linkedin-premium-membership-d4222bd1a0b3" # <--- Check for non properly aligned emojies }