diff --git a/core/medium_parser/__init__.py b/core/medium_parser/__init__.py index 7d6a968..a2b9f04 100644 --- a/core/medium_parser/__init__.py +++ b/core/medium_parser/__init__.py @@ -1,10 +1,8 @@ +import jinja2 from aiohttp_retry import ExponentialRetry -import jinja2 +from medium_parser import exceptions as exceptions +from medium_parser import exceptions as medium_parser_exceptions retry_options = ExponentialRetry(attempts=3) - -from . import exceptions as exceptions -from . import exceptions as medium_parser_exceptions - -jinja_env = jinja2.Environment() \ No newline at end of file +jinja_env = jinja2.Environment() diff --git a/core/medium_parser/core.py b/core/medium_parser/core.py index a348471..b3cd7a3 100644 --- a/core/medium_parser/core.py +++ b/core/medium_parser/core.py @@ -14,25 +14,25 @@ from rl_string_helper import RLStringHelper, parse_markups, split_overlapping_ra from . import jinja_env from .exceptions import InvalidMediumPostID, InvalidMediumPostURL, InvalidURL, MediumParserException, MediumPostQueryError -from .medium_api import query_post_by_id +from .medium_api import MediumApi from .models.html_result import HtmlResult from .time import convert_datetime_to_human_readable from .utils import correct_url, extract_hex_string, getting_percontage_of_match, is_has_valid_medium_post_id, is_valid_medium_url, is_valid_url, resolve_medium_url if typing.TYPE_CHECKING: - from database_lib import SQLiteCacheBackend + from database_lib import AbstractCacheBackend class MediumParser: - __slots__ = ("cache", "host_address", "jinja_template", "post_template", "timeout", "auth_cookies") + __slots__ = ("cache", "host_address", "jinja_template", "post_template", "timeout") - def __init__(self, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: Optional[str] = None, template_folder: str = "./templates"): - self.timeout = timeout - self.cache = cache - self.host_address = host_address - self.auth_cookies = auth_cookies - self.jinja_template = jinja2.Environment(loader=jinja2.FileSystemLoader(template_folder)) - self.post_template = self.jinja_template.get_template("post.html") + def __init__(self, cache: "AbstractCacheBackend", medium_api: MediumApi, timeout: int, host_address: str, template_folder: str = "./templates"): + self.timeout: int = timeout + self.cache: AbstractCacheBackend = cache + self.host_address: str = host_address + self.jinja_template: jinja2.Environment = jinja2.Environment(loader=jinja2.FileSystemLoader(template_folder)) + self.post_template: jinja2.Template = self.jinja_template.get_template("post.html") + self.medium_api: MediumApi = medium_api async def resolve(self, unknown: str) -> str: logger.debug(f"We got some unknown data: {unknown=}. Trying resolve them...///") @@ -83,7 +83,7 @@ class MediumParser: async def _get_from_api(): logger.debug("Using API to gather post data") try: - return await query_post_by_id(post_id, self.timeout, self.auth_cookies) + return await self.medium_api.query_post_by_id(post_id) except Exception as ex: logger.debug("Error while querying post data from Medium API") logger.exception(ex) @@ -218,20 +218,20 @@ class MediumParser: else: text_formater = None - for highlight in highlights: - for highlight_paragraph in highlight["paragraphs"]: - if highlight_paragraph["name"] == paragraph["name"]: - logger.trace("Apply highlight to this paragraph") - if highlight_paragraph["text"] != text_formater.get_text(): - logger.warning("Highlighted text and paragraph text are not the same! Skip...") - break - quote_markup_template = '{{ text }}' - text_formater.set_template( - highlight["startOffset"], - highlight["endOffset"], - quote_markup_template, - ) + for highlight in highlights: + for highlight_paragraph in highlight["paragraphs"]: + if highlight_paragraph["name"] == paragraph["name"]: + logger.trace("Apply highlight to this paragraph") + if highlight_paragraph["text"] != text_formater.get_text(): + logger.warning("Highlighted text and paragraph text are not the same! Skip...") break + quote_markup_template = '{{ text }}' + text_formater.set_template( + highlight["startOffset"], + highlight["endOffset"], + quote_markup_template, + ) + break if paragraph["type"] == "H2": css_class = [] diff --git a/core/medium_parser/exceptions.py b/core/medium_parser/exceptions.py index 4ae8f5d..abf438d 100644 --- a/core/medium_parser/exceptions.py +++ b/core/medium_parser/exceptions.py @@ -24,3 +24,15 @@ class InvalidMediumPostID(MediumParserException): class MediumPostQueryError(MediumParserException): pass + + +class MediumPostNotFound(MediumPostQueryError): + pass + + +class MediumPostUnavailable(MediumPostQueryError): + pass + + +class MediumPostDeleted(MediumPostQueryError): + pass diff --git a/core/medium_parser/medium_api.py b/core/medium_parser/medium_api.py index b594436..52c822c 100644 --- a/core/medium_parser/medium_api.py +++ b/core/medium_parser/medium_api.py @@ -1,76 +1,92 @@ -from typing import Optional +from typing import Optional, List import aiohttp import orjson +import random from aiohttp_retry import RetryClient from aiohttp_socks import ProxyConnector from loguru import logger -from . import retry_options -from .time import get_unix_ms -from .utils import generate_random_sha256_hash +from medium_parser import retry_options +from medium_parser.time import get_unix_ms +from medium_parser.utils import generate_random_sha256_hash socks_proxy = "socks5h://wgcf1:1080" -async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: Optional[str] = None, use_proxy: bool = True): - logger.debug(f"Starting request construction for post {post_id}") - auth_cookies = "" if auth_cookies is None else auth_cookies +class MediumApi: + __slots__ = ("auth_cookies", "proxy_list", "timeout") - connector = ProxyConnector.from_url(socks_proxy) if use_proxy else None + def __init__(self, auth_cookies: Optional[str] = None, proxy_list: Optional[List[str]] = None, timeout: int = 3): + self.auth_cookies = auth_cookies + self.proxy_list = proxy_list + self.timeout = timeout - headers = { - "X-APOLLO-OPERATION-ID": generate_random_sha256_hash(), - "X-APOLLO-OPERATION-NAME": "FullPostQuery", - "Accept": "multipart/mixed; deferSpec=20220824, application/json, application/json", - "Accept-Language": "en-US", - "X-Obvious-CID": "android", - "X-Xsrf-Token": "1", - "X-Client-Date": str(get_unix_ms()), - "User-Agent": "AdsBot-Google-Mobile", # "donkey/4.5.1187420", # <---- There is Medium version - "Cache-Control": "public, max-age=-1", - "Content-Type": "application/json", - "Connection": "Keep-Alive", - "Cookie": auth_cookies, - } + async def query_post_by_id(self, post_id: str): + return await self.query_post_graphql(post_id, self.timeout) - graphql_data = { - "operationName": "FullPostQuery", - "variables": { - "postId": post_id, - "postMeteringOptions": {}, - }, - "query": "query FullPostQuery($postId: ID!, $postMeteringOptions: PostMeteringOptions) { post(id: $postId) { __typename id ...FullPostData } meterPost(postId: $postId, postMeteringOptions: $postMeteringOptions) { __typename ...MeteringInfoData } } fragment UserFollowData on User { id socialStats { followingCount followerCount } viewerEdge { isFollowing } } fragment NewsletterData on NewsletterV3 { id viewerEdge { id isSubscribed } } fragment UserNewsletterData on User { id newsletterV3 { __typename ...NewsletterData } } fragment ImageMetadataData on ImageMetadata { id originalWidth originalHeight focusPercentX focusPercentY alt } fragment CollectionFollowData on Collection { id subscriberCount viewerEdge { isFollowing } } fragment CollectionNewsletterData on Collection { id newsletterV3 { __typename ...NewsletterData } } fragment BylineData on Post { id readingTime creator { __typename id imageId username name bio tippingLink viewerEdge { isUser } ...UserFollowData ...UserNewsletterData } collection { __typename id name avatar { __typename id ...ImageMetadataData } ...CollectionFollowData ...CollectionNewsletterData } isLocked firstPublishedAt latestPublishedVersion } fragment ResponseCountData on Post { postResponses { count } } fragment InResponseToPost on Post { id title creator { name } clapCount responsesCount isLocked } fragment PostVisibilityData on Post { id collection { viewerEdge { isEditor canEditPosts canEditOwnPosts } } creator { id } isLocked visibility } fragment PostMenuData on Post { id title creator { __typename ...UserFollowData } collection { __typename ...CollectionFollowData } } fragment PostMetaData on Post { __typename id title visibility ...ResponseCountData clapCount viewerEdge { clapCount } detectedLanguage mediumUrl readingTime updatedAt isLocked allowResponses isProxyPost latestPublishedVersion isSeries firstPublishedAt previewImage { id } inResponseToPostResult { __typename ...InResponseToPost } inResponseToMediaResource { mediumQuote { startOffset endOffset paragraphs { text type markups { type start end anchorType } } } } inResponseToEntityType canonicalUrl collection { id slug name shortDescription avatar { __typename id ...ImageMetadataData } viewerEdge { isFollowing isEditor canEditPosts canEditOwnPosts isMuting } } creator { id isFollowing name bio imageId mediumMemberAt twitterScreenName viewerEdge { isBlocking isMuting isUser } } previewContent { subtitle } pinnedByCreatorAt ...PostVisibilityData ...PostMenuData } fragment LinkMetadataList on Post { linkMetadataList { url alts { type url } } } fragment MediaResourceData on MediaResource { id iframeSrc thumbnailUrl } fragment IframeData on Iframe { iframeHeight iframeWidth mediaResource { __typename ...MediaResourceData } } fragment MarkupData on Markup { name type start end href title rel type anchorType userId creatorIds } fragment CatalogSummaryData on Catalog { id name description type visibility predefined responsesLocked creator { id name username imageId bio viewerEdge { isUser } } createdAt version itemsLastInsertedAt postItemsCount } fragment CatalogPreviewData on Catalog { __typename ...CatalogSummaryData id itemsConnection(pagingOptions: { limit: 10 } ) { items { entity { __typename ... on Post { id previewImage { id } } } } paging { count } } } fragment MixtapeMetadataData on MixtapeMetadata { mediaResourceId href thumbnailImageId mediaResource { mediumCatalog { __typename ...CatalogPreviewData } } } fragment ParagraphData on Paragraph { id name href text iframe { __typename ...IframeData } layout markups { __typename ...MarkupData } metadata { __typename ...ImageMetadataData } mixtapeMetadata { __typename ...MixtapeMetadataData } type hasDropCap dropCapImage { __typename ...ImageMetadataData } codeBlockMetadata { lang mode } } fragment QuoteData on Quote { id postId userId startOffset endOffset paragraphs { __typename id ...ParagraphData } quoteType } fragment HighlightsData on Post { id highlights { __typename ...QuoteData } } fragment PostFooterCountData on Post { __typename id clapCount viewerEdge { clapCount } ...ResponseCountData responsesLocked mediumUrl title collection { id viewerEdge { isMuting isFollowing } } creator { id viewerEdge { isMuting isFollowing } } } fragment TagNoViewerEdgeData on Tag { id normalizedTagSlug displayTitle followerCount postCount } fragment VideoMetadataData on VideoMetadata { videoId previewImageId originalWidth originalHeight } fragment SectionData on Section { name startIndex textLayout imageLayout videoLayout backgroundImage { __typename ...ImageMetadataData } backgroundVideo { __typename ...VideoMetadataData } } fragment PostBodyData on RichText { sections { __typename ...SectionData } paragraphs { __typename id ...ParagraphData } } fragment FullPostData on Post { __typename ...BylineData ...PostMetaData ...LinkMetadataList ...HighlightsData ...PostFooterCountData tags { __typename id ...TagNoViewerEdgeData } content(postMeteringOptions: $postMeteringOptions) { bodyModel { __typename ...PostBodyData } validatedShareKey } } fragment MeteringInfoData on MeteringInfo { maxUnlockCount unlocksRemaining postIds }", - } + async def query_post_graphql(self, post_id: str): + logger.debug(f"Starting request construction for post {post_id}") - response_data = None - exception = None + if self.proxy_list: + connector = ProxyConnector.from_url(random.choice(self.proxy_list)) + else: + connector = None - logger.debug(f"Request started...") + headers = { + "X-APOLLO-OPERATION-ID": generate_random_sha256_hash(), + "X-APOLLO-OPERATION-NAME": "FullPostQuery", + "Accept": "multipart/mixed; deferSpec=20220824, application/json, application/json", + "Accept-Language": "en-US", + "X-Obvious-CID": "android", + "X-Xsrf-Token": "1", + "X-Client-Date": str(get_unix_ms()), + "User-Agent": "AdsBot-Google-Mobile", # "donkey/4.5.1187420", # <---- There is Medium version + "Cache-Control": "public, max-age=-1", + "Content-Type": "application/json", + "Connection": "Keep-Alive", + } - async with aiohttp.ClientSession(connector=connector) as session: - async with RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) as retry_client: - async with retry_client.post( - "https://medium.com/_/graphql", - headers=headers, - json=graphql_data, - timeout=timeout, - ) as request: - if request.status != 200: - logger.error(f"Failed to fetch post by ID {post_id} with status code: {request.status}") - return None + if self.auth_cookies: + headers["Cookie"] = self.auth_cookies - try: - response_data = await request.json(loads=orjson.loads) - except Exception as ex: - logger.debug("Failed to parse response data as JSON") - logger.exception(ex) - exception = ex + graphql_data = { + "operationName": "FullPostQuery", + "variables": { + "postId": post_id, + "postMeteringOptions": {}, + }, + "query": "query FullPostQuery($postId: ID!, $postMeteringOptions: PostMeteringOptions) { post(id: $postId) { __typename id ...FullPostData } meterPost(postId: $postId, postMeteringOptions: $postMeteringOptions) { __typename ...MeteringInfoData } } fragment UserFollowData on User { id socialStats { followingCount followerCount } viewerEdge { isFollowing } } fragment NewsletterData on NewsletterV3 { id viewerEdge { id isSubscribed } } fragment UserNewsletterData on User { id newsletterV3 { __typename ...NewsletterData } } fragment ImageMetadataData on ImageMetadata { id originalWidth originalHeight focusPercentX focusPercentY alt } fragment CollectionFollowData on Collection { id subscriberCount viewerEdge { isFollowing } } fragment CollectionNewsletterData on Collection { id newsletterV3 { __typename ...NewsletterData } } fragment BylineData on Post { id readingTime creator { __typename id imageId username name bio tippingLink viewerEdge { isUser } ...UserFollowData ...UserNewsletterData } collection { __typename id name avatar { __typename id ...ImageMetadataData } ...CollectionFollowData ...CollectionNewsletterData } isLocked firstPublishedAt latestPublishedVersion } fragment ResponseCountData on Post { postResponses { count } } fragment InResponseToPost on Post { id title creator { name } clapCount responsesCount isLocked } fragment PostVisibilityData on Post { id collection { viewerEdge { isEditor canEditPosts canEditOwnPosts } } creator { id } isLocked visibility } fragment PostMenuData on Post { id title creator { __typename ...UserFollowData } collection { __typename ...CollectionFollowData } } fragment PostMetaData on Post { __typename id title visibility ...ResponseCountData clapCount viewerEdge { clapCount } detectedLanguage mediumUrl readingTime updatedAt isLocked allowResponses isProxyPost latestPublishedVersion isSeries firstPublishedAt previewImage { id } inResponseToPostResult { __typename ...InResponseToPost } inResponseToMediaResource { mediumQuote { startOffset endOffset paragraphs { text type markups { type start end anchorType } } } } inResponseToEntityType canonicalUrl collection { id slug name shortDescription avatar { __typename id ...ImageMetadataData } viewerEdge { isFollowing isEditor canEditPosts canEditOwnPosts isMuting } } creator { id isFollowing name bio imageId mediumMemberAt twitterScreenName viewerEdge { isBlocking isMuting isUser } } previewContent { subtitle } pinnedByCreatorAt ...PostVisibilityData ...PostMenuData } fragment LinkMetadataList on Post { linkMetadataList { url alts { type url } } } fragment MediaResourceData on MediaResource { id iframeSrc thumbnailUrl } fragment IframeData on Iframe { iframeHeight iframeWidth mediaResource { __typename ...MediaResourceData } } fragment MarkupData on Markup { name type start end href title rel type anchorType userId creatorIds } fragment CatalogSummaryData on Catalog { id name description type visibility predefined responsesLocked creator { id name username imageId bio viewerEdge { isUser } } createdAt version itemsLastInsertedAt postItemsCount } fragment CatalogPreviewData on Catalog { __typename ...CatalogSummaryData id itemsConnection(pagingOptions: { limit: 10 } ) { items { entity { __typename ... on Post { id previewImage { id } } } } paging { count } } } fragment MixtapeMetadataData on MixtapeMetadata { mediaResourceId href thumbnailImageId mediaResource { mediumCatalog { __typename ...CatalogPreviewData } } } fragment ParagraphData on Paragraph { id name href text iframe { __typename ...IframeData } layout markups { __typename ...MarkupData } metadata { __typename ...ImageMetadataData } mixtapeMetadata { __typename ...MixtapeMetadataData } type hasDropCap dropCapImage { __typename ...ImageMetadataData } codeBlockMetadata { lang mode } } fragment QuoteData on Quote { id postId userId startOffset endOffset paragraphs { __typename id ...ParagraphData } quoteType } fragment HighlightsData on Post { id highlights { __typename ...QuoteData } } fragment PostFooterCountData on Post { __typename id clapCount viewerEdge { clapCount } ...ResponseCountData responsesLocked mediumUrl title collection { id viewerEdge { isMuting isFollowing } } creator { id viewerEdge { isMuting isFollowing } } } fragment TagNoViewerEdgeData on Tag { id normalizedTagSlug displayTitle followerCount postCount } fragment VideoMetadataData on VideoMetadata { videoId previewImageId originalWidth originalHeight } fragment SectionData on Section { name startIndex textLayout imageLayout videoLayout backgroundImage { __typename ...ImageMetadataData } backgroundVideo { __typename ...VideoMetadataData } } fragment PostBodyData on RichText { sections { __typename ...SectionData } paragraphs { __typename id ...ParagraphData } } fragment FullPostData on Post { __typename ...BylineData ...PostMetaData ...LinkMetadataList ...HighlightsData ...PostFooterCountData tags { __typename id ...TagNoViewerEdgeData } content(postMeteringOptions: $postMeteringOptions) { bodyModel { __typename ...PostBodyData } validatedShareKey } } fragment MeteringInfoData on MeteringInfo { maxUnlockCount unlocksRemaining postIds }", + } - logger.debug(f"Request finished...") + response_data = None + exception = None - if exception: - logger.error(f"Exception occured while fetching post {post_id}, so let's just fuck it up") - raise exception + logger.debug(f"Request started...") - return response_data + async with aiohttp.ClientSession(connector=connector) as session: + async with RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) as retry_client: + async with retry_client.post( + "https://medium.com/_/graphql", + headers=headers, + json=graphql_data, + timeout=self.timeout, + ) as request: + if request.status != 200: + logger.error(f"Failed to fetch post by ID {post_id} with status code: {request.status}") + return None + + try: + response_data = await request.json(loads=orjson.loads) + except Exception as ex: + logger.debug("Failed to parse response data as JSON") + logger.exception(ex) + exception = ex + + logger.debug(f"Request finished...") + + if exception: + logger.error(f"Exception occured while fetching post {post_id}, so let's just fuck it up") + raise exception + + return response_data diff --git a/core/medium_parser/utils.py b/core/medium_parser/utils.py index c018a73..2339775 100644 --- a/core/medium_parser/utils.py +++ b/core/medium_parser/utils.py @@ -15,7 +15,7 @@ from async_lru import alru_cache from bs4 import BeautifulSoup from loguru import logger -from . import exceptions, retry_options +from medium_parser import exceptions, retry_options DEFAULT_URL_PROTOCOL = "https://" @@ -157,6 +157,7 @@ def unquerify_url(url: str) -> str: @lru_cache(maxsize=500) def un_wwwify(url: str): + # TODO: enhanced type checks if url.startswith("www."): return url.removeprefix("www.") return url diff --git a/database-lib/database_lib/__init__.py b/database-lib/database_lib/__init__.py index 8779748..1b102a3 100644 --- a/database-lib/database_lib/__init__.py +++ b/database-lib/database_lib/__init__.py @@ -1 +1,9 @@ -from .cache_db import SQLiteCacheBackend, PostgreSQLCacheBackend, migrate_to_postgres, execute_migrate_to_postgres_in_thread +from database_lib.main import AbstractCacheBackend, SQLiteCacheBackend, PostgreSQLCacheBackend, migrate_to_postgres, execute_migrate_to_postgres_in_thread + +__all__ = [ + "AbstractCacheBackend", + "SQLiteCacheBackend", + "PostgreSQLCacheBackend", + "migrate_to_postgres", + "execute_migrate_to_postgres_in_thread", +] diff --git a/database-lib/database_lib/cache_db.py b/database-lib/database_lib/main.py similarity index 85% rename from database-lib/database_lib/cache_db.py rename to database-lib/database_lib/main.py index e6ec70a..b8127f7 100644 --- a/database-lib/database_lib/cache_db.py +++ b/database-lib/database_lib/main.py @@ -4,6 +4,7 @@ import sqlite3 import threading from itertools import islice from typing import Union, Optional +from abc import ABC, abstractmethod import psycopg2 import sqlite_zstd @@ -12,7 +13,7 @@ from loguru import logger from psycopg2.extras import execute_batch -class CacheResponse: +class CacheData: __slots__ = ("data",) def __init__(self, data: str): @@ -28,7 +29,52 @@ class CacheResponse: return self.data -class SQLiteCacheBackend: +class CacheResponse: + __slots__ = ("key", "data") + + def __init__(self, key: str, data: Union[CacheData, str]): + self.key: str = key + self.data: CacheData = CacheData(data) if isinstance(data, str) else data + + def json(self): + return self.data.json() + + +class AbstractCacheBackend(ABC): + @abstractmethod + def init_db(self): + pass + + @abstractmethod + def all(self): + pass + + @abstractmethod + def all_length(self) -> int: + pass + + @abstractmethod + def random(self, size: int) -> list[CacheResponse]: + pass + + @abstractmethod + def pull(self, key: str) -> Union[CacheResponse, None]: + pass + + @abstractmethod + def push(self, key: str, value: Union[str, dict]) -> None: + pass + + @abstractmethod + def delete(self, key: str) -> None: + pass + + @abstractmethod + def close(self): + pass + + +class SQLiteCacheBackend(AbstractCacheBackend): __slots__ = ("connection", "cursor", "database", "lock") def __init__(self, database: str, zstd_enabled: bool = False): @@ -53,9 +99,10 @@ class SQLiteCacheBackend: with self.connection: return self.cursor.execute("SELECT COUNT(*) FROM cache").fetchone()[0] - def random(self, size: int): + def random(self, size: int) -> list[CacheResponse]: with self.connection: - return self.cursor.execute("SELECT * FROM cache ORDER BY RANDOM() LIMIT ?", (size,)).fetchall() + self.cursor.execute("SELECT key, value FROM cache ORDER BY RANDOM() LIMIT ?", (size,)) + return [CacheResponse(key, value) for key, value in self.cursor] def enable_zstd(self): if not self.zstd_enabled: @@ -65,7 +112,9 @@ class SQLiteCacheBackend: try: self.cursor.execute('SELECT zstd_enable_transparent(\'{"table": "cache", "column": "value", "compression_level": 9, "dict_chooser": "\'\'a\'\'"}\')') except Exception as error: - print(error) + logger.error(f"Error enabling ZSTD compression: {error}") + logger.exception(error) + self.connection.execute("PRAGMA auto_vacuum=full") def init_db(self): @@ -78,7 +127,7 @@ class SQLiteCacheBackend: cache = self.cursor.execute("SELECT value FROM cache WHERE key = :0", {"0": key}).fetchone() if cache: logger.debug("Value found in DB, returning it") - return CacheResponse(cache[0]) + return CacheResponse(key, cache[0]) else: logger.debug(f"No value found for key: {key}") return None @@ -161,7 +210,7 @@ class SQLiteCacheBackend: self.connection.close() -class PostgreSQLCacheBackend: +class PostgreSQLCacheBackend(AbstractCacheBackend): def __init__(self, connection_string: str): self.connection = psycopg2.connect(connection_string) self.cursor = self.connection.cursor() @@ -187,10 +236,10 @@ class PostgreSQLCacheBackend: self.cursor.execute("SELECT COUNT(*) FROM cache") return self.cursor.fetchone()[0] - def random(self, size: int): + def random(self, size: int) -> list[CacheResponse]: with self.connection: - self.cursor.execute("SELECT * FROM cache ORDER BY RANDOM() LIMIT %s", (size,)) - return self.cursor.fetchall() + self.cursor.execute("SELECT key, value FROM cache ORDER BY RANDOM() LIMIT %s", (size,)) + return [CacheResponse(key, value) for key, value in self.cursor] def pull(self, key: str) -> Union[CacheResponse, None]: with self.connection: @@ -198,7 +247,7 @@ class PostgreSQLCacheBackend: cache = self.cursor.fetchone() if cache: logger.debug("Value found in DB, returning it") - return CacheResponse(cache[0]) + return CacheResponse(key, cache[0]) else: logger.debug(f"No value found for key: {key}") return None @@ -258,7 +307,6 @@ def migrate_to_postgres(sqlite_db_path: str, pg_conn_string: str, chunk_size: in elapsed_time = time.time() - start_time rows_per_second = processed_rows / elapsed_time logger.info(f"Processed {processed_rows}/{total_rows} rows. Speed: {rows_per_second:.2f} rows/second") - except Exception as e: logger.error(f"An error occurred during migration: {e}") pg_db.connection.rollback() diff --git a/requirements-fast.txt b/requirements-fast.txt index 7efc3cb..bd0de70 100755 --- a/requirements-fast.txt +++ b/requirements-fast.txt @@ -1,2 +1 @@ orjson==3.10.6 -uvloop==0.19.0 diff --git a/requirements.txt b/requirements.txt index 8c35fed..18827d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ pickledb==0.9.2 html5lib==1.1 sentry-sdk[fastapi]==1.29.2 loguru==0.6.0 # due to: https://github.com/Delgan/loguru/issues/916 -uvicorn==0.27.1 +uvicorn[standard]==0.30.3 anyio<=4.0.0 # Workaround to: https://github.getafreenode.com/tiangolo/fastapi/discussions/11652 Jinja2==3.1.2 fastapi==0.108.0 diff --git a/server/config.py b/server/config.py index 857c11b..be13f84 100644 --- a/server/config.py +++ b/server/config.py @@ -17,7 +17,7 @@ MORE_LOGS = config("MORE_LOGS", cast=bool, default=False) DISABLE_EXTERNAL_DOCS = config("DISABLE_EXTERNAL_DOCS", cast=bool, default=True) DISABLE_RATE_LIMITER = config("DISABLE_RATE_LIMITER", cast=bool, default=True) -TIMEOUT = config("TIMEOUT", cast=int, default=25) +TIMEOUT = config("TIMEOUT", cast=int, default=8) REQUEST_TIMEOUT = config("REQUEST_TIMEOUT", cast=int, default=40) WORKER_TIMEOUT = config("WORKER_TIMEOUT", cast=int, default=120) diff --git a/server/handlers/main.py b/server/handlers/main.py index acde418..e22cef4 100644 --- a/server/handlers/main.py +++ b/server/handlers/main.py @@ -53,7 +53,7 @@ async def route_processing(path: str, request: Request): async def main_page(): homepage_template = await render_homepage(as_html=True) main_template_rendered = main_template.render(postleter=homepage_template) - base_template_rendered = base_template.render(body_template=main_template_rendered, HOST_ADDRESS=config.HOST_ADDRESS) + base_template_rendered = base_template.render(body_template=main_template_rendered, host_address=config.HOST_ADDRESS) parsed_template = parse(base_template_rendered) serialized_template = serialize(parsed_template, encoding='utf-8') return HTMLResponse(serialized_template) diff --git a/server/handlers/post.py b/server/handlers/post.py index 6727900..646cd1a 100644 --- a/server/handlers/post.py +++ b/server/handlers/post.py @@ -88,12 +88,13 @@ async def render_medium_post_link(path: str, use_cache: bool = True, use_redis: return await handle_exception(ex, status_code=500) else: base_context = { + "host_address": config.HOST_ADDRESS, "enable_ads_header": config.ENABLE_ADS_BANNER, "body_template": rendered_medium_post.data, "title": rendered_medium_post.title, "description": rendered_medium_post.description, } - rendered_post = base_template.render(base_context, HOST_ADDRESS=config.HOST_ADDRESS) + rendered_post = base_template.render(base_context) parsed_rendered_post = parse(rendered_post) serialized_rendered_post = serialize(parsed_rendered_post, encoding="utf-8") diff --git a/server/handlers/reverse_proxy.py b/server/handlers/reverse_proxy.py index 89691d7..46e73f1 100644 --- a/server/handlers/reverse_proxy.py +++ b/server/handlers/reverse_proxy.py @@ -30,13 +30,14 @@ async def iframe_proxy(iframe_id: str): return Response(content=request_content_soup.prettify(), media_type="text/html", headers=IFRAME_HEADERS) -async def miro_proxy(miro_data: str): - async with aiohttp.ClientSession() as client: - request = await client.get( - f"https://miro.medium.com/{miro_data}", - timeout=config.TIMEOUT, - headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}, - ) - request_content = await request.read() - content_type = request.headers["Content-Type"] - return Response(content=request_content, media_type=content_type) +# async def miro_proxy(miro_data: str): +# async with aiohttp.ClientSession() as client: +# request = await client.get( +# f"https://miro.medium.com/{miro_data}", +# timeout=config.TIMEOUT, +# headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}, +# ) +# request_content = await request.read() +# content_type = request.headers["Content-Type"] +# return Response(content=request_content, media_type=content_type) +# diff --git a/server/middlewares/logger.py b/server/middlewares/logger.py index 8cb72ea..3c76355 100644 --- a/server/middlewares/logger.py +++ b/server/middlewares/logger.py @@ -25,6 +25,7 @@ class LoggerMiddleware(BaseHTTPMiddleware): transponder_code = string_to_number_ascii(generated_id) transponder_code_correlation.set(transponder_code) url_correlation.set(request.url) + with logger.contextualize(id=generated_id): logger.debug(f"Current ID '{generated_id}' transponder code is '{transponder_code}'") logger.trace(request.__dict__) diff --git a/server/templates/base.html b/server/templates/base.html index 7dea80e..3c0d721 100644 --- a/server/templates/base.html +++ b/server/templates/base.html @@ -347,7 +347,7 @@ document.addEventListener('scroll', function () { if (element.tagName === "IMG" && element.hasAttribute("data-src")) { const srcAttribute = element.attributes["data-src"].value; if (srcAttribute.startsWith("https://miro.medium.com/v2/")) { - element.setAttribute("src", srcAttribute.replace("https://miro.medium.com/v2/", "{{HOST_ADDRESS}}/@miro/v2/")); + element.setAttribute("src", srcAttribute.replace("https://miro.medium.com/v2/", "{{host_address}}/@miro/v2/")); } } } diff --git a/server/utils/error.py b/server/utils/error.py index 529c3f3..5018602 100644 --- a/server/utils/error.py +++ b/server/utils/error.py @@ -1,3 +1,4 @@ +from typing import Optional import random from fastapi.responses import HTMLResponse @@ -28,18 +29,22 @@ ERROR_MSG_LIST = [ @trace -async def generate_error(error_msg: str = None, title: str = "Error", status_code: int = 500, quiet: bool = False): +async def generate_error(error_msg: Optional[str] = None, title: Optional[str] = None, status_code: int = 500, quiet: bool = False): if not error_msg: error_msg = random.choice(ERROR_MSG_LIST) + if not title: + title = "Opppps.." + if not quiet: send_message(f"📛 Error while processing url: {url_correlation.get()}, transponder_code: {transponder_code_correlation.get()}, error: {error_msg}") error_template_rendered = error_template.render(error_msg=error_msg, transponder_code=transponder_code_correlation.get()) base_context = { + "host_address": config.HOST_ADDRESS, "enable_ads_header": config.ENABLE_ADS_BANNER, "body_template": error_template_rendered, "title": title, } - base_template_rendered = base_template.render(base_context, HOST_ADDRESS=config.HOST_ADDRESS) + base_template_rendered = base_template.render(base_context) return HTMLResponse(base_template_rendered, status_code=status_code)