From 7638b5ef3fe93e911acf28852e362f06013b2359 Mon Sep 17 00:00:00 2001 From: ZhymabekRoman Date: Mon, 19 Feb 2024 09:28:05 +0600 Subject: [PATCH] Getting around of Cloudflare (Probably) Probably we need to use Selenium --- core/medium_parser/core.py | 18 ++++++++++++++---- core/medium_parser/medium_api.py | 14 ++++++++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/core/medium_parser/core.py b/core/medium_parser/core.py index e889530..f09eebb 100644 --- a/core/medium_parser/core.py +++ b/core/medium_parser/core.py @@ -1,4 +1,4 @@ -import os +import asyncio import math import textwrap import urllib.parse @@ -83,11 +83,21 @@ class MediumParser: logger.exception(ex) return None - async def query(self, use_cache: bool = True): + async def query(self, use_cache: bool = True, retry: int = 3): post_data = await self.get_post_data_from_cache() if use_cache else None - if not post_data: - post_data = await self.get_post_data_from_api() + attempt = 0 + while not post_data and attempt < retry: + try: + post_data = await self.get_post_data_from_api() + if post_data: + break + except Exception as e: + logger.error(f"Attempt {attempt + 1} failed with error: {e}") + finally: + logger.info(f"Retrying in {2 ** attempt} seconds...") + await asyncio.sleep(2 ** attempt) + attempt += 1 if not post_data or not isinstance(post_data, dict) or post_data.get("error") or not post_data.get("data") or not post_data.get("data").get("post"): raise MediumPostQueryError(f'Could not query post by ID from API: {self.post_id}') diff --git a/core/medium_parser/medium_api.py b/core/medium_parser/medium_api.py index 720c781..36a2f94 100644 --- a/core/medium_parser/medium_api.py +++ b/core/medium_parser/medium_api.py @@ -33,6 +33,7 @@ async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: str = " "query": "query FullPostQuery($postId: ID!, $postMeteringOptions: PostMeteringOptions) { post(id: $postId) { __typename id ...FullPostData } meterPost(postId: $postId, postMeteringOptions: $postMeteringOptions) { __typename ...MeteringInfoData } } fragment UserFollowData on User { id socialStats { followingCount followerCount } viewerEdge { isFollowing } } fragment NewsletterData on NewsletterV3 { id viewerEdge { id isSubscribed } } fragment UserNewsletterData on User { id newsletterV3 { __typename ...NewsletterData } } fragment ImageMetadataData on ImageMetadata { id originalWidth originalHeight focusPercentX focusPercentY alt } fragment CollectionFollowData on Collection { id subscriberCount viewerEdge { isFollowing } } fragment CollectionNewsletterData on Collection { id newsletterV3 { __typename ...NewsletterData } } fragment BylineData on Post { id readingTime creator { __typename id imageId username name bio tippingLink viewerEdge { isUser } ...UserFollowData ...UserNewsletterData } collection { __typename id name avatar { __typename id ...ImageMetadataData } ...CollectionFollowData ...CollectionNewsletterData } isLocked firstPublishedAt latestPublishedVersion } fragment ResponseCountData on Post { postResponses { count } } fragment InResponseToPost on Post { id title creator { name } clapCount responsesCount isLocked } fragment PostVisibilityData on Post { id collection { viewerEdge { isEditor canEditPosts canEditOwnPosts } } creator { id } isLocked visibility } fragment PostMenuData on Post { id title creator { __typename ...UserFollowData } collection { __typename ...CollectionFollowData } } fragment PostMetaData on Post { __typename id title visibility ...ResponseCountData clapCount viewerEdge { clapCount } detectedLanguage mediumUrl readingTime updatedAt isLocked allowResponses isProxyPost latestPublishedVersion isSeries firstPublishedAt previewImage { id } inResponseToPostResult { __typename ...InResponseToPost } inResponseToMediaResource { mediumQuote { startOffset endOffset paragraphs { text type markups { type start end anchorType } } } } inResponseToEntityType canonicalUrl collection { id slug name shortDescription avatar { __typename id ...ImageMetadataData } viewerEdge { isFollowing isEditor canEditPosts canEditOwnPosts isMuting } } creator { id isFollowing name bio imageId mediumMemberAt twitterScreenName viewerEdge { isBlocking isMuting isUser } } previewContent { subtitle } pinnedByCreatorAt ...PostVisibilityData ...PostMenuData } fragment LinkMetadataList on Post { linkMetadataList { url alts { type url } } } fragment MediaResourceData on MediaResource { id iframeSrc thumbnailUrl } fragment IframeData on Iframe { iframeHeight iframeWidth mediaResource { __typename ...MediaResourceData } } fragment MarkupData on Markup { name type start end href title rel type anchorType userId creatorIds } fragment CatalogSummaryData on Catalog { id name description type visibility predefined responsesLocked creator { id name username imageId bio viewerEdge { isUser } } createdAt version itemsLastInsertedAt postItemsCount } fragment CatalogPreviewData on Catalog { __typename ...CatalogSummaryData id itemsConnection(pagingOptions: { limit: 10 } ) { items { entity { __typename ... on Post { id previewImage { id } } } } paging { count } } } fragment MixtapeMetadataData on MixtapeMetadata { mediaResourceId href thumbnailImageId mediaResource { mediumCatalog { __typename ...CatalogPreviewData } } } fragment ParagraphData on Paragraph { id name href text iframe { __typename ...IframeData } layout markups { __typename ...MarkupData } metadata { __typename ...ImageMetadataData } mixtapeMetadata { __typename ...MixtapeMetadataData } type hasDropCap dropCapImage { __typename ...ImageMetadataData } codeBlockMetadata { lang mode } } fragment QuoteData on Quote { id postId userId startOffset endOffset paragraphs { __typename id ...ParagraphData } quoteType } fragment HighlightsData on Post { id highlights { __typename ...QuoteData } } fragment PostFooterCountData on Post { __typename id clapCount viewerEdge { clapCount } ...ResponseCountData responsesLocked mediumUrl title collection { id viewerEdge { isMuting isFollowing } } creator { id viewerEdge { isMuting isFollowing } } } fragment TagNoViewerEdgeData on Tag { id normalizedTagSlug displayTitle followerCount postCount } fragment VideoMetadataData on VideoMetadata { videoId previewImageId originalWidth originalHeight } fragment SectionData on Section { name startIndex textLayout imageLayout videoLayout backgroundImage { __typename ...ImageMetadataData } backgroundVideo { __typename ...VideoMetadataData } } fragment PostBodyData on RichText { sections { __typename ...SectionData } paragraphs { __typename id ...ParagraphData } } fragment FullPostData on Post { __typename ...BylineData ...PostMetaData ...LinkMetadataList ...HighlightsData ...PostFooterCountData tags { __typename id ...TagNoViewerEdgeData } content(postMeteringOptions: $postMeteringOptions) { bodyModel { __typename ...PostBodyData } validatedShareKey } } fragment MeteringInfoData on MeteringInfo { maxUnlockCount unlocksRemaining postIds }", } + response_data = None async with aiohttp.ClientSession() as session: retry_client = RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) request = await retry_client.post( @@ -41,8 +42,17 @@ async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: str = " json=json_data, timeout=timeout, ) - response = await request.json() + if request.status == 200: + try: + response_data = await request.json() + except Exception as ex: + logger.debug("Failed to parse response data as JSON") + logger.exception(ex) + else: + logger.error(f"Failed to fetch post by ID {post_id} with status code: {request.status}") + + response_data = await request.json() logger.trace(request.headers) - return response + return response_data