From ff187c5eabe2e17aa8b44517f1266509a7da84ab Mon Sep 17 00:00:00 2001 From: ZhymabekRoman Date: Tue, 6 Aug 2024 17:09:48 +0500 Subject: [PATCH] iframe & miro proxy: refactor & pass traffic proxy --- server/handlers/iframe.py | 40 ++++++++++++++++++++++++++++++ server/handlers/main.py | 3 ++- server/handlers/miro.py | 30 +++++++++++++++++++++++ server/handlers/reverse_proxy.py | 42 -------------------------------- 4 files changed, 72 insertions(+), 43 deletions(-) create mode 100644 server/handlers/iframe.py create mode 100644 server/handlers/miro.py delete mode 100644 server/handlers/reverse_proxy.py diff --git a/server/handlers/iframe.py b/server/handlers/iframe.py new file mode 100644 index 0000000..3218298 --- /dev/null +++ b/server/handlers/iframe.py @@ -0,0 +1,40 @@ +import aiohttp +from fastapi import Response +from aiohttp_retry import RetryClient +from aiohttp_socks import ProxyConnector +import random + +from server import config +from server.utils.logger_trace import trace +from medium_parser import retry_options + +from bs4 import BeautifulSoup + +IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"} + + +@trace +async def iframe_proxy(iframe_id: str): + connector = ProxyConnector.from_url(random.choice(config.PROXY_LIST)) if config.PROXY_LIST else None + + async with aiohttp.ClientSession(connector=connector) as session: + async with RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) as retry_client: + async with retry_client.get( + f"https://medium.com/media/{iframe_id}", + timeout=config.TIMEOUT, + headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}, + ) as request: + request_content = await request.text() + + patched_content = patch_iframe_content(request_content) + return Response(content=patched_content, media_type="text/html", headers=IFRAME_HEADERS) + + +def patch_iframe_content(content: str) -> str: + content = content.replace("document.domain = document.domain", 'console.log("[FREEDIUM] iframe workaround started")') + + soup = BeautifulSoup(content, "html.parser") + iframe_resizer_script = BeautifulSoup('', "html.parser").script + soup.head.append(iframe_resizer_script) + + return soup.prettify() diff --git a/server/handlers/main.py b/server/handlers/main.py index 2a41e86..68d7a0f 100644 --- a/server/handlers/main.py +++ b/server/handlers/main.py @@ -7,7 +7,8 @@ from loguru import logger from server import config from server.handlers.misc import delete_from_cache, report_problem from server.handlers.post import render_medium_post_link, render_homepage -from server.handlers.reverse_proxy import iframe_proxy, miro_proxy +from server.handlers.miro import miro_proxy +from server.handlers.iframe import iframe_proxy from server.services.jinja import base_template, main_template from server.utils.logger_trace import trace diff --git a/server/handlers/miro.py b/server/handlers/miro.py new file mode 100644 index 0000000..2e489c2 --- /dev/null +++ b/server/handlers/miro.py @@ -0,0 +1,30 @@ +import aiohttp +from fastapi import Response +from aiohttp_retry import RetryClient +from aiohttp_socks import ProxyConnector +import random + +from server import config +from medium_parser import retry_options + +IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"} + + +async def miro_proxy(miro_data: str, use_proxy: bool = False): + url = f"https://miro.medium.com/{miro_data}" + headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"} + + if use_proxy and config.PROXY_LIST: + proxy = random.choice(config.PROXY_LIST) + connector = ProxyConnector.from_url(proxy) + else: + connector = None + + async with aiohttp.ClientSession(connector=connector) as session: + client = session if not use_proxy else RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) + + async with client.get(url, timeout=config.TIMEOUT, headers=headers) as request: + request_content = await request.read() + content_type = request.headers["Content-Type"] + + return Response(content=request_content, media_type=content_type) diff --git a/server/handlers/reverse_proxy.py b/server/handlers/reverse_proxy.py deleted file mode 100644 index 6524e31..0000000 --- a/server/handlers/reverse_proxy.py +++ /dev/null @@ -1,42 +0,0 @@ -import aiohttp -from fastapi import Response - -from server import config -from server.utils.logger_trace import trace - -from bs4 import BeautifulSoup, Comment - -IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"} - - -@trace -async def iframe_proxy(iframe_id: str): - # How Medium embeds works: https://stackoverflow.com/questions/56594766/medium-embed-ly-notifyresize-does-not-work-on-safari - async with aiohttp.ClientSession() as client: - request = await client.get( - f"https://medium.com/media/{iframe_id}", - timeout=config.TIMEOUT, - headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}, - ) - request_content = await request.text() - request_content = request_content.replace("document.domain = document.domain", 'console.log("[FREEDIUM] iframe workaround started")') - - request_content_soup = BeautifulSoup(request_content, "html.parser") - iframe_hack_script = '' - new_script_tag = BeautifulSoup(iframe_hack_script, "html.parser").script - - request_content_soup.head.append(new_script_tag) - - return Response(content=request_content_soup.prettify(), media_type="text/html", headers=IFRAME_HEADERS) - - -async def miro_proxy(miro_data: str): - async with aiohttp.ClientSession() as client: - request = await client.get( - f"https://miro.medium.com/{miro_data}", - timeout=config.TIMEOUT, - headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}, - ) - request_content = await request.read() - content_type = request.headers["Content-Type"] - return Response(content=request_content, media_type=content_type)