iframe & miro proxy: refactor & pass traffic proxy

server/handlers/iframe.py

@@ -0,0 +1,40 @@
+import aiohttp
+from fastapi import Response
+from aiohttp_retry import RetryClient
+from aiohttp_socks import ProxyConnector
+import random
+
+from server import config
+from server.utils.logger_trace import trace
+from medium_parser import retry_options
+
+from bs4 import BeautifulSoup
+
+IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"}
+
+
+@trace
+async def iframe_proxy(iframe_id: str):
+    connector = ProxyConnector.from_url(random.choice(config.PROXY_LIST)) if config.PROXY_LIST else None
+
+    async with aiohttp.ClientSession(connector=connector) as session:
+        async with RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) as retry_client:
+            async with retry_client.get(
+                f"https://medium.com/media/{iframe_id}",
+                timeout=config.TIMEOUT,
+                headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"},
+            ) as request:
+                request_content = await request.text()
+
+    patched_content = patch_iframe_content(request_content)
+    return Response(content=patched_content, media_type="text/html", headers=IFRAME_HEADERS)
+
+
+def patch_iframe_content(content: str) -> str:
+    content = content.replace("document.domain = document.domain", 'console.log("[FREEDIUM] iframe workaround started")')
+
+    soup = BeautifulSoup(content, "html.parser")
+    iframe_resizer_script = BeautifulSoup('<script src="https://cdn.jsdelivr.net/npm/@iframe-resizer/child"></script>', "html.parser").script
+    soup.head.append(iframe_resizer_script)
+
+    return soup.prettify()

server/handlers/main.py

@@ -7,7 +7,8 @@ from loguru import logger
 from server import config
 from server.handlers.misc import delete_from_cache, report_problem
 from server.handlers.post import render_medium_post_link, render_homepage
-from server.handlers.reverse_proxy import iframe_proxy, miro_proxy
+from server.handlers.miro import miro_proxy
+from server.handlers.iframe import iframe_proxy
 from server.services.jinja import base_template, main_template
 from server.utils.logger_trace import trace
 

server/handlers/miro.py

@@ -0,0 +1,30 @@
+import aiohttp
+from fastapi import Response
+from aiohttp_retry import RetryClient
+from aiohttp_socks import ProxyConnector
+import random
+
+from server import config
+from medium_parser import retry_options
+
+IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"}
+
+
+async def miro_proxy(miro_data: str, use_proxy: bool = False):
+    url = f"https://miro.medium.com/{miro_data}"
+    headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}
+
+    if use_proxy and config.PROXY_LIST:
+        proxy = random.choice(config.PROXY_LIST)
+        connector = ProxyConnector.from_url(proxy)
+    else:
+        connector = None
+
+    async with aiohttp.ClientSession(connector=connector) as session:
+        client = session if not use_proxy else RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options)
+
+        async with client.get(url, timeout=config.TIMEOUT, headers=headers) as request:
+            request_content = await request.read()
+            content_type = request.headers["Content-Type"]
+
+    return Response(content=request_content, media_type=content_type)

server/handlers/reverse_proxy.py

@@ -1,42 +0,0 @@
-import aiohttp
-from fastapi import Response
-
-from server import config
-from server.utils.logger_trace import trace
-
-from bs4 import BeautifulSoup, Comment
-
-IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"}
-
-
-@trace
-async def iframe_proxy(iframe_id: str):
-    # How Medium embeds works: https://stackoverflow.com/questions/56594766/medium-embed-ly-notifyresize-does-not-work-on-safari
-    async with aiohttp.ClientSession() as client:
-        request = await client.get(
-            f"https://medium.com/media/{iframe_id}",
-            timeout=config.TIMEOUT,
-            headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"},
-        )
-        request_content = await request.text()
-        request_content = request_content.replace("document.domain = document.domain", 'console.log("[FREEDIUM] iframe workaround started")')
-
-    request_content_soup = BeautifulSoup(request_content, "html.parser")
-    iframe_hack_script = '<script src="https://cdn.jsdelivr.net/npm/@iframe-resizer/child"></script>'
-    new_script_tag = BeautifulSoup(iframe_hack_script, "html.parser").script
-
-    request_content_soup.head.append(new_script_tag)
-
-    return Response(content=request_content_soup.prettify(), media_type="text/html", headers=IFRAME_HEADERS)
-
-
-async def miro_proxy(miro_data: str):
-    async with aiohttp.ClientSession() as client:
-        request = await client.get(
-            f"https://miro.medium.com/{miro_data}",
-            timeout=config.TIMEOUT,
-            headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"},
-        )
-        request_content = await request.read()
-        content_type = request.headers["Content-Type"]
-    return Response(content=request_content, media_type=content_type)