{{code_block}}')
- code_block_template = jinja_env.from_string('{{ text }}')
+ pre_template = jinja_env.from_string('{{code_block}}')
+ code_block_template = jinja_env.from_string('{{ text }}')
code_css_class = []
if paragraph["codeBlockMetadata"] and paragraph["codeBlockMetadata"]["lang"] is not None:
code_css_class.append(f'language-{paragraph["codeBlockMetadata"]["lang"]}')
else:
code_css_class.append("nohighlight")
+ # code_css_class.append("auto")
code_list = []
_tmp_current_pos = current_pos
@@ -382,7 +382,9 @@ class MediumParser:
out_paragraphs.append(pre_template_rendered)
current_pos = _tmp_current_pos - 1
elif paragraph["type"] == "BQ":
- bq_template = jinja_env.from_string('') + bq_template = jinja_env.from_string( + '{{ text }}
' + ) bq_template_rendered = await bq_template.render_async(text=text_formater.get_text()) logger.trace(bq_template_rendered) out_paragraphs.append(bq_template_rendered) @@ -394,9 +396,7 @@ class MediumParser: elif paragraph["type"] == "MIXTAPE_EMBED": # TODO: redirect all Medium embeding articles to Fredium embed_template = jinja_env.from_string( - """ - -""" + '' ) if paragraph.get("mixtapeMetadata") is not None: url = paragraph["mixtapeMetadata"]["href"] @@ -437,7 +437,7 @@ class MediumParser: out_paragraphs.append(embed_template_rendered) elif paragraph["type"] == "IFRAME": iframe_template = jinja_env.from_string( - '' + '' ) iframe_template_rendered = await iframe_template.render_async(host_address=self.host_address, iframe_id=paragraph["iframe"]["mediaResource"]["id"]) out_paragraphs.append(iframe_template_rendered) diff --git a/core/medium_parser/utils.py b/core/medium_parser/utils.py index 744e401..e283dbc 100644 --- a/core/medium_parser/utils.py +++ b/core/medium_parser/utils.py @@ -49,7 +49,7 @@ KNOWN_MEDIUM_DOMAINS = ( "towardsdatascience.com", "thetaoist.online", "devopsquare.com", - "www.laceydearie.com", + "laceydearie.com", "bettermarketing.pub", "itnext.io", "eand.co", @@ -72,23 +72,20 @@ NOT_MEDIUM_DOMAINS = ( "yandex.ru", "yandex.kz", "youtube.com", - "www.nytimes.com", "nytimes.com", "wsj.com", - "www.forbes.com", - "www.wsj.com", "reddit.com", "elpais.com", "forbes.com", "bloomberg.com", - "www.lesechos.fr", - "www.otz.de", - "www.businessinsider.com", + "lesechos.fr", + "otz.de", + "businessinsider.com", "buff.ly", - "www.delish.com", - "www.economist.com", - "www.wired.com", - "www.rollingstone.com", + "delish.com", + "economist.com", + "wired.com", + "rollingstone.com", ) @@ -138,7 +135,7 @@ def get_unix_ms() -> int: return milliseconds_since_epoch -def unquerify_url(url): +def unquerify_url(url: str) -> str: """ Sanitizes a URL by removing all query parameters. @@ -157,6 +154,13 @@ def unquerify_url(url): return sanitized_url.removesuffix("/") +@lru_cache(maxsize=500) +def un_wwwify(url: str): + if url.startswith("www."): + return url.removeprefix("www.") + return url + + def correct_url(url: str) -> str: # Workaround for Safari bug. We don't known by what condition this happens, but sometimes we get # some broken URL, for example like "", and all of them based on user-agent comes from Safari browser engine, @@ -247,12 +251,13 @@ async def resolve_medium_short_link(short_url_id: str, timeout: int = 5) -> str: async def resolve_medium_url(url: str, timeout: int = 5) -> str: logger.debug(f"Trying resolve {url=}, with {timeout=}") parsed_url = urlparse(url) + parsed_netloc = un_wwwify(parsed_url.netloc) if parsed_url.path.startswith("/p/"): logger.debug("URL is Medium 'mobile' link") post_id = parsed_url.path.rsplit("/p/")[1] - elif parsed_url.netloc == "l.facebook.com" and parsed_url.path.startswith("/l.php"): + elif parsed_netloc == "l.facebook.com" and parsed_url.path.startswith("/l.php"): logger.debug("URL seems like is Facebook redirect (tracking) link") parsed_query = parse_qs(parsed_url.query) @@ -263,7 +268,7 @@ async def resolve_medium_url(url: str, timeout: int = 5) -> str: logger.debug("...but we get fucked up...") return False - elif parsed_url.netloc == "webcache.googleusercontent.com" and parsed_url.path.startswith("/search"): + elif parsed_netloc == "webcache.googleusercontent.com" and parsed_url.path.startswith("/search"): logger.debug("URL seems like is Google Web Archive page link") parsed_query = parse_qs(parsed_url.query) @@ -274,7 +279,7 @@ async def resolve_medium_url(url: str, timeout: int = 5) -> str: logger.debug("...but we get fucked up...") return False - elif parsed_url.netloc == "www.google.com" and parsed_url.path.startswith("/url"): + elif parsed_netloc == "google.com" and parsed_url.path.startswith("/url"): logger.debug("URL seems like is Google redirect (tracking) link") parsed_query = parse_qs(parsed_url.query) @@ -290,7 +295,7 @@ async def resolve_medium_url(url: str, timeout: int = 5) -> str: logger.debug("...but we get fucked up...") return False - elif parsed_url.netloc == "12ft.io": + elif parsed_netloc == "12ft.io": logger.debug("URL seems like is from our partner named 12ft.io") parsed_query = parse_qs(parsed_url.query) @@ -314,7 +319,7 @@ async def resolve_medium_url(url: str, timeout: int = 5) -> str: logger.debug("...but we get fucked up...") return False - elif parsed_url.netloc == "link.medium.com": + elif parsed_netloc == "link.medium.com": logger.debug("URL seems like is Medium short (SHORT) redirect (tracking) link. Make resolve them...") short_url_id = parsed_url.path.removeprefix("/") post_url = await resolve_medium_short_link(short_url_id, timeout) @@ -337,13 +342,16 @@ async def resolve_medium_url_old(url: str, timeout: int = 5) -> str: retry_client = RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options) request = await retry_client.get(url, timeout=timeout) response = await request.text() + soup = BeautifulSoup(response, "html.parser") type_meta_tag = soup.head.find("meta", property="og:type") if not type_meta_tag or type_meta_tag.get("content") != "article": return False + url_meta_tag = soup.head.find("meta", property="al:android:url") if not url_meta_tag or not url_meta_tag.get("content"): return False + parsed_url = urlparse(url_meta_tag["content"]) path = parsed_url.path.strip("/") parsed_value = path.split("/")[-1] @@ -394,16 +402,17 @@ async def is_valid_medium_url(url: str) -> bool: """ domain = get_fld(url) parsed_url = urlparse(url) + domain_netloc = un_wwwify(parsed_url.netloc) # TODO: http://freedium.cfd/https://www.google.com.vn/url?sa=i&url=https%3A%2F%2Fmedium.com%2F%40dugguRK%2Fabout-android-hardware-abstraction-layer-hal-5d191dafeb2c&psig=AOvVaw17KP0U_haPMmhAByeMTxSg&ust=1711354113283000&source=images&cd=vfe&opi=89978449&ved=0CBQQjhxqFwoTCMCM_oG5jIUDFQAAAAAdAAAAABAa if domain in ["12ft.io", "google.com", "facebook.com", "googleusercontent.com"]: return True - if domain in NOT_MEDIUM_DOMAINS or parsed_url.netloc in NOT_MEDIUM_DOMAINS: + if domain in NOT_MEDIUM_DOMAINS or domain_netloc in NOT_MEDIUM_DOMAINS: raise exceptions.NotValidMediumURL("100% not valid Medium URL") - if domain in KNOWN_MEDIUM_DOMAINS or parsed_url.netloc in KNOWN_MEDIUM_CUSTOM_DOMAINS: + if domain in KNOWN_MEDIUM_DOMAINS or domain_netloc in KNOWN_MEDIUM_CUSTOM_DOMAINS: return True logger.warning(f"url '{url}' wasn't detected in known Medium domains") @@ -411,7 +420,7 @@ async def is_valid_medium_url(url: str) -> bool: # XXX: Unfourtunately, for now we don't know ALL Medium's domains, so we need resolve links resolve_result = bool(await resolve_medium_url(url)) - # send_message(f"We found that {domain=}, {parsed_url.netloc=} is not listed in out known Medium database.\nURL: {url}") + # send_message(f"We found that {domain=}, {domain_netloc=} is not listed in out known Medium database.\nURL: {url}") return resolve_result diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 7041d9e..e042d88 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -34,6 +34,7 @@ services: # - ./.env:/app/.env # - ./server/user_data/logs:/app/server/user_data/logs - .:/app + - ./core/medium_parser/:/app/medium_parser develop: watch: - action: rebuild @@ -65,23 +66,23 @@ services: restart: always mem_limit: 2g - dragonfly: - image: 'docker.dragonflydb.io/dragonflydb/dragonfly' - ulimits: - memlock: -1 - expose: - - 6379 - networks: - - web_network - volumes: - - dragonflydata:/data - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 30s - start_period: 20s - timeout: 10s - retries: 3 - restart: always + # dragonfly: + # image: 'docker.dragonflydb.io/dragonflydb/dragonfly' + # ulimits: + # memlock: -1 + # expose: + # - 6379 + # networks: + # - web_network + # volumes: + # - dragonflydata:/data + # healthcheck: + # test: ["CMD", "redis-cli", "ping"] + # interval: 30s + # start_period: 20s + # timeout: 10s + # retries: 3 + # restart: always autoheal: restart: always diff --git a/server/handlers/reverse_proxy.py b/server/handlers/reverse_proxy.py index a296ae1..e47c938 100644 --- a/server/handlers/reverse_proxy.py +++ b/server/handlers/reverse_proxy.py @@ -4,8 +4,9 @@ from fastapi import Response from server import config from server.utils.logger_trace import trace -IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"} +from bs4 import BeautifulSoup, Comment +IFRAME_HEADERS = {"Access-Control-Allow-Origin": "*", "X-Frame-Options": "SAMEORIGIN"} @trace async def iframe_proxy(iframe_id): @@ -17,8 +18,15 @@ async def iframe_proxy(iframe_id): headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}, ) request_content = await request.text() - request_content = request_content.replace("document.domain = document.domain", "console.log('[FREEDIUM] iframe workaround')") - return Response(content=request_content, media_type="text/html", headers=IFRAME_HEADERS) + request_content = request_content.replace("document.domain = document.domain", 'console.log("[FREEDIUM] iframe workaround started")') + + request_content_soup = BeautifulSoup(request_content, "html.parser") + iframe_hack_script = '' + new_script_tag = BeautifulSoup(iframe_hack_script, 'html.parser').script + + request_content_soup.head.append(new_script_tag) + + return Response(content=request_content_soup.prettify(), media_type="text/html", headers=IFRAME_HEADERS) @trace @@ -31,4 +39,4 @@ async def miro_proxy(miro_data: str): ) request_content = await request.read() content_type = request.headers["Content-Type"] - return Response(content=request_content, media_type=content_type) \ No newline at end of file + return Response(content=request_content, media_type=content_type) diff --git a/server/templates/base.html b/server/templates/base.html index de9b6ae..3ec71ac 100644 --- a/server/templates/base.html +++ b/server/templates/base.html @@ -8,8 +8,9 @@ {% if creator %}{% endif %} + - + @@ -18,77 +19,13 @@ - - + + + - - - - - +{{ text }}