mirror of
https://codeberg.org/Freedium-cfd/web.git
synced 2026-03-11 09:04:37 +00:00
Refactor
This commit is contained in:
ZhymabekRoman
parent
6539d3a63e
commit
3f230cafa5
14 changed files with 373 additions and 171 deletions
|
|
@ -9,6 +9,7 @@ RUN apt-get update && \
|
|||
COPY ./requirements.txt ./
|
||||
COPY ./requirements-fast.txt ./
|
||||
|
||||
COPY ./server ./server
|
||||
COPY ./core ./core
|
||||
COPY ./rl_string_helper ./rl_string_helper
|
||||
COPY ./database-lib ./database-lib
|
||||
|
|
|
|||
|
|
@ -1,15 +1,7 @@
|
|||
|
||||
from aiohttp_retry import ExponentialRetry
|
||||
from loguru import logger
|
||||
|
||||
import jinja2
|
||||
from database_lib import SQLiteCacheBackend
|
||||
|
||||
cache = SQLiteCacheBackend('medium_db_cache.sqlite')
|
||||
cache.init_db()
|
||||
cache.enable_zstd()
|
||||
|
||||
logger.debug(f"Database length: {cache.all_length()}")
|
||||
|
||||
retry_options = ExponentialRetry(attempts=3)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,44 +1,63 @@
|
|||
import asyncio
|
||||
import math
|
||||
import textwrap
|
||||
import typing
|
||||
import urllib.parse
|
||||
from contextlib import suppress
|
||||
|
||||
import jinja2
|
||||
import tld
|
||||
from loguru import logger
|
||||
from contextlib import suppress
|
||||
|
||||
from rl_string_helper import RLStringHelper, parse_markups, split_overlapping_ranges
|
||||
from rl_string_helper import (RLStringHelper, parse_markups,
|
||||
split_overlapping_ranges)
|
||||
|
||||
from . import cache, jinja_env
|
||||
from .exceptions import InvalidMediumPostID, InvalidMediumPostURL, InvalidURL, MediumParserException, MediumPostQueryError
|
||||
from . import jinja_env
|
||||
from .exceptions import (InvalidMediumPostID, InvalidMediumPostURL, InvalidURL,
|
||||
MediumParserException, MediumPostQueryError)
|
||||
from .medium_api import query_post_by_id
|
||||
from .models.html_result import HtmlResult
|
||||
from .time import convert_datetime_to_human_readable
|
||||
from .utils import resolve_medium_url, getting_percontage_of_match, is_valid_medium_post_id_hexadecimal, is_valid_medium_url, is_valid_url, sanitize_url
|
||||
from .utils import (correct_url, getting_percontage_of_match,
|
||||
is_valid_medium_post_id_hexadecimal, is_valid_medium_url,
|
||||
is_valid_url, resolve_medium_url)
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from database_lib import SQLiteCacheBackend
|
||||
|
||||
class MediumParser:
|
||||
__slots__ = ("__post_id", "post_data", "jinja", "timeout", "host_address", "auth_cookies")
|
||||
__slots__ = ("__post_id", "auth_cookies", "cache", "host_address", "jinja", "post_data", "timeout")
|
||||
|
||||
def __init__(self, post_id: str, timeout: int, host_address: str, auth_cookies: str = None):
|
||||
def __init__(self, post_id: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None):
|
||||
self.timeout = timeout
|
||||
self.cache = cache
|
||||
self.host_address = host_address
|
||||
self.post_id = post_id
|
||||
self.post_data = None
|
||||
self.auth_cookies = auth_cookies
|
||||
|
||||
@classmethod
|
||||
async def from_url(cls, url: str, timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser":
|
||||
sanitized_url = sanitize_url(url)
|
||||
if is_valid_url(url) and not await is_valid_medium_url(sanitized_url, timeout):
|
||||
async def from_unknown(cls, unknown: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser":
|
||||
logger.debug(f"We got some unknown data: {unknown=}, with {cache=}, {timeout=}, {host_address=}, {auth_cookies=}. Trying resolve them...///")
|
||||
|
||||
if is_valid_medium_post_id_hexadecimal(unknown):
|
||||
logger.debug("Seems like it's valid post_id")
|
||||
return cls(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies)
|
||||
|
||||
logger.debug("...maybe it's URL. Let's checkout...")
|
||||
return await cls.from_url(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies)
|
||||
|
||||
@classmethod
|
||||
async def from_url(cls, url: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser":
|
||||
sanitized_url = correct_url(url)
|
||||
if not is_valid_url(url) or not await is_valid_medium_url(sanitized_url):
|
||||
raise InvalidURL(f"Invalid Medium URL: {sanitized_url}")
|
||||
|
||||
post_id = await resolve_medium_url(sanitized_url, timeout)
|
||||
if not post_id:
|
||||
raise InvalidMediumPostURL(f"Could not find Medium post ID for URL: {sanitized_url}")
|
||||
|
||||
return cls(post_id, timeout, host_address, auth_cookies)
|
||||
return cls(post_id, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies)
|
||||
|
||||
@property
|
||||
def post_id(self):
|
||||
|
|
@ -59,14 +78,14 @@ class MediumParser:
|
|||
if not post_id:
|
||||
post_id = self.post_id
|
||||
|
||||
cache.delete(post_id)
|
||||
self.cache.delete(post_id)
|
||||
|
||||
return True
|
||||
|
||||
async def get_post_data_from_cache(self):
|
||||
async def _get_from_cache():
|
||||
logger.debug("Using cache backend")
|
||||
post_data = cache.pull(self.post_id)
|
||||
post_data = self.cache.pull(self.post_id)
|
||||
if post_data:
|
||||
logger.debug("post query was found on cache")
|
||||
return post_data.json()
|
||||
|
|
@ -80,11 +99,11 @@ class MediumParser:
|
|||
|
||||
async def get_post_data_from_api(self):
|
||||
async def _get_from_api():
|
||||
logger.debug("Using API backend")
|
||||
logger.debug("Using API to gather post data")
|
||||
try:
|
||||
return await query_post_by_id(self.post_id, self.timeout, self.auth_cookies)
|
||||
except Exception as ex:
|
||||
logger.debug("Error while querying post by Medium API")
|
||||
logger.debug("Error while querying post data from Medium API")
|
||||
logger.exception(ex)
|
||||
return None
|
||||
|
||||
|
|
@ -127,11 +146,11 @@ class MediumParser:
|
|||
reason = "Post data missing 'data.post' key"
|
||||
|
||||
if reason is None:
|
||||
logger.debug("Post data was successfully queried")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Attempt {attempt + 1} failed with exception: {e}")
|
||||
finally:
|
||||
logger.info(f"Retrying in {2 ** attempt} seconds...")
|
||||
logger.debug(f"Retrying in {2 ** attempt} seconds...")
|
||||
await asyncio.sleep(2**attempt)
|
||||
attempt += 1
|
||||
else:
|
||||
|
|
@ -141,9 +160,11 @@ class MediumParser:
|
|||
raise MediumPostQueryError(f"Could not query post by ID from API: {self.post_id}. Reason: {reason}")
|
||||
|
||||
if not is_cache_used:
|
||||
cache.push(self.post_id, post_data)
|
||||
logger.debug("Pushing post data to cache")
|
||||
self.cache.push(self.post_id, post_data)
|
||||
|
||||
self.post_data = post_data
|
||||
logger.trace(f"Query: done")
|
||||
return post_data
|
||||
|
||||
async def _parse_and_render_content_html_post(self, content: dict, title: str, subtitle: str, preview_image_id: str, highlights: list, tags: list) -> tuple[list, str, str]:
|
||||
|
|
@ -179,7 +200,11 @@ class MediumParser:
|
|||
if current_pos in range(4):
|
||||
if paragraph["type"] in ["H3", "H4", "H2"]:
|
||||
if getting_percontage_of_match(paragraph["text"], title) > 80:
|
||||
logger.trace("Title was detected, ignore...")
|
||||
if title.endswith("…"):
|
||||
logger.trace("Title was detected, replace...")
|
||||
title = paragraph["text"]
|
||||
else:
|
||||
logger.trace("Title was detected, ignore...")
|
||||
current_pos += 1
|
||||
continue
|
||||
if paragraph["type"] in ["H4"]:
|
||||
|
|
@ -366,7 +391,7 @@ class MediumParser:
|
|||
logger.trace(pq_template_rendered)
|
||||
out_paragraphs.append(pq_template_rendered)
|
||||
elif paragraph["type"] == "MIXTAPE_EMBED":
|
||||
# TODO: redirect all Medium embeding artickles to Fredium
|
||||
# TODO: redirect all Medium embeding articles to Fredium
|
||||
embed_template = jinja_env.from_string(
|
||||
"""
|
||||
<div class="border border-gray-300 p-2 mt-7 items-center overflow-hidden"><a rel="noopener follow" href="{{ url }}" target="_blank"> <div class="flex flex-row justify-between p-2 overflow-hidden"><div class="flex flex-col justify-center p-2"><h2 class="text-black dark:text-gray-100 text-base font-bold">{{ embed_title }}</h2><div class="mt-2 block"><h3 class="text-grey-darker text-sm">{{ embed_description }}</h3></div><div class="mt-5" style=""><p class="text-grey-darker text-xs">{{ embed_site }}</p></div></div><div class="relative flex flew-row h-40 w-60"><div class="lazy absolute inset-0 bg-cover bg-center" data-bg="https://miro.medium.com/v2/resize:fit:320/{{ paragraph.mixtapeMetadata.thumbnailImageId }}"></div></div></div> </a></div>
|
||||
|
|
@ -389,8 +414,15 @@ class MediumParser:
|
|||
title_range = paragraph["markups"][1]
|
||||
description_range = paragraph["markups"][2]
|
||||
|
||||
logger.trace(f"{title_range=}")
|
||||
logger.trace(f"{description_range=}")
|
||||
|
||||
embed_title = text_raw[title_range["start"] : title_range["end"]]
|
||||
embed_description = text_raw[description_range["start"] : description_range["end"]]
|
||||
|
||||
logger.trace(f"{embed_title=}")
|
||||
logger.trace(f"{embed_description=}")
|
||||
|
||||
try:
|
||||
embed_site = tld.get_fld(url)
|
||||
except Exception as ex:
|
||||
|
|
@ -398,6 +430,8 @@ class MediumParser:
|
|||
parsed_url = urllib.parse.urlparse(url)
|
||||
embed_site = parsed_url.hostname
|
||||
|
||||
logger.trace(f"{embed_site=}")
|
||||
|
||||
embed_template_rendered = await embed_template.render_async(paragraph=paragraph, url=url, embed_title=embed_title, embed_description=embed_description, embed_site=embed_site)
|
||||
out_paragraphs.append(embed_template_rendered)
|
||||
elif paragraph["type"] == "IFRAME":
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ from .utils import generate_random_sha256_hash
|
|||
|
||||
# https://gist.github.com/vladar/a4e3afd608cfe8b13e5844d75447f0a4
|
||||
async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: str = ""):
|
||||
auth_cookies = "" if not auth_cookies else auth_cookies
|
||||
|
||||
headers = {
|
||||
"X-APOLLO-OPERATION-ID": generate_random_sha256_hash(),
|
||||
"X-APOLLO-OPERATION-NAME": "FullPostQuery",
|
||||
|
|
@ -21,7 +23,7 @@ async def query_post_by_id(post_id: str, timeout: int = 3, auth_cookies: str = "
|
|||
"Cache-Control": "public, max-age=-1",
|
||||
"Content-Type": "application/json",
|
||||
"Connection": "Keep-Alive",
|
||||
# "Cookie": auth_cookies,
|
||||
"Cookie": auth_cookies,
|
||||
}
|
||||
|
||||
json_data = {
|
||||
|
|
|
|||
|
|
@ -1,28 +1,104 @@
|
|||
import hashlib
|
||||
import secrets
|
||||
import difflib
|
||||
import urllib.parse
|
||||
from aiohttp_retry import RetryClient
|
||||
from datetime import datetime
|
||||
from loguru import logger
|
||||
from functools import lru_cache
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
import aiohttp
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import string
|
||||
import urllib.parse
|
||||
from datetime import datetime
|
||||
from functools import lru_cache
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from . import retry_options, exceptions
|
||||
|
||||
import aiohttp
|
||||
import tld
|
||||
from aiohttp_retry import RetryClient
|
||||
from async_lru import alru_cache
|
||||
from bs4 import BeautifulSoup
|
||||
from loguru import logger
|
||||
|
||||
from . import exceptions, retry_options
|
||||
|
||||
DEFAULT_URL_PROTOCOL = "https://"
|
||||
|
||||
VALID_ID_CHARS = set(string.ascii_letters + string.digits)
|
||||
|
||||
KNOWN_MEDIUM_NETLOC = ("javascript.plainenglish.io", "python.plainenglish.io", "blog.stackademic.com", "ai.gopubby.com", "blog.devops.dev", "levelup.gitconnected.com", "betterhumans.coach.me", "ai.plainenglish.io")
|
||||
KNOWN_MEDIUM_DOMAINS = ("medium.com", "uxplanet.org", "towardsdatascience.com", "thetaoist.online", "devopsquare.com", "bettermarketing.pub", "itnext.io", "eand.co", "betterprogramming.pub", "curiouse.co", "betterhumans.pub", "uxdesign.cc", "thebolditalic.com", "arcdigital.media", "codeburst.io", "psiloveyou.xyz", "writingcooperative.com", "entrepreneurshandbook.co", "prototypr.io", "theascent.pub")
|
||||
NOT_MEDIUM_DOMAINS = ("github.com", "yandex.ru", "yandex.kz", "youtube.com", "nytimes.com", "wsj.com", "reddit.com", "elpais.com", "forbes.com", "bloomberg.com")
|
||||
KNOWN_MEDIUM_CUSTOM_DOMAINS = (
|
||||
"javascript.plainenglish.io",
|
||||
"blog.llamaindex.ai",
|
||||
"code.likeagirl.io",
|
||||
"medium.datadriveninvestor.com",
|
||||
"blog.det.life",
|
||||
"python.plainenglish.io",
|
||||
"blog.stackademic.com",
|
||||
"ai.gopubby.com",
|
||||
"blog.devops.dev",
|
||||
"levelup.gitconnected.com",
|
||||
"betterhumans.coach.me",
|
||||
"ai.plainenglish.io",
|
||||
)
|
||||
KNOWN_MEDIUM_DOMAINS = (
|
||||
"medium.com",
|
||||
"uxplanet.org",
|
||||
"osintteam.blog",
|
||||
"ahmedelfakharany.com",
|
||||
"drlee.io",
|
||||
"artificialcorner.com",
|
||||
"generativeai.pub",
|
||||
"productcoalition.com",
|
||||
"towardsdev.com",
|
||||
"infosecwriteups.com",
|
||||
"towardsdatascience.com",
|
||||
"thetaoist.online",
|
||||
"devopsquare.com",
|
||||
"www.laceydearie.com",
|
||||
"bettermarketing.pub",
|
||||
"itnext.io",
|
||||
"eand.co",
|
||||
"betterprogramming.pub",
|
||||
"curiouse.co",
|
||||
"betterhumans.pub",
|
||||
"uxdesign.cc",
|
||||
"thebolditalic.com",
|
||||
"arcdigital.media",
|
||||
"codeburst.io",
|
||||
"psiloveyou.xyz",
|
||||
"writingcooperative.com",
|
||||
"entrepreneurshandbook.co",
|
||||
"prototypr.io",
|
||||
"theascent.pub",
|
||||
"storiusmag.com"
|
||||
)
|
||||
NOT_MEDIUM_DOMAINS = (
|
||||
"github.com",
|
||||
"yandex.ru",
|
||||
"yandex.kz",
|
||||
"youtube.com",
|
||||
"nytimes.com",
|
||||
"wsj.com",
|
||||
"www.wsj.com",
|
||||
"reddit.com",
|
||||
"elpais.com",
|
||||
"forbes.com",
|
||||
"bloomberg.com",
|
||||
"www.lesechos.fr",
|
||||
"www.otz.de",
|
||||
"buff.ly",
|
||||
"www.delish.com",
|
||||
"www.economist.com",
|
||||
"www.wired.com",
|
||||
"www.rollingstone.com",
|
||||
)
|
||||
|
||||
|
||||
def is_valid_url(url):
|
||||
"""
|
||||
Check if the given URL is valid by verifying if it has a valid scheme and netloc.
|
||||
|
||||
Parameters:
|
||||
url (str): The URL to be validated.
|
||||
|
||||
Returns:
|
||||
bool: True if the URL is valid, False otherwise.
|
||||
"""
|
||||
fld = get_fld(url)
|
||||
if not fld:
|
||||
return False
|
||||
|
|
@ -62,46 +138,81 @@ def get_unix_ms() -> int:
|
|||
def unquerify_url(url):
|
||||
"""
|
||||
Sanitizes a URL by removing all query parameters.
|
||||
|
||||
|
||||
Args:
|
||||
url: The URL to sanitize.
|
||||
|
||||
url: The URL to sanitize.
|
||||
|
||||
Returns:
|
||||
A sanitized URL.
|
||||
A sanitized URL.
|
||||
"""
|
||||
|
||||
|
||||
parsed_url = urllib.parse.urlparse(url)
|
||||
query = parsed_url.query
|
||||
if query:
|
||||
parsed_url = parsed_url._replace(query='')
|
||||
parsed_url = parsed_url._replace(query="")
|
||||
sanitized_url = urllib.parse.urlunparse(parsed_url)
|
||||
return sanitized_url.removesuffix("/")
|
||||
|
||||
|
||||
def sanitize_url(url):
|
||||
def correct_url(url: str) -> str:
|
||||
# Workaround for Safari bug. We don't known by what condition this happens, but sometimes we get
|
||||
# some broken URL, for example like "", and all of them based on user-agent comes from Safari browser engine,
|
||||
# from some kinda different platforms like Windows, and that's strange bcz does Windows has Safari browser? lmao
|
||||
|
||||
# TODO: fix
|
||||
|
||||
# unsafari_url = re.sub(r"https?://", DEFAULT_URL_PROTOCOL, url)
|
||||
# logger.debug(f"Is URL broken by Safari bug: {unsafari_url != url}")
|
||||
|
||||
unsafari_url = url
|
||||
|
||||
unquerified_url = unquerify_url(unsafari_url)
|
||||
logger.debug(f"Is URL has query data: {unquerified_url != unsafari_url}")
|
||||
|
||||
unplaginated_url = unplaginate_url(unquerified_url)
|
||||
logger.debug(f"Is URL has plagination: {unplaginated_url != unquerified_url}")
|
||||
|
||||
# parsed_url = urlparse(url)
|
||||
# if not bool(parsed_url.netloc and parsed_url.scheme):
|
||||
# return DEFAULT_PROTOCOL + url
|
||||
|
||||
# if not re.match(r'http[s]?://', url):
|
||||
# url = DEFAULT_PROTOCOL + url
|
||||
|
||||
return url
|
||||
|
||||
|
||||
def unplaginate_url(url):
|
||||
"""
|
||||
Removes page plaginations from URL
|
||||
"""
|
||||
sanitized_url = url.removesuffix("/page/2")
|
||||
return sanitized_url.removesuffix("/")
|
||||
|
||||
|
||||
@lru_cache(maxsize=100)
|
||||
def is_valid_medium_post_id_hexadecimal(hex_string: str) -> bool:
|
||||
# Check if the string is a valid hexadecimal string
|
||||
# isalnum()
|
||||
for char in hex_string:
|
||||
if char not in VALID_ID_CHARS:
|
||||
return False
|
||||
|
||||
# Unfortunately, this logic doesn't works correctly sometimes, because
|
||||
# there is some unique URLs that are has only digits, like this:
|
||||
# https://valeman.medium.com/python-vs-r-for-time-series-forecasting-395390432598
|
||||
|
||||
# Check if the string contains only lowercase hexadecimal characters
|
||||
# if not hex_string.islower():
|
||||
# return False
|
||||
|
||||
# Check if the length of the string is correct for a hexadecimal string (e.g., 10, 11 or 12 characters)
|
||||
if len(hex_string) not in range(8, 13):
|
||||
if len(hex_string) not in range(8, 12 + 1):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
async def resolve_medium_short_link_v1(short_url_id: str, timeout: int = 5) -> str:
|
||||
async def resolve_medium_short_link(short_url_id: str, timeout: int = 5) -> str:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
retry_client = RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options)
|
||||
request = await retry_client.get(
|
||||
|
|
@ -111,54 +222,93 @@ async def resolve_medium_short_link_v1(short_url_id: str, timeout: int = 5) -> s
|
|||
allow_redirects=False,
|
||||
)
|
||||
post_url = request.headers["Location"]
|
||||
return await resolve_medium_url(post_url)
|
||||
return post_url
|
||||
|
||||
|
||||
@alru_cache(maxsize=500)
|
||||
async def resolve_medium_url(url: str, timeout: int = 5) -> str:
|
||||
logger.debug(f"Trying resolve {url=}, with {timeout=}")
|
||||
parsed_url = urlparse(url)
|
||||
|
||||
if parsed_url.path.startswith("/p/"):
|
||||
logger.debug("URL is Medium 'mobile' link")
|
||||
post_id = parsed_url.path.rsplit("/p/")[1]
|
||||
|
||||
elif parsed_url.netloc == "l.facebook.com" and parsed_url.path.startswith("/l.php"):
|
||||
logger.debug("URL seems like is Facebook redirect (tracking) link")
|
||||
|
||||
parsed_query = parse_qs(parsed_url.query)
|
||||
if parsed_query.get("u") and len(parsed_query["u"]) == 1:
|
||||
post_url = parsed_query["u"][0]
|
||||
return await resolve_medium_url(post_url)
|
||||
|
||||
logger.debug("...but we get fucked up...")
|
||||
return False
|
||||
|
||||
elif parsed_url.netloc == "webcache.googleusercontent.com" and parsed_url.path.startswith("/search"):
|
||||
logger.debug("URL seems like is Google Web Archive page link")
|
||||
|
||||
parsed_query = parse_qs(parsed_url.query)
|
||||
if parsed_query.get("q") and len(parsed_query["q"]) == 1:
|
||||
post_url = parsed_query["q"][0].removeprefix("cache:")
|
||||
return await resolve_medium_url(post_url)
|
||||
|
||||
logger.debug("...but we get fucked up...")
|
||||
return False
|
||||
|
||||
elif parsed_url.netloc == "www.google.com" and parsed_url.path.startswith("/url"):
|
||||
logger.debug("URL seems like is Google redirect (tracking) link")
|
||||
|
||||
parsed_query = parse_qs(parsed_url.query)
|
||||
if parsed_query.get("url") and len(parsed_query["url"]) == 1:
|
||||
logger.debug("..and we got 'url' passed param. Make resolve them....")
|
||||
post_url = parsed_query["url"][0]
|
||||
return await resolve_medium_url(post_url)
|
||||
elif parsed_query.get("q") and len(parsed_query["q"]) == 1:
|
||||
logger.debug("..and we got 'q' passed param. Make resolve them....")
|
||||
post_url = parsed_query["q"][0]
|
||||
return await resolve_medium_url(post_url)
|
||||
|
||||
logger.debug("...but we get fucked up...")
|
||||
return False
|
||||
|
||||
elif parsed_url.netloc == "12ft.io":
|
||||
logger.debug("URL seems like is from our partner named 12ft.io")
|
||||
|
||||
parsed_query = parse_qs(parsed_url.query)
|
||||
if parsed_query.get("q") and len(parsed_query["q"]) == 1:
|
||||
logger.debug("..and we got 'q' passed param. Make resolve them....")
|
||||
post_url = parsed_query["q"][0]
|
||||
return await resolve_medium_url(post_url)
|
||||
|
||||
logger.debug("...but we get fucked up...")
|
||||
return False
|
||||
|
||||
elif parsed_url.path.startswith("/m/global-identity-2"):
|
||||
logger.debug("URL seems like is Medium redirect (tracking) link. Possibly from email subscription")
|
||||
|
||||
parsed_query = parse_qs(parsed_url.query)
|
||||
if parsed_query.get("redirectUrl") and len(parsed_query["redirectUrl"]) == 1:
|
||||
logger.debug("..and we got 'redirectUrl' passed param. Make resolve them....")
|
||||
post_url = parsed_query["redirectUrl"][0]
|
||||
return await resolve_medium_url(post_url)
|
||||
|
||||
logger.debug("...but we get fucked up...")
|
||||
return False
|
||||
|
||||
elif parsed_url.netloc == "link.medium.com":
|
||||
logger.debug("URL seems like is Medium short (SHORT) redirect (tracking) link. Make resolve them...")
|
||||
short_url_id = parsed_url.path.removeprefix("/")
|
||||
return await resolve_medium_short_link_v1(short_url_id, timeout)
|
||||
post_url = await resolve_medium_short_link(short_url_id, timeout)
|
||||
return await resolve_medium_url(post_url)
|
||||
|
||||
else:
|
||||
logger.debug("We can't determine the URL type. Let's just try to extract the post_id...")
|
||||
post_url = parsed_url.path.split("/")[-1]
|
||||
post_id = post_url.split("-")[-1]
|
||||
|
||||
if not is_valid_medium_post_id_hexadecimal(post_id):
|
||||
logger.warning(f"...but hoops, that's invalid post_id: {post_id}")
|
||||
return False
|
||||
|
||||
return post_id
|
||||
|
|
@ -182,40 +332,7 @@ async def resolve_medium_url_old(url: str, timeout: int = 5) -> str:
|
|||
return parsed_value
|
||||
|
||||
|
||||
@lru_cache(maxsize=200)
|
||||
def get_fld(url: str):
|
||||
try:
|
||||
fld = tld.get_fld(url)
|
||||
except Exception as ex:
|
||||
logger.trace(ex)
|
||||
return None
|
||||
else:
|
||||
return fld
|
||||
|
||||
|
||||
async def is_valid_medium_url(url: str, timeout: int = 5) -> bool:
|
||||
"""
|
||||
Check if the url is a valid medium.com url
|
||||
|
||||
First stage of url validation is checking if the domain is in the known medium.com url list. If the domain is in the list, then the url is valid
|
||||
Second stage is checking if the url is valid Medium site by performing a GET request to the url and checking the site name meta tag. If the site name meta tag is Medium, then the url is valid
|
||||
"""
|
||||
# First stage
|
||||
domain = get_fld(url)
|
||||
parsed_url = urlparse(url)
|
||||
|
||||
if domain in ["12ft.io", "google.com", "facebook.com", "googleusercontent.com"]:
|
||||
return True
|
||||
|
||||
if domain in NOT_MEDIUM_DOMAINS:
|
||||
raise exceptions.NotValidMediumURL("100% not valid Medium URL")
|
||||
|
||||
if domain in KNOWN_MEDIUM_DOMAINS or parsed_url.netloc in KNOWN_MEDIUM_NETLOC:
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"url '{url}' wasn't detected in known medium domains")
|
||||
|
||||
# Second stage
|
||||
async def is_valid_medium_url_old(url: str, timeout: int = 5):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
retry_client = RetryClient(client_session=session, raise_for_status=False, retry_options=retry_options)
|
||||
|
||||
|
|
@ -237,3 +354,47 @@ async def is_valid_medium_url(url: str, timeout: int = 5) -> bool:
|
|||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
@lru_cache(maxsize=500)
|
||||
def get_fld(url: str):
|
||||
try:
|
||||
fld = tld.get_fld(url)
|
||||
except Exception as ex:
|
||||
logger.trace(ex)
|
||||
return None
|
||||
else:
|
||||
return fld
|
||||
|
||||
|
||||
@alru_cache(maxsize=100)
|
||||
async def is_valid_medium_url(url: str) -> bool:
|
||||
"""
|
||||
Check if the url is a valid Medium article page
|
||||
|
||||
Check if the domain is in the known Medium domains and subdomains list. If the doman/subdomain is in the list, then the url is valid
|
||||
"""
|
||||
domain = get_fld(url)
|
||||
parsed_url = urlparse(url)
|
||||
|
||||
# TODO: http://freedium.cfd/https://www.google.com.vn/url?sa=i&url=https%3A%2F%2Fmedium.com%2F%40dugguRK%2Fabout-android-hardware-abstraction-layer-hal-5d191dafeb2c&psig=AOvVaw17KP0U_haPMmhAByeMTxSg&ust=1711354113283000&source=images&cd=vfe&opi=89978449&ved=0CBQQjhxqFwoTCMCM_oG5jIUDFQAAAAAdAAAAABAa
|
||||
|
||||
if domain in ["12ft.io", "google.com", "facebook.com", "googleusercontent.com"]:
|
||||
return True
|
||||
|
||||
if domain in NOT_MEDIUM_DOMAINS or parsed_url.netloc in NOT_MEDIUM_DOMAINS:
|
||||
raise exceptions.NotValidMediumURL("100% not valid Medium URL")
|
||||
|
||||
if domain in KNOWN_MEDIUM_DOMAINS or parsed_url.netloc in KNOWN_MEDIUM_CUSTOM_DOMAINS:
|
||||
return True
|
||||
|
||||
logger.warning(f"url '{url}' wasn't detected in known Medium domains")
|
||||
|
||||
# XXX: Unfourtunately, for now we don't know ALL Medium's domains, so we need resolve links
|
||||
resolve_result = bool(await resolve_medium_url(url))
|
||||
|
||||
# send_message(f"We found that {domain=}, {parsed_url.netloc=} is not listed in out known Medium database.\nURL: {url}")
|
||||
|
||||
return resolve_result
|
||||
|
||||
# return False
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@ rl_string_helper==0.1.0
|
|||
database_lib==0.1.0
|
||||
|
||||
loguru==0.6.0
|
||||
aiohttp==3.8.5
|
||||
aiohttp==3.9.4
|
||||
aiohttp-retry==2.8.3
|
||||
tld==0.13
|
||||
bs4==0.0.1
|
||||
Jinja2==3.1.2
|
||||
beautifulsoup4==4.12.2
|
||||
async-lru==2.0.4
|
||||
|
|
@ -31,7 +31,27 @@ services:
|
|||
dockerfile: ./Dockerfile
|
||||
command: python3 -m server server
|
||||
volumes:
|
||||
# - ./.env:/app/.env
|
||||
# - ./server/user_data/logs:/app/server/user_data/logs
|
||||
- .:/app
|
||||
develop:
|
||||
watch:
|
||||
- action: rebuild
|
||||
path: ./server
|
||||
target: /app/server
|
||||
- action: rebuild
|
||||
path: ./core
|
||||
target: /app/core
|
||||
- action: rebuild
|
||||
path: ./rl_string_helper
|
||||
target: /app/rl_string_helper
|
||||
- action: rebuild
|
||||
path: ./database-lib
|
||||
target: /app/database-lib
|
||||
- action: rebuild
|
||||
path: "**/requirements.txt"
|
||||
- action: rebuild
|
||||
path: "**/requirements-fast.txt"
|
||||
expose:
|
||||
- 7080
|
||||
networks:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import datetime as dt
|
||||
from loguru import logger
|
||||
import pickledb
|
||||
from multiprocessing import Value
|
||||
import logging
|
||||
|
|
@ -9,10 +10,15 @@ import redis.asyncio as redis
|
|||
from xkcdpass import xkcd_password as xp
|
||||
|
||||
from server.utils.loguru_handler import InterceptHandler
|
||||
from database_lib import SQLiteCacheBackend
|
||||
|
||||
medium_cache = SQLiteCacheBackend('medium_db_cache.sqlite')
|
||||
medium_cache.init_db()
|
||||
medium_cache.enable_zstd()
|
||||
logger.debug(f"Database length: {medium_cache.all_length()}")
|
||||
|
||||
redis_storage = redis.Redis(host="dragonfly", port=6379, db=0)
|
||||
|
||||
|
||||
logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True)
|
||||
|
||||
url_correlation: ContextVar[Optional[str]] = ContextVar("url_correlation", default="UNKNOWN_URL")
|
||||
|
|
|
|||
|
|
@ -16,4 +16,4 @@ REQUEST_TIMEOUT = config("REQUEST_TIMEOUT", cast=int, default=40)
|
|||
WORKER_TIMEOUT = config("WORKER_TIMEOUT", cast=int, default=120)
|
||||
SENTRY_SDK_DSN = config("SENTRY_SDK_DSN", default=None)
|
||||
ENABLE_ADS_BANNER = config("ENABLE_ADS_BANNER", cast=bool, default=False)
|
||||
CACHE_LIFE_TIME = config("CACHE_LIFE_TIME", cast=int, default=60 * 60 * 24)
|
||||
CACHE_LIFE_TIME = config("CACHE_LIFE_TIME", cast=int, default=60 * 60 * 5)
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
from html5lib.html5parser import parse
|
||||
from html5lib import serialize
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
from html5lib import serialize
|
||||
from html5lib.html5parser import parse
|
||||
from loguru import logger
|
||||
|
||||
from server import config
|
||||
from server.services.jinja import base_template, main_template
|
||||
from server.handlers.misc import delete_from_cache, report_problem
|
||||
from server.handlers.post import render_medium_post_link, render_postleter
|
||||
from server.handlers.reverse_proxy import miro_proxy, iframe_proxy
|
||||
from server.handlers.misc import report_problem, delete_from_cache
|
||||
from server.handlers.reverse_proxy import iframe_proxy, miro_proxy
|
||||
from server.services.jinja import base_template, main_template
|
||||
from server.utils.logger_trace import trace
|
||||
|
||||
|
||||
|
|
@ -16,31 +16,29 @@ from server.utils.logger_trace import trace
|
|||
async def route_processing(path: str, request: Request):
|
||||
if not path:
|
||||
return await main_page()
|
||||
if request.scope.get("query_string"):
|
||||
path = request.url.path + "?" + request.scope["query_string"].decode()
|
||||
else:
|
||||
path = request.url.path
|
||||
path = path.removeprefix("/")
|
||||
|
||||
if path.startswith("render-no-cache/"):
|
||||
query_params = request.query_params
|
||||
redis = not "no-redis" in query_params
|
||||
db_cache = not "no-db-cache" in query_params
|
||||
|
||||
logger.trace(f"no_cache: {db_cache}, no_redis: {redis}")
|
||||
|
||||
path = request.url.path.removeprefix("/")
|
||||
|
||||
if not db_cache or not redis:
|
||||
key_data = request.headers.get("ADMIN_SECRET_KEY")
|
||||
|
||||
if key_data != config.ADMIN_SECRET_KEY:
|
||||
return JSONResponse({"message": f"Wrong secret key: {key_data}"}, status_code=403)
|
||||
|
||||
path = path.removeprefix("render-no-cache/")
|
||||
if path.startswith("/no-redis/"):
|
||||
path = path.removeprefix("/no-redis/")
|
||||
return await render_medium_post_link(path, True, False)
|
||||
return await render_medium_post_link(path, False)
|
||||
elif path.startswith("@miro/"):
|
||||
if path.startswith("@miro/"):
|
||||
miro_data = path.removeprefix("@miro/")
|
||||
return await miro_proxy(miro_data)
|
||||
elif path.startswith("render_iframe/"):
|
||||
iframe_id = path.removeprefix("render_iframe/")
|
||||
return await iframe_proxy(iframe_id)
|
||||
|
||||
return await render_medium_post_link(path)
|
||||
return await render_medium_post_link(path, db_cache, redis)
|
||||
|
||||
|
||||
@trace
|
||||
|
|
|
|||
|
|
@ -1,23 +1,21 @@
|
|||
from fastapi.responses import HTMLResponse
|
||||
|
||||
import asyncio
|
||||
import pickle
|
||||
from html5lib.html5parser import parse
|
||||
from html5lib import serialize
|
||||
from loguru import logger
|
||||
|
||||
from server import config, url_correlation, redis_storage, home_page_process, transponder_code_correlation
|
||||
from fastapi.responses import HTMLResponse
|
||||
from html5lib import serialize
|
||||
from html5lib.html5parser import parse
|
||||
from loguru import logger
|
||||
from medium_parser import medium_parser_exceptions
|
||||
from medium_parser.core import MediumParser
|
||||
|
||||
from server import config, home_page_process, medium_cache, redis_storage, transponder_code_correlation
|
||||
from server.services.jinja import base_template, postleter_template
|
||||
from server.utils.error import generate_error
|
||||
from server.utils.cache import aio_redis_cache
|
||||
from server.utils.exceptions import handle_exception
|
||||
from server.utils.logger_trace import trace
|
||||
from server.utils.notify import send_message
|
||||
from server.utils.cache import aio_redis_cache
|
||||
from server.utils.utils import correct_url, safe_check_redis_connection
|
||||
from server.utils.exceptions import handle_exception
|
||||
from server.utils.utils import safe_check_redis_connection
|
||||
|
||||
from medium_parser import medium_parser_exceptions
|
||||
from medium_parser import cache as medium_cache
|
||||
from medium_parser.core import MediumParser
|
||||
from medium_parser.utils import is_valid_medium_post_id_hexadecimal
|
||||
|
||||
@trace
|
||||
@aio_redis_cache(10 * 60)
|
||||
|
|
@ -26,14 +24,21 @@ async def render_postleter(limit: int = 30, as_html: bool = False):
|
|||
home_page_process[transponder_code_correlation.get()] = random_post_id_list
|
||||
|
||||
outlet_posts_list = []
|
||||
tasks = []
|
||||
for post_id in random_post_id_list:
|
||||
try:
|
||||
post = MediumParser(post_id, timeout=3, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES)
|
||||
await post.query(force_cache=True, retry=1)
|
||||
post_metadata = await post.generate_metadata(as_dict=True)
|
||||
outlet_posts_list.append(post_metadata)
|
||||
except Exception as ex:
|
||||
await handle_exception(ex, message=f"Couldn't render post_id for postleter: {post_id}")
|
||||
async def fetch_post_metadata(post_id):
|
||||
try:
|
||||
post = MediumParser(post_id, cache=medium_cache, timeout=3, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES)
|
||||
await post.query(force_cache=True, retry=1)
|
||||
post_metadata = await post.generate_metadata(as_dict=True)
|
||||
outlet_posts_list.append(post_metadata)
|
||||
except Exception as ex:
|
||||
await handle_exception(ex, message=f"Couldn't render post_id for postleter: {post_id}")
|
||||
|
||||
task = fetch_post_metadata(post_id)
|
||||
tasks.append(task)
|
||||
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
postleter_template_rendered = await postleter_template.render_async(post_list=outlet_posts_list)
|
||||
if as_html:
|
||||
|
|
@ -44,25 +49,31 @@ async def render_postleter(limit: int = 30, as_html: bool = False):
|
|||
@trace
|
||||
async def render_medium_post_link(path: str, use_cache: bool = True, use_redis: bool = True):
|
||||
redis_available = await safe_check_redis_connection(redis_storage)
|
||||
logger.debug("Redis available: {}", redis_available)
|
||||
|
||||
try:
|
||||
if is_valid_medium_post_id_hexadecimal(path):
|
||||
medium_parser = MediumParser(path, timeout=config.TIMEOUT, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES)
|
||||
else:
|
||||
url = correct_url(path)
|
||||
medium_parser = await MediumParser.from_url(url, timeout=config.TIMEOUT, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES)
|
||||
medium_post_id = medium_parser.post_id
|
||||
medium_parser = await MediumParser.from_unknown(path, cache=medium_cache, timeout=config.TIMEOUT, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES)
|
||||
logger.debug("MediumParser initialized for path: {}", path)
|
||||
redis_result = None
|
||||
if redis_available and use_cache and use_redis:
|
||||
redis_result = await redis_storage.get(medium_post_id)
|
||||
else:
|
||||
redis_result = None
|
||||
redis_result = await redis_storage.get(medium_parser.post_id)
|
||||
logger.debug("Redis cache hit for post_id: {}", medium_parser.post_id)
|
||||
|
||||
if not redis_result:
|
||||
logger.debug("No cache found, querying MediumParser")
|
||||
await medium_parser.query(use_cache=use_cache)
|
||||
rendered_medium_post = await medium_parser.render_as_html("server/templates")
|
||||
logger.debug("Rendered Medium post from HTML template")
|
||||
if redis_available and use_redis:
|
||||
await redis_storage.setex(medium_parser.post_id, config.CACHE_LIFE_TIME, pickle.dumps(rendered_medium_post))
|
||||
logger.debug("Stored rendered post in Redis cache")
|
||||
else:
|
||||
rendered_medium_post = pickle.loads(redis_result)
|
||||
logger.debug("Loaded rendered post from Redis cache")
|
||||
|
||||
except medium_parser_exceptions.InvalidURL as ex:
|
||||
return await handle_exception(ex,
|
||||
return await handle_exception(
|
||||
ex,
|
||||
"Unable to identify the Medium article URL.",
|
||||
status_code=404,
|
||||
)
|
||||
|
|
@ -87,13 +98,7 @@ async def render_medium_post_link(path: str, use_cache: bool = True, use_redis:
|
|||
}
|
||||
rendered_post = await base_template.render_async(base_context, HOST_ADDRESS=config.HOST_ADDRESS)
|
||||
parsed_rendered_post = parse(rendered_post)
|
||||
serialized_rendered_post = serialize(parsed_rendered_post, encoding='utf-8')
|
||||
|
||||
if not redis_result:
|
||||
if not redis_available:
|
||||
send_message("ERROR: Redis is not available. Please check your configuration.")
|
||||
elif use_redis:
|
||||
await redis_storage.setex(medium_post_id, config.CACHE_LIFE_TIME, pickle.dumps(rendered_medium_post))
|
||||
send_message(f"✅ Successfully rendered post: {url_correlation.get()}", True, "GOOD")
|
||||
serialized_rendered_post = serialize(parsed_rendered_post, encoding="utf-8")
|
||||
|
||||
send_message(f"✅ Successfully rendered post: {path}", True, "GOOD")
|
||||
return HTMLResponse(serialized_rendered_post)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from datetime import datetime
|
|||
|
||||
from server import maintenance_mode
|
||||
from server.utils.notify import send_message
|
||||
from medium_parser import cache as medium_cache
|
||||
from server import medium_cache
|
||||
|
||||
from time import sleep
|
||||
from loguru import logger
|
||||
|
|
|
|||
|
|
@ -1,26 +1,8 @@
|
|||
import random
|
||||
import re
|
||||
import socket
|
||||
|
||||
from server.utils.logger_trace import trace
|
||||
|
||||
DEFAULT_PROTOCOL = "https://"
|
||||
|
||||
|
||||
@trace
|
||||
def correct_url(url: str) -> str:
|
||||
# Workaround for Safari bug
|
||||
url = re.sub(r"https?://?", DEFAULT_PROTOCOL, url)
|
||||
|
||||
# parsed_url = urlparse(url)
|
||||
# if not bool(parsed_url.netloc and parsed_url.scheme):
|
||||
# return DEFAULT_PROTOCOL + url
|
||||
|
||||
# if not re.match(r'http[s]?://', url):
|
||||
# url = DEFAULT_PROTOCOL + url
|
||||
|
||||
return url
|
||||
|
||||
|
||||
def string_to_number_ascii(input_str: str, key_number: int = None):
|
||||
if not key_number:
|
||||
|
|
|
|||
4
test.py
4
test.py
|
|
@ -1,6 +1,6 @@
|
|||
import requests
|
||||
|
||||
server_url = input("Enter your Freedium instance server URL: ")
|
||||
server_url = input("Enter your Freedium instance server URL (for example: http://localhost:6752/ ): ")
|
||||
|
||||
# List of some problematic Medium posts
|
||||
url_for_test = {
|
||||
|
|
@ -13,7 +13,7 @@ url_for_test = {
|
|||
"https://valeman.medium.com/python-vs-r-for-time-series-forecasting-395390432598",
|
||||
"https://medium.com/@aleb/how-to-generate-random-user-agents-with-an-api-22aad3d232cb",
|
||||
"https://medium.com/angular-in-depth/the-best-way-to-unsubscribe-rxjs-observable-in-the-angular-applications-d8f9aa42f6a0",
|
||||
"515dd5a43948",
|
||||
"515dd5a43948", # full address: https://medium.com/macoclock/12-macos-apps-so-good-you-will-wonder-how-they-are-free-515dd5a43948
|
||||
"https://anudeep-vysyaraju.medium.com/how-any-gitamite-can-get-free-linkedin-premium-membership-d4222bd1a0b3" # <--- Check for non properly aligned emojies
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue