diff --git a/core/medium_parser/utils.py b/core/medium_parser/utils.py index a5fc17b..755b79b 100644 --- a/core/medium_parser/utils.py +++ b/core/medium_parser/utils.py @@ -17,9 +17,8 @@ from bs4 import BeautifulSoup VALID_ID_CHARS = set(string.ascii_letters + string.digits) -KNOWN_MEDIUM_NETLOC = ("javascript.plainenglish.io", "python.plainenglish.io", "levelup.gitconnected.com") -KNOWN_MEDIUM_DOMAINS = ("medium.com", "uxplanet.org", "towardsdatascience.com", "itnext.io", "eand.co", "betterprogramming.pub", "curiouse.co", "betterhumans.pub", "uxdesign.cc") - +KNOWN_MEDIUM_NETLOC = ("javascript.plainenglish.io", "python.plainenglish.io", "levelup.gitconnected.com", "betterhumans.coach.me") +KNOWN_MEDIUM_DOMAINS = ("medium.com", "uxplanet.org", "towardsdatascience.com", "itnext.io", "eand.co", "betterprogramming.pub", "curiouse.co", "betterhumans.pub", "uxdesign.cc", "thebolditalic.com", "arcdigital.media", "codeburst.io", "psiloveyou.xyz", "writingcooperative.com", "entrepreneurshandbook.co", "prototypr.io", "theascent.pub") NOT_MEDIUM_DOMAINS = ("github.com", "yandex.ru", "yandex.kz", "youtube.com", "nytimes.com", "wsj.com", "reddit.com", "elpais.com", "forbes.com", "bloomberg.com") @@ -61,22 +60,22 @@ def get_unix_ms() -> int: def unquerify_url(url): - """ - Sanitizes a URL by removing all query parameters. - - Args: - url: The URL to sanitize. - - Returns: - A sanitized URL. - """ - - parsed_url = urllib.parse.urlparse(url) - query = parsed_url.query - if query: - parsed_url = parsed_url._replace(query='') - sanitized_url = urllib.parse.urlunparse(parsed_url) - return sanitized_url.removesuffix("/") + """ + Sanitizes a URL by removing all query parameters. + + Args: + url: The URL to sanitize. + + Returns: + A sanitized URL. + """ + + parsed_url = urllib.parse.urlparse(url) + query = parsed_url.query + if query: + parsed_url = parsed_url._replace(query='') + sanitized_url = urllib.parse.urlunparse(parsed_url) + return sanitized_url.removesuffix("/") def sanitize_url(url): @@ -85,21 +84,21 @@ def sanitize_url(url): def is_valid_medium_post_id_hexadecimal(hex_string: str) -> bool: - # Check if the string is a valid hexadecimal string - # isalnum() - for char in hex_string: - if char not in VALID_ID_CHARS: - return False - - # Check if the string contains only lowercase hexadecimal characters - # if not hex_string.islower(): - # return False - - # Check if the length of the string is correct for a hexadecimal string (e.g., 10, 11 or 12 characters) - if len(hex_string) not in range(8, 13): + # Check if the string is a valid hexadecimal string + # isalnum() + for char in hex_string: + if char not in VALID_ID_CHARS: return False - return True + # Check if the string contains only lowercase hexadecimal characters + # if not hex_string.islower(): + # return False + + # Check if the length of the string is correct for a hexadecimal string (e.g., 10, 11 or 12 characters) + if len(hex_string) not in range(8, 13): + return False + + return True async def resolve_medium_short_link_v1(short_url_id: str, timeout: int = 5) -> str: diff --git a/core/tests/example_test.py b/core/tests/example_test.py index 620c516..6c23460 100644 --- a/core/tests/example_test.py +++ b/core/tests/example_test.py @@ -23,7 +23,7 @@ async def main(): logger.add(sys.stderr, level="TRACE") # dl = await MediumParser.from_url("") - dl = MediumParser("3d8e0ba02d10", 8, "localhost") + dl = MediumParser("1059aeab90d", 8, "localhost") query_result = await dl.query(use_cache=False) with open("query_result.json", "w") as f: diff --git a/core/templates/post.html b/core/tests/templates/post.html similarity index 100% rename from core/templates/post.html rename to core/tests/templates/post.html diff --git a/server/config.py b/server/config.py index cb3729b..3089bfb 100644 --- a/server/config.py +++ b/server/config.py @@ -11,7 +11,7 @@ LOG_LEVEL_NAME = config("LOG_LEVEL_NAME", default="INFO") MORE_LOGS = config("MORE_LOGS", cast=bool, default=False) DISABLE_EXTERNAL_DOCS = config("DISABLE_EXTERNAL_DOCS", cast=bool, default=True) DISABLE_RATE_LIMITER = config("DISABLE_RATE_LIMITER", cast=bool, default=True) -TIMEOUT = config("TIMEOUT", cast=int, default=15) +TIMEOUT = config("TIMEOUT", cast=int, default=25) REQUEST_TIMEOUT = config("REQUEST_TIMEOUT", cast=int, default=40) WORKER_TIMEOUT = config("WORKER_TIMEOUT", cast=int, default=120) SENTRY_SDK_DSN = config("SENTRY_SDK_DSN", default=None) diff --git a/server/middlewares/__init__.py b/server/middlewares/__init__.py index 85cef45..79de8ff 100644 --- a/server/middlewares/__init__.py +++ b/server/middlewares/__init__.py @@ -1,10 +1,10 @@ from starlette.middleware.cors import CORSMiddleware from server.middlewares.logger import LoggerMiddleware +origins = ["*"] def register_middlewares(app): app.add_middleware(LoggerMiddleware) - origins = ["*"] app.add_middleware( CORSMiddleware, allow_origins=origins,