diff --git a/core/medium_parser/core.py b/core/medium_parser/core.py
index f5e058d..a348471 100644
--- a/core/medium_parser/core.py
+++ b/core/medium_parser/core.py
@@ -3,10 +3,11 @@ import math
import textwrap
import typing
import urllib.parse
-from contextlib import suppress
+from typing import Optional
import jinja2
import tld
+from asyncer import asyncify
from loguru import logger
from rl_string_helper import RLStringHelper, parse_markups, split_overlapping_ranges
@@ -16,111 +17,99 @@ from .exceptions import InvalidMediumPostID, InvalidMediumPostURL, InvalidURL, M
from .medium_api import query_post_by_id
from .models.html_result import HtmlResult
from .time import convert_datetime_to_human_readable
-from .utils import correct_url, getting_percontage_of_match, is_has_valid_medium_post_id, is_valid_medium_url, is_valid_url, resolve_medium_url, extract_hex_string
+from .utils import correct_url, extract_hex_string, getting_percontage_of_match, is_has_valid_medium_post_id, is_valid_medium_url, is_valid_url, resolve_medium_url
if typing.TYPE_CHECKING:
from database_lib import SQLiteCacheBackend
class MediumParser:
- __slots__ = ("__post_id", "auth_cookies", "cache", "host_address", "jinja", "post_data", "timeout")
+ __slots__ = ("cache", "host_address", "jinja_template", "post_template", "timeout", "auth_cookies")
- def __init__(self, post_id: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None):
+ def __init__(self, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: Optional[str] = None, template_folder: str = "./templates"):
self.timeout = timeout
self.cache = cache
self.host_address = host_address
- self.post_id = post_id
- self.post_data = None
self.auth_cookies = auth_cookies
+ self.jinja_template = jinja2.Environment(loader=jinja2.FileSystemLoader(template_folder))
+ self.post_template = self.jinja_template.get_template("post.html")
- @classmethod
- async def from_unknown(cls, unknown: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser":
- logger.debug(f"We got some unknown data: {unknown=}, with {cache=}, {timeout=}, {host_address=}, {auth_cookies=}. Trying resolve them...///")
+ async def resolve(self, unknown: str) -> str:
+ logger.debug(f"We got some unknown data: {unknown=}. Trying resolve them...///")
if is_has_valid_medium_post_id(unknown):
logger.debug("Seems like it's valid post_id")
- return cls(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies)
+ return extract_hex_string(unknown)
logger.debug("...maybe it's URL. Let's checkout...")
- return await cls.from_url(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies)
+ post_id = await self.resolve_url(unknown)
+ return post_id
- @classmethod
- async def from_url(cls, url: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser":
+ async def resolve_url(self, url: str) -> str:
sanitized_url = correct_url(url)
if not is_valid_url(url) or not await is_valid_medium_url(sanitized_url):
raise InvalidURL(f"Invalid Medium URL: {sanitized_url}")
- post_id = await resolve_medium_url(sanitized_url, timeout)
+ post_id = await resolve_medium_url(sanitized_url, self.timeout)
if not post_id:
raise InvalidMediumPostURL(f"Could not find Medium post ID for URL: {sanitized_url}")
- return cls(post_id, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies)
-
- @property
- def post_id(self):
- return self.__post_id
-
- @post_id.setter
- def post_id(self, value):
- if not is_has_valid_medium_post_id(value):
- raise InvalidMediumPostID(f"Invalid medium post ID: {value}")
-
- self.__post_id = extract_hex_string(value)
-
- @post_id.getter
- def post_id(self):
- return self.__post_id
-
- async def delete_from_cache(self, post_id: str = None):
- if not post_id:
- post_id = self.post_id
+ return post_id
+ async def delete_from_cache(self, post_id: str):
self.cache.delete(post_id)
-
return True
- async def get_post_data_from_cache(self):
+ async def get_post_data_from_cache(self, post_id: str):
async def _get_from_cache():
logger.debug("Using cache backend")
- post_data = self.cache.pull(self.post_id)
+ post_data = self.cache.pull(post_id)
if post_data:
logger.debug("post query was found on cache")
return post_data.json()
- logger.debug(f"No data found in cache by {self.post_id}")
+ logger.debug(f"No data found in cache by {post_id}")
return None
- with suppress(Exception):
- return await asyncio.wait_for(_get_from_cache(), timeout=3)
+ try:
+ return await asyncio.wait_for(_get_from_cache(), timeout=self.timeout)
+ except asyncio.TimeoutError:
+ logger.debug("Timeout while waiting for cache")
+ return None
+ except Exception as e:
+ logger.error(f"Error while waiting for cache: {e}")
+ return None
- return None
-
- async def get_post_data_from_api(self):
+ async def get_post_data_from_api(self, post_id: str):
async def _get_from_api():
logger.debug("Using API to gather post data")
try:
- return await query_post_by_id(self.post_id, self.timeout, self.auth_cookies)
+ return await query_post_by_id(post_id, self.timeout, self.auth_cookies)
except Exception as ex:
logger.debug("Error while querying post data from Medium API")
logger.exception(ex)
return None
- with suppress(Exception):
- return await asyncio.wait_for(_get_from_api(), timeout=self.timeout + 1)
+ try:
+ return await asyncio.wait_for(_get_from_api(), timeout=self.timeout)
+ except asyncio.TimeoutError:
+ logger.debug("Timeout while waiting for cache")
+ return None
+ except Exception as e:
+ logger.error(f"Error while waiting for cache: {e}")
+ return None
- return None
-
- async def query_get(self, use_cache: bool, force_cache: bool = False):
+ async def query_get(self, post_id: str, use_cache: bool, force_cache: bool = False):
cache_used = True
- post_data = await self.get_post_data_from_cache() if use_cache else None
+ post_data = await self.get_post_data_from_cache(post_id) if use_cache else None
if not post_data and not force_cache:
logger.debug("Getting value from cache failed, using API")
cache_used = False
- post_data = await self.get_post_data_from_api()
+ post_data = await self.get_post_data_from_api(post_id)
return post_data, cache_used
- async def query(self, use_cache: bool = True, retry: int = 2, force_cache: bool = False):
+ async def query(self, post_id: str, use_cache: bool = True, retry: int = 2, force_cache: bool = False):
logger.debug(f"Medium QUERY: {use_cache=}, {retry=}, {force_cache=}")
post_data, is_cache_used = None, False
@@ -129,7 +118,7 @@ class MediumParser:
reason = None
while not post_data and attempt < retry:
try:
- post_data, is_cache_used = await self.query_get(use_cache, force_cache)
+ post_data, is_cache_used = await self.query_get(post_id, use_cache, force_cache)
if not post_data:
reason = "No post data returned"
@@ -139,7 +128,7 @@ class MediumParser:
reason = f"Post data contains an error: {post_data=}"
elif not post_data.get("data"):
reason = f"Post data missing 'data' key: {post_data=}"
- elif not post_data.get("data").get("post"):
+ elif not post_data.get("data", {}).get("post"):
reason = f"Post data missing 'data.post' key: {post_data=}"
if reason is None:
@@ -149,18 +138,18 @@ class MediumParser:
logger.error(f"Attempt {attempt + 1} failed with exception: {e}")
logger.debug(f"Retrying in {2 ** attempt} seconds...")
await asyncio.sleep(2**attempt)
+ finally:
attempt += 1
else:
if not reason:
reason = "Unknown"
- raise MediumPostQueryError(f"Could not query post by ID from API: {self.post_id}. Reason: {reason}")
+ raise MediumPostQueryError(f"Could not query post by ID from API: {post_id}. Reason: {reason}")
if not is_cache_used:
logger.debug("Pushing post data to cache")
- self.cache.push(self.post_id, post_data)
+ self.cache.push(post_id, post_data)
- self.post_data = post_data
logger.trace(f"Query: done")
return post_data
@@ -217,7 +206,7 @@ class MediumParser:
current_pos += 1
continue
elif subtitle and subtitle.endswith("…") and len(paragraph["text"]) > 100:
- subtitle = None
+ subtitle = ""
elif paragraph["type"] == "IMG":
if paragraph["metadata"] and paragraph["metadata"]["id"] == preview_image_id:
logger.trace("Preview image was detected, ignore...")
@@ -446,33 +435,33 @@ class MediumParser:
return out_paragraphs, title, subtitle
- async def render_as_html(self, template_folder: str = "./templates"):
+ async def render_as_html(self, post_id: str):
+ post_data = await self.query(post_id)
try:
- result = await self._render_as_html(template_folder)
+ result = await self._render_as_html(post_data, post_id)
except Exception as ex:
raise ex
- # raise MediumParserException(ex) from ex
else:
return result
- async def generate_metadata(self, as_dict: bool = False) -> tuple:
- title = RLStringHelper(self.post_data["data"]["post"]["title"], ["minimal"]).get_text()
- subtitle = RLStringHelper(self.post_data["data"]["post"]["previewContent"]["subtitle"]).get_text()
+ async def generate_metadata(self, post_data: dict, post_id: str, as_dict: bool = False) -> tuple:
+ title = RLStringHelper(post_data["data"]["post"]["title"], ["minimal"]).get_text()
+ subtitle = RLStringHelper(post_data["data"]["post"]["previewContent"]["subtitle"]).get_text()
description = RLStringHelper(textwrap.shorten(subtitle, width=100, placeholder="...")).get_text()
- preview_image_id = self.post_data["data"]["post"]["previewImage"]["id"]
- creator = self.post_data["data"]["post"]["creator"]
- collection = self.post_data["data"]["post"]["collection"]
- url = self.post_data["data"]["post"]["mediumUrl"]
+ preview_image_id = post_data["data"]["post"]["previewImage"]["id"]
+ creator = post_data["data"]["post"]["creator"]
+ collection = post_data["data"]["post"]["collection"]
+ url = post_data["data"]["post"]["mediumUrl"]
- reading_time = math.ceil(self.post_data["data"]["post"]["readingTime"])
- free_access = "No" if self.post_data["data"]["post"]["isLocked"] else "Yes"
- updated_at = convert_datetime_to_human_readable(self.post_data["data"]["post"]["updatedAt"])
- first_published_at = convert_datetime_to_human_readable(self.post_data["data"]["post"]["firstPublishedAt"])
- tags = self.post_data["data"]["post"]["tags"]
+ reading_time = math.ceil(post_data["data"]["post"]["readingTime"])
+ free_access = "No" if post_data["data"]["post"]["isLocked"] else "Yes"
+ updated_at = convert_datetime_to_human_readable(post_data["data"]["post"]["updatedAt"])
+ first_published_at = convert_datetime_to_human_readable(post_data["data"]["post"]["firstPublishedAt"])
+ tags = post_data["data"]["post"]["tags"]
if as_dict:
return {
- "post_id": self.post_id,
+ "post_id": post_id,
"title": title,
"subtitle": subtitle,
"description": description,
@@ -489,27 +478,18 @@ class MediumParser:
return title, subtitle, description, url, creator, collection, reading_time, free_access, updated_at, first_published_at, preview_image_id, tags
- async def _render_as_html(self, template_folder: str = "./templates") -> "HtmlResult":
- if not self.post_data:
- logger.warning(f"No post data found for post ID: {self.post_id}. Querying...")
- await self.query()
-
- # Load templates once at the start
- jinja_template = jinja2.Environment(loader=jinja2.FileSystemLoader(template_folder))
- post_template = jinja_template.get_template("post.html")
-
+ async def _render_as_html(self, post_data: dict, post_id: str) -> "HtmlResult":
# Generate metadata in parallel
- metadata_task = asyncio.create_task(self.generate_metadata())
+ metadata_task = asyncio.create_task(self.generate_metadata(post_data, post_id))
# Parse and render content in parallel
- content, title, subtitle = await asyncio.to_thread(
- self._parse_and_render_content_html_post,
- self.post_data["data"]["post"]["content"],
- self.post_data["data"]["post"]["title"],
- self.post_data["data"]["post"]["previewContent"]["subtitle"],
- self.post_data["data"]["post"]["previewImage"]["id"],
- self.post_data["data"]["post"]["highlights"],
- self.post_data["data"]["post"]["tags"],
+ content, title, subtitle = await asyncify(self._parse_and_render_content_html_post)(
+ post_data["data"]["post"]["content"],
+ post_data["data"]["post"]["title"],
+ post_data["data"]["post"]["previewContent"]["subtitle"],
+ post_data["data"]["post"]["previewImage"]["id"],
+ post_data["data"]["post"]["highlights"],
+ post_data["data"]["post"]["tags"],
)
# Await metadata
@@ -535,7 +515,7 @@ class MediumParser:
"content": content,
"tags": tags,
}
- post_template_rendered = post_template.render(post_context)
+ post_template_rendered = self.post_template.render(post_context)
return HtmlResult(post_page_title_rendered, description, url, post_template_rendered)
diff --git a/core/medium_parser/utils.py b/core/medium_parser/utils.py
index 2e1d027..c018a73 100644
--- a/core/medium_parser/utils.py
+++ b/core/medium_parser/utils.py
@@ -107,9 +107,10 @@ def is_valid_url(url):
return bool(parsed_url.scheme and parsed_url.netloc)
-def getting_percontage_of_match(string: str, matched_string: str) -> int:
+def getting_percontage_of_match(string: str, matched_string: str) -> float:
if string is None or matched_string is None:
- return 0
+ return 0.0
+
return difflib.SequenceMatcher(None, string, matched_string).ratio() * 100
diff --git a/core/requirements.txt b/core/requirements.txt
index bad65f0..e3006b2 100644
--- a/core/requirements.txt
+++ b/core/requirements.txt
@@ -8,4 +8,5 @@ tld==0.13
bs4==0.0.1
Jinja2==3.1.2
beautifulsoup4==4.12.2
-async-lru==2.0.4
\ No newline at end of file
+async-lru==2.0.4
+asyncer
\ No newline at end of file
diff --git a/database-lib/database_lib/cache_db.py b/database-lib/database_lib/cache_db.py
index f32ef6e..e6ec70a 100644
--- a/database-lib/database_lib/cache_db.py
+++ b/database-lib/database_lib/cache_db.py
@@ -1,4 +1,4 @@
-import json
+import orjson as json
import time
import sqlite3
import threading
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index cbd43f4..49f908d 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -55,8 +55,8 @@ services:
# path: "**/requirements.txt"
# - action: rebuild
# path: "**/requirements-fast.txt"
- expose:
- - 7080
+ ports:
+ - "7080:7080"
networks:
- freedium_net
# healthcheck:
@@ -69,29 +69,10 @@ services:
mem_limit: 4g
stop_grace_period: 2m
- redis_service:
- image: redis:latest
- networks:
- - freedium_net
- healthcheck:
- test: ["CMD", "redis-cli", "ping"]
- interval: 30s
- start_period: 20s
- timeout: 10s
- retries: 3
- restart: always
- stop_grace_period: 2m
-
# redis_service:
- # image: 'docker.dragonflydb.io/dragonflydb/dragonfly'
- # ulimits:
- # memlock: -1
- # expose:
- # - 6379
+ # image: redis:latest
# networks:
- # - web_network
- # volumes:
- # - dragonflydata:/data
+ # - freedium_net
# healthcheck:
# test: ["CMD", "redis-cli", "ping"]
# interval: 30s
@@ -101,6 +82,26 @@ services:
# restart: always
# stop_grace_period: 2m
+ redis_service:
+ image: 'docker.dragonflydb.io/dragonflydb/dragonfly'
+ ulimits:
+ memlock: -1
+ # expose:
+ # - 6379
+ networks:
+ - freedium_net
+ volumes:
+ - dragonflydata:/data
+ healthcheck:
+ test: ["CMD", "redis-cli", "ping"]
+ interval: 30s
+ start_period: 20s
+ timeout: 10s
+ retries: 3
+ restart: always
+ stop_grace_period: 2m
+ # mem_limit: 1g
+
postgres:
image: postgres:16.3-alpine3.20
networks:
@@ -108,7 +109,8 @@ services:
ports:
- 5432:5432
volumes:
- - ./postgres:/var/lib/postgresql/data
+ - ~/apps/postgres:/var/lib/postgresql/data
+ # - ./postgres:/var/lib/postgresql/data
environment:
- POSTGRES_PASSWORD=postgres
- POSTGRES_USER=postgres
diff --git a/requirements.txt b/requirements.txt
index 14a3837..8c35fed 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,4 +9,4 @@ fastapi==0.108.0
gunicorn==21.2.0
redis[hiredis]==4.6.0
xkcdpass==1.19.3
-apscheduler==3.10.4
+apscheduler==3.10.4
\ No newline at end of file
diff --git a/rl_string_helper/rl_string_helper/string_helper.py b/rl_string_helper/rl_string_helper/string_helper.py
index 6b8b0b1..b696db6 100644
--- a/rl_string_helper/rl_string_helper/string_helper.py
+++ b/rl_string_helper/rl_string_helper/string_helper.py
@@ -1,37 +1,25 @@
from loguru import logger
-
from .logger_trace import trace
from .utils import quote_html, quote_symbol
-
from jinja2 import Environment, DebugUndefined, Template
jinja_env = Environment(undefined=DebugUndefined)
-
-# TODO: doc!
-class StringAsignmentMix:
+class StringAssignmentMix:
__slots__ = ("string", "string_list")
def __init__(self, string: str):
- if isinstance(string, str):
- self.string = string
- elif isinstance(string, StringAsignmentMix):
- self.string = string.string
- else:
- raise ValueError(f"Incorrect string type: {type(string)}")
-
+ self.string = str(string) if isinstance(string, StringAssignmentMix) else string
self.string_list = list(self.string)
def __render_string(self):
self.string = "".join(self.string_list)
def __len__(self):
- self.__render_string()
- return len(self.string)
+ return len(self.string_list)
def pop(self, key):
self.string_list.pop(key)
- # self.__render_string()
return self
def encode(self, encoding: str):
@@ -40,26 +28,20 @@ class StringAsignmentMix:
def insert(self, key: int, value):
self.string_list.insert(key, value)
- # self.__render_string()
return self
def __setitem__(self, key, value):
- logger.trace(f"Calling __setitem__ with {key=}, {value=}")
self.string_list[key] = value
return self
def __getitem__(self, key):
- logger.trace(f"Calling __getitem__ with {key=}")
- str_list_res = self.string_list[key]
- return "".join(str_list_res)
+ return "".join(self.string_list[key])
def __str__(self):
self.__render_string()
return self.string
- def __repr__(self):
- self.__render_string()
- return self.__str__()
+ __repr__ = __str__
# TODO: more clarified description
@@ -77,18 +59,19 @@ class RLStringHelper:
__slots__ = ("string", "templates", "replaces", "quote_html_type", "quote_replaces", "_default_bang_char")
def __init__(self, string: str, quote_html_type: list[str] = ["full"], _default_bang_char: str = "R"):
- self.string = StringAsignmentMix(quote_symbol(string))
+ self.string = StringAssignmentMix(quote_symbol(string))
self.templates = []
self.quote_replaces = []
self.replaces = []
self.quote_html_type = quote_html_type
self._default_bang_char = _default_bang_char
+
@trace
def pre_utf_16_bang(self, string: str, string_pos_matrix: list):
utf_16_bang_list = []
string_len_utf_16 = len(string.encode("utf-16-le")) // 2
if string_len_utf_16 == len(string):
- logger.trace("String is doesn't contain multibyte characters")
+ logger.trace("String doesn't contain multibyte characters")
return string, string_pos_matrix, utf_16_bang_list
i = 0
@@ -98,30 +81,18 @@ class RLStringHelper:
char_len = len(char.encode("utf-16-le")) // 2
if char_len == 2:
char_len_dif = char_len - 1
- logger.trace(char_len_dif)
- logger.trace(f"'{char}' char is two bytes")
char_present = self._default_bang_char * char_len_dif
- logger.trace(f"{char_present=}")
string, string_pos_matrix = self._paste_char(string, string_pos_matrix, new_i + 1, char_present)
i += 1
utf_16_bang_list.append((i, char_len_dif, i))
- elif char_len == 1:
- logger.trace(f"'{char}' char is single byte")
- else:
- logger.warning(f"{char=} looks like is multibyte: {char_len}")
- ValueError(f"Invalid char length: {char}")
-
i += 1
- logger.trace(utf_16_bang_list)
- logger.trace(string_pos_matrix)
- logger.trace(len(string))
return string, string_pos_matrix, utf_16_bang_list
def _paste_char(self, string: str, string_pos_matrix: list, pos: int, char: str):
char_len = len(char)
string_pos_matrix.insert(pos, string_pos_matrix[pos])
- for matrix_i, matrix in enumerate(string_pos_matrix[pos + 1:], pos + 1):
+ for matrix_i in range(pos + 1, len(string_pos_matrix)):
string_pos_matrix[matrix_i] += char_len
string.insert(pos, char)
return string, string_pos_matrix
@@ -129,7 +100,7 @@ class RLStringHelper:
def _delete_char(self, string: str, string_pos_matrix: list, pos: int, char_len: int, old_pos: int):
string.pop(pos)
string_pos_matrix.pop(old_pos)
- for matrix_i, matrix in enumerate(string_pos_matrix[pos:], pos):
+ for matrix_i in range(pos, len(string_pos_matrix)):
if isinstance(string_pos_matrix[matrix_i], int):
string_pos_matrix[matrix_i] -= char_len
elif isinstance(string_pos_matrix[matrix_i], tuple):
@@ -138,77 +109,47 @@ class RLStringHelper:
@trace
def post_utf_16_bang(self, string: str, string_pos_matrix: list, utf_16_bang_list: list):
- string = StringAsignmentMix(string)
-
+ string = StringAssignmentMix(string)
post_transbang = 0
for bang_pos, char_len, old_pos in utf_16_bang_list:
string, string_pos_matrix = self._delete_char(string, string_pos_matrix, bang_pos - post_transbang, char_len, old_pos - post_transbang)
post_transbang += char_len
-
- logger.trace(utf_16_bang_list)
- logger.trace(string_pos_matrix)
return string, string_pos_matrix
@trace
def set_template(self, start: int, end: int, template: str):
if not isinstance(template, Template):
template = jinja_env.from_string(template)
- lazy_template = (start, end), template
- self.templates.append(lazy_template)
- logger.trace(self.templates)
+ self.templates.append(((start, end), template))
@trace
def set_replace(self, start: int, end: int, replace_with: str):
- lazy_replace = (start, end), replace_with
- self.replaces.append(lazy_replace)
- logger.trace(self.replaces)
+ self.replaces.append(((start, end), replace_with))
+ @trace
def _render_templates(self, string: str, string_pos_matrix: list, utf_16_bang_list: list):
if not self.templates:
return string, string_pos_matrix, utf_16_bang_list
- templates = self.templates
- templates.reverse()
-
- older_text = string
+ templates = reversed(self.templates)
updated_text = string
- logger.trace(string_pos_matrix)
-
@trace
def _get_prefix_len(template_raw: Template, inner_char: str = "{"):
- prefix_len = 0
template = template_raw.render()
- for i in range(len(template)):
- if template[i] == inner_char:
- return prefix_len
- prefix_len += 1
- else:
- raise ValueError(f"Invalid template: {template}")
+ return template.find(inner_char)
@trace
def _get_suffix_len(template_raw: Template, outer_char: str = "}"):
- suffix_len = 0
template = template_raw.render()
- for i in range(len(template) - 1, -1, -1):
- if template[i] == outer_char:
- return suffix_len
- suffix_len += 1
- else:
- raise ValueError(f"Invalid template: {template}")
+ return len(template) - template.rfind(outer_char) - 1
@trace
def update_nested_positions(start, end, prefix_len, suffix_len):
- logger.trace(len(self.string) == len(string_pos_matrix))
- logger.trace(f"{len(self.string)=}")
for i in range(end, len(string_pos_matrix)):
- logger.trace(f"{i=}")
- logger.trace(f"{string_pos_matrix[i]=}")
- string_pos_matrix[i] = string_pos_matrix[i] + suffix_len + prefix_len
-
+ string_pos_matrix[i] += suffix_len + prefix_len
for i in range(start, end):
- string_pos_matrix[i] = string_pos_matrix[i] + prefix_len
-
+ string_pos_matrix[i] += prefix_len
for n in range(len(utf_16_bang_list)):
utf_16_bang = utf_16_bang_list[n]
if utf_16_bang[2] > end:
@@ -216,62 +157,24 @@ class RLStringHelper:
elif utf_16_bang[2] > start:
utf_16_bang_list[n] = (utf_16_bang[0] + prefix_len, utf_16_bang[1], utf_16_bang[2])
- logger.trace(string_pos_matrix)
- logger.trace(utf_16_bang_list)
-
- logger.trace(string_pos_matrix)
-
for (start, end), template in templates:
- logger.trace(older_text == updated_text)
- logger.trace(f"{updated_text}")
-
- logger.trace(f"{start=}, {end=}, template={str(template)}")
-
- if start >= len(string_pos_matrix):
- logger.warning("Start position is out of range. Ignore...")
+ if start >= len(string_pos_matrix) or end - 1 >= len(string_pos_matrix):
continue
- elif end - 1 >= len(string_pos_matrix):
- logger.warning("End position is out of range. Using workaround.")
- while end - 1 >= len(string_pos_matrix):
- end -= 1
-
if start == end:
- logger.trace("Start and end positions are the same")
continue
- logger.trace(f"{len(string_pos_matrix)=}")
-
- new_start, new_end = (
- string_pos_matrix[start],
- string_pos_matrix[end - 1] + 1,
- )
-
+ new_start, new_end = string_pos_matrix[start], string_pos_matrix[end - 1] + 1
if new_end < new_start:
- logger.error(f"Invalid negative range: {new_start=} {new_end=}. Ignore.....")
- # we had to ignore this error since we need to release new version
- # raise ValueError(f"Invalid negative range: {new_start=} {new_end=}")
continue
- logger.trace(f"{new_start=}, {new_end=}")
-
- logger.trace(updated_text[new_start:new_end])
-
- older_text = updated_text
- logger.trace(f"{older_text=}")
-
- context_text = template.render(text=older_text[new_start:new_end])
- logger.trace(context_text)
+ context_text = template.render(text=updated_text[new_start:new_end])
updated_text_template = jinja_env.from_string("{{ updated_text[:new_start] }}{{ context_text }}{{updated_text[new_end:]}}")
updated_text = updated_text_template.render(updated_text=updated_text, context_text=context_text, new_start=new_start, new_end=new_end)
- logger.trace(updated_text)
prefix_len = _get_prefix_len(template)
suffix_len = _get_suffix_len(template)
-
update_nested_positions(start, end, prefix_len, suffix_len)
- logger.trace(string_pos_matrix)
-
return updated_text, string_pos_matrix, utf_16_bang_list
@trace
@@ -279,24 +182,20 @@ class RLStringHelper:
if not self.replaces and not self.quote_replaces:
return string, string_pos_matrix, utf_16_bang_list
- string = StringAsignmentMix(string)
+ string = StringAssignmentMix(string)
replaces = self.replaces + self.quote_replaces
@trace
def update_positions(start: int, end: int, replace_len: int, new_start: int, new_end: int):
- pos_len = len(range(start, end))
- logger.trace(pos_len)
- pos_len_diff = replace_len - pos_len
- logger.trace(pos_len_diff)
- for pos_index, pos_matrix in enumerate(string_pos_matrix[end:], end):
- if isinstance(pos_matrix, int):
+ pos_len_diff = replace_len - (end - start)
+ for pos_index in range(end, len(string_pos_matrix)):
+ if isinstance(string_pos_matrix[pos_index], int):
string_pos_matrix[pos_index] += pos_len_diff
- elif isinstance(pos_matrix, tuple):
+ elif isinstance(string_pos_matrix[pos_index], tuple):
string_pos_matrix[pos_index] = (
string_pos_matrix[pos_index][0] + pos_len_diff,
string_pos_matrix[pos_index][1] + pos_len_diff,
)
-
if pos_len_diff != 0:
for i in range(start, end):
if isinstance(string_pos_matrix[i], int):
@@ -309,61 +208,35 @@ class RLStringHelper:
string_pos_matrix[i][0] + replace_len,
string_pos_matrix[i][1] + replace_len,
)
-
for n in range(len(utf_16_bang_list)):
utf_16_bang = utf_16_bang_list[n]
if utf_16_bang[0] > end:
utf_16_bang_list[n] = (utf_16_bang[0] + pos_len_diff, utf_16_bang[1], utf_16_bang[2])
- logger.trace(string_pos_matrix)
-
for (start, end), replace_with in replaces:
new_start, new_end = string_pos_matrix[start], string_pos_matrix[end - 1]
if isinstance(new_end, int):
new_end += 1
if isinstance(new_start, tuple) or isinstance(new_end, tuple):
- if isinstance(new_start, tuple):
- new_start_tmp = list(range(new_start[0], new_start[1] + 1))
- else:
- new_start_tmp = [new_start]
-
- if isinstance(new_end, tuple):
- new_end_tmp = list(range(new_end[0], new_end[1] + 1))
- else:
- new_end_tmp = [new_end]
-
- new_range = new_start_tmp + new_end_tmp
- logger.trace(new_range)
- new_start, new_end = min(new_range), max(new_range)
-
- logger.trace(f"{new_start=}, {new_end=}")
-
- logger.trace(string[new_start:new_end])
+ new_start = min(new_start) if isinstance(new_start, tuple) else new_start
+ new_end = max(new_end) if isinstance(new_end, tuple) else new_end
string[new_start:new_end] = replace_with
- logger.trace(string)
-
update_positions(start, end, len(replace_with), new_start, new_end)
- logger.trace(string_pos_matrix)
return string, string_pos_matrix, utf_16_bang_list
@trace
def __str__(self):
- string = StringAsignmentMix(self.string)
-
- string_pos_matrix = [pos for pos in range(len(string))]
+ string = StringAssignmentMix(self.string)
+ string_pos_matrix = list(range(len(string)))
updated_text, string_pos_matrix, utf_16_bang_list = self.pre_utf_16_bang(string, string_pos_matrix)
if self.quote_html_type:
- self.quote_replaces = []
- html_quote_replaces = quote_html(str(updated_text), self.quote_html_type)
- for html_quote in html_quote_replaces:
- self.quote_replaces.append(html_quote)
+ self.quote_replaces = list(quote_html(str(updated_text), self.quote_html_type))
if not self.templates and not self.replaces and not self.quote_replaces:
- logger.trace("No templates, no replaces, no quote_replaces")
return str(self.string)
updated_text, string_pos_matrix, utf_16_bang_list = self._render_templates(updated_text, string_pos_matrix, utf_16_bang_list)
@@ -374,37 +247,26 @@ class RLStringHelper:
def get_text(self):
return self.__str__()
-
def split_overlapping_ranges(markups, _retry_count: int = 7):
- last_fixed_markup = markups
for _ in range(len(markups) * _retry_count):
- markups = split_overlapping_range_position(markups)
- if last_fixed_markup and len(last_fixed_markup) == len(markups):
+ new_markups = split_overlapping_range_position(markups)
+ if len(new_markups) == len(markups):
break
- last_fixed_markup = markups
- return last_fixed_markup
-
+ markups = new_markups
+ return markups
def split_overlapping_range_position(positions):
if not positions:
return []
positions.sort(key=lambda x: x["start"])
- logger.debug(f"Sorted positions: {positions}")
-
result = [positions[0]]
- logger.debug(f"Initial result: {result}")
for pos in positions[1:]:
- logger.debug(f"Processing position: {pos}")
last = result[-1]
-
if pos["start"] < last["end"]:
- logger.debug("Overlap detected")
if pos["type"] != last["type"]:
- logger.debug("Different type")
if pos["end"] <= last["end"]:
- logger.debug("Case 1: Different type, ends before or at last")
result[-1] = {
"start": last["start"],
"end": pos["start"],
@@ -422,7 +284,6 @@ def split_overlapping_range_position(positions):
}
)
else:
- logger.debug("Case 2: Different type, ends after last")
result[-1] = {
"start": last["start"],
"end": pos["start"],
@@ -431,32 +292,22 @@ def split_overlapping_range_position(positions):
}
result.append(pos.copy())
else:
- logger.debug("Case 3: Same type, update end")
result[-1]["end"] = max(last["end"], pos["end"])
else:
- logger.debug("Case 4: No overlap, add new position")
result.append(pos.copy())
- logger.debug(f"Updated result: {result}")
-
- logger.debug(f"Final result: {result}")
return result
-
def raw_render(**kwargs):
for key, value in kwargs.items():
if isinstance(value, str):
kwargs[key] = f"{{% raw %}}{value}{{% endraw %}}"
return kwargs
-
def parse_markups(markups: list[str]):
- logger.trace(f"Given {markups=}")
markups_out = []
for markup in markups:
- logger.trace(f"Processing {markups=}")
- logger.trace(markup)
if markup["type"] == "A":
if markup["anchorType"] == "LINK":
template = jinja_env.from_string('{{text}}')
@@ -465,7 +316,6 @@ def parse_markups(markups: list[str]):
template = jinja_env.from_string('{{text}}')
template = template.render(userId=markup["userId"])
else:
- logger.error(f"Can't proccess 'anchorType': {markup['anchorType']}")
continue
elif markup["type"] == "STRONG":
template = "{{text}}"
@@ -474,11 +324,9 @@ def parse_markups(markups: list[str]):
elif markup["type"] == "CODE":
template = "{{text}}"
else:
- logger.error(f"Unknown markup type: {markup}")
continue
template = jinja_env.from_string(template)
-
markup["template"] = template
markups_out.append(markup)
diff --git a/server/__init__.py b/server/__init__.py
index 844c9be..ef4f46c 100644
--- a/server/__init__.py
+++ b/server/__init__.py
@@ -1,3 +1,10 @@
+import sentry_sdk
+from server import config
+
+if config.SENTRY_SDK_DSN:
+ sentry_sdk.init(dsn=config.SENTRY_SDK_DSN, traces_sample_rate=config.SENTRY_TRACES_SAMPLE_RATE, profiles_sample_rate=config.SENTRY_PROFILES_SAMPLE_RATE)
+
+
from contextvars import ContextVar
from multiprocessing import Value
from typing import Optional
@@ -10,8 +17,8 @@ from database_lib import PostgreSQLCacheBackend, migrate_to_postgres, execute_mi
from loguru import logger
from xkcdpass import xkcd_password as xp
-from server import config
from server.utils.logger import configure_logger
+from medium_parser.core import MediumParser
from server.utils.loguru_handler import InterceptHandler
@@ -38,22 +45,27 @@ configure_logger()
medium_cache = PostgreSQLCacheBackend("postgresql://postgres:postgres@postgres:5432/postgres")
medium_cache.init_db()
-
# migrate_to_postgres_thread = execute_migrate_to_postgres_in_thread("medium_db_cache.sqlite", "postgresql://postgres:postgres@postgres:5432/postgres")
-
logger.debug(f"Database length: {medium_cache.all_length()}")
-redis_storage = redis.Redis(host=config.REDIS_HOST, port=config.REDIS_PORT, db=0)
+medium_parser = MediumParser(cache=medium_cache, timeout=3, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES, template_folder="server/templates")
+
+redis_storage = redis.Redis(
+ host=config.REDIS_HOST,
+ port=config.REDIS_PORT,
+ db=0,
+ socket_timeout=config.REDIS_TIMEOUT,
+ socket_connect_timeout=config.REDIS_TIMEOUT,
+ # decode_responses=True
+)
url_correlation: ContextVar[Optional[str]] = ContextVar("url_correlation", default="UNKNOWN_URL")
transponder_code_correlation: ContextVar[Optional[str]] = ContextVar("transponder_code_correlation", default="unknown transponder location... Beep!")
-home_page_process = {}
-
-ban_db = pickledb.load('ban_post_list.db', True)
+ban_db = pickledb.load("ban_post_list.db", True)
WORDS_LIST_FILE = "xkcdpass/static/legac"
xkcd_passwd = xp.generate_wordlist(wordfile=WORDS_LIST_FILE, min_length=5, max_length=8)
-maintenance_mode = Value('b', False)
+maintenance_mode = Value("b", False)
diff --git a/server/config.py b/server/config.py
index 848add0..3f064ca 100644
--- a/server/config.py
+++ b/server/config.py
@@ -3,21 +3,33 @@ from starlette.config import Config
config = Config(".env")
HOST_ADDRESS = config("HOST_ADDRESS", default="https://freedium.cfd")
+
MEDIUM_AUTH_COOKIES = config("MEDIUM_AUTH_COOKIES", default=None)
-TELEGRAM_ADMIN_ID = config("TELEGRAM_ADMIN_ID", cast=int, default=0)
+
ADMIN_SECRET_KEY = config("ADMIN_SECRET_KEY")
+
+TELEGRAM_ADMIN_ID = config("TELEGRAM_ADMIN_ID", cast=int, default=0)
TELEGRAM_BOT_TOKEN = config("TELEGRAM_BOT_TOKEN", default=None)
+
LOG_LEVEL_NAME = config("LOG_LEVEL_NAME", default="INFO")
MORE_LOGS = config("MORE_LOGS", cast=bool, default=False)
+
DISABLE_EXTERNAL_DOCS = config("DISABLE_EXTERNAL_DOCS", cast=bool, default=True)
DISABLE_RATE_LIMITER = config("DISABLE_RATE_LIMITER", cast=bool, default=True)
+
TIMEOUT = config("TIMEOUT", cast=int, default=25)
REQUEST_TIMEOUT = config("REQUEST_TIMEOUT", cast=int, default=40)
WORKER_TIMEOUT = config("WORKER_TIMEOUT", cast=int, default=120)
-SENTRY_SDK_DSN = config("SENTRY_SDK_DSN", default=None)
-ENABLE_ADS_BANNER = config("ENABLE_ADS_BANNER", cast=bool, default=False)
+
CACHE_LIFE_TIME = config("CACHE_LIFE_TIME", cast=int, default=60 * 60 * 5)
-HOME_PAGE_MAX_POSTS = config("HOME_PAGE_MAX_POSTS", cast=int, default=30)
+
+HOME_PAGE_MAX_POSTS = config("HOME_PAGE_MAX_POSTS", cast=int, default=15)
+ENABLE_ADS_BANNER = config("ENABLE_ADS_BANNER", cast=bool, default=False)
REDIS_HOST = config("REDIS_HOST", default="redis_service")
REDIS_PORT = config("REDIS_PORT", cast=int, default=6379)
+REDIS_TIMEOUT = config("REDIS_TIMEOUT", cast=int, default=0.75)
+
+SENTRY_SDK_DSN = config("SENTRY_SDK_DSN", default=None)
+SENTRY_TRACES_SAMPLE_RATE = config("SENTRY_TRACES_SAMPLE_RATE", cast=float, default=0.2)
+SENTRY_PROFILES_SAMPLE_RATE = config("SENTRY_PROFILES_SAMPLE_RATE", cast=float, default=0.2)
diff --git a/server/handlers/misc.py b/server/handlers/misc.py
index d9e00b9..f4aed91 100644
--- a/server/handlers/misc.py
+++ b/server/handlers/misc.py
@@ -2,9 +2,7 @@ from loguru import logger
from pydantic import BaseModel
from fastapi.responses import JSONResponse
-from medium_parser.core import MediumParser
-
-from server import config, ban_db
+from server import config, ban_db, medium_parser
from server.utils.notify import send_message
from server.utils.logger_trace import trace
@@ -30,11 +28,10 @@ async def delete_from_cache(key_data: DeleteFromCache):
return JSONResponse({"message": f"Wrong secret key: {key_data.ADMIN_SECRET_KEY}"}, status_code=403)
try:
- post = MediumParser(key_data.key, timeout=config.TIMEOUT, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES)
- await post.delete_from_cache()
+ await medium_parser.delete_from_cache(key_data.key)
except Exception as ex:
logger.exception(ex)
return JSONResponse({"message": f"Couldn't delete from cache: {ex}"}, status_code=500)
else:
ban_db.set(key_data.key, 1)
- return JSONResponse({"message": "OK"}, status_code=200)
\ No newline at end of file
+ return JSONResponse({"message": "OK"}, status_code=200)
diff --git a/server/handlers/post.py b/server/handlers/post.py
index aeaf9e2..6727900 100644
--- a/server/handlers/post.py
+++ b/server/handlers/post.py
@@ -7,9 +7,8 @@ from html5lib.html5parser import parse
from async_lru import alru_cache
from loguru import logger
from medium_parser import medium_parser_exceptions
-from medium_parser.core import MediumParser
-from server import config, home_page_process, medium_cache, redis_storage, transponder_code_correlation
+from server import config, medium_cache, redis_storage, medium_parser
from server.services.jinja import base_template, homepage_template
from server.utils.cache import aio_redis_cache
from server.utils.exceptions import handle_exception
@@ -21,17 +20,16 @@ from server.utils.utils import safe_check_redis_connection
@trace
@aio_redis_cache(10 * 60)
async def render_homepage(limit: int = config.HOME_PAGE_MAX_POSTS, as_html: bool = False):
- random_post_id_list = [i[0] for i in medium_cache.random(limit)]
- home_page_process[transponder_code_correlation.get()] = random_post_id_list
+ random_post_id_list = list(set([i[0] for i in medium_cache.random(limit)]))
outlet_posts_list = []
tasks = []
for post_id in random_post_id_list:
async def fetch_post_metadata(post_id):
try:
- post = MediumParser(post_id, cache=medium_cache, timeout=3, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES)
- await post.query(force_cache=True, retry=1)
- post_metadata = await post.generate_metadata(as_dict=True)
+ logger.debug(f"Fetching post_id: {post_id}")
+ post_data = await medium_parser.query(post_id, force_cache=True, retry=1)
+ post_metadata = await medium_parser.generate_metadata(post_data, post_id, as_dict=True)
outlet_posts_list.append(post_metadata)
except Exception as ex:
await handle_exception(ex, message=f"Couldn't render post_id for postleter: {post_id}. Just ignore that")
@@ -44,29 +42,28 @@ async def render_homepage(limit: int = config.HOME_PAGE_MAX_POSTS, as_html: bool
homepage_template_rendered = homepage_template.render(post_list=outlet_posts_list)
if as_html:
return homepage_template_rendered
+
return HTMLResponse(homepage_template_rendered)
async def render_medium_post_link(path: str, use_cache: bool = True, use_redis: bool = True):
redis_available = await safe_check_redis_connection(redis_storage)
- logger.debug("Redis available: {}", redis_available)
+ logger.debug(f"Redis available: {redis_available}")
try:
- medium_parser = await MediumParser.from_unknown(path, cache=medium_cache, timeout=config.TIMEOUT, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES)
- logger.debug("MediumParser initialized for path: {}", path)
+ post_id = await medium_parser.resolve(path)
redis_result = None
if redis_available and use_cache and use_redis:
- redis_result = await redis_storage.get(medium_parser.post_id)
- logger.debug("Redis cache hit for post_id: {}", medium_parser.post_id)
+ redis_result = await redis_storage.get(post_id)
+ logger.debug(f"Redis cache hit for post_id: {post_id}")
if not redis_result:
- logger.debug("No cache found, querying MediumParser")
- await medium_parser.query(use_cache=use_cache)
- rendered_medium_post = await medium_parser.render_as_html("server/templates")
+ logger.debug(f"No cache found, querying...: {post_id}")
+ rendered_medium_post = await medium_parser.render_as_html(post_id)
logger.debug("Rendered Medium post from HTML template")
if redis_available and use_redis:
- await redis_storage.setex(medium_parser.post_id, config.CACHE_LIFE_TIME, pickle.dumps(rendered_medium_post))
- logger.debug("Stored rendered post in Redis cache")
+ await redis_storage.setex(post_id, config.CACHE_LIFE_TIME, pickle.dumps(rendered_medium_post))
+ logger.debug(f"Stored rendered post in Redis cache: {post_id}")
else:
rendered_medium_post = pickle.loads(redis_result)
logger.debug("Loaded rendered post from Redis cache")
diff --git a/server/main.py b/server/main.py
index 2effb19..983cbfc 100644
--- a/server/main.py
+++ b/server/main.py
@@ -22,9 +22,6 @@ FASTAPI_APPLICATION_CONFIG = {"title": APP_TITLE, "version": APP_VERSION}
if config.DISABLE_EXTERNAL_DOCS:
FASTAPI_APPLICATION_CONFIG.update({"openapi_url": None, "docs_url": None, "redoc_url": None})
-if config.SENTRY_SDK_DSN:
- sentry_sdk.init(dsn=config.SENTRY_SDK_DSN, traces_sample_rate=1.0)
-
async def limiter_callback(request, response, pexpire: int):
expire = ceil(pexpire / 1000)
diff --git a/server/middlewares/logger.py b/server/middlewares/logger.py
index b10f3a7..8cb72ea 100644
--- a/server/middlewares/logger.py
+++ b/server/middlewares/logger.py
@@ -8,7 +8,7 @@ from starlette.requests import Request
from starlette.responses import Response, StreamingResponse
from starlette.types import Message
-from server import transponder_code_correlation, url_correlation, xkcd_passwd, xp, config, home_page_process
+from server import transponder_code_correlation, url_correlation, xkcd_passwd, xp, config
from server.utils.anti_bot import filter_bots
from server.utils.notify import send_message
from server.utils.error import generate_error
@@ -51,7 +51,9 @@ class LoggerMiddleware(BaseHTTPMiddleware):
except Exception as ex:
exception_class = type(ex)
logger.exception(ex)
- send_message(f"Error while processing url: {url_correlation.get()}, transponder_id: {generated_id}, transponder_code: {transponder_code_correlation.get()}, error: {ex}. exception: {exception_class.__name__}. {home_page_process.get(transponder_code_correlation.get(), '')}")
+ send_message(
+ f"Error while processing url: {url_correlation.get()}, transponder_id: {generated_id}, transponder_code: {transponder_code_correlation.get()}, error: {ex}. exception: {exception_class.__name__}."
+ )
response = await generate_error()
logger.trace(response.__dict__)
diff --git a/server/utils/exceptions.py b/server/utils/exceptions.py
index a596945..8d7a52b 100644
--- a/server/utils/exceptions.py
+++ b/server/utils/exceptions.py
@@ -1,4 +1,5 @@
import sentry_sdk
+import asyncio
from loguru import logger
from server.utils.error import generate_error
@@ -6,7 +7,8 @@ from server.utils.error import generate_error
async def handle_exception(ex, message="An error occurred", status_code=500, quiet: bool = False):
logger.exception(ex)
- if not quiet:
- sentry_sdk.capture_exception(ex)
+ # TODO: optimize
+ # if not quiet:
+ # sentry_sdk.capture_exception(ex)
return await generate_error(message, status_code=status_code, quiet=quiet)