mirror of
https://codeberg.org/Freedium-cfd/web.git
synced 2026-03-11 09:04:37 +00:00
prod: optimize & refactor
This commit is contained in:
ZhymabekRoman
parent
234744283c
commit
f5333e7399
14 changed files with 203 additions and 352 deletions
|
|
@ -3,10 +3,11 @@ import math
|
|||
import textwrap
|
||||
import typing
|
||||
import urllib.parse
|
||||
from contextlib import suppress
|
||||
from typing import Optional
|
||||
|
||||
import jinja2
|
||||
import tld
|
||||
from asyncer import asyncify
|
||||
from loguru import logger
|
||||
|
||||
from rl_string_helper import RLStringHelper, parse_markups, split_overlapping_ranges
|
||||
|
|
@ -16,111 +17,99 @@ from .exceptions import InvalidMediumPostID, InvalidMediumPostURL, InvalidURL, M
|
|||
from .medium_api import query_post_by_id
|
||||
from .models.html_result import HtmlResult
|
||||
from .time import convert_datetime_to_human_readable
|
||||
from .utils import correct_url, getting_percontage_of_match, is_has_valid_medium_post_id, is_valid_medium_url, is_valid_url, resolve_medium_url, extract_hex_string
|
||||
from .utils import correct_url, extract_hex_string, getting_percontage_of_match, is_has_valid_medium_post_id, is_valid_medium_url, is_valid_url, resolve_medium_url
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from database_lib import SQLiteCacheBackend
|
||||
|
||||
|
||||
class MediumParser:
|
||||
__slots__ = ("__post_id", "auth_cookies", "cache", "host_address", "jinja", "post_data", "timeout")
|
||||
__slots__ = ("cache", "host_address", "jinja_template", "post_template", "timeout", "auth_cookies")
|
||||
|
||||
def __init__(self, post_id: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None):
|
||||
def __init__(self, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: Optional[str] = None, template_folder: str = "./templates"):
|
||||
self.timeout = timeout
|
||||
self.cache = cache
|
||||
self.host_address = host_address
|
||||
self.post_id = post_id
|
||||
self.post_data = None
|
||||
self.auth_cookies = auth_cookies
|
||||
self.jinja_template = jinja2.Environment(loader=jinja2.FileSystemLoader(template_folder))
|
||||
self.post_template = self.jinja_template.get_template("post.html")
|
||||
|
||||
@classmethod
|
||||
async def from_unknown(cls, unknown: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser":
|
||||
logger.debug(f"We got some unknown data: {unknown=}, with {cache=}, {timeout=}, {host_address=}, {auth_cookies=}. Trying resolve them...///")
|
||||
async def resolve(self, unknown: str) -> str:
|
||||
logger.debug(f"We got some unknown data: {unknown=}. Trying resolve them...///")
|
||||
|
||||
if is_has_valid_medium_post_id(unknown):
|
||||
logger.debug("Seems like it's valid post_id")
|
||||
return cls(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies)
|
||||
return extract_hex_string(unknown)
|
||||
|
||||
logger.debug("...maybe it's URL. Let's checkout...")
|
||||
return await cls.from_url(unknown, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies)
|
||||
post_id = await self.resolve_url(unknown)
|
||||
return post_id
|
||||
|
||||
@classmethod
|
||||
async def from_url(cls, url: str, cache: "SQLiteCacheBackend", timeout: int, host_address: str, auth_cookies: str = None) -> "MediumParser":
|
||||
async def resolve_url(self, url: str) -> str:
|
||||
sanitized_url = correct_url(url)
|
||||
if not is_valid_url(url) or not await is_valid_medium_url(sanitized_url):
|
||||
raise InvalidURL(f"Invalid Medium URL: {sanitized_url}")
|
||||
|
||||
post_id = await resolve_medium_url(sanitized_url, timeout)
|
||||
post_id = await resolve_medium_url(sanitized_url, self.timeout)
|
||||
if not post_id:
|
||||
raise InvalidMediumPostURL(f"Could not find Medium post ID for URL: {sanitized_url}")
|
||||
|
||||
return cls(post_id, cache=cache, timeout=timeout, host_address=host_address, auth_cookies=auth_cookies)
|
||||
|
||||
@property
|
||||
def post_id(self):
|
||||
return self.__post_id
|
||||
|
||||
@post_id.setter
|
||||
def post_id(self, value):
|
||||
if not is_has_valid_medium_post_id(value):
|
||||
raise InvalidMediumPostID(f"Invalid medium post ID: {value}")
|
||||
|
||||
self.__post_id = extract_hex_string(value)
|
||||
|
||||
@post_id.getter
|
||||
def post_id(self):
|
||||
return self.__post_id
|
||||
|
||||
async def delete_from_cache(self, post_id: str = None):
|
||||
if not post_id:
|
||||
post_id = self.post_id
|
||||
return post_id
|
||||
|
||||
async def delete_from_cache(self, post_id: str):
|
||||
self.cache.delete(post_id)
|
||||
|
||||
return True
|
||||
|
||||
async def get_post_data_from_cache(self):
|
||||
async def get_post_data_from_cache(self, post_id: str):
|
||||
async def _get_from_cache():
|
||||
logger.debug("Using cache backend")
|
||||
post_data = self.cache.pull(self.post_id)
|
||||
post_data = self.cache.pull(post_id)
|
||||
if post_data:
|
||||
logger.debug("post query was found on cache")
|
||||
return post_data.json()
|
||||
logger.debug(f"No data found in cache by {self.post_id}")
|
||||
logger.debug(f"No data found in cache by {post_id}")
|
||||
return None
|
||||
|
||||
with suppress(Exception):
|
||||
return await asyncio.wait_for(_get_from_cache(), timeout=3)
|
||||
try:
|
||||
return await asyncio.wait_for(_get_from_cache(), timeout=self.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
logger.debug("Timeout while waiting for cache")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error while waiting for cache: {e}")
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
async def get_post_data_from_api(self):
|
||||
async def get_post_data_from_api(self, post_id: str):
|
||||
async def _get_from_api():
|
||||
logger.debug("Using API to gather post data")
|
||||
try:
|
||||
return await query_post_by_id(self.post_id, self.timeout, self.auth_cookies)
|
||||
return await query_post_by_id(post_id, self.timeout, self.auth_cookies)
|
||||
except Exception as ex:
|
||||
logger.debug("Error while querying post data from Medium API")
|
||||
logger.exception(ex)
|
||||
return None
|
||||
|
||||
with suppress(Exception):
|
||||
return await asyncio.wait_for(_get_from_api(), timeout=self.timeout + 1)
|
||||
try:
|
||||
return await asyncio.wait_for(_get_from_api(), timeout=self.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
logger.debug("Timeout while waiting for cache")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error while waiting for cache: {e}")
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
async def query_get(self, use_cache: bool, force_cache: bool = False):
|
||||
async def query_get(self, post_id: str, use_cache: bool, force_cache: bool = False):
|
||||
cache_used = True
|
||||
post_data = await self.get_post_data_from_cache() if use_cache else None
|
||||
post_data = await self.get_post_data_from_cache(post_id) if use_cache else None
|
||||
|
||||
if not post_data and not force_cache:
|
||||
logger.debug("Getting value from cache failed, using API")
|
||||
cache_used = False
|
||||
post_data = await self.get_post_data_from_api()
|
||||
post_data = await self.get_post_data_from_api(post_id)
|
||||
|
||||
return post_data, cache_used
|
||||
|
||||
async def query(self, use_cache: bool = True, retry: int = 2, force_cache: bool = False):
|
||||
async def query(self, post_id: str, use_cache: bool = True, retry: int = 2, force_cache: bool = False):
|
||||
logger.debug(f"Medium QUERY: {use_cache=}, {retry=}, {force_cache=}")
|
||||
|
||||
post_data, is_cache_used = None, False
|
||||
|
|
@ -129,7 +118,7 @@ class MediumParser:
|
|||
reason = None
|
||||
while not post_data and attempt < retry:
|
||||
try:
|
||||
post_data, is_cache_used = await self.query_get(use_cache, force_cache)
|
||||
post_data, is_cache_used = await self.query_get(post_id, use_cache, force_cache)
|
||||
|
||||
if not post_data:
|
||||
reason = "No post data returned"
|
||||
|
|
@ -139,7 +128,7 @@ class MediumParser:
|
|||
reason = f"Post data contains an error: {post_data=}"
|
||||
elif not post_data.get("data"):
|
||||
reason = f"Post data missing 'data' key: {post_data=}"
|
||||
elif not post_data.get("data").get("post"):
|
||||
elif not post_data.get("data", {}).get("post"):
|
||||
reason = f"Post data missing 'data.post' key: {post_data=}"
|
||||
|
||||
if reason is None:
|
||||
|
|
@ -149,18 +138,18 @@ class MediumParser:
|
|||
logger.error(f"Attempt {attempt + 1} failed with exception: {e}")
|
||||
logger.debug(f"Retrying in {2 ** attempt} seconds...")
|
||||
await asyncio.sleep(2**attempt)
|
||||
finally:
|
||||
attempt += 1
|
||||
else:
|
||||
if not reason:
|
||||
reason = "Unknown"
|
||||
|
||||
raise MediumPostQueryError(f"Could not query post by ID from API: {self.post_id}. Reason: {reason}")
|
||||
raise MediumPostQueryError(f"Could not query post by ID from API: {post_id}. Reason: {reason}")
|
||||
|
||||
if not is_cache_used:
|
||||
logger.debug("Pushing post data to cache")
|
||||
self.cache.push(self.post_id, post_data)
|
||||
self.cache.push(post_id, post_data)
|
||||
|
||||
self.post_data = post_data
|
||||
logger.trace(f"Query: done")
|
||||
return post_data
|
||||
|
||||
|
|
@ -217,7 +206,7 @@ class MediumParser:
|
|||
current_pos += 1
|
||||
continue
|
||||
elif subtitle and subtitle.endswith("…") and len(paragraph["text"]) > 100:
|
||||
subtitle = None
|
||||
subtitle = ""
|
||||
elif paragraph["type"] == "IMG":
|
||||
if paragraph["metadata"] and paragraph["metadata"]["id"] == preview_image_id:
|
||||
logger.trace("Preview image was detected, ignore...")
|
||||
|
|
@ -446,33 +435,33 @@ class MediumParser:
|
|||
|
||||
return out_paragraphs, title, subtitle
|
||||
|
||||
async def render_as_html(self, template_folder: str = "./templates"):
|
||||
async def render_as_html(self, post_id: str):
|
||||
post_data = await self.query(post_id)
|
||||
try:
|
||||
result = await self._render_as_html(template_folder)
|
||||
result = await self._render_as_html(post_data, post_id)
|
||||
except Exception as ex:
|
||||
raise ex
|
||||
# raise MediumParserException(ex) from ex
|
||||
else:
|
||||
return result
|
||||
|
||||
async def generate_metadata(self, as_dict: bool = False) -> tuple:
|
||||
title = RLStringHelper(self.post_data["data"]["post"]["title"], ["minimal"]).get_text()
|
||||
subtitle = RLStringHelper(self.post_data["data"]["post"]["previewContent"]["subtitle"]).get_text()
|
||||
async def generate_metadata(self, post_data: dict, post_id: str, as_dict: bool = False) -> tuple:
|
||||
title = RLStringHelper(post_data["data"]["post"]["title"], ["minimal"]).get_text()
|
||||
subtitle = RLStringHelper(post_data["data"]["post"]["previewContent"]["subtitle"]).get_text()
|
||||
description = RLStringHelper(textwrap.shorten(subtitle, width=100, placeholder="...")).get_text()
|
||||
preview_image_id = self.post_data["data"]["post"]["previewImage"]["id"]
|
||||
creator = self.post_data["data"]["post"]["creator"]
|
||||
collection = self.post_data["data"]["post"]["collection"]
|
||||
url = self.post_data["data"]["post"]["mediumUrl"]
|
||||
preview_image_id = post_data["data"]["post"]["previewImage"]["id"]
|
||||
creator = post_data["data"]["post"]["creator"]
|
||||
collection = post_data["data"]["post"]["collection"]
|
||||
url = post_data["data"]["post"]["mediumUrl"]
|
||||
|
||||
reading_time = math.ceil(self.post_data["data"]["post"]["readingTime"])
|
||||
free_access = "No" if self.post_data["data"]["post"]["isLocked"] else "Yes"
|
||||
updated_at = convert_datetime_to_human_readable(self.post_data["data"]["post"]["updatedAt"])
|
||||
first_published_at = convert_datetime_to_human_readable(self.post_data["data"]["post"]["firstPublishedAt"])
|
||||
tags = self.post_data["data"]["post"]["tags"]
|
||||
reading_time = math.ceil(post_data["data"]["post"]["readingTime"])
|
||||
free_access = "No" if post_data["data"]["post"]["isLocked"] else "Yes"
|
||||
updated_at = convert_datetime_to_human_readable(post_data["data"]["post"]["updatedAt"])
|
||||
first_published_at = convert_datetime_to_human_readable(post_data["data"]["post"]["firstPublishedAt"])
|
||||
tags = post_data["data"]["post"]["tags"]
|
||||
|
||||
if as_dict:
|
||||
return {
|
||||
"post_id": self.post_id,
|
||||
"post_id": post_id,
|
||||
"title": title,
|
||||
"subtitle": subtitle,
|
||||
"description": description,
|
||||
|
|
@ -489,27 +478,18 @@ class MediumParser:
|
|||
|
||||
return title, subtitle, description, url, creator, collection, reading_time, free_access, updated_at, first_published_at, preview_image_id, tags
|
||||
|
||||
async def _render_as_html(self, template_folder: str = "./templates") -> "HtmlResult":
|
||||
if not self.post_data:
|
||||
logger.warning(f"No post data found for post ID: {self.post_id}. Querying...")
|
||||
await self.query()
|
||||
|
||||
# Load templates once at the start
|
||||
jinja_template = jinja2.Environment(loader=jinja2.FileSystemLoader(template_folder))
|
||||
post_template = jinja_template.get_template("post.html")
|
||||
|
||||
async def _render_as_html(self, post_data: dict, post_id: str) -> "HtmlResult":
|
||||
# Generate metadata in parallel
|
||||
metadata_task = asyncio.create_task(self.generate_metadata())
|
||||
metadata_task = asyncio.create_task(self.generate_metadata(post_data, post_id))
|
||||
|
||||
# Parse and render content in parallel
|
||||
content, title, subtitle = await asyncio.to_thread(
|
||||
self._parse_and_render_content_html_post,
|
||||
self.post_data["data"]["post"]["content"],
|
||||
self.post_data["data"]["post"]["title"],
|
||||
self.post_data["data"]["post"]["previewContent"]["subtitle"],
|
||||
self.post_data["data"]["post"]["previewImage"]["id"],
|
||||
self.post_data["data"]["post"]["highlights"],
|
||||
self.post_data["data"]["post"]["tags"],
|
||||
content, title, subtitle = await asyncify(self._parse_and_render_content_html_post)(
|
||||
post_data["data"]["post"]["content"],
|
||||
post_data["data"]["post"]["title"],
|
||||
post_data["data"]["post"]["previewContent"]["subtitle"],
|
||||
post_data["data"]["post"]["previewImage"]["id"],
|
||||
post_data["data"]["post"]["highlights"],
|
||||
post_data["data"]["post"]["tags"],
|
||||
)
|
||||
|
||||
# Await metadata
|
||||
|
|
@ -535,7 +515,7 @@ class MediumParser:
|
|||
"content": content,
|
||||
"tags": tags,
|
||||
}
|
||||
post_template_rendered = post_template.render(post_context)
|
||||
post_template_rendered = self.post_template.render(post_context)
|
||||
|
||||
return HtmlResult(post_page_title_rendered, description, url, post_template_rendered)
|
||||
|
||||
|
|
|
|||
|
|
@ -107,9 +107,10 @@ def is_valid_url(url):
|
|||
return bool(parsed_url.scheme and parsed_url.netloc)
|
||||
|
||||
|
||||
def getting_percontage_of_match(string: str, matched_string: str) -> int:
|
||||
def getting_percontage_of_match(string: str, matched_string: str) -> float:
|
||||
if string is None or matched_string is None:
|
||||
return 0
|
||||
return 0.0
|
||||
|
||||
return difflib.SequenceMatcher(None, string, matched_string).ratio() * 100
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,4 +8,5 @@ tld==0.13
|
|||
bs4==0.0.1
|
||||
Jinja2==3.1.2
|
||||
beautifulsoup4==4.12.2
|
||||
async-lru==2.0.4
|
||||
async-lru==2.0.4
|
||||
asyncer
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import json
|
||||
import orjson as json
|
||||
import time
|
||||
import sqlite3
|
||||
import threading
|
||||
|
|
|
|||
|
|
@ -55,8 +55,8 @@ services:
|
|||
# path: "**/requirements.txt"
|
||||
# - action: rebuild
|
||||
# path: "**/requirements-fast.txt"
|
||||
expose:
|
||||
- 7080
|
||||
ports:
|
||||
- "7080:7080"
|
||||
networks:
|
||||
- freedium_net
|
||||
# healthcheck:
|
||||
|
|
@ -69,29 +69,10 @@ services:
|
|||
mem_limit: 4g
|
||||
stop_grace_period: 2m
|
||||
|
||||
redis_service:
|
||||
image: redis:latest
|
||||
networks:
|
||||
- freedium_net
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 30s
|
||||
start_period: 20s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
restart: always
|
||||
stop_grace_period: 2m
|
||||
|
||||
# redis_service:
|
||||
# image: 'docker.dragonflydb.io/dragonflydb/dragonfly'
|
||||
# ulimits:
|
||||
# memlock: -1
|
||||
# expose:
|
||||
# - 6379
|
||||
# image: redis:latest
|
||||
# networks:
|
||||
# - web_network
|
||||
# volumes:
|
||||
# - dragonflydata:/data
|
||||
# - freedium_net
|
||||
# healthcheck:
|
||||
# test: ["CMD", "redis-cli", "ping"]
|
||||
# interval: 30s
|
||||
|
|
@ -101,6 +82,26 @@ services:
|
|||
# restart: always
|
||||
# stop_grace_period: 2m
|
||||
|
||||
redis_service:
|
||||
image: 'docker.dragonflydb.io/dragonflydb/dragonfly'
|
||||
ulimits:
|
||||
memlock: -1
|
||||
# expose:
|
||||
# - 6379
|
||||
networks:
|
||||
- freedium_net
|
||||
volumes:
|
||||
- dragonflydata:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 30s
|
||||
start_period: 20s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
restart: always
|
||||
stop_grace_period: 2m
|
||||
# mem_limit: 1g
|
||||
|
||||
postgres:
|
||||
image: postgres:16.3-alpine3.20
|
||||
networks:
|
||||
|
|
@ -108,7 +109,8 @@ services:
|
|||
ports:
|
||||
- 5432:5432
|
||||
volumes:
|
||||
- ./postgres:/var/lib/postgresql/data
|
||||
- ~/apps/postgres:/var/lib/postgresql/data
|
||||
# - ./postgres:/var/lib/postgresql/data
|
||||
environment:
|
||||
- POSTGRES_PASSWORD=postgres
|
||||
- POSTGRES_USER=postgres
|
||||
|
|
|
|||
|
|
@ -9,4 +9,4 @@ fastapi==0.108.0
|
|||
gunicorn==21.2.0
|
||||
redis[hiredis]==4.6.0
|
||||
xkcdpass==1.19.3
|
||||
apscheduler==3.10.4
|
||||
apscheduler==3.10.4
|
||||
|
|
@ -1,37 +1,25 @@
|
|||
from loguru import logger
|
||||
|
||||
from .logger_trace import trace
|
||||
from .utils import quote_html, quote_symbol
|
||||
|
||||
from jinja2 import Environment, DebugUndefined, Template
|
||||
|
||||
jinja_env = Environment(undefined=DebugUndefined)
|
||||
|
||||
|
||||
# TODO: doc!
|
||||
class StringAsignmentMix:
|
||||
class StringAssignmentMix:
|
||||
__slots__ = ("string", "string_list")
|
||||
|
||||
def __init__(self, string: str):
|
||||
if isinstance(string, str):
|
||||
self.string = string
|
||||
elif isinstance(string, StringAsignmentMix):
|
||||
self.string = string.string
|
||||
else:
|
||||
raise ValueError(f"Incorrect string type: {type(string)}")
|
||||
|
||||
self.string = str(string) if isinstance(string, StringAssignmentMix) else string
|
||||
self.string_list = list(self.string)
|
||||
|
||||
def __render_string(self):
|
||||
self.string = "".join(self.string_list)
|
||||
|
||||
def __len__(self):
|
||||
self.__render_string()
|
||||
return len(self.string)
|
||||
return len(self.string_list)
|
||||
|
||||
def pop(self, key):
|
||||
self.string_list.pop(key)
|
||||
# self.__render_string()
|
||||
return self
|
||||
|
||||
def encode(self, encoding: str):
|
||||
|
|
@ -40,26 +28,20 @@ class StringAsignmentMix:
|
|||
|
||||
def insert(self, key: int, value):
|
||||
self.string_list.insert(key, value)
|
||||
# self.__render_string()
|
||||
return self
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
logger.trace(f"Calling __setitem__ with {key=}, {value=}")
|
||||
self.string_list[key] = value
|
||||
return self
|
||||
|
||||
def __getitem__(self, key):
|
||||
logger.trace(f"Calling __getitem__ with {key=}")
|
||||
str_list_res = self.string_list[key]
|
||||
return "".join(str_list_res)
|
||||
return "".join(self.string_list[key])
|
||||
|
||||
def __str__(self):
|
||||
self.__render_string()
|
||||
return self.string
|
||||
|
||||
def __repr__(self):
|
||||
self.__render_string()
|
||||
return self.__str__()
|
||||
__repr__ = __str__
|
||||
|
||||
|
||||
# TODO: more clarified description
|
||||
|
|
@ -77,18 +59,19 @@ class RLStringHelper:
|
|||
__slots__ = ("string", "templates", "replaces", "quote_html_type", "quote_replaces", "_default_bang_char")
|
||||
|
||||
def __init__(self, string: str, quote_html_type: list[str] = ["full"], _default_bang_char: str = "R"):
|
||||
self.string = StringAsignmentMix(quote_symbol(string))
|
||||
self.string = StringAssignmentMix(quote_symbol(string))
|
||||
self.templates = []
|
||||
self.quote_replaces = []
|
||||
self.replaces = []
|
||||
self.quote_html_type = quote_html_type
|
||||
self._default_bang_char = _default_bang_char
|
||||
|
||||
@trace
|
||||
def pre_utf_16_bang(self, string: str, string_pos_matrix: list):
|
||||
utf_16_bang_list = []
|
||||
string_len_utf_16 = len(string.encode("utf-16-le")) // 2
|
||||
if string_len_utf_16 == len(string):
|
||||
logger.trace("String is doesn't contain multibyte characters")
|
||||
logger.trace("String doesn't contain multibyte characters")
|
||||
return string, string_pos_matrix, utf_16_bang_list
|
||||
|
||||
i = 0
|
||||
|
|
@ -98,30 +81,18 @@ class RLStringHelper:
|
|||
char_len = len(char.encode("utf-16-le")) // 2
|
||||
if char_len == 2:
|
||||
char_len_dif = char_len - 1
|
||||
logger.trace(char_len_dif)
|
||||
logger.trace(f"'{char}' char is two bytes")
|
||||
char_present = self._default_bang_char * char_len_dif
|
||||
logger.trace(f"{char_present=}")
|
||||
string, string_pos_matrix = self._paste_char(string, string_pos_matrix, new_i + 1, char_present)
|
||||
i += 1
|
||||
utf_16_bang_list.append((i, char_len_dif, i))
|
||||
elif char_len == 1:
|
||||
logger.trace(f"'{char}' char is single byte")
|
||||
else:
|
||||
logger.warning(f"{char=} looks like is multibyte: {char_len}")
|
||||
ValueError(f"Invalid char length: {char}")
|
||||
|
||||
i += 1
|
||||
|
||||
logger.trace(utf_16_bang_list)
|
||||
logger.trace(string_pos_matrix)
|
||||
logger.trace(len(string))
|
||||
return string, string_pos_matrix, utf_16_bang_list
|
||||
|
||||
def _paste_char(self, string: str, string_pos_matrix: list, pos: int, char: str):
|
||||
char_len = len(char)
|
||||
string_pos_matrix.insert(pos, string_pos_matrix[pos])
|
||||
for matrix_i, matrix in enumerate(string_pos_matrix[pos + 1:], pos + 1):
|
||||
for matrix_i in range(pos + 1, len(string_pos_matrix)):
|
||||
string_pos_matrix[matrix_i] += char_len
|
||||
string.insert(pos, char)
|
||||
return string, string_pos_matrix
|
||||
|
|
@ -129,7 +100,7 @@ class RLStringHelper:
|
|||
def _delete_char(self, string: str, string_pos_matrix: list, pos: int, char_len: int, old_pos: int):
|
||||
string.pop(pos)
|
||||
string_pos_matrix.pop(old_pos)
|
||||
for matrix_i, matrix in enumerate(string_pos_matrix[pos:], pos):
|
||||
for matrix_i in range(pos, len(string_pos_matrix)):
|
||||
if isinstance(string_pos_matrix[matrix_i], int):
|
||||
string_pos_matrix[matrix_i] -= char_len
|
||||
elif isinstance(string_pos_matrix[matrix_i], tuple):
|
||||
|
|
@ -138,77 +109,47 @@ class RLStringHelper:
|
|||
|
||||
@trace
|
||||
def post_utf_16_bang(self, string: str, string_pos_matrix: list, utf_16_bang_list: list):
|
||||
string = StringAsignmentMix(string)
|
||||
|
||||
string = StringAssignmentMix(string)
|
||||
post_transbang = 0
|
||||
for bang_pos, char_len, old_pos in utf_16_bang_list:
|
||||
string, string_pos_matrix = self._delete_char(string, string_pos_matrix, bang_pos - post_transbang, char_len, old_pos - post_transbang)
|
||||
post_transbang += char_len
|
||||
|
||||
logger.trace(utf_16_bang_list)
|
||||
logger.trace(string_pos_matrix)
|
||||
return string, string_pos_matrix
|
||||
|
||||
@trace
|
||||
def set_template(self, start: int, end: int, template: str):
|
||||
if not isinstance(template, Template):
|
||||
template = jinja_env.from_string(template)
|
||||
lazy_template = (start, end), template
|
||||
self.templates.append(lazy_template)
|
||||
logger.trace(self.templates)
|
||||
self.templates.append(((start, end), template))
|
||||
|
||||
@trace
|
||||
def set_replace(self, start: int, end: int, replace_with: str):
|
||||
lazy_replace = (start, end), replace_with
|
||||
self.replaces.append(lazy_replace)
|
||||
logger.trace(self.replaces)
|
||||
self.replaces.append(((start, end), replace_with))
|
||||
|
||||
@trace
|
||||
def _render_templates(self, string: str, string_pos_matrix: list, utf_16_bang_list: list):
|
||||
if not self.templates:
|
||||
return string, string_pos_matrix, utf_16_bang_list
|
||||
|
||||
templates = self.templates
|
||||
templates.reverse()
|
||||
|
||||
older_text = string
|
||||
templates = reversed(self.templates)
|
||||
updated_text = string
|
||||
|
||||
logger.trace(string_pos_matrix)
|
||||
|
||||
@trace
|
||||
def _get_prefix_len(template_raw: Template, inner_char: str = "{"):
|
||||
prefix_len = 0
|
||||
template = template_raw.render()
|
||||
for i in range(len(template)):
|
||||
if template[i] == inner_char:
|
||||
return prefix_len
|
||||
prefix_len += 1
|
||||
else:
|
||||
raise ValueError(f"Invalid template: {template}")
|
||||
return template.find(inner_char)
|
||||
|
||||
@trace
|
||||
def _get_suffix_len(template_raw: Template, outer_char: str = "}"):
|
||||
suffix_len = 0
|
||||
template = template_raw.render()
|
||||
for i in range(len(template) - 1, -1, -1):
|
||||
if template[i] == outer_char:
|
||||
return suffix_len
|
||||
suffix_len += 1
|
||||
else:
|
||||
raise ValueError(f"Invalid template: {template}")
|
||||
return len(template) - template.rfind(outer_char) - 1
|
||||
|
||||
@trace
|
||||
def update_nested_positions(start, end, prefix_len, suffix_len):
|
||||
logger.trace(len(self.string) == len(string_pos_matrix))
|
||||
logger.trace(f"{len(self.string)=}")
|
||||
for i in range(end, len(string_pos_matrix)):
|
||||
logger.trace(f"{i=}")
|
||||
logger.trace(f"{string_pos_matrix[i]=}")
|
||||
string_pos_matrix[i] = string_pos_matrix[i] + suffix_len + prefix_len
|
||||
|
||||
string_pos_matrix[i] += suffix_len + prefix_len
|
||||
for i in range(start, end):
|
||||
string_pos_matrix[i] = string_pos_matrix[i] + prefix_len
|
||||
|
||||
string_pos_matrix[i] += prefix_len
|
||||
for n in range(len(utf_16_bang_list)):
|
||||
utf_16_bang = utf_16_bang_list[n]
|
||||
if utf_16_bang[2] > end:
|
||||
|
|
@ -216,62 +157,24 @@ class RLStringHelper:
|
|||
elif utf_16_bang[2] > start:
|
||||
utf_16_bang_list[n] = (utf_16_bang[0] + prefix_len, utf_16_bang[1], utf_16_bang[2])
|
||||
|
||||
logger.trace(string_pos_matrix)
|
||||
logger.trace(utf_16_bang_list)
|
||||
|
||||
logger.trace(string_pos_matrix)
|
||||
|
||||
for (start, end), template in templates:
|
||||
logger.trace(older_text == updated_text)
|
||||
logger.trace(f"{updated_text}")
|
||||
|
||||
logger.trace(f"{start=}, {end=}, template={str(template)}")
|
||||
|
||||
if start >= len(string_pos_matrix):
|
||||
logger.warning("Start position is out of range. Ignore...")
|
||||
if start >= len(string_pos_matrix) or end - 1 >= len(string_pos_matrix):
|
||||
continue
|
||||
elif end - 1 >= len(string_pos_matrix):
|
||||
logger.warning("End position is out of range. Using workaround.")
|
||||
while end - 1 >= len(string_pos_matrix):
|
||||
end -= 1
|
||||
|
||||
if start == end:
|
||||
logger.trace("Start and end positions are the same")
|
||||
continue
|
||||
|
||||
logger.trace(f"{len(string_pos_matrix)=}")
|
||||
|
||||
new_start, new_end = (
|
||||
string_pos_matrix[start],
|
||||
string_pos_matrix[end - 1] + 1,
|
||||
)
|
||||
|
||||
new_start, new_end = string_pos_matrix[start], string_pos_matrix[end - 1] + 1
|
||||
if new_end < new_start:
|
||||
logger.error(f"Invalid negative range: {new_start=} {new_end=}. Ignore.....")
|
||||
# we had to ignore this error since we need to release new version
|
||||
# raise ValueError(f"Invalid negative range: {new_start=} {new_end=}")
|
||||
continue
|
||||
|
||||
logger.trace(f"{new_start=}, {new_end=}")
|
||||
|
||||
logger.trace(updated_text[new_start:new_end])
|
||||
|
||||
older_text = updated_text
|
||||
logger.trace(f"{older_text=}")
|
||||
|
||||
context_text = template.render(text=older_text[new_start:new_end])
|
||||
logger.trace(context_text)
|
||||
context_text = template.render(text=updated_text[new_start:new_end])
|
||||
updated_text_template = jinja_env.from_string("{{ updated_text[:new_start] }}{{ context_text }}{{updated_text[new_end:]}}")
|
||||
updated_text = updated_text_template.render(updated_text=updated_text, context_text=context_text, new_start=new_start, new_end=new_end)
|
||||
logger.trace(updated_text)
|
||||
|
||||
prefix_len = _get_prefix_len(template)
|
||||
suffix_len = _get_suffix_len(template)
|
||||
|
||||
update_nested_positions(start, end, prefix_len, suffix_len)
|
||||
|
||||
logger.trace(string_pos_matrix)
|
||||
|
||||
return updated_text, string_pos_matrix, utf_16_bang_list
|
||||
|
||||
@trace
|
||||
|
|
@ -279,24 +182,20 @@ class RLStringHelper:
|
|||
if not self.replaces and not self.quote_replaces:
|
||||
return string, string_pos_matrix, utf_16_bang_list
|
||||
|
||||
string = StringAsignmentMix(string)
|
||||
string = StringAssignmentMix(string)
|
||||
replaces = self.replaces + self.quote_replaces
|
||||
|
||||
@trace
|
||||
def update_positions(start: int, end: int, replace_len: int, new_start: int, new_end: int):
|
||||
pos_len = len(range(start, end))
|
||||
logger.trace(pos_len)
|
||||
pos_len_diff = replace_len - pos_len
|
||||
logger.trace(pos_len_diff)
|
||||
for pos_index, pos_matrix in enumerate(string_pos_matrix[end:], end):
|
||||
if isinstance(pos_matrix, int):
|
||||
pos_len_diff = replace_len - (end - start)
|
||||
for pos_index in range(end, len(string_pos_matrix)):
|
||||
if isinstance(string_pos_matrix[pos_index], int):
|
||||
string_pos_matrix[pos_index] += pos_len_diff
|
||||
elif isinstance(pos_matrix, tuple):
|
||||
elif isinstance(string_pos_matrix[pos_index], tuple):
|
||||
string_pos_matrix[pos_index] = (
|
||||
string_pos_matrix[pos_index][0] + pos_len_diff,
|
||||
string_pos_matrix[pos_index][1] + pos_len_diff,
|
||||
)
|
||||
|
||||
if pos_len_diff != 0:
|
||||
for i in range(start, end):
|
||||
if isinstance(string_pos_matrix[i], int):
|
||||
|
|
@ -309,61 +208,35 @@ class RLStringHelper:
|
|||
string_pos_matrix[i][0] + replace_len,
|
||||
string_pos_matrix[i][1] + replace_len,
|
||||
)
|
||||
|
||||
for n in range(len(utf_16_bang_list)):
|
||||
utf_16_bang = utf_16_bang_list[n]
|
||||
if utf_16_bang[0] > end:
|
||||
utf_16_bang_list[n] = (utf_16_bang[0] + pos_len_diff, utf_16_bang[1], utf_16_bang[2])
|
||||
|
||||
logger.trace(string_pos_matrix)
|
||||
|
||||
for (start, end), replace_with in replaces:
|
||||
new_start, new_end = string_pos_matrix[start], string_pos_matrix[end - 1]
|
||||
if isinstance(new_end, int):
|
||||
new_end += 1
|
||||
|
||||
if isinstance(new_start, tuple) or isinstance(new_end, tuple):
|
||||
if isinstance(new_start, tuple):
|
||||
new_start_tmp = list(range(new_start[0], new_start[1] + 1))
|
||||
else:
|
||||
new_start_tmp = [new_start]
|
||||
|
||||
if isinstance(new_end, tuple):
|
||||
new_end_tmp = list(range(new_end[0], new_end[1] + 1))
|
||||
else:
|
||||
new_end_tmp = [new_end]
|
||||
|
||||
new_range = new_start_tmp + new_end_tmp
|
||||
logger.trace(new_range)
|
||||
new_start, new_end = min(new_range), max(new_range)
|
||||
|
||||
logger.trace(f"{new_start=}, {new_end=}")
|
||||
|
||||
logger.trace(string[new_start:new_end])
|
||||
new_start = min(new_start) if isinstance(new_start, tuple) else new_start
|
||||
new_end = max(new_end) if isinstance(new_end, tuple) else new_end
|
||||
|
||||
string[new_start:new_end] = replace_with
|
||||
logger.trace(string)
|
||||
|
||||
update_positions(start, end, len(replace_with), new_start, new_end)
|
||||
logger.trace(string_pos_matrix)
|
||||
|
||||
return string, string_pos_matrix, utf_16_bang_list
|
||||
|
||||
@trace
|
||||
def __str__(self):
|
||||
string = StringAsignmentMix(self.string)
|
||||
|
||||
string_pos_matrix = [pos for pos in range(len(string))]
|
||||
string = StringAssignmentMix(self.string)
|
||||
string_pos_matrix = list(range(len(string)))
|
||||
updated_text, string_pos_matrix, utf_16_bang_list = self.pre_utf_16_bang(string, string_pos_matrix)
|
||||
|
||||
if self.quote_html_type:
|
||||
self.quote_replaces = []
|
||||
html_quote_replaces = quote_html(str(updated_text), self.quote_html_type)
|
||||
for html_quote in html_quote_replaces:
|
||||
self.quote_replaces.append(html_quote)
|
||||
self.quote_replaces = list(quote_html(str(updated_text), self.quote_html_type))
|
||||
|
||||
if not self.templates and not self.replaces and not self.quote_replaces:
|
||||
logger.trace("No templates, no replaces, no quote_replaces")
|
||||
return str(self.string)
|
||||
|
||||
updated_text, string_pos_matrix, utf_16_bang_list = self._render_templates(updated_text, string_pos_matrix, utf_16_bang_list)
|
||||
|
|
@ -374,37 +247,26 @@ class RLStringHelper:
|
|||
def get_text(self):
|
||||
return self.__str__()
|
||||
|
||||
|
||||
def split_overlapping_ranges(markups, _retry_count: int = 7):
|
||||
last_fixed_markup = markups
|
||||
for _ in range(len(markups) * _retry_count):
|
||||
markups = split_overlapping_range_position(markups)
|
||||
if last_fixed_markup and len(last_fixed_markup) == len(markups):
|
||||
new_markups = split_overlapping_range_position(markups)
|
||||
if len(new_markups) == len(markups):
|
||||
break
|
||||
last_fixed_markup = markups
|
||||
return last_fixed_markup
|
||||
|
||||
markups = new_markups
|
||||
return markups
|
||||
|
||||
def split_overlapping_range_position(positions):
|
||||
if not positions:
|
||||
return []
|
||||
|
||||
positions.sort(key=lambda x: x["start"])
|
||||
logger.debug(f"Sorted positions: {positions}")
|
||||
|
||||
result = [positions[0]]
|
||||
logger.debug(f"Initial result: {result}")
|
||||
|
||||
for pos in positions[1:]:
|
||||
logger.debug(f"Processing position: {pos}")
|
||||
last = result[-1]
|
||||
|
||||
if pos["start"] < last["end"]:
|
||||
logger.debug("Overlap detected")
|
||||
if pos["type"] != last["type"]:
|
||||
logger.debug("Different type")
|
||||
if pos["end"] <= last["end"]:
|
||||
logger.debug("Case 1: Different type, ends before or at last")
|
||||
result[-1] = {
|
||||
"start": last["start"],
|
||||
"end": pos["start"],
|
||||
|
|
@ -422,7 +284,6 @@ def split_overlapping_range_position(positions):
|
|||
}
|
||||
)
|
||||
else:
|
||||
logger.debug("Case 2: Different type, ends after last")
|
||||
result[-1] = {
|
||||
"start": last["start"],
|
||||
"end": pos["start"],
|
||||
|
|
@ -431,32 +292,22 @@ def split_overlapping_range_position(positions):
|
|||
}
|
||||
result.append(pos.copy())
|
||||
else:
|
||||
logger.debug("Case 3: Same type, update end")
|
||||
result[-1]["end"] = max(last["end"], pos["end"])
|
||||
else:
|
||||
logger.debug("Case 4: No overlap, add new position")
|
||||
result.append(pos.copy())
|
||||
|
||||
logger.debug(f"Updated result: {result}")
|
||||
|
||||
logger.debug(f"Final result: {result}")
|
||||
return result
|
||||
|
||||
|
||||
def raw_render(**kwargs):
|
||||
for key, value in kwargs.items():
|
||||
if isinstance(value, str):
|
||||
kwargs[key] = f"{{% raw %}}{value}{{% endraw %}}"
|
||||
return kwargs
|
||||
|
||||
|
||||
def parse_markups(markups: list[str]):
|
||||
logger.trace(f"Given {markups=}")
|
||||
markups_out = []
|
||||
|
||||
for markup in markups:
|
||||
logger.trace(f"Processing {markups=}")
|
||||
logger.trace(markup)
|
||||
if markup["type"] == "A":
|
||||
if markup["anchorType"] == "LINK":
|
||||
template = jinja_env.from_string('<a style="text-decoration: underline;" rel="{{rel}}" title="{{title}}" href="{{href}}" target="_blank">{{text}}</a>')
|
||||
|
|
@ -465,7 +316,6 @@ def parse_markups(markups: list[str]):
|
|||
template = jinja_env.from_string('<a style="text-decoration: underline;" href="https://medium.com/u/{{userId}}">{{text}}</a>')
|
||||
template = template.render(userId=markup["userId"])
|
||||
else:
|
||||
logger.error(f"Can't proccess 'anchorType': {markup['anchorType']}")
|
||||
continue
|
||||
elif markup["type"] == "STRONG":
|
||||
template = "<strong>{{text}}</strong>"
|
||||
|
|
@ -474,11 +324,9 @@ def parse_markups(markups: list[str]):
|
|||
elif markup["type"] == "CODE":
|
||||
template = "<code class='p-1.5 bg-gray-300 dark:bg-gray-600'>{{text}}</code>"
|
||||
else:
|
||||
logger.error(f"Unknown markup type: {markup}")
|
||||
continue
|
||||
|
||||
template = jinja_env.from_string(template)
|
||||
|
||||
markup["template"] = template
|
||||
markups_out.append(markup)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,10 @@
|
|||
import sentry_sdk
|
||||
from server import config
|
||||
|
||||
if config.SENTRY_SDK_DSN:
|
||||
sentry_sdk.init(dsn=config.SENTRY_SDK_DSN, traces_sample_rate=config.SENTRY_TRACES_SAMPLE_RATE, profiles_sample_rate=config.SENTRY_PROFILES_SAMPLE_RATE)
|
||||
|
||||
|
||||
from contextvars import ContextVar
|
||||
from multiprocessing import Value
|
||||
from typing import Optional
|
||||
|
|
@ -10,8 +17,8 @@ from database_lib import PostgreSQLCacheBackend, migrate_to_postgres, execute_mi
|
|||
from loguru import logger
|
||||
from xkcdpass import xkcd_password as xp
|
||||
|
||||
from server import config
|
||||
from server.utils.logger import configure_logger
|
||||
from medium_parser.core import MediumParser
|
||||
from server.utils.loguru_handler import InterceptHandler
|
||||
|
||||
|
||||
|
|
@ -38,22 +45,27 @@ configure_logger()
|
|||
|
||||
medium_cache = PostgreSQLCacheBackend("postgresql://postgres:postgres@postgres:5432/postgres")
|
||||
medium_cache.init_db()
|
||||
|
||||
# migrate_to_postgres_thread = execute_migrate_to_postgres_in_thread("medium_db_cache.sqlite", "postgresql://postgres:postgres@postgres:5432/postgres")
|
||||
|
||||
logger.debug(f"Database length: {medium_cache.all_length()}")
|
||||
|
||||
redis_storage = redis.Redis(host=config.REDIS_HOST, port=config.REDIS_PORT, db=0)
|
||||
medium_parser = MediumParser(cache=medium_cache, timeout=3, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES, template_folder="server/templates")
|
||||
|
||||
redis_storage = redis.Redis(
|
||||
host=config.REDIS_HOST,
|
||||
port=config.REDIS_PORT,
|
||||
db=0,
|
||||
socket_timeout=config.REDIS_TIMEOUT,
|
||||
socket_connect_timeout=config.REDIS_TIMEOUT,
|
||||
# decode_responses=True
|
||||
)
|
||||
|
||||
url_correlation: ContextVar[Optional[str]] = ContextVar("url_correlation", default="UNKNOWN_URL")
|
||||
transponder_code_correlation: ContextVar[Optional[str]] = ContextVar("transponder_code_correlation", default="unknown transponder location... Beep!")
|
||||
|
||||
home_page_process = {}
|
||||
|
||||
ban_db = pickledb.load('ban_post_list.db', True)
|
||||
ban_db = pickledb.load("ban_post_list.db", True)
|
||||
|
||||
WORDS_LIST_FILE = "xkcdpass/static/legac"
|
||||
|
||||
xkcd_passwd = xp.generate_wordlist(wordfile=WORDS_LIST_FILE, min_length=5, max_length=8)
|
||||
|
||||
maintenance_mode = Value('b', False)
|
||||
maintenance_mode = Value("b", False)
|
||||
|
|
|
|||
|
|
@ -3,21 +3,33 @@ from starlette.config import Config
|
|||
config = Config(".env")
|
||||
|
||||
HOST_ADDRESS = config("HOST_ADDRESS", default="https://freedium.cfd")
|
||||
|
||||
MEDIUM_AUTH_COOKIES = config("MEDIUM_AUTH_COOKIES", default=None)
|
||||
TELEGRAM_ADMIN_ID = config("TELEGRAM_ADMIN_ID", cast=int, default=0)
|
||||
|
||||
ADMIN_SECRET_KEY = config("ADMIN_SECRET_KEY")
|
||||
|
||||
TELEGRAM_ADMIN_ID = config("TELEGRAM_ADMIN_ID", cast=int, default=0)
|
||||
TELEGRAM_BOT_TOKEN = config("TELEGRAM_BOT_TOKEN", default=None)
|
||||
|
||||
LOG_LEVEL_NAME = config("LOG_LEVEL_NAME", default="INFO")
|
||||
MORE_LOGS = config("MORE_LOGS", cast=bool, default=False)
|
||||
|
||||
DISABLE_EXTERNAL_DOCS = config("DISABLE_EXTERNAL_DOCS", cast=bool, default=True)
|
||||
DISABLE_RATE_LIMITER = config("DISABLE_RATE_LIMITER", cast=bool, default=True)
|
||||
|
||||
TIMEOUT = config("TIMEOUT", cast=int, default=25)
|
||||
REQUEST_TIMEOUT = config("REQUEST_TIMEOUT", cast=int, default=40)
|
||||
WORKER_TIMEOUT = config("WORKER_TIMEOUT", cast=int, default=120)
|
||||
SENTRY_SDK_DSN = config("SENTRY_SDK_DSN", default=None)
|
||||
ENABLE_ADS_BANNER = config("ENABLE_ADS_BANNER", cast=bool, default=False)
|
||||
|
||||
CACHE_LIFE_TIME = config("CACHE_LIFE_TIME", cast=int, default=60 * 60 * 5)
|
||||
HOME_PAGE_MAX_POSTS = config("HOME_PAGE_MAX_POSTS", cast=int, default=30)
|
||||
|
||||
HOME_PAGE_MAX_POSTS = config("HOME_PAGE_MAX_POSTS", cast=int, default=15)
|
||||
ENABLE_ADS_BANNER = config("ENABLE_ADS_BANNER", cast=bool, default=False)
|
||||
|
||||
REDIS_HOST = config("REDIS_HOST", default="redis_service")
|
||||
REDIS_PORT = config("REDIS_PORT", cast=int, default=6379)
|
||||
REDIS_TIMEOUT = config("REDIS_TIMEOUT", cast=int, default=0.75)
|
||||
|
||||
SENTRY_SDK_DSN = config("SENTRY_SDK_DSN", default=None)
|
||||
SENTRY_TRACES_SAMPLE_RATE = config("SENTRY_TRACES_SAMPLE_RATE", cast=float, default=0.2)
|
||||
SENTRY_PROFILES_SAMPLE_RATE = config("SENTRY_PROFILES_SAMPLE_RATE", cast=float, default=0.2)
|
||||
|
|
|
|||
|
|
@ -2,9 +2,7 @@ from loguru import logger
|
|||
from pydantic import BaseModel
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from medium_parser.core import MediumParser
|
||||
|
||||
from server import config, ban_db
|
||||
from server import config, ban_db, medium_parser
|
||||
from server.utils.notify import send_message
|
||||
from server.utils.logger_trace import trace
|
||||
|
||||
|
|
@ -30,11 +28,10 @@ async def delete_from_cache(key_data: DeleteFromCache):
|
|||
return JSONResponse({"message": f"Wrong secret key: {key_data.ADMIN_SECRET_KEY}"}, status_code=403)
|
||||
|
||||
try:
|
||||
post = MediumParser(key_data.key, timeout=config.TIMEOUT, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES)
|
||||
await post.delete_from_cache()
|
||||
await medium_parser.delete_from_cache(key_data.key)
|
||||
except Exception as ex:
|
||||
logger.exception(ex)
|
||||
return JSONResponse({"message": f"Couldn't delete from cache: {ex}"}, status_code=500)
|
||||
else:
|
||||
ban_db.set(key_data.key, 1)
|
||||
return JSONResponse({"message": "OK"}, status_code=200)
|
||||
return JSONResponse({"message": "OK"}, status_code=200)
|
||||
|
|
|
|||
|
|
@ -7,9 +7,8 @@ from html5lib.html5parser import parse
|
|||
from async_lru import alru_cache
|
||||
from loguru import logger
|
||||
from medium_parser import medium_parser_exceptions
|
||||
from medium_parser.core import MediumParser
|
||||
|
||||
from server import config, home_page_process, medium_cache, redis_storage, transponder_code_correlation
|
||||
from server import config, medium_cache, redis_storage, medium_parser
|
||||
from server.services.jinja import base_template, homepage_template
|
||||
from server.utils.cache import aio_redis_cache
|
||||
from server.utils.exceptions import handle_exception
|
||||
|
|
@ -21,17 +20,16 @@ from server.utils.utils import safe_check_redis_connection
|
|||
@trace
|
||||
@aio_redis_cache(10 * 60)
|
||||
async def render_homepage(limit: int = config.HOME_PAGE_MAX_POSTS, as_html: bool = False):
|
||||
random_post_id_list = [i[0] for i in medium_cache.random(limit)]
|
||||
home_page_process[transponder_code_correlation.get()] = random_post_id_list
|
||||
random_post_id_list = list(set([i[0] for i in medium_cache.random(limit)]))
|
||||
|
||||
outlet_posts_list = []
|
||||
tasks = []
|
||||
for post_id in random_post_id_list:
|
||||
async def fetch_post_metadata(post_id):
|
||||
try:
|
||||
post = MediumParser(post_id, cache=medium_cache, timeout=3, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES)
|
||||
await post.query(force_cache=True, retry=1)
|
||||
post_metadata = await post.generate_metadata(as_dict=True)
|
||||
logger.debug(f"Fetching post_id: {post_id}")
|
||||
post_data = await medium_parser.query(post_id, force_cache=True, retry=1)
|
||||
post_metadata = await medium_parser.generate_metadata(post_data, post_id, as_dict=True)
|
||||
outlet_posts_list.append(post_metadata)
|
||||
except Exception as ex:
|
||||
await handle_exception(ex, message=f"Couldn't render post_id for postleter: {post_id}. Just ignore that")
|
||||
|
|
@ -44,29 +42,28 @@ async def render_homepage(limit: int = config.HOME_PAGE_MAX_POSTS, as_html: bool
|
|||
homepage_template_rendered = homepage_template.render(post_list=outlet_posts_list)
|
||||
if as_html:
|
||||
return homepage_template_rendered
|
||||
|
||||
return HTMLResponse(homepage_template_rendered)
|
||||
|
||||
|
||||
async def render_medium_post_link(path: str, use_cache: bool = True, use_redis: bool = True):
|
||||
redis_available = await safe_check_redis_connection(redis_storage)
|
||||
logger.debug("Redis available: {}", redis_available)
|
||||
logger.debug(f"Redis available: {redis_available}")
|
||||
|
||||
try:
|
||||
medium_parser = await MediumParser.from_unknown(path, cache=medium_cache, timeout=config.TIMEOUT, host_address=config.HOST_ADDRESS, auth_cookies=config.MEDIUM_AUTH_COOKIES)
|
||||
logger.debug("MediumParser initialized for path: {}", path)
|
||||
post_id = await medium_parser.resolve(path)
|
||||
redis_result = None
|
||||
if redis_available and use_cache and use_redis:
|
||||
redis_result = await redis_storage.get(medium_parser.post_id)
|
||||
logger.debug("Redis cache hit for post_id: {}", medium_parser.post_id)
|
||||
redis_result = await redis_storage.get(post_id)
|
||||
logger.debug(f"Redis cache hit for post_id: {post_id}")
|
||||
|
||||
if not redis_result:
|
||||
logger.debug("No cache found, querying MediumParser")
|
||||
await medium_parser.query(use_cache=use_cache)
|
||||
rendered_medium_post = await medium_parser.render_as_html("server/templates")
|
||||
logger.debug(f"No cache found, querying...: {post_id}")
|
||||
rendered_medium_post = await medium_parser.render_as_html(post_id)
|
||||
logger.debug("Rendered Medium post from HTML template")
|
||||
if redis_available and use_redis:
|
||||
await redis_storage.setex(medium_parser.post_id, config.CACHE_LIFE_TIME, pickle.dumps(rendered_medium_post))
|
||||
logger.debug("Stored rendered post in Redis cache")
|
||||
await redis_storage.setex(post_id, config.CACHE_LIFE_TIME, pickle.dumps(rendered_medium_post))
|
||||
logger.debug(f"Stored rendered post in Redis cache: {post_id}")
|
||||
else:
|
||||
rendered_medium_post = pickle.loads(redis_result)
|
||||
logger.debug("Loaded rendered post from Redis cache")
|
||||
|
|
|
|||
|
|
@ -22,9 +22,6 @@ FASTAPI_APPLICATION_CONFIG = {"title": APP_TITLE, "version": APP_VERSION}
|
|||
if config.DISABLE_EXTERNAL_DOCS:
|
||||
FASTAPI_APPLICATION_CONFIG.update({"openapi_url": None, "docs_url": None, "redoc_url": None})
|
||||
|
||||
if config.SENTRY_SDK_DSN:
|
||||
sentry_sdk.init(dsn=config.SENTRY_SDK_DSN, traces_sample_rate=1.0)
|
||||
|
||||
|
||||
async def limiter_callback(request, response, pexpire: int):
|
||||
expire = ceil(pexpire / 1000)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from starlette.requests import Request
|
|||
from starlette.responses import Response, StreamingResponse
|
||||
from starlette.types import Message
|
||||
|
||||
from server import transponder_code_correlation, url_correlation, xkcd_passwd, xp, config, home_page_process
|
||||
from server import transponder_code_correlation, url_correlation, xkcd_passwd, xp, config
|
||||
from server.utils.anti_bot import filter_bots
|
||||
from server.utils.notify import send_message
|
||||
from server.utils.error import generate_error
|
||||
|
|
@ -51,7 +51,9 @@ class LoggerMiddleware(BaseHTTPMiddleware):
|
|||
except Exception as ex:
|
||||
exception_class = type(ex)
|
||||
logger.exception(ex)
|
||||
send_message(f"Error while processing url: <code>{url_correlation.get()}</code>, transponder_id: <code>{generated_id}</code>, transponder_code: <code>{transponder_code_correlation.get()}</code>, error: <code>{ex}</code>. exception: <code>{exception_class.__name__}</code>. {home_page_process.get(transponder_code_correlation.get(), '')}")
|
||||
send_message(
|
||||
f"Error while processing url: <code>{url_correlation.get()}</code>, transponder_id: <code>{generated_id}</code>, transponder_code: <code>{transponder_code_correlation.get()}</code>, error: <code>{ex}</code>. exception: <code>{exception_class.__name__}</code>."
|
||||
)
|
||||
response = await generate_error()
|
||||
|
||||
logger.trace(response.__dict__)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import sentry_sdk
|
||||
import asyncio
|
||||
from loguru import logger
|
||||
|
||||
from server.utils.error import generate_error
|
||||
|
|
@ -6,7 +7,8 @@ from server.utils.error import generate_error
|
|||
|
||||
async def handle_exception(ex, message="An error occurred", status_code=500, quiet: bool = False):
|
||||
logger.exception(ex)
|
||||
if not quiet:
|
||||
sentry_sdk.capture_exception(ex)
|
||||
# TODO: optimize
|
||||
# if not quiet:
|
||||
# sentry_sdk.capture_exception(ex)
|
||||
|
||||
return await generate_error(message, status_code=status_code, quiet=quiet)
|
||||
|
|
|
|||
Loading…
Reference in a new issue