prehistoric-systems/web
1
0
Fork 0
mirror of https://codeberg.org/Freedium-cfd/web.git synced 2026-03-11 09:04:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Trying to fix some strange DB bug

Browse source
This commit is contained in:
ZhymabekRoman 2024-02-23 10:49:43 +06:00
parent d3a95ef11a
commit c0ef176bfd
3 changed files with 14 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
core/medium_parser/cache_db.py
View file

@ -1,6 +1,7 @@
from typing import Union
import sqlite3
import json
from loguru import logger
from warnings import warn
try:
import sqlite_zstd
@ -9,7 +10,7 @@ except ImportError:
sqlite_zstd = None
class CacheResponse:
__slots__ = ('data')
__slots__ = ('data',)
def __init__(self, data: str):
self.data = data
@ -28,8 +29,8 @@ class SQLiteCacheBackend:
self.connection = sqlite3.connect(database)
self.connection.enable_load_extension(True) # Enable loading of extensions
self.connection.execute("PRAGMA foreign_keys = ON") # Need for working with foreign keys in db
self.connection.execute("PRAGMA journal_mode=WAL")
self.connection.execute("PRAGMA auto_vacuum=full")
self.connection.execute("PRAGMA journal_mode=WAL") # Need to properly work with ZSTD compression
self.connection.execute("PRAGMA auto_vacuum=full") # Same as above thing
self.cursor = self.connection.cursor()
if sqlite_zstd is not None:
@ -45,8 +46,8 @@ class SQLiteCacheBackend:
def random(self, size: int):
with self.connection:
return self.cursor.execute("SELECT * FROM cache ORDER BY RANDOM() LIMIT :0", {'0': size}).fetchall()
return self.cursor.execute("SELECT * FROM cache ORDER BY RANDOM() LIMIT ?", (size,)).fetchall()
def enable_zstd(self):
if sqlite_zstd is None:
raise ValueError("Can't use zstd compression. Please install 'sqlite_zstd' package")
@ -68,6 +69,7 @@ class SQLiteCacheBackend:
with self.connection:
cache = self.cursor.execute("SELECT value FROM cache WHERE key = :0", {'0': key}).fetchone()
if cache:
logger.debug("Value found in DB, returning it")
return CacheResponse(cache[0])
def push(self, key: str, value: str) -> None:

5
core/medium_parser/core.py
View file

@ -72,6 +72,7 @@ class MediumParser:
if post_data:
logger.debug("post query was found on cache")
return post_data.json()
logger.debug(f"No data found in cache by {self.post_id}")
return None
async def get_post_data_from_api(self):
@ -84,8 +85,12 @@ class MediumParser:
return None
async def query(self, use_cache: bool = True, retry: int = 3):
logger.debug(f"Medium QUERY: {use_cache=}, {retry=}")
post_data = await self.get_post_data_from_cache() if use_cache else None
if not post_data:
logger.debug("Getting value from cache failed, using API")
attempt = 0
while not post_data and attempt < retry:
try:

4
server/middlewares/logger.py
View file

@ -34,7 +34,7 @@ class LoggerMiddleware(BaseHTTPMiddleware):
transponder_code_correlation.set(transponder_code)
url_correlation.set(request.url)
with logger.contextualize(id=generated_id):
logger.trace(f"Current ID '{generated_id}' transponder code is '{transponder_code}'")
logger.debug(f"Current ID '{generated_id}' transponder code is '{transponder_code}'")
logger.trace(request.__dict__)
await request.body()
@ -64,7 +64,7 @@ class LoggerMiddleware(BaseHTTPMiddleware):
except Exception as ex:
exception_class = type(ex)
logger.exception(ex)
send_message(f"Error while processing url: <code>{url_correlation.get()}</code>, transponder_code: <code>{transponder_code_correlation.get()}</code>, error: <code>{ex}</code>. exception: <code>{exception_class.__name__}</code>. {home_page_process.get(transponder_code_correlation.get(), '')}")
send_message(f"Error while processing url: <code>{url_correlation.get()}</code>, transponder_id: <code>{generated_id}</code>, transponder_code: <code>{transponder_code_correlation.get()}</code>, error: <code>{ex}</code>. exception: <code>{exception_class.__name__}</code>. {home_page_process.get(transponder_code_correlation.get(), '')}")
response = await generate_error()
logger.trace(response.__dict__)