diff --git a/core/tests/example_test.py b/core/tests/example_test.py index 6c23460..38d305e 100644 --- a/core/tests/example_test.py +++ b/core/tests/example_test.py @@ -5,6 +5,7 @@ import sys import jinja2 from loguru import logger from medium_parser.core import MediumParser +from database_lib import SQLiteCacheBackend jinja2_env = jinja2.Environment( loader=jinja2.FileSystemLoader("./"), @@ -19,11 +20,14 @@ async def safe_main(): async def main(): logger.remove() - # logger.add(sys.stderr, level="INFO") - logger.add(sys.stderr, level="TRACE") + logger.add(sys.stderr, level="INFO") + # logger.add(sys.stderr, level="TRACE") + logger.add("trace.log", level="TRACE") # dl = await MediumParser.from_url("") - dl = MediumParser("1059aeab90d", 8, "localhost") + sqlite = SQLiteCacheBackend("test_db.sqlite") + sqlite.init_db() + dl = MediumParser("ef85d8e72883", sqlite, 8, "localhost") query_result = await dl.query(use_cache=False) with open("query_result.json", "w") as f: diff --git a/rl_string_helper/rl_string_helper/string_helper.py b/rl_string_helper/rl_string_helper/string_helper.py index 38c5d15..b2c5564 100644 --- a/rl_string_helper/rl_string_helper/string_helper.py +++ b/rl_string_helper/rl_string_helper/string_helper.py @@ -225,7 +225,7 @@ class RLStringHelper: logger.trace(older_text == updated_text) logger.trace(f"{updated_text}") - logger.trace(f"{start=}, {end=}, {template=}") + logger.trace(f"{start=}, {end=}, template={str(template)}") if start >= len(string_pos_matrix): logger.warning("Start position is out of range. Ignore...") @@ -375,9 +375,9 @@ class RLStringHelper: return self.__str__() -def split_overlapping_ranges(markups): +def split_overlapping_ranges(markups, _retry_count: int = 7): last_fixed_markup = markups - for _ in range(len(markups) * 7): + for _ in range(len(markups) * _retry_count): markups = split_overlapping_range_position(markups) if last_fixed_markup and len(last_fixed_markup) == len(markups): break @@ -389,101 +389,58 @@ def split_overlapping_range_position(positions): if not positions: return [] - # Sort the positions by start positions.sort(key=lambda x: x["start"]) - logger.trace(positions) + logger.debug(f"Sorted positions: {positions}") - # Initialize the result list with the first position result = [positions[0]] - logger.trace(result) + logger.debug(f"Initial result: {result}") for pos in positions[1:]: - logger.trace(pos) + logger.debug(f"Processing position: {pos}") last = result[-1] - # If the current position overlaps with the last one in the result if pos["start"] < last["end"]: - logger.trace(0) - # If the current position has a different markup and ends before the last one - if pos["type"] != last["type"] and pos["end"] < last["end"]: - logger.trace(1) - # Split the last position into three - result[-1] = { - "start": last["start"], - "end": pos["start"], - "type": last["type"], - "template": last["template"], - } - logger.trace(result) - result.append( - { - "start": pos["start"], - "end": pos["end"], - "type": pos["type"], - "template": pos["template"], - } - ) - logger.trace(result) - result.append( - { - "start": pos["start"], - "end": pos["end"], + logger.debug("Overlap detected") + if pos["type"] != last["type"]: + logger.debug("Different type") + if pos["end"] <= last["end"]: + logger.debug("Case 1: Different type, ends before or at last") + result[-1] = { + "start": last["start"], + "end": pos["start"], "type": last["type"], "template": last["template"], } - ) - logger.trace(result) - result.append( - { - "start": pos["end"], - "end": last["end"], + result.append(pos.copy()) + if pos["end"] < last["end"]: + result.append( + { + "start": pos["end"], + "end": last["end"], + "type": last["type"], + "template": last["template"], + } + ) + else: + logger.debug("Case 2: Different type, ends after last") + result[-1] = { + "start": last["start"], + "end": pos["start"], "type": last["type"], "template": last["template"], } - ) - logger.trace(result) - elif pos["type"] != last["type"]: - logger.trace(2) - # Split the last position into two, updating end of the last position - result[-1] = { - "start": last["start"], - "end": pos["start"], - "type": last["type"], - "template": last["template"], - } - logger.trace(result) - result.append( - { - "start": pos["start"], - "end": pos["end"], - "type": pos["type"], - "template": pos["template"], - } - ) - logger.trace(result) - result.append( - { - "start": pos["start"], - "end": last["end"], - "type": last["type"], - "template": last["template"], - } - ) - logger.trace(result) + result.append(pos.copy()) else: - logger.trace(3) - # Update the end of the last position in the result + logger.debug("Case 3: Same type, update end") result[-1]["end"] = max(last["end"], pos["end"]) - logger.trace(result) else: - logger.trace(4) - # Add the current position to the result - result.append(pos) - logger.trace(result) + logger.debug("Case 4: No overlap, add new position") + result.append(pos.copy()) + logger.debug(f"Updated result: {result}") + + logger.debug(f"Final result: {result}") return result - - def raw_render(**kwargs): for key, value in kwargs.items(): if isinstance(value, str): @@ -491,10 +448,12 @@ def raw_render(**kwargs): return kwargs -def parse_markups(markups: list): +def parse_markups(markups: list[str]): + logger.trace(f"Given {markups=}") markups_out = [] for markup in markups: + logger.trace(f"Processing {markups=}") logger.trace(markup) if markup["type"] == "A": if markup["anchorType"] == "LINK": diff --git a/rl_string_helper/tests/test_rl_string_helper.py b/rl_string_helper/tests/test_rl_string_helper.py index e4d36f0..d4c0cba 100644 --- a/rl_string_helper/tests/test_rl_string_helper.py +++ b/rl_string_helper/tests/test_rl_string_helper.py @@ -1,7 +1,7 @@ import sys - +import re from loguru import logger -from rl_string_helper import RLStringHelper, quote_html, parse_markups +from rl_string_helper import RLStringHelper, quote_html, parse_markups, split_overlapping_ranges class TestRLStringHelper: @@ -10,26 +10,26 @@ class TestRLStringHelper: logger.add(sys.stdout, level="TRACE") def test_html_quote(self): - quoted_string_1 = [i for i in quote_html("")] - assert quoted_string_1 == [((0, 1), '<'), ((12, 13), '>')] + quoted_string_1 = [i for i in quote_html("", "full")] + assert quoted_string_1 == [((0, 1), "<"), ((12, 13), ">")] # Test with standard HTML characters html = '
Hello & World
' - result = list(quote_html(html)) - expected = [((0, 1), '<'), ((11, 12), '"'), ((16, 17), '"'), ((17, 18), '>'), ((24, 25), '&'), ((31, 32), '<'), ((36, 37), '>')] - assert result == expected + result = list(quote_html(html, "full")) + expected = [((0, 1), "<"), ((11, 12), """), ((16, 17), """), ((17, 18), ">"), ((24, 25), "&"), ((31, 32), "<"), ((36, 37), ">")] + assert sorted(result) == sorted(expected) # Test with extra characters html = '
\nHello & World
' - result = list(quote_html(html, True)) - expected = [((0, 1), '<'), ((11, 12), '"'), ((16, 17), '"'), ((17, 18), '>'), ((25, 26), '&'), ((32, 33), '<'), ((37, 38), '>'), ((18, 19), '
')] - assert result == expected + result = list(quote_html(html, "extra")) + expected = [((0, 1), "<"), ((11, 12), """), ((16, 17), """), ((17, 18), ">"), ((25, 26), "&"), ((32, 33), "<"), ((37, 38), ">"), ((18, 19), "
")] + assert sorted(result) == sorted(expected) # Test with quote characters html = '
Hello & \'World\'
' - result = list(quote_html(html)) - expected = [((0, 1), '<'), ((11, 12), '"'), ((16, 17), '"'), ((17, 18), '>'), ((24, 25), '&'), ((26, 27), '''), ((32, 33), '''), ((33, 34), '<'), ((38, 39), '>')] - assert result == expected + result = list(quote_html(html, "full")) + expected = [((0, 1), "<"), ((11, 12), """), ((16, 17), """), ((17, 18), ">"), ((24, 25), "&"), ((26, 27), "'"), ((32, 33), "'"), ((33, 34), "<"), ((38, 39), ">")] + assert sorted(result) == sorted(expected) def test_basic_template(self): helper = RLStringHelper("Hello world") @@ -42,6 +42,27 @@ class TestRLStringHelper: helper.set_template(0, 11, "{{text}}") assert str(helper) == "Hello world" + def test_super_duper_overlapsing(self): + # https://medium.com/google-cloud/implementing-semantic-caching-a-step-by-step-guide-to-faster-cost-effective-genai-workflows-ef85d8e72883#bypass + text = "Note: The patterns and ideas discussed in this post are broadly applicable and can be adopted for other cloud providers." + helper = RLStringHelper(text) + markups = ( + [ + {"__typename": "Markup", "name": None, "type": "CODE", "start": 0, "end": 5, "href": None, "title": None, "rel": None, "anchorType": None, "userId": None, "creatorIds": None}, + {"__typename": "Markup", "name": None, "type": "STRONG", "start": 0, "end": 6, "href": None, "title": None, "rel": None, "anchorType": None, "userId": None, "creatorIds": None}, + {"__typename": "Markup", "name": None, "type": "EM", "start": 0, "end": 6, "href": None, "title": None, "rel": None, "anchorType": None, "userId": None, "creatorIds": None}, + ], + ) + parsed_markups = parse_markups(markups[0]) + logger.debug(parsed_markups) + parsed_markups = split_overlapping_ranges(parsed_markups) + logger.debug(parsed_markups) + for markup in parsed_markups: + helper.set_template(markup["start"], markup["end"], markup["template"]) + + expected_pattern = r"]*>Note: The patterns and ideas discussed in this post are broadly applicable and can be adopted for other cloud providers\." + assert re.match(expected_pattern, str(helper)) + def test_basic_replace(self): # Replace A to B - ONE to ONE char helper = RLStringHelper("ABC") @@ -111,7 +132,8 @@ class TestRLStringHelper: helper = RLStringHelper("Hello world") markups = parse_markups([href_markup]) - for markup in markups: + parsed_markups = split_overlapping_ranges(markups) + for markup in parsed_markups: helper.set_template(markup["start"], markup["end"], markup["template"]) assert helper.get_text() == 'Hello world' diff --git a/server/templates/post.html b/server/templates/post.html index 3f6d368..aa7d510 100644 --- a/server/templates/post.html +++ b/server/templates/post.html @@ -1,8 +1,6 @@
- {% if enable_ads_header %} -
- {% endif %} + {% if enable_ads_header %}
{% endif %}

< Go to the original