fixing multiple overlapsing position in different tags

core/tests/example_test.py

@@ -5,6 +5,7 @@ import sys
 import jinja2
 from loguru import logger
 from medium_parser.core import MediumParser
+from database_lib import SQLiteCacheBackend
 
 jinja2_env = jinja2.Environment(
     loader=jinja2.FileSystemLoader("./"),
@@ -19,11 +20,14 @@ async def safe_main():
 
 async def main():
     logger.remove()
-    # logger.add(sys.stderr, level="INFO")
-    logger.add(sys.stderr, level="TRACE")
+    logger.add(sys.stderr, level="INFO")
+    # logger.add(sys.stderr, level="TRACE")
+    logger.add("trace.log", level="TRACE")
 
     # dl = await MediumParser.from_url("")
-    dl = MediumParser("1059aeab90d", 8, "localhost")
+    sqlite = SQLiteCacheBackend("test_db.sqlite")
+    sqlite.init_db()
+    dl = MediumParser("ef85d8e72883", sqlite, 8, "localhost")
     query_result = await dl.query(use_cache=False)
 
     with open("query_result.json", "w") as f:

rl_string_helper/rl_string_helper/string_helper.py

@@ -225,7 +225,7 @@ class RLStringHelper:
             logger.trace(older_text == updated_text)
             logger.trace(f"{updated_text}")
 
-            logger.trace(f"{start=}, {end=}, {template=}")
+            logger.trace(f"{start=}, {end=}, template={str(template)}")
 
             if start >= len(string_pos_matrix):
                 logger.warning("Start position is out of range. Ignore...")
@@ -375,9 +375,9 @@ class RLStringHelper:
         return self.__str__()
 
 
-def split_overlapping_ranges(markups):
+def split_overlapping_ranges(markups, _retry_count: int = 7):
     last_fixed_markup = markups
-    for _ in range(len(markups) * 7):
+    for _ in range(len(markups) * _retry_count):
         markups = split_overlapping_range_position(markups)
         if last_fixed_markup and len(last_fixed_markup) == len(markups):
             break
@@ -389,101 +389,58 @@ def split_overlapping_range_position(positions):
     if not positions:
         return []
 
-    # Sort the positions by start
     positions.sort(key=lambda x: x["start"])
-    logger.trace(positions)
+    logger.debug(f"Sorted positions: {positions}")
 
-    # Initialize the result list with the first position
     result = [positions[0]]
-    logger.trace(result)
+    logger.debug(f"Initial result: {result}")
 
     for pos in positions[1:]:
-        logger.trace(pos)
+        logger.debug(f"Processing position: {pos}")
         last = result[-1]
 
-        # If the current position overlaps with the last one in the result
         if pos["start"] < last["end"]:
-            logger.trace(0)
-            # If the current position has a different markup and ends before the last one
-            if pos["type"] != last["type"] and pos["end"] < last["end"]:
-                logger.trace(1)
-                # Split the last position into three
-                result[-1] = {
-                    "start": last["start"],
-                    "end": pos["start"],
-                    "type": last["type"],
-                    "template": last["template"],
-                }
-                logger.trace(result)
-                result.append(
-                    {
-                        "start": pos["start"],
-                        "end": pos["end"],
-                        "type": pos["type"],
-                        "template": pos["template"],
-                    }
-                )
-                logger.trace(result)
-                result.append(
-                    {
-                        "start": pos["start"],
-                        "end": pos["end"],
+            logger.debug("Overlap detected")
+            if pos["type"] != last["type"]:
+                logger.debug("Different type")
+                if pos["end"] <= last["end"]:
+                    logger.debug("Case 1: Different type, ends before or at last")
+                    result[-1] = {
+                        "start": last["start"],
+                        "end": pos["start"],
                         "type": last["type"],
                         "template": last["template"],
                     }
-                )
-                logger.trace(result)
-                result.append(
-                    {
-                        "start": pos["end"],
-                        "end": last["end"],
+                    result.append(pos.copy())
+                    if pos["end"] < last["end"]:
+                        result.append(
+                            {
+                                "start": pos["end"],
+                                "end": last["end"],
+                                "type": last["type"],
+                                "template": last["template"],
+                            }
+                        )
+                else:
+                    logger.debug("Case 2: Different type, ends after last")
+                    result[-1] = {
+                        "start": last["start"],
+                        "end": pos["start"],
                         "type": last["type"],
                         "template": last["template"],
                     }
-                )
-                logger.trace(result)
-            elif pos["type"] != last["type"]:
-                logger.trace(2)
-                # Split the last position into two, updating end of the last position
-                result[-1] = {
-                    "start": last["start"],
-                    "end": pos["start"],
-                    "type": last["type"],
-                    "template": last["template"],
-                }
-                logger.trace(result)
-                result.append(
-                    {
-                        "start": pos["start"],
-                        "end": pos["end"],
-                        "type": pos["type"],
-                        "template": pos["template"],
-                    }
-                )
-                logger.trace(result)
-                result.append(
-                    {
-                        "start": pos["start"],
-                        "end": last["end"],
-                        "type": last["type"],
-                        "template": last["template"],
-                    }
-                )
-                logger.trace(result)
+                    result.append(pos.copy())
             else:
-                logger.trace(3)
-                # Update the end of the last position in the result
+                logger.debug("Case 3: Same type, update end")
                 result[-1]["end"] = max(last["end"], pos["end"])
-                logger.trace(result)
         else:
-            logger.trace(4)
-            # Add the current position to the result
-            result.append(pos)
-            logger.trace(result)
+            logger.debug("Case 4: No overlap, add new position")
+            result.append(pos.copy())
 
+        logger.debug(f"Updated result: {result}")
+
+    logger.debug(f"Final result: {result}")
     return result
-
-
 def raw_render(**kwargs):
     for key, value in kwargs.items():
         if isinstance(value, str):
@@ -491,10 +448,12 @@ def raw_render(**kwargs):
     return kwargs
 
 
-def parse_markups(markups: list):
+def parse_markups(markups: list[str]):
+    logger.trace(f"Given {markups=}")
     markups_out = []
 
     for markup in markups:
+        logger.trace(f"Processing {markups=}")
         logger.trace(markup)
         if markup["type"] == "A":
             if markup["anchorType"] == "LINK":

rl_string_helper/tests/test_rl_string_helper.py

@@ -1,7 +1,7 @@
 import sys
-
+import re
 from loguru import logger
-from rl_string_helper import RLStringHelper, quote_html, parse_markups
+from rl_string_helper import RLStringHelper, quote_html, parse_markups, split_overlapping_ranges
 
 
 class TestRLStringHelper:
@@ -10,26 +10,26 @@ class TestRLStringHelper:
         logger.add(sys.stdout, level="TRACE")
 
     def test_html_quote(self):
-        quoted_string_1 = [i for i in quote_html("<Hello world>")]
-        assert quoted_string_1 == [((0, 1), '&lt;'), ((12, 13), '&gt;')]
+        quoted_string_1 = [i for i in quote_html("<Hello world>", "full")]
+        assert quoted_string_1 == [((0, 1), "&lt;"), ((12, 13), "&gt;")]
 
         # Test with standard HTML characters
         html = '<div class="test">Hello & World</div>'
-        result = list(quote_html(html))
-        expected = [((0, 1), '&lt;'), ((11, 12), '&quot;'), ((16, 17), '&quot;'), ((17, 18), '&gt;'), ((24, 25), '&amp;'), ((31, 32), '&lt;'), ((36, 37), '&gt;')]
-        assert result == expected
+        result = list(quote_html(html, "full"))
+        expected = [((0, 1), "&lt;"), ((11, 12), "&quot;"), ((16, 17), "&quot;"), ((17, 18), "&gt;"), ((24, 25), "&amp;"), ((31, 32), "&lt;"), ((36, 37), "&gt;")]
+        assert sorted(result) == sorted(expected)
 
         # Test with extra characters
         html = '<div class="test">\nHello & World</div>'
-        result = list(quote_html(html, True))
-        expected = [((0, 1), '&lt;'), ((11, 12), '&quot;'), ((16, 17), '&quot;'), ((17, 18), '&gt;'), ((25, 26), '&amp;'), ((32, 33), '&lt;'), ((37, 38), '&gt;'), ((18, 19), '<br />')]
-        assert result == expected
+        result = list(quote_html(html, "extra"))
+        expected = [((0, 1), "&lt;"), ((11, 12), "&quot;"), ((16, 17), "&quot;"), ((17, 18), "&gt;"), ((25, 26), "&amp;"), ((32, 33), "&lt;"), ((37, 38), "&gt;"), ((18, 19), "<br />")]
+        assert sorted(result) == sorted(expected)
 
         # Test with quote characters
         html = '<div class="test">Hello & \'World\'</div>'
-        result = list(quote_html(html))
-        expected = [((0, 1), '&lt;'), ((11, 12), '&quot;'), ((16, 17), '&quot;'), ((17, 18), '&gt;'), ((24, 25), '&amp;'), ((26, 27), '&#39'), ((32, 33), '&#39'), ((33, 34), '&lt;'), ((38, 39), '&gt;')]
-        assert result == expected
+        result = list(quote_html(html, "full"))
+        expected = [((0, 1), "&lt;"), ((11, 12), "&quot;"), ((16, 17), "&quot;"), ((17, 18), "&gt;"), ((24, 25), "&amp;"), ((26, 27), "&#39"), ((32, 33), "&#39"), ((33, 34), "&lt;"), ((38, 39), "&gt;")]
+        assert sorted(result) == sorted(expected)
 
     def test_basic_template(self):
         helper = RLStringHelper("Hello world")
@@ -42,6 +42,27 @@ class TestRLStringHelper:
         helper.set_template(0, 11, "<i>{{text}}</i>")
         assert str(helper) == "<i><a>Hello</a> <b>world</b></i>"
 
+    def test_super_duper_overlapsing(self):
+        # https://medium.com/google-cloud/implementing-semantic-caching-a-step-by-step-guide-to-faster-cost-effective-genai-workflows-ef85d8e72883#bypass
+        text = "Note: The patterns and ideas discussed in this post are broadly applicable and can be adopted for other cloud providers."
+        helper = RLStringHelper(text)
+        markups = (
+            [
+                {"__typename": "Markup", "name": None, "type": "CODE", "start": 0, "end": 5, "href": None, "title": None, "rel": None, "anchorType": None, "userId": None, "creatorIds": None},
+                {"__typename": "Markup", "name": None, "type": "STRONG", "start": 0, "end": 6, "href": None, "title": None, "rel": None, "anchorType": None, "userId": None, "creatorIds": None},
+                {"__typename": "Markup", "name": None, "type": "EM", "start": 0, "end": 6, "href": None, "title": None, "rel": None, "anchorType": None, "userId": None, "creatorIds": None},
+            ],
+        )
+        parsed_markups = parse_markups(markups[0])
+        logger.debug(parsed_markups)
+        parsed_markups = split_overlapping_ranges(parsed_markups)
+        logger.debug(parsed_markups)
+        for markup in parsed_markups:
+            helper.set_template(markup["start"], markup["end"], markup["template"])
+
+        expected_pattern = r"<em><strong><code[^>]*>Note:</code> </strong></em>The patterns and ideas discussed in this post are broadly applicable and can be adopted for other cloud providers\."
+        assert re.match(expected_pattern, str(helper))
+
     def test_basic_replace(self):
         # Replace A to B - ONE to ONE char
         helper = RLStringHelper("ABC")
@@ -111,7 +132,8 @@ class TestRLStringHelper:
 
         helper = RLStringHelper("Hello world")
         markups = parse_markups([href_markup])
-        for markup in markups:
+        parsed_markups = split_overlapping_ranges(markups)
+        for markup in parsed_markups:
             helper.set_template(markup["start"], markup["end"], markup["template"])
         assert helper.get_text() == '<a style="text-decoration: underline;" rel="nofollow" title="" href="https://readwise.io/bookreview/{{book_id" target="_blank">Hello world</a>'
 

server/templates/post.html

@@ -1,8 +1,6 @@
 <div class="container w-full md:max-w-3xl mx-auto pt-20 break-words text-gray-900 dark:text-gray-200 bg-white dark:bg-gray-800">
     <div class="w-full px-4 md:px-6 text-xl text-gray-800 dark:text-gray-100 leading-normal" style="font-family:Georgia,serif;">
-        {% if enable_ads_header %}
-        <div class="pt-8"></div>
-        {% endif %}
+        {% if enable_ads_header %}<div class="pt-8"></div>{% endif %}
         <div class="font-sans">
             <p class="text-base md:text-sm text-green-500 font-bold pb-3">
                 <a href="{{ url }}#bypass" class="text-sm md:text-sm text-green-500 font-bold no-underline hover:underline ">&lt; Go to the original</a>