Add highlights to newly saved pages if they contain highlight markers

packages/api/src/generated/graphql.ts

@@ -1747,6 +1747,7 @@ export type SaveError = {
 };
 
 export enum SaveErrorCode {
+  EmbeddedHighlightFailed = 'EMBEDDED_HIGHLIGHT_FAILED',
   Unauthorized = 'UNAUTHORIZED',
   Unknown = 'UNKNOWN'
 }

packages/api/src/generated/schema.graphql

@@ -1242,6 +1242,7 @@ type SaveError {
 }
 
 enum SaveErrorCode {
+  EMBEDDED_HIGHLIGHT_FAILED
   UNAUTHORIZED
   UNKNOWN
 }

packages/api/src/schema.ts

@@ -490,6 +490,7 @@ const schema = gql`
   enum SaveErrorCode {
     UNKNOWN
     UNAUTHORIZED
+    EMBEDDED_HIGHLIGHT_FAILED
   }
 
   type SaveError {

packages/api/src/services/save_page.ts

@@ -14,6 +14,7 @@ import normalizeUrl from 'normalize-url'
 import { createPageSaveRequest } from './create_page_save_request'
 import { ArticleSavingRequestStatus, Page } from '../elastic/types'
 import { createPage, getPageByParam, updatePage } from '../elastic/pages'
+import { addHighlightToPage } from '../elastic/highlights'
 
 type SaveContext = {
   pubsub: PubsubClient
@@ -101,12 +102,15 @@ export const savePage = async (
     savedAt: new Date(),
   }
 
+  let pageId: string | undefined = undefined
   const existingPage = await getPageByParam({
     userId: saver.userId,
     url: articleToSave.url,
     state: ArticleSavingRequestStatus.Succeeded,
   })
+
   if (existingPage) {
+    pageId = existingPage.id
     if (
       !(await updatePage(
         existingPage.id,
@@ -139,7 +143,8 @@ export const savePage = async (
       }
     }
   } else {
-    if (!(await createPage(articleToSave, ctx))) {
+    pageId = await createPage(articleToSave, ctx)
+    if (!pageId) {
       return {
         errorCodes: [SaveErrorCode.Unknown],
         message: 'Failed to create new page',
@@ -147,6 +152,28 @@ export const savePage = async (
     }
   }
 
+  if (pageId && parseResult.highlightData) {
+    const highlight = {
+      updatedAt: new Date(),
+      createdAt: new Date(),
+      userId: ctx.uid,
+      elasticPageId: pageId,
+      ...parseResult.highlightData,
+    }
+
+    if (
+      !(await addHighlightToPage(pageId, highlight, {
+        pubsub: ctx.pubsub,
+        uid: ctx.uid,
+      }))
+    ) {
+      return {
+        errorCodes: [SaveErrorCode.EmbeddedHighlightFailed],
+        message: 'Failed to save highlight',
+      }
+    }
+  }
+
   return {
     clientRequestId: input.clientRequestId,
     url: `${homePageURL()}/${saver.username}/${slug}`,

packages/api/src/utils/highlightGenerator.ts

@@ -0,0 +1,328 @@
+import { diff_match_patch as DiffMatchPatch } from 'diff-match-patch'
+import { interpolationSearch } from './interpolationSearch'
+import { v4 as uuidv4 } from 'uuid'
+import { nanoid } from 'nanoid'
+
+const highlightTag = 'omnivore_highlight'
+export const maxHighlightLength = 2000
+
+const nonParagraphTagsRegEx =
+  /^(a|b|basefont|bdo|big|em|font|i|s|small|span|strike|strong|su[bp]|tt|u|code|mark)$/i
+const highlightContentRegex = new RegExp(
+  `<${highlightTag}>([\\s\\S]*)<\\/${highlightTag}>`,
+  'i'
+)
+const maxDeepPatchDistance = 4000
+const maxDeepPatchThreshhold = 0.5
+const maxSurroundingTextLength = 2000
+
+type TextNode = {
+  startIndex: number
+  node: Node
+  isParagraphStart?: boolean
+}
+
+type ArticleTextContent = {
+  textNodes: TextNode[]
+  articleText: string
+}
+
+export type EmbeddedHighlightData = {
+  prefix: string
+  suffix: string
+  quote: string
+  id: string
+  shortId: string
+  patch: string
+}
+
+function getTextNodesBetween(rootNode: Node, startNode: Node, endNode: Node) {
+  let textNodeStartingPoint = 0
+  let articleText = ''
+  let newParagraph = false
+  const textNodes: TextNode[] = []
+  let pastStartNode = false,
+    reachedEndNode = false
+
+  function pushNode(node: Node) {
+    textNodes.push({
+      node,
+      startIndex: textNodeStartingPoint,
+      isParagraphStart: newParagraph,
+    })
+    textNodeStartingPoint += node.nodeValue?.length || 0
+    articleText += node.nodeValue
+    newParagraph = false
+  }
+
+  function getTextNodes(node: Node) {
+    if (node == startNode) {
+      pastStartNode = true
+    }
+
+    if (node.nodeType == 3) {
+      if (
+        pastStartNode &&
+        !reachedEndNode &&
+        !/^\s*$/.test(node.nodeValue || '')
+      ) {
+        pushNode(node)
+      }
+    } else {
+      if (!nonParagraphTagsRegEx.test((node as Element).tagName))
+        newParagraph = true
+    }
+
+    for (
+      var i = 0, len = node.childNodes.length;
+      !reachedEndNode && i < len;
+      ++i
+    ) {
+      getTextNodes(node.childNodes[i])
+    }
+
+    if (node == endNode) {
+      reachedEndNode = true
+    }
+  }
+
+  getTextNodes(rootNode)
+
+  return {
+    textNodes,
+    articleText,
+  }
+}
+
+export function findEmbeddedHighlight(
+  dom: Element
+): EmbeddedHighlightData | undefined {
+  const startNode = dom.querySelector(
+    'span[data-omnivore-highlight-start="true"]'
+  )
+  const endNode = dom.querySelector('span[data-omnivore-highlight-end="true"]')
+
+  const articleContentElement = dom
+  if (!articleContentElement || !startNode || !endNode) {
+    return undefined
+  }
+
+  const beforeNodes = getTextNodesBetween(dom, articleContentElement, startNode)
+  const highlightNodes = getTextNodesBetween(dom, startNode, endNode)
+  const afterNodes = getTextNodesBetween(dom, endNode, articleContentElement)
+  const allArticleNodes = getTextNodesBetween(
+    dom,
+    articleContentElement,
+    articleContentElement
+  )
+
+  const patch = generateDiffPatch(
+    allArticleNodes,
+    beforeNodes,
+    highlightNodes,
+    afterNodes
+  )
+
+  const id = uuidv4()
+  const shortId = nanoid(8)
+  const info = getPrefixAndSuffix(allArticleNodes, patch)
+  const quote = getQuoteText(highlightNodes)
+
+  return {
+    id,
+    shortId,
+    quote,
+    patch,
+    prefix: info.prefix,
+    suffix: info.suffix,
+  }
+}
+
+const getQuoteText = (highlight: ArticleTextContent): string => {
+  let quote = ''
+
+  highlight.textNodes.forEach((textNode, i) => {
+    if (textNode.isParagraphStart && i > 0) {
+      quote += '\n'
+    }
+    quote += textNode.node.textContent
+  })
+
+  return quote
+}
+
+function generateDiffPatch(
+  allArticleNodes: ArticleTextContent,
+  beforeNodes: ArticleTextContent,
+  highlightNodes: ArticleTextContent,
+  afterNodes: ArticleTextContent
+): string {
+  const textWithTags = `${beforeNodes.articleText}<${highlightTag}>${highlightNodes.articleText}</${highlightTag}>${afterNodes.articleText}`
+
+  const diffMatchPatch = new DiffMatchPatch()
+  const patch = diffMatchPatch.patch_toText(
+    diffMatchPatch.patch_make(allArticleNodes.articleText, textWithTags)
+  )
+
+  if (!patch) throw new Error('Invalid patch')
+  return patch
+}
+
+function getPrefixAndSuffix(
+  articleTextNodes: ArticleTextContent,
+  patch: string
+): {
+  prefix: string
+  suffix: string
+  highlightTextStart: number
+  highlightTextEnd: number
+  textNodes: TextNode[]
+  textNodeIndex: number
+} {
+  if (!patch) throw new Error('Invalid patch')
+  const textNodes = articleTextNodes.textNodes
+
+  const { highlightTextStart, highlightTextEnd } = selectionOffsetsFromPatch(
+    articleTextNodes.articleText,
+    patch
+  )
+
+  // Searching for the starting text node using interpolation search algorithm
+  const textNodeIndex = interpolationSearch(
+    textNodes.map(({ startIndex: startIndex }) => startIndex),
+    highlightTextStart
+  )
+  const endTextNodeIndex = interpolationSearch(
+    textNodes.map(({ startIndex: startIndex }) => startIndex),
+    highlightTextEnd
+  )
+
+  const prefix = getSurroundingText({
+    textNodes,
+    startingTextNodeIndex: textNodeIndex,
+    startingOffset: highlightTextStart - textNodes[textNodeIndex].startIndex,
+    side: 'prefix',
+  })
+  const suffix = getSurroundingText({
+    textNodes,
+    startingTextNodeIndex: endTextNodeIndex,
+    startingOffset: highlightTextEnd - textNodes[endTextNodeIndex].startIndex,
+    side: 'suffix',
+  })
+  return {
+    prefix,
+    suffix,
+    highlightTextStart,
+    highlightTextEnd,
+    textNodes,
+    textNodeIndex,
+  }
+}
+
+/**
+ * Gets the part of text from the starting point to the paragraph ending from the
+ * specified side
+ * @param param0 - Object that includes textNodes array, starting point and the
+ * way of movement (prefix, suffix)
+ * @returns String of text to fulfill the paragraph that surrounds the
+ * highlight from either starting or ending point
+ */
+const getSurroundingText = ({
+  textNodes,
+  startingTextNodeIndex,
+  startingOffset,
+  side,
+}: {
+  textNodes: TextNode[]
+  startingTextNodeIndex: number
+  startingOffset: number
+  side: 'prefix' | 'suffix'
+}): string => {
+  const isPrefix = side === 'prefix'
+  let i = startingTextNodeIndex
+  const getTextPart = (): string => {
+    i += isPrefix ? -1 : 1
+    const { node, isParagraphStart: startsParagraph } = textNodes[i]
+    const text = node.nodeValue || ''
+
+    if (isPrefix) {
+      if (startsParagraph) return text
+      if (text.length > maxSurroundingTextLength) return text
+      return getTextPart() + text
+    } else {
+      if (!textNodes[i + 1] || textNodes[i + 1].isParagraphStart) return text
+      if (text.length > maxSurroundingTextLength) return text
+      return text + getTextPart()
+    }
+  }
+  const truncateText = (str: string): string => {
+    if (str.length <= maxSurroundingTextLength) return str
+    if (isPrefix) {
+      return str.slice(str.length - maxSurroundingTextLength)
+    }
+    return str.substring(0, maxSurroundingTextLength)
+  }
+
+  const { isParagraphStart: startsParagraph, node } =
+    textNodes[startingTextNodeIndex]
+  const nodeText = node.nodeValue || ''
+
+  const text = isPrefix
+    ? nodeText.substring(0, startingOffset)
+    : nodeText.substring(startingOffset)
+
+  if (isPrefix) {
+    return truncateText(startsParagraph ? text : getTextPart() + text)
+  } else {
+    return truncateText(
+      !textNodes[i + 1] || textNodes[i + 1].isParagraphStart
+        ? text
+        : text + getTextPart()
+    )
+  }
+}
+
+const selectionOffsetsFromPatch = (
+  articleText: string,
+  patch: string
+): {
+  highlightTextStart: number
+  highlightTextEnd: number
+  matchingHighlightContent: RegExpExecArray
+} => {
+  if (!patch) throw new Error('Invalid patch')
+  const dmp = new DiffMatchPatch()
+  // Applying a patch to the whole article text to find the selection content via regexp
+  const appliedPatch = dmp.patch_apply(dmp.patch_fromText(patch), articleText)
+
+  let matchingHighlightContent
+  if (!appliedPatch[1][0]) {
+    dmp.Match_Threshold = maxDeepPatchThreshhold
+    dmp.Match_Distance = maxDeepPatchDistance
+    const deeperAppliedPatch = dmp.patch_apply(
+      dmp.patch_fromText(patch),
+      articleText
+    )
+    if (!deeperAppliedPatch[1][0]) {
+      throw new Error('Unable to find the highlight')
+    } else {
+      matchingHighlightContent = highlightContentRegex.exec(
+        deeperAppliedPatch[0]
+      )
+    }
+  } else {
+    matchingHighlightContent = highlightContentRegex.exec(appliedPatch[0])
+  }
+
+  if (!matchingHighlightContent)
+    throw new Error('Unable to find the highlight from patch')
+
+  const highlightTextStart = matchingHighlightContent.index
+  const highlightTextEnd =
+    highlightTextStart + matchingHighlightContent[1].length
+  return {
+    highlightTextStart,
+    highlightTextEnd,
+    matchingHighlightContent,
+  }
+}

packages/api/src/utils/parser.ts

@@ -16,6 +16,11 @@ import { ILike } from 'typeorm'
 import { v4 as uuid } from 'uuid'
 import addressparser from 'addressparser'
 import { preParseContent } from '@omnivore/content-handler'
+import {
+  findEmbeddedHighlight,
+  EmbeddedHighlightData,
+} from './highlightGenerator'
+import { HighlightData } from '../datalayer/highlight/model'
 
 const logger = buildLogger('utils.parse')
 
@@ -70,6 +75,7 @@ export type ParsedContentPuppeteer = {
   parsedContent: Readability.ParseResult | null
   canonicalUrl?: string | null
   pageType: PageType
+  highlightData?: EmbeddedHighlightData
 }
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
@@ -178,6 +184,7 @@ export const parsePreparedContent = async (
   }
 
   let article = null
+  let highlightData = undefined
   const { document, pageInfo } = preparedDocument
 
   // Checking for content type acceptance or if there are no contentType
@@ -234,6 +241,10 @@ export const parsePreparedContent = async (
         article.content = article.dom.outerHTML
       }
 
+      if (article?.dom) {
+        highlightData = findEmbeddedHighlight(article?.dom)
+      }
+
       const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [
         'omnivore-highlight-id',
         'data-twitter-tweet-id',
@@ -315,6 +326,7 @@ export const parsePreparedContent = async (
     parsedContent: article,
     canonicalUrl,
     pageType: parseOriginalContent(dom),
+    highlightData,
   }
 }
 

packages/api/test/utils/parser.test.ts

@@ -38,9 +38,9 @@ describe('parseMetadata', async () => {
 
 describe('parsePreparedContent', async () => {
   it('gets published date when JSONLD fails to load', async () => {
-    nock('https://stratechery.com:443', {"encodedQueryParams":true})
+    nock('https://stratechery.com:443', { encodedQueryParams: true })
       .get('/wp-json/oembed/1.0/embed')
-      .query({"url":"https%3A%2F%2Fstratechery.com%2F2016%2Fits-a-tesla%2F"})
+      .query({ url: 'https%3A%2F%2Fstratechery.com%2F2016%2Fits-a-tesla%2F' })
       .reply(401)
 
     const html = load('./test/utils/data/stratechery-blog-post.html')
@@ -53,6 +53,23 @@ describe('parsePreparedContent', async () => {
       new Date('2016-04-05T15:27:51+00:00').getTime()
     )
   })
+  it('returns a highlight range if markers are found in the HTML', async () => {
+    const html = `
+      <html>
+        <body>
+          <span data-omnivore-highlight-start="true"></span>This is some text within the highlight markers<span data-omnivore-highlight-end="true"></span>
+        </body>
+      </html>
+    `
+    const result = await parsePreparedContent('https://blog.omnivore.app/', {
+      document: html,
+      pageInfo: {},
+    })
+
+    expect(result.highlightData?.quote).to.eq(
+      'This is some text within the highlight markers'
+    )
+  })
 })
 
 describe('parsePreparedContent', async () => {