diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 965d4c9cc..0d46ef642 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -92,6 +92,7 @@ import { } from '../../utils/helpers' import { createImageProxyUrl } from '../../utils/imageproxy' import { + contentConverter, getDistillerResult, htmlToMarkdown, ParsedContentPuppeteer, @@ -106,10 +107,11 @@ import { import { WithDataSourcesContext } from '../types' import { pageTypeForContentType } from '../upload_files' -enum ArticleFormat { +export enum ArticleFormat { Markdown = 'markdown', Html = 'html', Distiller = 'distiller', + HighlightedMarkdown = 'highlightedMarkdown', } export type PartialArticle = Omit< @@ -943,9 +945,13 @@ export const searchResolver = authorized< if (siteIcon && !isBase64Image(siteIcon)) { siteIcon = createImageProxyUrl(siteIcon, 128, 128) } - - if (params.includeContent && params.format === 'markdown' && r.content) { - r.content = htmlToMarkdown(r.content) + if (params.includeContent && r.content) { + // convert html to the requested format + const format = params.format || ArticleFormat.Html + const converter = contentConverter(format) + if (converter) { + r.content = converter(r.content, r.highlights) + } } return { diff --git a/packages/api/src/utils/highlightGenerator.ts b/packages/api/src/utils/highlightGenerator.ts index 52aa86a4c..e9c8e05b5 100644 --- a/packages/api/src/utils/highlightGenerator.ts +++ b/packages/api/src/utils/highlightGenerator.ts @@ -1,10 +1,11 @@ import { diff_match_patch as DiffMatchPatch } from 'diff-match-patch' -import { interpolationSearch } from './interpolationSearch' -import { v4 as uuidv4 } from 'uuid' import { nanoid } from 'nanoid' +import { v4 as uuidv4 } from 'uuid' +import { interpolationSearch } from './interpolationSearch' const highlightTag = 'omnivore_highlight' export const maxHighlightLength = 2000 +export const highlightIdAttribute = 'omnivore-highlight-id' const nonParagraphTagsRegEx = /^(a|b|basefont|bdo|big|em|font|i|s|small|span|strike|strong|su[bp]|tt|u|code|mark)$/i @@ -36,6 +37,15 @@ export type EmbeddedHighlightData = { patch: string } +type FillNodeResponse = { + node: Node + textPartsToHighlight: { + text: string + highlight: boolean + }[] + isParagraphStart?: boolean +} + function getTextNodesBetween(rootNode: Node, startNode: Node, endNode: Node) { let textNodeStartingPoint = 0 let articleText = '' @@ -325,3 +335,97 @@ const selectionOffsetsFromPatch = ( matchingHighlightContent, } } + +const fillHighlight = ({ + textNodes, + startingTextNodeIndex, + highlightTextStart, + highlightTextEnd, +}: { + textNodes: TextNode[] + startingTextNodeIndex: number + highlightTextStart: number + highlightTextEnd: number +}): FillNodeResponse => { + const { + node, + startIndex: startIndex, + isParagraphStart, + } = textNodes[startingTextNodeIndex] + const text = node.nodeValue || '' + + const textBeforeHighlightLength = highlightTextStart - startIndex + const textAfterHighlightLength = highlightTextEnd - startIndex + + const textPartsToHighlight = [] + textBeforeHighlightLength > 0 && + textPartsToHighlight.push({ + text: text.substring(0, textBeforeHighlightLength), + highlight: false, + }) + textPartsToHighlight.push({ + text: text.substring(textBeforeHighlightLength, textAfterHighlightLength), + highlight: true, + }) + textAfterHighlightLength <= text.length && + textPartsToHighlight.push({ + text: text.substring(textAfterHighlightLength), + highlight: false, + }) + return { + node, + textPartsToHighlight, + isParagraphStart, + } +} + +export function makeHighlightNodeAttributes( + id: string, + patch: string, + document: Document +) { + const rootNode = document.documentElement + + const allArticleNodes = getTextNodesBetween(rootNode, rootNode, rootNode) + const { highlightTextStart, highlightTextEnd, textNodes, textNodeIndex } = + getPrefixAndSuffix(allArticleNodes, patch) + + let startingTextNodeIndex = textNodeIndex + let quote = '' + + while (highlightTextEnd > textNodes[startingTextNodeIndex].startIndex) { + const { node, textPartsToHighlight, isParagraphStart } = fillHighlight({ + textNodes, + startingTextNodeIndex, + highlightTextStart, + highlightTextEnd, + }) + const { parentNode, nextSibling } = node + + // check if the node is a
 tag
+    const isPre = node.parentElement?.tagName === 'PRE'
+
+    parentNode?.removeChild(node)
+    textPartsToHighlight.forEach(({ highlight, text: rawText }, i) => {
+      // If we are not in preformatted text, prevent hardcoded \n,
+      // we'll create new-lines based on the startsParagraph data
+      const text = isPre ? rawText : rawText.replace(/\n/g, '')
+      const newTextNode = document.createTextNode(rawText)
+
+      if (!highlight) {
+        return parentNode?.insertBefore(newTextNode, nextSibling)
+      } else {
+        if (text) {
+          isParagraphStart && !i && quote && (quote += '\n')
+          quote += text
+        }
+
+        const newHighlightSpan = document.createElement('span')
+        newHighlightSpan.setAttribute(highlightIdAttribute, id)
+        newHighlightSpan.appendChild(newTextNode)
+        return parentNode?.insertBefore(newHighlightSpan, nextSibling)
+      }
+    })
+    startingTextNodeIndex++
+  }
+}
diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts
index 8c894d968..a3046d94f 100644
--- a/packages/api/src/utils/parser.ts
+++ b/packages/api/src/utils/parser.ts
@@ -10,17 +10,21 @@ import * as hljs from 'highlightjs'
 import { decode } from 'html-entities'
 import * as jwt from 'jsonwebtoken'
 import { parseHTML } from 'linkedom'
-import { NodeHtmlMarkdown } from 'node-html-markdown'
+import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
 import { ILike } from 'typeorm'
 import { promisify } from 'util'
 import { v4 as uuid } from 'uuid'
+import { Highlight } from '../elastic/types'
 import { User } from '../entity/user'
 import { getRepository } from '../entity/utils'
 import { env } from '../env'
 import { PageType, PreparedDocumentInput } from '../generated/graphql'
+import { ArticleFormat } from '../resolvers/article'
 import {
   EmbeddedHighlightData,
   findEmbeddedHighlight,
+  highlightIdAttribute,
+  makeHighlightNodeAttributes,
 } from './highlightGenerator'
 import { createImageProxyUrl } from './imageproxy'
 import { buildLogger, LogRecord } from './logger'
@@ -488,16 +492,73 @@ export const fetchFavicon = async (
   }
 }
 
+// custom transformer to wrap  tags in markdown highlight tags `==`
+export const highlightTranslators: TranslatorConfigObject = {
+  span: ({ node }) => {
+    const id = node.getAttribute(highlightIdAttribute)
+    if (!id) return {}
+
+    return {
+      prefix: '==',
+      postfix: '==',
+    }
+  },
+}
+
 /* ********************************************************* *
  * Re-use
  * If using it several times, creating an instance saves time
  * ********************************************************* */
 const nhm = new NodeHtmlMarkdown(
   /* options (optional) */ {},
-  /* customTransformers (optional) */ undefined,
+  /* customTransformers (optional) */ highlightTranslators,
   /* customCodeBlockTranslators (optional) */ undefined
 )
 
+type contentConverterFunc = (html: string, highlights?: Highlight[]) => string
+
+export const contentConverter = (
+  format: string
+): contentConverterFunc | undefined => {
+  switch (format) {
+    case ArticleFormat.Markdown:
+      return htmlToMarkdown
+    case ArticleFormat.HighlightedMarkdown:
+      return htmlToHighlightedMarkdown
+    case ArticleFormat.Html:
+    default:
+      return undefined
+  }
+}
+
+export const htmlToHighlightedMarkdown = (
+  html: string,
+  highlights?: Highlight[]
+): string => {
+  if (!highlights) {
+    return nhm.translate(/* html */ html)
+  }
+
+  const document = parseHTML(html).document
+  // wrap highlights in special tags
+  highlights
+    .filter((h) => h.type == 'HIGHLIGHT' && h.patch)
+    .forEach((highlight) => {
+      try {
+        makeHighlightNodeAttributes(
+          highlight.id,
+          highlight.patch as string,
+          document
+        )
+      } catch (err) {
+        console.error(err)
+      }
+    })
+  html = document.documentElement.outerHTML
+
+  return nhm.translate(/* html */ html)
+}
+
 export const htmlToMarkdown = (html: string) => {
   return nhm.translate(/* html */ html)
 }