Merge pull request #2190 from omnivore-app/feature/export-highlighted-markdown-content

Allow content to be converted into highlightedMarkdown format

packages/api/src/resolvers/article/index.ts

@@ -92,6 +92,7 @@ import {
 } from '../../utils/helpers'
 import { createImageProxyUrl } from '../../utils/imageproxy'
 import {
+  contentConverter,
   getDistillerResult,
   htmlToMarkdown,
   ParsedContentPuppeteer,
@@ -106,10 +107,11 @@ import {
 import { WithDataSourcesContext } from '../types'
 import { pageTypeForContentType } from '../upload_files'
 
-enum ArticleFormat {
+export enum ArticleFormat {
   Markdown = 'markdown',
   Html = 'html',
   Distiller = 'distiller',
+  HighlightedMarkdown = 'highlightedMarkdown',
 }
 
 export type PartialArticle = Omit<
@@ -943,9 +945,13 @@ export const searchResolver = authorized<
     if (siteIcon && !isBase64Image(siteIcon)) {
       siteIcon = createImageProxyUrl(siteIcon, 128, 128)
     }
-
-    if (params.includeContent && params.format === 'markdown' && r.content) {
-      r.content = htmlToMarkdown(r.content)
+    if (params.includeContent && r.content) {
+      // convert html to the requested format
+      const format = params.format || ArticleFormat.Html
+      const converter = contentConverter(format)
+      if (converter) {
+        r.content = converter(r.content, r.highlights)
+      }
     }
 
     return {

packages/api/src/utils/highlightGenerator.ts

@@ -1,10 +1,11 @@
 import { diff_match_patch as DiffMatchPatch } from 'diff-match-patch'
-import { interpolationSearch } from './interpolationSearch'
-import { v4 as uuidv4 } from 'uuid'
 import { nanoid } from 'nanoid'
+import { v4 as uuidv4 } from 'uuid'
+import { interpolationSearch } from './interpolationSearch'
 
 const highlightTag = 'omnivore_highlight'
 export const maxHighlightLength = 2000
+export const highlightIdAttribute = 'omnivore-highlight-id'
 
 const nonParagraphTagsRegEx =
   /^(a|b|basefont|bdo|big|em|font|i|s|small|span|strike|strong|su[bp]|tt|u|code|mark)$/i
@@ -36,6 +37,15 @@ export type EmbeddedHighlightData = {
   patch: string
 }
 
+type FillNodeResponse = {
+  node: Node
+  textPartsToHighlight: {
+    text: string
+    highlight: boolean
+  }[]
+  isParagraphStart?: boolean
+}
+
 function getTextNodesBetween(rootNode: Node, startNode: Node, endNode: Node) {
   let textNodeStartingPoint = 0
   let articleText = ''
@@ -325,3 +335,97 @@ const selectionOffsetsFromPatch = (
     matchingHighlightContent,
   }
 }
+
+const fillHighlight = ({
+  textNodes,
+  startingTextNodeIndex,
+  highlightTextStart,
+  highlightTextEnd,
+}: {
+  textNodes: TextNode[]
+  startingTextNodeIndex: number
+  highlightTextStart: number
+  highlightTextEnd: number
+}): FillNodeResponse => {
+  const {
+    node,
+    startIndex: startIndex,
+    isParagraphStart,
+  } = textNodes[startingTextNodeIndex]
+  const text = node.nodeValue || ''
+
+  const textBeforeHighlightLength = highlightTextStart - startIndex
+  const textAfterHighlightLength = highlightTextEnd - startIndex
+
+  const textPartsToHighlight = []
+  textBeforeHighlightLength > 0 &&
+    textPartsToHighlight.push({
+      text: text.substring(0, textBeforeHighlightLength),
+      highlight: false,
+    })
+  textPartsToHighlight.push({
+    text: text.substring(textBeforeHighlightLength, textAfterHighlightLength),
+    highlight: true,
+  })
+  textAfterHighlightLength <= text.length &&
+    textPartsToHighlight.push({
+      text: text.substring(textAfterHighlightLength),
+      highlight: false,
+    })
+  return {
+    node,
+    textPartsToHighlight,
+    isParagraphStart,
+  }
+}
+
+export function makeHighlightNodeAttributes(
+  id: string,
+  patch: string,
+  document: Document
+) {
+  const rootNode = document.documentElement
+
+  const allArticleNodes = getTextNodesBetween(rootNode, rootNode, rootNode)
+  const { highlightTextStart, highlightTextEnd, textNodes, textNodeIndex } =
+    getPrefixAndSuffix(allArticleNodes, patch)
+
+  let startingTextNodeIndex = textNodeIndex
+  let quote = ''
+
+  while (highlightTextEnd > textNodes[startingTextNodeIndex].startIndex) {
+    const { node, textPartsToHighlight, isParagraphStart } = fillHighlight({
+      textNodes,
+      startingTextNodeIndex,
+      highlightTextStart,
+      highlightTextEnd,
+    })
+    const { parentNode, nextSibling } = node
+
+    // check if the node is a <pre> tag
+    const isPre = node.parentElement?.tagName === 'PRE'
+
+    parentNode?.removeChild(node)
+    textPartsToHighlight.forEach(({ highlight, text: rawText }, i) => {
+      // If we are not in preformatted text, prevent hardcoded \n,
+      // we'll create new-lines based on the startsParagraph data
+      const text = isPre ? rawText : rawText.replace(/\n/g, '')
+      const newTextNode = document.createTextNode(rawText)
+
+      if (!highlight) {
+        return parentNode?.insertBefore(newTextNode, nextSibling)
+      } else {
+        if (text) {
+          isParagraphStart && !i && quote && (quote += '\n')
+          quote += text
+        }
+
+        const newHighlightSpan = document.createElement('span')
+        newHighlightSpan.setAttribute(highlightIdAttribute, id)
+        newHighlightSpan.appendChild(newTextNode)
+        return parentNode?.insertBefore(newHighlightSpan, nextSibling)
+      }
+    })
+    startingTextNodeIndex++
+  }
+}

packages/api/src/utils/parser.ts

@@ -10,17 +10,21 @@ import * as hljs from 'highlightjs'
 import { decode } from 'html-entities'
 import * as jwt from 'jsonwebtoken'
 import { parseHTML } from 'linkedom'
-import { NodeHtmlMarkdown } from 'node-html-markdown'
+import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
 import { ILike } from 'typeorm'
 import { promisify } from 'util'
 import { v4 as uuid } from 'uuid'
+import { Highlight } from '../elastic/types'
 import { User } from '../entity/user'
 import { getRepository } from '../entity/utils'
 import { env } from '../env'
 import { PageType, PreparedDocumentInput } from '../generated/graphql'
+import { ArticleFormat } from '../resolvers/article'
 import {
   EmbeddedHighlightData,
   findEmbeddedHighlight,
+  highlightIdAttribute,
+  makeHighlightNodeAttributes,
 } from './highlightGenerator'
 import { createImageProxyUrl } from './imageproxy'
 import { buildLogger, LogRecord } from './logger'
@@ -488,16 +492,73 @@ export const fetchFavicon = async (
   }
 }
 
+// custom transformer to wrap <span class="highlight"> tags in markdown highlight tags `==`
+export const highlightTranslators: TranslatorConfigObject = {
+  span: ({ node }) => {
+    const id = node.getAttribute(highlightIdAttribute)
+    if (!id) return {}
+
+    return {
+      prefix: '==',
+      postfix: '==',
+    }
+  },
+}
+
 /* ********************************************************* *
  * Re-use
  * If using it several times, creating an instance saves time
  * ********************************************************* */
 const nhm = new NodeHtmlMarkdown(
   /* options (optional) */ {},
-  /* customTransformers (optional) */ undefined,
+  /* customTransformers (optional) */ highlightTranslators,
   /* customCodeBlockTranslators (optional) */ undefined
 )
 
+type contentConverterFunc = (html: string, highlights?: Highlight[]) => string
+
+export const contentConverter = (
+  format: string
+): contentConverterFunc | undefined => {
+  switch (format) {
+    case ArticleFormat.Markdown:
+      return htmlToMarkdown
+    case ArticleFormat.HighlightedMarkdown:
+      return htmlToHighlightedMarkdown
+    case ArticleFormat.Html:
+    default:
+      return undefined
+  }
+}
+
+export const htmlToHighlightedMarkdown = (
+  html: string,
+  highlights?: Highlight[]
+): string => {
+  if (!highlights) {
+    return nhm.translate(/* html */ html)
+  }
+
+  const document = parseHTML(html).document
+  // wrap highlights in special tags
+  highlights
+    .filter((h) => h.type == 'HIGHLIGHT' && h.patch)
+    .forEach((highlight) => {
+      try {
+        makeHighlightNodeAttributes(
+          highlight.id,
+          highlight.patch as string,
+          document
+        )
+      } catch (err) {
+        console.error(err)
+      }
+    })
+  html = document.documentElement.outerHTML
+
+  return nhm.translate(/* html */ html)
+}
+
 export const htmlToMarkdown = (html: string) => {
   return nhm.translate(/* html */ html)
 }