diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 965d4c9cc..0d46ef642 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -92,6 +92,7 @@ import { } from '../../utils/helpers' import { createImageProxyUrl } from '../../utils/imageproxy' import { + contentConverter, getDistillerResult, htmlToMarkdown, ParsedContentPuppeteer, @@ -106,10 +107,11 @@ import { import { WithDataSourcesContext } from '../types' import { pageTypeForContentType } from '../upload_files' -enum ArticleFormat { +export enum ArticleFormat { Markdown = 'markdown', Html = 'html', Distiller = 'distiller', + HighlightedMarkdown = 'highlightedMarkdown', } export type PartialArticle = Omit< @@ -943,9 +945,13 @@ export const searchResolver = authorized< if (siteIcon && !isBase64Image(siteIcon)) { siteIcon = createImageProxyUrl(siteIcon, 128, 128) } - - if (params.includeContent && params.format === 'markdown' && r.content) { - r.content = htmlToMarkdown(r.content) + if (params.includeContent && r.content) { + // convert html to the requested format + const format = params.format || ArticleFormat.Html + const converter = contentConverter(format) + if (converter) { + r.content = converter(r.content, r.highlights) + } } return { diff --git a/packages/api/src/utils/highlightGenerator.ts b/packages/api/src/utils/highlightGenerator.ts index 52aa86a4c..e9c8e05b5 100644 --- a/packages/api/src/utils/highlightGenerator.ts +++ b/packages/api/src/utils/highlightGenerator.ts @@ -1,10 +1,11 @@ import { diff_match_patch as DiffMatchPatch } from 'diff-match-patch' -import { interpolationSearch } from './interpolationSearch' -import { v4 as uuidv4 } from 'uuid' import { nanoid } from 'nanoid' +import { v4 as uuidv4 } from 'uuid' +import { interpolationSearch } from './interpolationSearch' const highlightTag = 'omnivore_highlight' export const maxHighlightLength = 2000 +export const highlightIdAttribute = 'omnivore-highlight-id' const nonParagraphTagsRegEx = /^(a|b|basefont|bdo|big|em|font|i|s|small|span|strike|strong|su[bp]|tt|u|code|mark)$/i @@ -36,6 +37,15 @@ export type EmbeddedHighlightData = { patch: string } +type FillNodeResponse = { + node: Node + textPartsToHighlight: { + text: string + highlight: boolean + }[] + isParagraphStart?: boolean +} + function getTextNodesBetween(rootNode: Node, startNode: Node, endNode: Node) { let textNodeStartingPoint = 0 let articleText = '' @@ -325,3 +335,97 @@ const selectionOffsetsFromPatch = ( matchingHighlightContent, } } + +const fillHighlight = ({ + textNodes, + startingTextNodeIndex, + highlightTextStart, + highlightTextEnd, +}: { + textNodes: TextNode[] + startingTextNodeIndex: number + highlightTextStart: number + highlightTextEnd: number +}): FillNodeResponse => { + const { + node, + startIndex: startIndex, + isParagraphStart, + } = textNodes[startingTextNodeIndex] + const text = node.nodeValue || '' + + const textBeforeHighlightLength = highlightTextStart - startIndex + const textAfterHighlightLength = highlightTextEnd - startIndex + + const textPartsToHighlight = [] + textBeforeHighlightLength > 0 && + textPartsToHighlight.push({ + text: text.substring(0, textBeforeHighlightLength), + highlight: false, + }) + textPartsToHighlight.push({ + text: text.substring(textBeforeHighlightLength, textAfterHighlightLength), + highlight: true, + }) + textAfterHighlightLength <= text.length && + textPartsToHighlight.push({ + text: text.substring(textAfterHighlightLength), + highlight: false, + }) + return { + node, + textPartsToHighlight, + isParagraphStart, + } +} + +export function makeHighlightNodeAttributes( + id: string, + patch: string, + document: Document +) { + const rootNode = document.documentElement + + const allArticleNodes = getTextNodesBetween(rootNode, rootNode, rootNode) + const { highlightTextStart, highlightTextEnd, textNodes, textNodeIndex } = + getPrefixAndSuffix(allArticleNodes, patch) + + let startingTextNodeIndex = textNodeIndex + let quote = '' + + while (highlightTextEnd > textNodes[startingTextNodeIndex].startIndex) { + const { node, textPartsToHighlight, isParagraphStart } = fillHighlight({ + textNodes, + startingTextNodeIndex, + highlightTextStart, + highlightTextEnd, + }) + const { parentNode, nextSibling } = node + + // check if the node is a
tag
+ const isPre = node.parentElement?.tagName === 'PRE'
+
+ parentNode?.removeChild(node)
+ textPartsToHighlight.forEach(({ highlight, text: rawText }, i) => {
+ // If we are not in preformatted text, prevent hardcoded \n,
+ // we'll create new-lines based on the startsParagraph data
+ const text = isPre ? rawText : rawText.replace(/\n/g, '')
+ const newTextNode = document.createTextNode(rawText)
+
+ if (!highlight) {
+ return parentNode?.insertBefore(newTextNode, nextSibling)
+ } else {
+ if (text) {
+ isParagraphStart && !i && quote && (quote += '\n')
+ quote += text
+ }
+
+ const newHighlightSpan = document.createElement('span')
+ newHighlightSpan.setAttribute(highlightIdAttribute, id)
+ newHighlightSpan.appendChild(newTextNode)
+ return parentNode?.insertBefore(newHighlightSpan, nextSibling)
+ }
+ })
+ startingTextNodeIndex++
+ }
+}
diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts
index 8c894d968..a3046d94f 100644
--- a/packages/api/src/utils/parser.ts
+++ b/packages/api/src/utils/parser.ts
@@ -10,17 +10,21 @@ import * as hljs from 'highlightjs'
import { decode } from 'html-entities'
import * as jwt from 'jsonwebtoken'
import { parseHTML } from 'linkedom'
-import { NodeHtmlMarkdown } from 'node-html-markdown'
+import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
import { ILike } from 'typeorm'
import { promisify } from 'util'
import { v4 as uuid } from 'uuid'
+import { Highlight } from '../elastic/types'
import { User } from '../entity/user'
import { getRepository } from '../entity/utils'
import { env } from '../env'
import { PageType, PreparedDocumentInput } from '../generated/graphql'
+import { ArticleFormat } from '../resolvers/article'
import {
EmbeddedHighlightData,
findEmbeddedHighlight,
+ highlightIdAttribute,
+ makeHighlightNodeAttributes,
} from './highlightGenerator'
import { createImageProxyUrl } from './imageproxy'
import { buildLogger, LogRecord } from './logger'
@@ -488,16 +492,73 @@ export const fetchFavicon = async (
}
}
+// custom transformer to wrap tags in markdown highlight tags `==`
+export const highlightTranslators: TranslatorConfigObject = {
+ span: ({ node }) => {
+ const id = node.getAttribute(highlightIdAttribute)
+ if (!id) return {}
+
+ return {
+ prefix: '==',
+ postfix: '==',
+ }
+ },
+}
+
/* ********************************************************* *
* Re-use
* If using it several times, creating an instance saves time
* ********************************************************* */
const nhm = new NodeHtmlMarkdown(
/* options (optional) */ {},
- /* customTransformers (optional) */ undefined,
+ /* customTransformers (optional) */ highlightTranslators,
/* customCodeBlockTranslators (optional) */ undefined
)
+type contentConverterFunc = (html: string, highlights?: Highlight[]) => string
+
+export const contentConverter = (
+ format: string
+): contentConverterFunc | undefined => {
+ switch (format) {
+ case ArticleFormat.Markdown:
+ return htmlToMarkdown
+ case ArticleFormat.HighlightedMarkdown:
+ return htmlToHighlightedMarkdown
+ case ArticleFormat.Html:
+ default:
+ return undefined
+ }
+}
+
+export const htmlToHighlightedMarkdown = (
+ html: string,
+ highlights?: Highlight[]
+): string => {
+ if (!highlights) {
+ return nhm.translate(/* html */ html)
+ }
+
+ const document = parseHTML(html).document
+ // wrap highlights in special tags
+ highlights
+ .filter((h) => h.type == 'HIGHLIGHT' && h.patch)
+ .forEach((highlight) => {
+ try {
+ makeHighlightNodeAttributes(
+ highlight.id,
+ highlight.patch as string,
+ document
+ )
+ } catch (err) {
+ console.error(err)
+ }
+ })
+ html = document.documentElement.outerHTML
+
+ return nhm.translate(/* html */ html)
+}
+
export const htmlToMarkdown = (html: string) => {
return nhm.translate(/* html */ html)
}