Merge pull request #1152 from omnivore-app/feat/recursive-ssml-to-html

Update the HTML to SSML function to use child text items instead of using the textContent attribute'

packages/text-to-speech/src/htmlToSsml.ts

@@ -0,0 +1,185 @@
+import { parseHTML } from 'linkedom'
+
+// this code needs to be kept in sync with the
+// frontend code in: useReadingProgressAnchor
+
+const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [
+  'omnivore-highlight-id',
+  'data-twitter-tweet-id',
+  'data-instagram-id',
+]
+
+function ssmlTagsForTopLevelElement() {
+  return {
+    opening: `<p>`,
+    closing: `</p>`,
+  }
+}
+
+function parseDomTree(pageNode: Element) {
+  if (!pageNode || pageNode.childNodes.length == 0) {
+    console.log(' no child nodes found')
+    return []
+  }
+
+  const nodesToVisitStack = [pageNode]
+  const visitedNodeList = []
+
+  while (nodesToVisitStack.length > 0) {
+    const currentNode = nodesToVisitStack.pop()
+    if (
+      currentNode?.nodeType !== 1 /* Node.ELEMENT_NODE */ ||
+      // Avoiding dynamic elements from being counted as anchor-allowed elements
+      ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES.some((attrib) =>
+        currentNode.hasAttribute(attrib)
+      )
+    ) {
+      continue
+    }
+
+    visitedNodeList.push(currentNode)
+    ;[].slice
+      .call(currentNode.childNodes)
+      .reverse()
+      .forEach(function (node) {
+        nodesToVisitStack.push(node)
+      })
+  }
+
+  visitedNodeList.shift()
+  visitedNodeList.forEach((node, index) => {
+    // start from index 1, index 0 reserved for anchor unknown.
+    node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString())
+  })
+  return visitedNodeList
+}
+
+function emit(textItems: string[], text: string) {
+  textItems.push(text)
+}
+
+function cleanTextNode(textNode: ChildNode): string {
+  return (textNode.textContent ?? '').replace(/\s+/g, ' ')
+}
+
+function emitTextNode(
+  textItems: string[],
+  cleanedText: string,
+  textNode: ChildNode
+) {
+  const ssmlElement =
+    textNode.parentNode?.nodeName === 'B' ? 'emphasis' : undefined
+  if (!cleanedText) {
+    return
+  }
+
+  if (ssmlElement) {
+    emit(textItems, `<${ssmlElement}>`)
+  }
+  emit(textItems, `${cleanedText}`)
+  if (ssmlElement) {
+    emit(textItems, `</${ssmlElement}>`)
+  }
+}
+
+function emitElement(
+  textItems: string[],
+  element: Element,
+  isTopLevel: boolean
+) {
+  const SKIP_TAGS = ['SCRIPT', 'STYLE', 'IMG', 'FIGURE', 'FIGCAPTION', 'IFRAME']
+
+  const topLevelTags = ssmlTagsForTopLevelElement()
+  const idx = element.getAttribute('data-omnivore-anchor-idx')
+  let maxVisitedIdx = Number(idx)
+
+  if (isTopLevel) {
+    emit(textItems, topLevelTags.opening)
+  }
+
+  for (const child of Array.from(element.childNodes)) {
+    if (SKIP_TAGS.indexOf(child.nodeName) >= 0) {
+      continue
+    }
+
+    if (
+      child.nodeType == 3 /* Node.TEXT_NODE */ &&
+      (child.textContent?.length ?? 0) > 0
+    ) {
+      const cleanedText = cleanTextNode(child)
+      if (idx && cleanedText.length > 1) {
+        // Make sure its more than just a space
+        emit(textItems, `<bookmark mark="${idx}" />`)
+      }
+      emitTextNode(textItems, cleanedText, child)
+    }
+    if (child.nodeType == 1 /* Node.ELEMENT_NODE */) {
+      maxVisitedIdx = emitElement(textItems, child as HTMLElement, false)
+    }
+  }
+
+  if (isTopLevel) {
+    emit(textItems, topLevelTags.closing)
+  }
+
+  return Number(maxVisitedIdx)
+}
+
+export type SSMLItem = {
+  open: string
+  close: string
+  textItems: string[]
+}
+
+export type SSMLOptions = {
+  primaryVoice: string
+  secondaryVoice: string
+  rate: string
+  language: string
+}
+
+const startSsml = (element: Element, options: SSMLOptions): string => {
+  const voice =
+    element.nodeName === 'BLOCKQUOTE'
+      ? options.secondaryVoice
+      : options.primaryVoice
+  return `
+<speak xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="http://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" version="1.0" xml:lang="${options.language}"><voice name="${voice}"><prosody rate="${options.rate}" pitch="default">
+  `
+}
+
+const endSsml = (): string => {
+  return `</prosody></voice></speak>`
+}
+
+export const ssmlItemText = (item: SSMLItem): string => {
+  return [item.open, ...item.textItems, item.close].join('')
+}
+
+export const htmlToSsml = (html: string, options: SSMLOptions): SSMLItem[] => {
+  const dom = parseHTML(html)
+  const body = dom.document.querySelector('#readability-page-1')
+  if (!body) {
+    throw new Error('Unable to parse HTML document')
+  }
+
+  const parsedNodes = parseDomTree(body)
+  if (parsedNodes.length < 1) {
+    throw new Error('No HTML nodes found')
+  }
+
+  const items: SSMLItem[] = []
+  for (let i = 1; i < parsedNodes.length + 1; i++) {
+    const textItems: string[] = []
+    const node = parsedNodes[i - 1]
+
+    i = emitElement(textItems, node, true)
+    items.push({
+      open: startSsml(node, options),
+      close: endSsml(),
+      textItems: textItems,
+    })
+  }
+
+  return items
+}

packages/text-to-speech/src/index.ts

@@ -18,6 +18,7 @@ import {
 import axios from 'axios'
 import * as jwt from 'jsonwebtoken'
 import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
+import { htmlToSsml, ssmlItemText } from './htmlToSsml'
 dotenv.config()
 
 interface TextToSpeechInput {
@@ -215,35 +216,23 @@ const synthesizeTextToSpeech = async (
       currentTextChunk = ''
     }
   } else {
-    const document = parseHTML(input.text).document
-    const elements = document.querySelectorAll(
-      'h1, h2, h3, p, ul, ol, blockquote'
-    )
-    // convert html elements to the ssml document
-    for (const e of Array.from(elements)) {
-      const htmlElement = e as HTMLElement
-      if (htmlElement.innerText) {
-        // use complimentary voice for blockquote, hardcoded for now
-        const voice =
-          htmlElement.tagName.toLowerCase() === 'blockquote'
-            ? input.complimentaryVoice || 'en-US-AriaNeural'
-            : input.voice
-        const ssml = htmlElementToSsml({
-          htmlElement: e,
-          language: input.languageCode,
-          rate: input.rate,
-          volume: input.volume,
-          voice,
-        })
-        console.debug(`synthesizing ${ssml}`)
-        const result = await speakSsmlAsyncPromise(ssml)
-        // if (result.reason === ResultReason.Canceled) {
-        //   synthesizer.close()
-        //   throw new Error(result.errorDetails)
-        // }
-        timeOffset = timeOffset + result.audioDuration
-        // characterOffset = characterOffset + htmlElement.innerText.length
-      }
+    const ssmlItems = htmlToSsml(input.text, {
+      primaryVoice: speechConfig.speechSynthesisVoiceName,
+      secondaryVoice: 'en-US-GuyNeural',
+      language: speechConfig.speechSynthesisLanguage,
+      rate: '1',
+    })
+
+    for (const ssmlItem of Array.from(ssmlItems)) {
+      const ssml = ssmlItemText(ssmlItem)
+      console.debug(`synthesizing ${ssml}`)
+      const result = await speakSsmlAsyncPromise(ssml)
+      // if (result.reason === ResultReason.Canceled) {
+      //   synthesizer.close()
+      //   throw new Error(result.errorDetails)
+      // }
+      timeOffset = timeOffset + result.audioDuration
+      // characterOffset = characterOffset + htmlElement.innerText.length
     }
   }
   writeStream.end()

packages/text-to-speech/test/htmlToSsml.test.ts

@@ -0,0 +1,99 @@
+import 'mocha'
+import { expect } from 'chai'
+
+import fs  from 'fs'
+import { glob } from 'glob'
+import { htmlToSsml } from '../src/htmlToSsml'
+
+describe('htmlToSsml', () => {
+  const TEST_OPTIONS = {
+    primaryVoice: 'test-primary',
+    secondaryVoice: 'test-secondary',
+    language: 'en-US',
+    rate: '1'
+  }
+
+  describe('a simple html file', () => {
+    it('should convert Html to SSML', async () => {
+      const ssml = htmlToSsml(`
+        <div class="page" id="readability-page-1">
+          <p data-omnivore-anchor-idx="1">this is some text</p>
+        </div>
+      `, TEST_OPTIONS
+      )
+      const text = ssml[0].textItems.join('').trim()
+      expect(text).to.equal(
+        `<p><bookmark mark="1" />this is some text</p>`
+      )
+    })
+  })
+  describe('a file with nested elements', () => {
+    it('should convert Html to SSML', async () => {
+      const ssml = htmlToSsml(`
+<div class="page" id="readability-page-1">
+<p>
+this is in the first paragraph
+<span>this is in the second span</span>
+this is also in the first paragraph
+</p>
+</div>
+      `, TEST_OPTIONS
+      )
+      const text = ssml[0].textItems.join('').trim()
+      expect(text).to.equal(
+        `<p><bookmark mark="1" /> this is in the first paragraph <bookmark mark="2" />this is in the second span<bookmark mark="1" /> this is also in the first paragraph </p>`.trim()
+      )
+    })
+  })
+  describe('a file with blockquotes', () => {
+    it('should convert Html to SSML with complimentary voices', async () => {
+      const ssml = htmlToSsml(`
+<div class="page" id="readability-page-1">
+<p>first</p>
+<blockquote>second</blockquote>
+<p>third</p>
+</div>
+      `, TEST_OPTIONS
+      )
+      const first = ssml[0].textItems.join('').trim()
+      const second = ssml[1].textItems.join('').trim()
+      const third = ssml[2].textItems.join('').trim()
+
+      expect(first).to.equal(
+        `<p><bookmark mark="1" />first</p>`
+      )
+      expect(second).to.equal(
+        `<p><bookmark mark="2" />second</p>`
+      )
+      expect(third).to.equal(
+        `<p><bookmark mark="3" />third</p>`
+      )
+
+      expect(ssml[0].open.trim()).to.equal(
+        `<speak xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="http://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" version="1.0" xml:lang="en-US"><voice name="test-primary"><prosody rate="1" pitch="default">`
+      )
+      expect(ssml[1].open.trim()).to.equal(
+        `<speak xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="http://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" version="1.0" xml:lang="en-US"><voice name="test-secondary"><prosody rate="1" pitch="default">`
+      )
+      expect(ssml[2].open.trim()).to.equal(
+        `<speak xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="http://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" version="1.0" xml:lang="en-US"><voice name="test-primary"><prosody rate="1" pitch="default">`
+      )
+    })
+  })
+  // For local testing:
+  // describe('readability test files', () => {
+  //   it('should convert Html to SSML without throwing', async () => {
+  //     const g = new glob.GlobSync('../readabilityjs/test/test-pages/*')
+  //     console.log('glob: ', glob)
+  //     for (const f of g.found) {
+  //       const readablePath = `${f}/expected.html`
+  //       if (!fs.existsSync(readablePath)) {
+  //         continue
+  //       }
+  //       const html = fs.readFileSync(readablePath, { encoding: 'utf-8' })
+  //       const ssmlItems = htmlToSsml(html, TEST_OPTIONS)
+  //       console.log('SSML ITEMS', ssmlItems)
+  //     }
+  //   })
+  // })
+})