Replace emphasis elements in HTML

packages/api/src/routers/article_router.ts

@@ -18,7 +18,6 @@ import { Claims } from '../resolvers/types'
 import { getRepository } from '../entity/utils'
 import { Speech } from '../entity/speech'
 import { getPageById } from '../elastic/pages'
-import { parseHTML } from 'linkedom'
 import { synthesizeTextToSpeech } from '../utils/textToSpeech'
 import { UserPersonalization } from '../entity/user_personalization'
 
@@ -117,18 +116,19 @@ export function articleRouter() {
         return res.status(404).send('Page not found')
       }
 
-      const text = parseHTML(page.content).document.documentElement.innerText
-      if (!text) {
-        return res.status(404).send('Page has no text')
-      }
+      // const text = parseHTML(page.content).document.documentElement.innerText
+      // if (!text) {
+      //   return res.status(404).send('Page has no text')
+      // }
 
       try {
         const startTime = Date.now()
         const speechOutput = await synthesizeTextToSpeech({
           id,
-          text,
+          text: page.content,
           languageCode: page.language,
           voice: userPersonalization.speechVoice,
+          textType: 'ssml',
         })
         logger.info('Created speech', {
           audioUrl: speechOutput.audioUrl,

packages/api/src/routers/svc/speech.ts

@@ -5,7 +5,6 @@ import { getRepository } from '../../entity/utils'
 import { getPageById } from '../../elastic/pages'
 import { synthesizeTextToSpeech } from '../../utils/textToSpeech'
 import { Speech } from '../../entity/speech'
-import { parseHTML } from 'linkedom'
 import { UserPersonalization } from '../../entity/user_personalization'
 import { buildLogger } from '../../utils/logger'
 
@@ -39,12 +38,10 @@ export function speechServiceRouter() {
     if (!page) {
       return res.status(200).send('Page not found')
     }
-
-    const text = parseHTML(page.content).document.documentElement.innerText
-    if (!text) {
-      return res.status(200).send('Page has no text')
-    }
-
+    // const text = parseHTML(page.content).document.documentElement.innerText
+    // if (!text) {
+    //   return res.status(200).send('Page has no text')
+    // }
     logger.info(`Create article speech`, {
       body: {
         userId,
@@ -59,9 +56,10 @@ export function speechServiceRouter() {
       const startTime = Date.now()
       const speechOutput = await synthesizeTextToSpeech({
         id: pageId,
-        text,
+        text: page.content,
         languageCode: page.language,
         voice: userPersonalization.speechVoice,
+        textType: 'ssml',
       })
       logger.info('Created speech', {
         audioUrl: speechOutput.audioUrl,

packages/api/src/utils/textToSpeech.ts

@@ -107,7 +107,7 @@ export const synthesizeTextToSpeech = async (
   }
 
   synthesizer.bookmarkReached = (s, e) => {
-    logger.info(
+    logger.debug(
       `(Bookmark reached), Audio offset: ${
         e.audioOffset / 10000
       }ms, bookmark text: ${e.text}`
@@ -170,20 +170,20 @@ export const synthesizeTextToSpeech = async (
     }
   } else {
     const document = parseHTML(input.text).document
-    const elements = document.querySelectorAll('h1, h2, h3, p, li')
+    const elements = document.querySelectorAll('h1, h2, h3, p, ul, ol')
     // convert html elements to the ssml document
     for (const e of Array.from(elements)) {
       const htmlElement = e as HTMLElement
       if (htmlElement.innerText) {
-        const result = await speakSsmlAsyncPromise(
-          htmlElementToSsml(
-            htmlElement,
-            input.languageCode,
-            input.voice,
-            input.rate,
-            input.volume
-          )
+        const ssml = htmlElementToSsml(
+          e,
+          input.languageCode,
+          input.voice,
+          input.rate,
+          input.volume
         )
+        logger.debug(`synthesizing ${ssml}`)
+        const result = await speakSsmlAsyncPromise(ssml)
         timeOffset = timeOffset + result.audioDuration
         characterOffset = characterOffset + htmlElement.innerText.length
       }
@@ -211,12 +211,32 @@ export const synthesizeTextToSpeech = async (
 }
 
 export const htmlElementToSsml = (
-  htmlElement: HTMLElement,
+  htmlElement: Element,
   language = 'en-US',
   voice = 'en-US-JennyNeural',
   rate = 1,
   volume = 100
 ): string => {
+  const appendBookmarkElement = (parent: Element, element: Element) => {
+    const id = element.getAttribute('data-omnivore-anchor-idx')
+    if (id) {
+      const bookMark = ssml.createElement('bookmark')
+      bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`)
+      parent.appendChild(bookMark)
+    }
+  }
+
+  const replaceEmphasisElement = (element: Element, level: string) => {
+    logger.debug(`replaceEmphasisElement: ${element.innerHTML}`)
+    const parent = ssml.createDocumentFragment() as unknown as Element
+    appendBookmarkElement(parent, element)
+    const emphasisElement = ssml.createElement('emphasis')
+    emphasisElement.setAttribute('level', level)
+    emphasisElement.innerHTML = element.innerHTML.trim()
+    parent.appendChild(emphasisElement)
+    element?.parentNode?.replaceChild(parent, element)
+  }
+
   // create new ssml document
   const ssml = parseHTML('').document
   const speakElement = ssml.createElement('speak')
@@ -231,14 +251,53 @@ export const htmlElementToSsml = (
   prosodyElement.setAttribute('volume', volume.toString())
   voiceElement.appendChild(prosodyElement)
   // add each paragraph to the ssml document
-  const id = htmlElement.getAttribute('data-omnivore-anchor-idx')
-  if (id) {
-    const text = htmlElement.innerText
-    const bookMark = ssml.createElement('bookmark')
-    bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`)
-    prosodyElement.appendChild(bookMark)
-    prosodyElement.appendChild(ssml.createTextNode(text))
-  }
+  appendBookmarkElement(prosodyElement, htmlElement)
+  // add text to the ssml document
+  htmlElement.querySelectorAll('*').forEach((e) => {
+    switch (e.tagName.toLowerCase()) {
+      case 's':
+        replaceEmphasisElement(e, 'reduced')
+        break
+      case 'sub':
+        if (e.getAttribute('alias') === null) {
+          replaceEmphasisElement(e, 'reduced')
+        }
+        break
+      case 'i':
+      case 'em':
+      case 'q':
+      case 'blockquote':
+      case 'cite':
+      case 'del':
+      case 'strike':
+      case 'sup':
+      case 'summary':
+      case 'caption':
+      case 'figcaption':
+        replaceEmphasisElement(e, 'reduced')
+        break
+      case 'b':
+      case 'strong':
+      case 'dt':
+      case 'dfn':
+      case 'u':
+      case 'li':
+      case 'mark':
+      case 'th':
+      case 'title':
+      case 'var':
+        replaceEmphasisElement(e, 'moderate')
+        break
+      default: {
+        const text = (e as HTMLElement).innerText.trim()
+        if (text) {
+          const textElement = ssml.createTextNode(text)
+          e.parentNode?.replaceChild(textElement, e)
+        }
+      }
+    }
+  })
+  prosodyElement.appendChild(htmlElement)
 
   return speakElement.outerHTML
 }

packages/api/test/utils/textToSpeech.test.ts

@@ -7,16 +7,22 @@ import {
 import { expect } from 'chai'
 import { generateFakeUuid } from '../util'
 import { parseHTML } from 'linkedom'
+import fs from 'fs'
 
 describe('textToSpeech', () => {
+  const load = (path: string): string => {
+    return fs.readFileSync(path, 'utf8')
+  }
+
   describe('synthesizeTextToSpeech', () => {
     it('should create an audio file with speech marks', async () => {
+      const html = load('./test/utils/data/text-to-speech.html')
       const input: TextToSpeechInput = {
         id: generateFakeUuid(),
-        text: 'Marry had a little lamb',
+        text: html,
         languageCode: 'en-US',
         voice: 'en-US-JennyNeural',
-        textType: 'text',
+        textType: 'ssml',
       }
       const output = await synthesizeTextToSpeech(input)
       expect(output.audioUrl).to.be.a('string')
@@ -31,7 +37,7 @@ describe('textToSpeech', () => {
       ).document.documentElement
       const ssml = htmlElementToSsml(htmlElement)
       expect(ssml).to.equal(
-        `<speak xml:lang="en-US" xmlns="http://www.w3.org/2001/10/synthesis" version="1.0"><voice name="en-US-JennyNeural"><prosody volume="100" rate="1"><bookmark mark="data-omnivore-anchor-idx-1"></bookmark>Marry had a little lamb</prosody></voice></speak>`
+        `<speak xml:lang="en-US" xmlns="http://www.w3.org/2001/10/synthesis" version="1.0"><voice name="en-US-JennyNeural"><prosody volume="100" rate="1"><bookmark mark="data-omnivore-anchor-idx-1"></bookmark><p data-omnivore-anchor-idx="1">Marry had a little lamb</p></prosody></voice></speak>`
       )
     })
   })