Merge pull request #1170 from omnivore-app/feature/parse-utterances

feature/parse utterances

packages/api/package.json

@@ -61,7 +61,6 @@
     "graphql-shield": "^7.5.0",
     "highlightjs": "^9.16.2",
     "html-entities": "^2.3.2",
-    "html-to-text": "^8.2.1",
     "intercom-client": "^3.1.4",
     "jsonwebtoken": "^8.5.1",
     "jwks-rsa": "^2.0.3",
@@ -71,7 +70,6 @@
     "luxon": "^2.3.1",
     "microsoft-cognitiveservices-speech-sdk": "^1.22.0",
     "nanoid": "^3.1.25",
-    "natural": "^5.2.3",
     "nodemailer": "^6.7.3",
     "normalize-url": "^6.1.0",
     "oauth": "^0.9.15",
@@ -107,13 +105,11 @@
     "@types/express": "^4.17.7",
     "@types/graphql-fields": "^1.3.4",
     "@types/highlightjs": "^9.12.2",
-    "@types/html-to-text": "^8.1.1",
     "@types/intercom-client": "^2.11.8",
     "@types/jsonwebtoken": "^8.5.0",
     "@types/luxon": "^1.25.0",
     "@types/mocha": "^8.2.2",
     "@types/nanoid": "^3.0.0",
-    "@types/natural": "^5.1.1",
     "@types/nodemailer": "^6.4.4",
     "@types/oauth": "^0.9.1",
     "@types/private-ip": "^1.0.0",

packages/api/src/routers/article_router.ts

@@ -22,28 +22,15 @@ import { getPageById, updatePage } from '../elastic/pages'
 import { generateDownloadSignedUrl } from '../utils/uploads'
 import { enqueueTextToSpeech } from '../utils/createTask'
 import { createPubSubClient } from '../datalayer/pubsub'
-import { htmlToSsmlItems, SSMLItem } from '@omnivore/text-to-speech-handler'
-import { WordPunctTokenizer } from 'natural'
-import { htmlToText } from 'html-to-text'
+import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler'
 
-interface Utterance {
-  wordOffset: number
-  wordCount: number
+interface SpeechInput {
   voice?: string
-  text: string
-  idx: number
+  secondaryVoice?: string
+  priority?: 'low' | 'high'
 }
-
-interface SSMLOutput {
-  wordCount: number
-  averageWPM: number
-  language: string
-  defaultVoice: string
-  utterances: Utterance[]
-}
-
+const outputFormats = ['mp3', 'speech-marks', 'speech-file']
 const logger = buildLogger('app.dispatch')
-const WORDS_PER_MINUTE = 200
 
 export function articleRouter() {
   const router = express.Router()
@@ -93,18 +80,13 @@ export function articleRouter() {
   })
 
   router.get(
-    '/:id/:outputFormat/:priority/:voice?',
+    '/:id/:outputFormat',
     cors<express.Request>(corsConfig),
     async (req, res) => {
       const articleId = req.params.id
       const outputFormat = req.params.outputFormat
-      const voice = req.params.voice || 'en-US-JennyNeural'
-      const priority = req.params.priority
-      if (
-        !articleId ||
-        !['mp3', 'speech-marks', 'ssml'].includes(outputFormat) ||
-        !['low', 'high'].includes(priority)
-      ) {
+      const { voice, priority, secondaryVoice } = req.query as SpeechInput
+      if (!articleId || outputFormats.indexOf(outputFormat) === -1) {
         return res.status(400).send('Invalid data')
       }
       const token = req.cookies?.auth || req.headers?.authorization
@@ -120,26 +102,17 @@ export function articleRouter() {
         },
       })
 
-      if (outputFormat === 'ssml') {
+      if (outputFormat === 'speech-file') {
         const page = await getPageById(articleId)
         if (!page) {
           return res.status(404).send('Page not found')
         }
-        const ssmlItems = htmlToSsmlItems(page.content, {
+        const speechFile = htmlToSpeechFile(page.content, {
           primaryVoice: voice,
-          secondaryVoice: 'en-US-GuyNeural',
-          rate: '1',
-          language: page.language || 'en-US',
+          secondaryVoice: secondaryVoice,
+          language: page.language,
         })
-        const [utterances, wordCount] = ssmlItemsToUtterances(ssmlItems)
-        const ssmlOutput: SSMLOutput = {
-          wordCount,
-          averageWPM: WORDS_PER_MINUTE,
-          language: page.language || 'en-US',
-          defaultVoice: voice,
-          utterances,
-        }
-        return res.send(ssmlOutput)
+        return res.send(speechFile)
       }
 
       const existingSpeech = await getRepository(Speech).findOne({
@@ -199,7 +172,7 @@ export function articleRouter() {
         speechId: speech.id,
         text: page.content,
         voice: speech.voice,
-        priority: priority as 'low' | 'high',
+        priority: priority || 'high',
       })
       logger.info('Start Text to speech task', { taskName })
       res.status(202).send('Text to speech task started')
@@ -219,24 +192,3 @@ const redirectUrl = async (speech: Speech, outputFormat: string) => {
       return generateDownloadSignedUrl(speech.audioFileName)
   }
 }
-
-const ssmlItemsToUtterances = (items: SSMLItem[]): [Utterance[], number] => {
-  const tokenizer = new WordPunctTokenizer()
-  let wordOffset = 0
-  return [
-    items.map((item) => {
-      const text = htmlToText(item.textItems.join(''), { wordwrap: false })
-      const wordCount = tokenizer.tokenize(text).length
-      const utterance: Utterance = {
-        wordOffset,
-        wordCount,
-        text,
-        voice: item.voice,
-        idx: item.idx,
-      }
-      wordOffset += wordCount
-      return utterance
-    }),
-    wordOffset,
-  ]
-}

packages/api/src/textToSpeech.d.ts

@@ -1,21 +1,29 @@
 declare module '@omnivore/text-to-speech-handler' {
-  export function htmlToSsmlItems(
+  export function htmlToSpeechFile(
     html: string,
     options: SSMLOptions
-  ): SSMLItem[]
+  ): SpeechFile
 
   export interface SSMLOptions {
-    primaryVoice: string
-    secondaryVoice: string
-    rate: string
-    language: string
+    primaryVoice?: string
+    secondaryVoice?: string
+    rate?: number
+    language?: string
   }
 
-  export interface SSMLItem {
-    open: string
-    close: string
-    textItems: string[]
+  interface Utterance {
     idx: number
+    wordOffset: number
+    wordCount: number
     voice?: string
+    text: string
+  }
+
+  export interface SpeechFile {
+    wordCount: number
+    averageWPM: number
+    language: string
+    defaultVoice: string
+    utterances: Utterance[]
   }
 }

packages/text-to-speech/package.json

@@ -23,7 +23,9 @@
   "devDependencies": {
     "@types/node": "^14.11.2",
     "@types/underscore": "^1.11.4",
-    "eslint-plugin-prettier": "^4.0.0"
+    "eslint-plugin-prettier": "^4.0.0",
+    "@types/html-to-text": "^8.1.1",
+    "@types/natural": "^5.1.1"
   },
   "dependencies": {
     "@google-cloud/functions-framework": "3.1.2",
@@ -34,6 +36,8 @@
     "jsonwebtoken": "^8.5.1",
     "linkedom": "^0.14.12",
     "microsoft-cognitiveservices-speech-sdk": "^1.22.0",
-    "underscore": "^1.13.4"
+    "underscore": "^1.13.4",
+    "natural": "^5.2.3",
+    "html-to-text": "^8.2.1"
   }
 }

packages/text-to-speech/src/htmlToSsml.ts

@@ -1,9 +1,47 @@
 import { parseHTML } from 'linkedom'
 import * as _ from 'underscore'
+import { WordPunctTokenizer } from 'natural'
+import { htmlToText } from 'html-to-text'
 
 // this code needs to be kept in sync with the
 // frontend code in: useReadingProgressAnchor
 
+export interface Utterance {
+  idx: number
+  text: string
+  wordOffset: number
+  wordCount: number
+  voice?: string
+}
+
+export interface SpeechFile {
+  wordCount: number
+  averageWPM: number
+  language: string
+  defaultVoice: string
+  utterances: Utterance[]
+}
+
+export type SSMLItem = {
+  open: string
+  close: string
+  textItems: string[]
+  idx: number
+  voice?: string
+}
+
+export type SSMLOptions = {
+  primaryVoice?: string
+  secondaryVoice?: string
+  rate?: number
+  language?: string
+}
+
+const WORDS_PER_MINUTE = 200
+const DEFAULT_LANGUAGE = 'en-US'
+const DEFAULT_VOICE = 'en-US-JennyNeural'
+const DEFAULT_RATE = 1.25
+
 const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [
   'omnivore-highlight-id',
   'data-twitter-tweet-id',
@@ -140,30 +178,16 @@ function emitElement(
   return Number(maxVisitedIdx)
 }
 
-export type SSMLItem = {
-  open: string
-  close: string
-  textItems: string[]
-  idx: number
-  voice?: string
-}
-
-export type SSMLOptions = {
-  primaryVoice: string
-  secondaryVoice: string
-  rate: number
-  language: string
-}
-
-export const startSsml = (
-  element: Element | null,
-  options: SSMLOptions
-): string => {
+export const startSsml = (options: SSMLOptions, element?: Element): string => {
   const voice =
     element?.nodeName === 'BLOCKQUOTE'
       ? options.secondaryVoice
       : options.primaryVoice
-  return `<speak xmlns="http://www.w3.org/2001/10/synthesis" version="1.0" xml:lang="${options.language}"><voice name="${voice}"><prosody rate="${options.rate}">`
+  return `<speak xmlns="http://www.w3.org/2001/10/synthesis" version="1.0" xml:lang="${
+    options.language || DEFAULT_LANGUAGE
+  }"><voice name="${voice || DEFAULT_VOICE}"><prosody rate="${
+    options.rate || DEFAULT_RATE
+  }">`
 }
 
 export const endSsml = (): string => {
@@ -210,7 +234,7 @@ export const htmlToSsmlItems = (
       const idx = i
       i = emitElement(textItems, node, true)
       items.push({
-        open: startSsml(node, options),
+        open: startSsml(options, node),
         close: endSsml(),
         textItems: textItems,
         idx,
@@ -222,3 +246,69 @@ export const htmlToSsmlItems = (
 
   return items
 }
+
+const htmlToUtterance = (
+  tokenizer: WordPunctTokenizer,
+  idx: number,
+  htmlItems: string[],
+  wordOffset: number,
+  voice?: string
+): Utterance => {
+  const text = htmlToText(htmlItems.join(''), { wordwrap: false })
+  const wordCount = tokenizer.tokenize(text).length
+  return {
+    idx,
+    text,
+    wordOffset,
+    wordCount,
+    voice,
+  }
+}
+
+export const htmlToSpeechFile = (
+  html: string,
+  options: SSMLOptions
+): SpeechFile => {
+  console.debug('creating speech file with options', options)
+
+  const dom = parseHTML(html)
+  const body = dom.document.querySelector('#readability-page-1')
+  if (!body) {
+    throw new Error('Unable to parse HTML document')
+  }
+
+  const parsedNodes = parseDomTree(body)
+  if (parsedNodes.length < 1) {
+    throw new Error('No HTML nodes found')
+  }
+
+  const tokenizer = new WordPunctTokenizer()
+  const utterances: Utterance[] = []
+  let wordOffset = 0
+  for (let i = 2; i < parsedNodes.length + 2; i++) {
+    const textItems: string[] = []
+    const node = parsedNodes[i - 2]
+
+    if (TOP_LEVEL_TAGS.includes(node.nodeName) || hasSignificantText(node)) {
+      const idx = i
+      i = emitElement(textItems, node, true)
+      const utterance = htmlToUtterance(
+        tokenizer,
+        idx,
+        textItems,
+        wordOffset,
+        node.nodeName === 'BLOCKQUOTE' ? options.secondaryVoice : undefined
+      )
+      utterances.push(utterance)
+      wordOffset += utterance.wordCount
+    }
+  }
+
+  return {
+    wordCount: wordOffset,
+    averageWPM: WORDS_PER_MINUTE,
+    language: options.language || DEFAULT_LANGUAGE,
+    defaultVoice: options.primaryVoice || DEFAULT_VOICE,
+    utterances,
+  }
+}

packages/text-to-speech/src/index.ts

@@ -9,11 +9,7 @@ import * as jwt from 'jsonwebtoken'
 import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
 import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech'
 import { File, Storage } from '@google-cloud/storage'
-import { htmlToSsmlItems } from './htmlToSsml'
-
-interface SSMLInput {
-  text: string
-}
+import { htmlToSpeechFile } from './htmlToSsml'
 
 interface UtteranceInput {
   voice?: string
@@ -176,9 +172,9 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
         return res.status(500).send({ errorCode: 'SYNTHESIZER_ERROR' })
       }
       res.send({
+        idx: utteranceInput.idx,
         audioData: audioData.toString('hex'),
         speechMarks,
-        idx: utteranceInput.idx,
       })
     } catch (e) {
       console.error('Text to speech streaming error', e)
@@ -188,7 +184,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
 )
 
 module.exports = {
-  htmlToSsmlItems,
+  htmlToSpeechFile,
   textToSpeechStreamingHandler,
   textToSpeechHandler,
 }

packages/text-to-speech/src/textToSpeech.ts

@@ -13,9 +13,9 @@ export interface TextToSpeechInput {
   text: string
   voice?: string
   language?: string
-  textType?: 'html' | 'ssml' | 'utterance'
+  textType?: 'html' | 'utterance'
   rate?: number
-  complimentaryVoice?: string
+  secondaryVoice?: string
   audioStream?: NodeJS.ReadWriteStream
 }
 
@@ -51,6 +51,7 @@ export const synthesizeTextToSpeech = async (
   const synthesizer = new SpeechSynthesizer(speechConfig)
   const speechMarks: SpeechMark[] = []
   let timeOffset = 0
+  let wordOffset = 0
 
   synthesizer.synthesizing = function (s, e) {
     // convert arrayBuffer to stream and write to stream
@@ -93,7 +94,7 @@ export const synthesizeTextToSpeech = async (
     speechMarks.push({
       word: e.text,
       time: (timeOffset + e.audioOffset) / 10000,
-      start: e.textOffset,
+      start: wordOffset + e.textOffset,
       length: e.wordLength,
       type: 'word',
     })
@@ -130,10 +131,10 @@ export const synthesizeTextToSpeech = async (
 
   try {
     const ssmlOptions = {
-      primaryVoice: input.voice || 'en-US-JennyNeural',
-      secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural',
-      language: input.language || 'en-US',
-      rate: input.rate || 1.25,
+      primaryVoice: input.voice,
+      secondaryVoice: input.secondaryVoice,
+      language: input.language,
+      rate: input.rate,
     }
     if (textType === 'html') {
       const ssmlItems = htmlToSsmlItems(input.text, ssmlOptions)
@@ -142,15 +143,19 @@ export const synthesizeTextToSpeech = async (
         const result = await speakSsmlAsyncPromise(ssml)
         timeOffset = timeOffset + result.audioDuration
       }
-    } else {
-      // assemble ssml
-      const ssml = `${startSsml(null, ssmlOptions)}${input.text}${endSsml()}`
-      const result = await speakSsmlAsyncPromise(ssml)
       return {
-        audioData: Buffer.from(result.audioData),
         speechMarks,
       }
     }
+    // for utterance
+    const start = startSsml(ssmlOptions)
+    wordOffset = -start.length
+    const ssml = `${start}${input.text}${endSsml()}`
+    const result = await speakSsmlAsyncPromise(ssml)
+    return {
+      audioData: Buffer.from(result.audioData),
+      speechMarks,
+    }
   } catch (error) {
     console.error('synthesis error', error)
     throw error
@@ -160,8 +165,4 @@ export const synthesizeTextToSpeech = async (
     synthesizer.close()
     console.debug('synthesizer closed')
   }
-
-  return {
-    speechMarks,
-  }
 }