Return utterances in the API if output format is ssml

packages/api/src/routers/article_router.ts

@@ -2,6 +2,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 /* eslint-disable @typescript-eslint/explicit-module-boundary-types */
+/* eslint-disable @typescript-eslint/no-unsafe-call */
 import express from 'express'
 import { CreateArticleErrorCode } from '../generated/graphql'
 import { isSiteBlockedForParse } from '../utils/blocked'
@@ -21,9 +22,28 @@ import { getPageById, updatePage } from '../elastic/pages'
 import { generateDownloadSignedUrl } from '../utils/uploads'
 import { enqueueTextToSpeech } from '../utils/createTask'
 import { createPubSubClient } from '../datalayer/pubsub'
-import { htmlToSsml } from '@omnivore/text-to-speech-handler'
+import { htmlToSsmlItems, SSMLItem } from '@omnivore/text-to-speech-handler'
+import { WordPunctTokenizer } from 'natural'
+import { htmlToText } from 'html-to-text'
+
+interface Utterance {
+  wordOffset: number
+  wordsCount: number
+  voice?: string
+  text: string
+  idx: number
+}
+
+interface SSMLOutput {
+  wordsCount: number
+  averageWPM: number
+  language: string
+  defaultVoice: string
+  utterances: Utterance[]
+}
 
 const logger = buildLogger('app.dispatch')
+const WORDS_PER_MINUTE = 200
 
 export function articleRouter() {
   const router = express.Router()
@@ -105,14 +125,21 @@ export function articleRouter() {
         if (!page) {
           return res.status(404).send('Page not found')
         }
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-call
-        const ssmlItems = htmlToSsml(page.content, {
+        const ssmlItems = htmlToSsmlItems(page.content, {
           primaryVoice: voice,
           secondaryVoice: 'en-US-GuyNeural',
           rate: '1',
           language: page.language || 'en-US',
         })
-        return res.send({ ssmlItems })
+        const [utterances, wordsCount] = ssmlItemsToUtterances(ssmlItems)
+        const ssmlOutput: SSMLOutput = {
+          wordsCount,
+          averageWPM: WORDS_PER_MINUTE,
+          language: page.language || 'en-US',
+          defaultVoice: voice,
+          utterances,
+        }
+        return res.send(ssmlOutput)
       }
 
       const existingSpeech = await getRepository(Speech).findOne({
@@ -192,3 +219,24 @@ const redirectUrl = async (speech: Speech, outputFormat: string) => {
       return generateDownloadSignedUrl(speech.audioFileName)
   }
 }
+
+const ssmlItemsToUtterances = (items: SSMLItem[]): [Utterance[], number] => {
+  const tokenizer = new WordPunctTokenizer()
+  let wordOffset = 0
+  return [
+    items.map((item) => {
+      const text = htmlToText(item.textItems.join(''), { wordwrap: false })
+      const wordsCount = tokenizer.tokenize(text).length
+      const utterance: Utterance = {
+        wordOffset,
+        wordsCount,
+        text,
+        voice: item.voice,
+        idx: item.idx,
+      }
+      wordOffset += wordsCount
+      return utterance
+    }),
+    wordOffset,
+  ]
+}

packages/api/src/textToSpeech.d.ts

@@ -1,12 +1,21 @@
 declare module '@omnivore/text-to-speech-handler' {
-  function htmlToSsml(html: string, options: SSMLOptions): string[]
+  export function htmlToSsmlItems(
+    html: string,
+    options: SSMLOptions
+  ): SSMLItem[]
 
-  interface SSMLOptions {
+  export interface SSMLOptions {
     primaryVoice: string
     secondaryVoice: string
     rate: string
     language: string
   }
 
-  export { htmlToSsml }
+  export interface SSMLItem {
+    open: string
+    close: string
+    textItems: string[]
+    idx: number
+    voice?: string
+  }
 }

packages/text-to-speech/src/index.ts

@@ -9,7 +9,7 @@ import * as jwt from 'jsonwebtoken'
 import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
 import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech'
 import { File, Storage } from '@google-cloud/storage'
-import { htmlToSsml } from './htmlToSsml'
+import { htmlToSsmlItems } from './htmlToSsml'
 
 interface SSMLInput {
   text: string
@@ -20,6 +20,7 @@ interface UtteranceInput {
   rate?: number
   language?: string
   text: string
+  idx: string
 }
 
 interface HTMLInput {
@@ -177,6 +178,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
       res.send({
         audioData: audioData.toString('hex'),
         speechMarks,
+        idx: utteranceInput.idx,
       })
     } catch (e) {
       console.error('Text to speech streaming error', e)
@@ -186,7 +188,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
 )
 
 module.exports = {
-  htmlToSsml,
+  htmlToSsmlItems,
   textToSpeechStreamingHandler,
   textToSpeechHandler,
 }