Read Utterance input

packages/text-to-speech/src/htmlToSsml.ts

@@ -4,6 +4,13 @@ import * as _ from 'underscore'
 // this code needs to be kept in sync with the
 // frontend code in: useReadingProgressAnchor
 
+export interface Utterance {
+  wordOffset: number
+  voice?: string
+  words: string[]
+  text: string
+}
+
 const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [
   'omnivore-highlight-id',
   'data-twitter-tweet-id',
@@ -153,15 +160,18 @@ export type SSMLOptions = {
   language: string
 }
 
-const startSsml = (element: Element, options: SSMLOptions): string => {
+export const startSsml = (
+  element: Element | null,
+  options: SSMLOptions
+): string => {
   const voice =
-    element.nodeName === 'BLOCKQUOTE'
+    element?.nodeName === 'BLOCKQUOTE'
       ? options.secondaryVoice
       : options.primaryVoice
   return `<speak xmlns="http://www.w3.org/2001/10/synthesis" version="1.0" xml:lang="${options.language}"><voice name="${voice}"><prosody rate="${options.rate}">`
 }
 
-const endSsml = (): string => {
+export const endSsml = (): string => {
   return `</prosody></voice></speak>`
 }
 

packages/text-to-speech/src/index.ts

@@ -9,9 +9,28 @@ import * as jwt from 'jsonwebtoken'
 import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
 import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech'
 import { File, Storage } from '@google-cloud/storage'
-import { PassThrough } from 'stream'
 import { htmlToSsml } from './htmlToSsml'
-import * as fs from 'fs'
+
+interface SSMLInput {
+  text: string
+}
+
+interface UtteranceInput {
+  voice?: string
+  rate?: number
+  language?: string
+  text: string
+}
+
+interface HTMLInput {
+  id: string
+  text: string
+  voice?: string
+  language?: string
+  rate?: number
+  complimentaryVoice?: string
+  bucket: string
+}
 
 dotenv.config()
 Sentry.GCPFunction.init({
@@ -71,7 +90,7 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction(
       console.error(e)
       return res.status(200).send('UNAUTHENTICATED')
     }
-    const input = req.body as TextToSpeechInput
+    const input = req.body as HTMLInput
     const id = input.id
     const bucket = input.bucket
     if (!id || !bucket) {
@@ -130,41 +149,41 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
     }
     const token = (req.query.token || req.headers.authorization) as string
     if (!token) {
-      return res.status(200).send({ errorCode: 'UNAUTHORIZED' })
+      return res.status(401).send({ errorCode: 'UNAUTHORIZED' })
     }
     try {
       jwt.verify(token, process.env.JWT_SECRET)
     } catch (e) {
       console.error(e)
-      return res.status(200).send({ errorCode: 'UNAUTHORIZED' })
+      return res.status(401).send({ errorCode: 'UNAUTHORIZED' })
     }
 
     try {
       // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-      // const ssmlItems = req.body.ssmlItems as string[]
-      // if (!ssmlItems || ssmlItems.length === 0) {
-      //   return res.status(200).send({ errorCode: 'INVALID_DATA' })
-      // }
-      // hardcoded for now
-      const ssml = fs.readFileSync('./data/ssml.xml', 'utf8')
-      const audioStream = new PassThrough()
-      const speechMarksStream = new PassThrough()
-      const input: TextToSpeechInput = {
-        text: '',
-        textType: 'ssml',
-        audioStream,
-        ssmlItems: [ssml],
-        speechMarksStream,
+      const utteranceInput = req.body as UtteranceInput
+      if (!utteranceInput.text) {
+        return res.status(400).send({ errorCode: 'INVALID_DATA' })
       }
-      res.set({
-        'Content-Type': 'audio/mpeg',
-        'Transfer-Encoding': 'chunked',
-      })
-
-      console.info('Text to speech starts streaming')
-      audioStream.pipe(res)
-
-      await synthesizeTextToSpeech(input)
+      const input: TextToSpeechInput = {
+        ...utteranceInput,
+        textType: 'utterance',
+      }
+      const { audioStream, speechMarks } = await synthesizeTextToSpeech(input)
+      // const readStream = new Readable()
+      // readStream.push(JSON.stringify({ audioData, speechMarks }))
+      //
+      // res.set({
+      //   'Content-Type': 'application/json',
+      //   'Transfer-Encoding': 'chunked',
+      // })
+      // console.info('Text to speech starts streaming')
+      // pipeline(readStream, res, (err) => {
+      //   if (err) {
+      //     console.error('Text to speech streaming error', err)
+      //     res.status(500).send({ errorCode: 'STREAMING_ERROR' })
+      //   }
+      // })
+      res.send({ audioData: audioStream.read(), speechMarks })
     } catch (e) {
       console.error('Text to speech streaming error', e)
       return res.status(500).send({ errorCodes: 'SYNTHESIZER_ERROR' })

packages/text-to-speech/src/textToSpeech.ts

@@ -7,24 +7,22 @@ import {
   SpeechSynthesisResult,
   SpeechSynthesizer,
 } from 'microsoft-cognitiveservices-speech-sdk'
-import { htmlToSsmlItems, ssmlItemText } from './htmlToSsml'
+import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml'
+import { PassThrough } from 'stream'
 
 export interface TextToSpeechInput {
-  id?: string
   text: string
   voice?: string
-  languageCode?: string
-  textType?: 'html' | 'ssml'
+  language?: string
+  textType?: 'html' | 'ssml' | 'utterance'
   rate?: number
-  volume?: number
   complimentaryVoice?: string
-  bucket?: string
-  audioStream: NodeJS.ReadWriteStream
-  ssmlItems?: string[]
-  speechMarksStream: NodeJS.ReadWriteStream
+  audioStream?: NodeJS.ReadWriteStream
+  speechMarksStream?: NodeJS.ReadWriteStream
 }
 
 export interface TextToSpeechOutput {
+  audioStream: NodeJS.ReadWriteStream
   speechMarks: SpeechMark[]
 }
 
@@ -43,8 +41,8 @@ export const synthesizeTextToSpeech = async (
     throw new Error('Azure Speech Key or Region not set')
   }
   const textType = input.textType || 'html'
-  const audioStream = input.audioStream
-  const speechMarksStream = input.speechMarksStream
+  const audioStream = input.audioStream || new PassThrough()
+  const speechMarksStream = input.speechMarksStream || new PassThrough()
   const speechConfig = SpeechConfig.fromSubscription(
     process.env.AZURE_SPEECH_KEY,
     process.env.AZURE_SPEECH_REGION
@@ -95,17 +93,15 @@ export const synthesizeTextToSpeech = async (
         e.text
       }`
     )
-    speechMarksStream.write(
-      Buffer.from(
-        JSON.stringify({
-          word: e.text,
-          time: (timeOffset + e.audioOffset) / 10000,
-          start: e.textOffset,
-          length: e.wordLength,
-          type: 'word',
-        })
-      )
-    )
+    const speechMark: SpeechMark = {
+      word: e.text,
+      time: (timeOffset + e.audioOffset) / 10000,
+      start: e.textOffset,
+      length: e.wordLength,
+      type: 'word',
+    }
+    speechMarks.push(speechMark)
+    speechMarksStream.write(Buffer.from(JSON.stringify(speechMark)))
   }
 
   synthesizer.bookmarkReached = (s, e) => {
@@ -114,15 +110,13 @@ export const synthesizeTextToSpeech = async (
         e.audioOffset / 10000
       }ms, bookmark text: ${e.text}`
     )
-    speechMarksStream.write(
-      Buffer.from(
-        JSON.stringify({
-          word: e.text,
-          time: (timeOffset + e.audioOffset) / 10000,
-          type: 'bookmark',
-        })
-      )
-    )
+    const speechMark: SpeechMark = {
+      word: e.text,
+      time: (timeOffset + e.audioOffset) / 10000,
+      type: 'bookmark',
+    }
+    speechMarks.push(speechMark)
+    speechMarksStream.write(Buffer.from(JSON.stringify(speechMark)))
   }
 
   const speakSsmlAsyncPromise = (
@@ -142,22 +136,22 @@ export const synthesizeTextToSpeech = async (
   }
 
   try {
+    const ssmlOptions = {
+      primaryVoice: input.voice || 'en-US-JennyNeural',
+      secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural',
+      language: input.language || 'en-US',
+      rate: '1.333',
+    }
     if (textType === 'html') {
-      const ssmlItems = htmlToSsmlItems(input.text, {
-        primaryVoice: input.voice || 'en-US-JennyNeural',
-        secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural',
-        language: input.languageCode || 'en-US',
-        rate: '1.333',
-      })
+      const ssmlItems = htmlToSsmlItems(input.text, ssmlOptions)
       for (const ssmlItem of ssmlItems) {
         const ssml = ssmlItemText(ssmlItem)
         const result = await speakSsmlAsyncPromise(ssml)
         timeOffset = timeOffset + result.audioDuration
       }
     } else {
-      for (const ssmlItem of input.ssmlItems || []) {
-        await speakSsmlAsyncPromise(ssmlItem)
-      }
+      const ssml = `${startSsml(null, ssmlOptions)}${input.text}${endSsml()}`
+      await speakSsmlAsyncPromise(ssml)
     }
   } catch (error) {
     console.error('synthesis error', error)
@@ -171,6 +165,7 @@ export const synthesizeTextToSpeech = async (
   }
 
   return {
+    audioStream,
     speechMarks,
   }
 }