diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts
index e65755cfc..ed9d13b7a 100644
--- a/packages/text-to-speech/src/htmlToSsml.ts
+++ b/packages/text-to-speech/src/htmlToSsml.ts
@@ -4,6 +4,13 @@ import * as _ from 'underscore'
// this code needs to be kept in sync with the
// frontend code in: useReadingProgressAnchor
+export interface Utterance {
+ wordOffset: number
+ voice?: string
+ words: string[]
+ text: string
+}
+
const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [
'omnivore-highlight-id',
'data-twitter-tweet-id',
@@ -153,15 +160,18 @@ export type SSMLOptions = {
language: string
}
-const startSsml = (element: Element, options: SSMLOptions): string => {
+export const startSsml = (
+ element: Element | null,
+ options: SSMLOptions
+): string => {
const voice =
- element.nodeName === 'BLOCKQUOTE'
+ element?.nodeName === 'BLOCKQUOTE'
? options.secondaryVoice
: options.primaryVoice
return ``
}
-const endSsml = (): string => {
+export const endSsml = (): string => {
return ``
}
diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts
index ce2acb4b2..53d34d25c 100644
--- a/packages/text-to-speech/src/index.ts
+++ b/packages/text-to-speech/src/index.ts
@@ -9,9 +9,28 @@ import * as jwt from 'jsonwebtoken'
import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech'
import { File, Storage } from '@google-cloud/storage'
-import { PassThrough } from 'stream'
import { htmlToSsml } from './htmlToSsml'
-import * as fs from 'fs'
+
+interface SSMLInput {
+ text: string
+}
+
+interface UtteranceInput {
+ voice?: string
+ rate?: number
+ language?: string
+ text: string
+}
+
+interface HTMLInput {
+ id: string
+ text: string
+ voice?: string
+ language?: string
+ rate?: number
+ complimentaryVoice?: string
+ bucket: string
+}
dotenv.config()
Sentry.GCPFunction.init({
@@ -71,7 +90,7 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction(
console.error(e)
return res.status(200).send('UNAUTHENTICATED')
}
- const input = req.body as TextToSpeechInput
+ const input = req.body as HTMLInput
const id = input.id
const bucket = input.bucket
if (!id || !bucket) {
@@ -130,41 +149,41 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
}
const token = (req.query.token || req.headers.authorization) as string
if (!token) {
- return res.status(200).send({ errorCode: 'UNAUTHORIZED' })
+ return res.status(401).send({ errorCode: 'UNAUTHORIZED' })
}
try {
jwt.verify(token, process.env.JWT_SECRET)
} catch (e) {
console.error(e)
- return res.status(200).send({ errorCode: 'UNAUTHORIZED' })
+ return res.status(401).send({ errorCode: 'UNAUTHORIZED' })
}
try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
- // const ssmlItems = req.body.ssmlItems as string[]
- // if (!ssmlItems || ssmlItems.length === 0) {
- // return res.status(200).send({ errorCode: 'INVALID_DATA' })
- // }
- // hardcoded for now
- const ssml = fs.readFileSync('./data/ssml.xml', 'utf8')
- const audioStream = new PassThrough()
- const speechMarksStream = new PassThrough()
- const input: TextToSpeechInput = {
- text: '',
- textType: 'ssml',
- audioStream,
- ssmlItems: [ssml],
- speechMarksStream,
+ const utteranceInput = req.body as UtteranceInput
+ if (!utteranceInput.text) {
+ return res.status(400).send({ errorCode: 'INVALID_DATA' })
}
- res.set({
- 'Content-Type': 'audio/mpeg',
- 'Transfer-Encoding': 'chunked',
- })
-
- console.info('Text to speech starts streaming')
- audioStream.pipe(res)
-
- await synthesizeTextToSpeech(input)
+ const input: TextToSpeechInput = {
+ ...utteranceInput,
+ textType: 'utterance',
+ }
+ const { audioStream, speechMarks } = await synthesizeTextToSpeech(input)
+ // const readStream = new Readable()
+ // readStream.push(JSON.stringify({ audioData, speechMarks }))
+ //
+ // res.set({
+ // 'Content-Type': 'application/json',
+ // 'Transfer-Encoding': 'chunked',
+ // })
+ // console.info('Text to speech starts streaming')
+ // pipeline(readStream, res, (err) => {
+ // if (err) {
+ // console.error('Text to speech streaming error', err)
+ // res.status(500).send({ errorCode: 'STREAMING_ERROR' })
+ // }
+ // })
+ res.send({ audioData: audioStream.read(), speechMarks })
} catch (e) {
console.error('Text to speech streaming error', e)
return res.status(500).send({ errorCodes: 'SYNTHESIZER_ERROR' })
diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts
index 9c6356566..f9836973a 100644
--- a/packages/text-to-speech/src/textToSpeech.ts
+++ b/packages/text-to-speech/src/textToSpeech.ts
@@ -7,24 +7,22 @@ import {
SpeechSynthesisResult,
SpeechSynthesizer,
} from 'microsoft-cognitiveservices-speech-sdk'
-import { htmlToSsmlItems, ssmlItemText } from './htmlToSsml'
+import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml'
+import { PassThrough } from 'stream'
export interface TextToSpeechInput {
- id?: string
text: string
voice?: string
- languageCode?: string
- textType?: 'html' | 'ssml'
+ language?: string
+ textType?: 'html' | 'ssml' | 'utterance'
rate?: number
- volume?: number
complimentaryVoice?: string
- bucket?: string
- audioStream: NodeJS.ReadWriteStream
- ssmlItems?: string[]
- speechMarksStream: NodeJS.ReadWriteStream
+ audioStream?: NodeJS.ReadWriteStream
+ speechMarksStream?: NodeJS.ReadWriteStream
}
export interface TextToSpeechOutput {
+ audioStream: NodeJS.ReadWriteStream
speechMarks: SpeechMark[]
}
@@ -43,8 +41,8 @@ export const synthesizeTextToSpeech = async (
throw new Error('Azure Speech Key or Region not set')
}
const textType = input.textType || 'html'
- const audioStream = input.audioStream
- const speechMarksStream = input.speechMarksStream
+ const audioStream = input.audioStream || new PassThrough()
+ const speechMarksStream = input.speechMarksStream || new PassThrough()
const speechConfig = SpeechConfig.fromSubscription(
process.env.AZURE_SPEECH_KEY,
process.env.AZURE_SPEECH_REGION
@@ -95,17 +93,15 @@ export const synthesizeTextToSpeech = async (
e.text
}`
)
- speechMarksStream.write(
- Buffer.from(
- JSON.stringify({
- word: e.text,
- time: (timeOffset + e.audioOffset) / 10000,
- start: e.textOffset,
- length: e.wordLength,
- type: 'word',
- })
- )
- )
+ const speechMark: SpeechMark = {
+ word: e.text,
+ time: (timeOffset + e.audioOffset) / 10000,
+ start: e.textOffset,
+ length: e.wordLength,
+ type: 'word',
+ }
+ speechMarks.push(speechMark)
+ speechMarksStream.write(Buffer.from(JSON.stringify(speechMark)))
}
synthesizer.bookmarkReached = (s, e) => {
@@ -114,15 +110,13 @@ export const synthesizeTextToSpeech = async (
e.audioOffset / 10000
}ms, bookmark text: ${e.text}`
)
- speechMarksStream.write(
- Buffer.from(
- JSON.stringify({
- word: e.text,
- time: (timeOffset + e.audioOffset) / 10000,
- type: 'bookmark',
- })
- )
- )
+ const speechMark: SpeechMark = {
+ word: e.text,
+ time: (timeOffset + e.audioOffset) / 10000,
+ type: 'bookmark',
+ }
+ speechMarks.push(speechMark)
+ speechMarksStream.write(Buffer.from(JSON.stringify(speechMark)))
}
const speakSsmlAsyncPromise = (
@@ -142,22 +136,22 @@ export const synthesizeTextToSpeech = async (
}
try {
+ const ssmlOptions = {
+ primaryVoice: input.voice || 'en-US-JennyNeural',
+ secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural',
+ language: input.language || 'en-US',
+ rate: '1.333',
+ }
if (textType === 'html') {
- const ssmlItems = htmlToSsmlItems(input.text, {
- primaryVoice: input.voice || 'en-US-JennyNeural',
- secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural',
- language: input.languageCode || 'en-US',
- rate: '1.333',
- })
+ const ssmlItems = htmlToSsmlItems(input.text, ssmlOptions)
for (const ssmlItem of ssmlItems) {
const ssml = ssmlItemText(ssmlItem)
const result = await speakSsmlAsyncPromise(ssml)
timeOffset = timeOffset + result.audioDuration
}
} else {
- for (const ssmlItem of input.ssmlItems || []) {
- await speakSsmlAsyncPromise(ssmlItem)
- }
+ const ssml = `${startSsml(null, ssmlOptions)}${input.text}${endSsml()}`
+ await speakSsmlAsyncPromise(ssml)
}
} catch (error) {
console.error('synthesis error', error)
@@ -171,6 +165,7 @@ export const synthesizeTextToSpeech = async (
}
return {
+ audioStream,
speechMarks,
}
}