From c4221f3bf507c37f4ad402aca929da5ece3b0401 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 6 Sep 2022 15:23:39 +0800 Subject: [PATCH] Read Utterance input --- packages/text-to-speech/src/htmlToSsml.ts | 16 ++++- packages/text-to-speech/src/index.ts | 75 +++++++++++++-------- packages/text-to-speech/src/textToSpeech.ts | 75 ++++++++++----------- 3 files changed, 95 insertions(+), 71 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index e65755cfc..ed9d13b7a 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -4,6 +4,13 @@ import * as _ from 'underscore' // this code needs to be kept in sync with the // frontend code in: useReadingProgressAnchor +export interface Utterance { + wordOffset: number + voice?: string + words: string[] + text: string +} + const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ 'omnivore-highlight-id', 'data-twitter-tweet-id', @@ -153,15 +160,18 @@ export type SSMLOptions = { language: string } -const startSsml = (element: Element, options: SSMLOptions): string => { +export const startSsml = ( + element: Element | null, + options: SSMLOptions +): string => { const voice = - element.nodeName === 'BLOCKQUOTE' + element?.nodeName === 'BLOCKQUOTE' ? options.secondaryVoice : options.primaryVoice return `` } -const endSsml = (): string => { +export const endSsml = (): string => { return `` } diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index ce2acb4b2..53d34d25c 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -9,9 +9,28 @@ import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' import { File, Storage } from '@google-cloud/storage' -import { PassThrough } from 'stream' import { htmlToSsml } from './htmlToSsml' -import * as fs from 'fs' + +interface SSMLInput { + text: string +} + +interface UtteranceInput { + voice?: string + rate?: number + language?: string + text: string +} + +interface HTMLInput { + id: string + text: string + voice?: string + language?: string + rate?: number + complimentaryVoice?: string + bucket: string +} dotenv.config() Sentry.GCPFunction.init({ @@ -71,7 +90,7 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( console.error(e) return res.status(200).send('UNAUTHENTICATED') } - const input = req.body as TextToSpeechInput + const input = req.body as HTMLInput const id = input.id const bucket = input.bucket if (!id || !bucket) { @@ -130,41 +149,41 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( } const token = (req.query.token || req.headers.authorization) as string if (!token) { - return res.status(200).send({ errorCode: 'UNAUTHORIZED' }) + return res.status(401).send({ errorCode: 'UNAUTHORIZED' }) } try { jwt.verify(token, process.env.JWT_SECRET) } catch (e) { console.error(e) - return res.status(200).send({ errorCode: 'UNAUTHORIZED' }) + return res.status(401).send({ errorCode: 'UNAUTHORIZED' }) } try { // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - // const ssmlItems = req.body.ssmlItems as string[] - // if (!ssmlItems || ssmlItems.length === 0) { - // return res.status(200).send({ errorCode: 'INVALID_DATA' }) - // } - // hardcoded for now - const ssml = fs.readFileSync('./data/ssml.xml', 'utf8') - const audioStream = new PassThrough() - const speechMarksStream = new PassThrough() - const input: TextToSpeechInput = { - text: '', - textType: 'ssml', - audioStream, - ssmlItems: [ssml], - speechMarksStream, + const utteranceInput = req.body as UtteranceInput + if (!utteranceInput.text) { + return res.status(400).send({ errorCode: 'INVALID_DATA' }) } - res.set({ - 'Content-Type': 'audio/mpeg', - 'Transfer-Encoding': 'chunked', - }) - - console.info('Text to speech starts streaming') - audioStream.pipe(res) - - await synthesizeTextToSpeech(input) + const input: TextToSpeechInput = { + ...utteranceInput, + textType: 'utterance', + } + const { audioStream, speechMarks } = await synthesizeTextToSpeech(input) + // const readStream = new Readable() + // readStream.push(JSON.stringify({ audioData, speechMarks })) + // + // res.set({ + // 'Content-Type': 'application/json', + // 'Transfer-Encoding': 'chunked', + // }) + // console.info('Text to speech starts streaming') + // pipeline(readStream, res, (err) => { + // if (err) { + // console.error('Text to speech streaming error', err) + // res.status(500).send({ errorCode: 'STREAMING_ERROR' }) + // } + // }) + res.send({ audioData: audioStream.read(), speechMarks }) } catch (e) { console.error('Text to speech streaming error', e) return res.status(500).send({ errorCodes: 'SYNTHESIZER_ERROR' }) diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 9c6356566..f9836973a 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -7,24 +7,22 @@ import { SpeechSynthesisResult, SpeechSynthesizer, } from 'microsoft-cognitiveservices-speech-sdk' -import { htmlToSsmlItems, ssmlItemText } from './htmlToSsml' +import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml' +import { PassThrough } from 'stream' export interface TextToSpeechInput { - id?: string text: string voice?: string - languageCode?: string - textType?: 'html' | 'ssml' + language?: string + textType?: 'html' | 'ssml' | 'utterance' rate?: number - volume?: number complimentaryVoice?: string - bucket?: string - audioStream: NodeJS.ReadWriteStream - ssmlItems?: string[] - speechMarksStream: NodeJS.ReadWriteStream + audioStream?: NodeJS.ReadWriteStream + speechMarksStream?: NodeJS.ReadWriteStream } export interface TextToSpeechOutput { + audioStream: NodeJS.ReadWriteStream speechMarks: SpeechMark[] } @@ -43,8 +41,8 @@ export const synthesizeTextToSpeech = async ( throw new Error('Azure Speech Key or Region not set') } const textType = input.textType || 'html' - const audioStream = input.audioStream - const speechMarksStream = input.speechMarksStream + const audioStream = input.audioStream || new PassThrough() + const speechMarksStream = input.speechMarksStream || new PassThrough() const speechConfig = SpeechConfig.fromSubscription( process.env.AZURE_SPEECH_KEY, process.env.AZURE_SPEECH_REGION @@ -95,17 +93,15 @@ export const synthesizeTextToSpeech = async ( e.text }` ) - speechMarksStream.write( - Buffer.from( - JSON.stringify({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - start: e.textOffset, - length: e.wordLength, - type: 'word', - }) - ) - ) + const speechMark: SpeechMark = { + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + start: e.textOffset, + length: e.wordLength, + type: 'word', + } + speechMarks.push(speechMark) + speechMarksStream.write(Buffer.from(JSON.stringify(speechMark))) } synthesizer.bookmarkReached = (s, e) => { @@ -114,15 +110,13 @@ export const synthesizeTextToSpeech = async ( e.audioOffset / 10000 }ms, bookmark text: ${e.text}` ) - speechMarksStream.write( - Buffer.from( - JSON.stringify({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - type: 'bookmark', - }) - ) - ) + const speechMark: SpeechMark = { + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + type: 'bookmark', + } + speechMarks.push(speechMark) + speechMarksStream.write(Buffer.from(JSON.stringify(speechMark))) } const speakSsmlAsyncPromise = ( @@ -142,22 +136,22 @@ export const synthesizeTextToSpeech = async ( } try { + const ssmlOptions = { + primaryVoice: input.voice || 'en-US-JennyNeural', + secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural', + language: input.language || 'en-US', + rate: '1.333', + } if (textType === 'html') { - const ssmlItems = htmlToSsmlItems(input.text, { - primaryVoice: input.voice || 'en-US-JennyNeural', - secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural', - language: input.languageCode || 'en-US', - rate: '1.333', - }) + const ssmlItems = htmlToSsmlItems(input.text, ssmlOptions) for (const ssmlItem of ssmlItems) { const ssml = ssmlItemText(ssmlItem) const result = await speakSsmlAsyncPromise(ssml) timeOffset = timeOffset + result.audioDuration } } else { - for (const ssmlItem of input.ssmlItems || []) { - await speakSsmlAsyncPromise(ssmlItem) - } + const ssml = `${startSsml(null, ssmlOptions)}${input.text}${endSsml()}` + await speakSsmlAsyncPromise(ssml) } } catch (error) { console.error('synthesis error', error) @@ -171,6 +165,7 @@ export const synthesizeTextToSpeech = async ( } return { + audioStream, speechMarks, } }