diff --git a/packages/text-to-speech/data/test.ssml b/packages/text-to-speech/data/test.ssml new file mode 100644 index 000000000..e69de29bb diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index a213d8b42..8c51640aa 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -4,49 +4,15 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import * as Sentry from '@sentry/serverless' -import { parseHTML } from 'linkedom' -import { File, Storage } from '@google-cloud/storage' -import { - CancellationDetails, - CancellationReason, - ResultReason, - SpeechConfig, - SpeechSynthesisOutputFormat, - SpeechSynthesisResult, - SpeechSynthesizer, -} from 'microsoft-cognitiveservices-speech-sdk' import axios from 'axios' import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import -import { htmlToSsml, ssmlItemText } from './htmlToSsml' +import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' +import { File, Storage } from '@google-cloud/storage' +import { createWriteStream } from 'fs' dotenv.config() -interface TextToSpeechInput { - id: string - text: string - voice?: string - languageCode?: string - textType?: 'text' | 'ssml' - rate?: number - volume?: number - complimentaryVoice?: string - bucket: string -} - -interface TextToSpeechOutput { - audioFileName: string - speechMarksFileName: string -} - -interface SpeechMark { - time: number - start?: number - length?: number - word: string - type: 'word' | 'bookmark' -} - const storage = new Storage() const uploadToBucket = async ( @@ -85,293 +51,10 @@ const updateSpeech = async ( return response.status === 200 } -const synthesizeTextToSpeech = async ( - input: TextToSpeechInput -): Promise => { - if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) { - throw new Error('Azure Speech Key or Region not set') - } - const audioFileName = `speech/${input.id}.mp3` - const audioFile = createGCSFile(input.bucket, audioFileName) - const writeStream = audioFile.createWriteStream({ - resumable: true, - }) - const speechConfig = SpeechConfig.fromSubscription( - process.env.AZURE_SPEECH_KEY, - process.env.AZURE_SPEECH_REGION - ) - const textType = input.textType || 'text' - if (textType === 'text') { - speechConfig.speechSynthesisLanguage = input.languageCode || 'en-US' - speechConfig.speechSynthesisVoiceName = input.voice || 'en-US-JennyNeural' - } - speechConfig.speechSynthesisOutputFormat = - SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 - - // Create the speech synthesizer. - const synthesizer = new SpeechSynthesizer(speechConfig) - const speechMarks: SpeechMark[] = [] - let timeOffset = 0 - let characterOffset = 0 - - synthesizer.synthesizing = function (s, e) { - // convert arrayBuffer to stream and write to gcs file - writeStream.write(Buffer.from(e.result.audioData)) - } - - // The event synthesis completed signals that the synthesis is completed. - synthesizer.synthesisCompleted = (s, e) => { - console.info( - `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${ - e.result.audioData.byteLength - }` - ) - } - - // The synthesis started event signals that the synthesis is started. - synthesizer.synthesisStarted = (s, e) => { - console.info('(synthesis started)') - } - - // The event signals that the service has stopped processing speech. - // This can happen when an error is encountered. - synthesizer.SynthesisCanceled = (s, e) => { - const cancellationDetails = CancellationDetails.fromResult(e.result) - let str = - '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason] - if (cancellationDetails.reason === CancellationReason.Error) { - str += ': ' + e.result.errorDetails - } - console.info(str) - } - - // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. - synthesizer.wordBoundary = (s, e) => { - speechMarks.push({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - start: characterOffset + e.textOffset, - length: e.wordLength, - type: 'word', - }) - } - - synthesizer.bookmarkReached = (s, e) => { - console.debug( - `(Bookmark reached), Audio offset: ${ - e.audioOffset / 10000 - }ms, bookmark text: ${e.text}` - ) - speechMarks.push({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - type: 'bookmark', - }) - } - - const speakTextAsyncPromise = ( - text: string - ): Promise => { - return new Promise((resolve, reject) => { - synthesizer.speakTextAsync( - text, - (result) => { - resolve(result) - }, - (error) => { - reject(error) - } - ) - }) - } - - const speakSsmlAsyncPromise = ( - text: string - ): Promise => { - return new Promise((resolve, reject) => { - synthesizer.speakSsmlAsync( - text, - (result) => { - resolve(result) - }, - (error) => { - reject(error) - } - ) - }) - } - - if (textType === 'text') { - // slice the text into chunks of 5,000 characters - let currentTextChunk = '' - const textChunks = input.text.split('\n') - for (let i = 0; i < textChunks.length; i++) { - currentTextChunk += textChunks[i] + '\n' - if (currentTextChunk.length < 5000 && i < textChunks.length - 1) { - continue - } - console.debug(`synthesizing ${currentTextChunk}`) - const result = await speakTextAsyncPromise(currentTextChunk) - timeOffset = timeOffset + result.audioDuration - characterOffset = characterOffset + currentTextChunk.length - currentTextChunk = '' - } - } else { - const ssmlItems = htmlToSsml(input.text, { - primaryVoice: input.voice || 'en-US-JennyNeural', - secondaryVoice: 'en-US-GuyNeural', - language: input.languageCode || 'en-US', - rate: '1', - }) - - for (const ssmlItem of Array.from(ssmlItems)) { - const ssml = ssmlItemText(ssmlItem) - console.debug(`synthesizing ${ssml}`) - const result = await speakSsmlAsyncPromise(ssml) - if (result.reason === ResultReason.Canceled) { - writeStream.end() - synthesizer.close() - throw new Error(result.errorDetails) - } - timeOffset = timeOffset + result.audioDuration - // characterOffset = characterOffset + htmlElement.innerText.length - } - } - writeStream.end() - synthesizer.close() - - console.debug(`audio file: ${audioFileName}`) - - // upload Speech Marks file to GCS - const speechMarksFileName = `speech/${input.id}.json` - await uploadToBucket( - speechMarksFileName, - Buffer.from(JSON.stringify(speechMarks)), - input.bucket - ) - - return { - audioFileName, - speechMarksFileName, - } -} - -const htmlElementToSsml = ({ - htmlElement, - language = 'en-US', - voice = 'en-US-JennyNeural', - rate = 1, - volume = 100, -}: { - htmlElement: Element - language?: string - voice?: string - rate?: number - volume?: number -}): string => { - const replaceElement = (newElement: Element, oldElement: Element) => { - const id = oldElement.getAttribute('data-omnivore-anchor-idx') - if (id) { - const e = htmlElement.querySelector(`[data-omnivore-anchor-idx="${id}"]`) - e?.parentNode?.replaceChild(newElement, e) - } - } - - const appendBookmarkElement = (parent: Element, element: Element) => { - const id = element.getAttribute('data-omnivore-anchor-idx') - if (id) { - const bookMark = ssml.createElement('bookmark') - bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`) - parent.appendChild(bookMark) - } - } - - const replaceWithEmphasis = (element: Element, level: string) => { - const parent = ssml.createDocumentFragment() as unknown as Element - appendBookmarkElement(parent, element) - const emphasisElement = ssml.createElement('emphasis') - emphasisElement.setAttribute('level', level) - emphasisElement.innerHTML = element.innerHTML.trim() - parent.appendChild(emphasisElement) - replaceElement(parent, element) - } - - const replaceWithSentence = (element: Element) => { - const parent = ssml.createDocumentFragment() as unknown as Element - appendBookmarkElement(parent, element) - const sentenceElement = ssml.createElement('s') - sentenceElement.innerHTML = element.innerHTML.trim() - parent.appendChild(sentenceElement) - replaceElement(parent, element) - } - - // create new ssml document - const ssml = parseHTML('').document - const speakElement = ssml.createElement('speak') - speakElement.setAttribute('version', '1.0') - speakElement.setAttribute('xmlns', 'http://www.w3.org/2001/10/synthesis') - speakElement.setAttribute('xml:lang', language) - const voiceElement = ssml.createElement('voice') - voiceElement.setAttribute('name', voice) - speakElement.appendChild(voiceElement) - const prosodyElement = ssml.createElement('prosody') - prosodyElement.setAttribute('rate', `${rate}`) - prosodyElement.setAttribute('volume', volume.toString()) - voiceElement.appendChild(prosodyElement) - // add each paragraph to the ssml document - appendBookmarkElement(prosodyElement, htmlElement) - // replace emphasis elements with ssml - htmlElement.querySelectorAll('*').forEach((e) => { - switch (e.tagName.toLowerCase()) { - case 's': - replaceWithEmphasis(e, 'moderate') - break - case 'sub': - if (e.getAttribute('alias') === null) { - replaceWithEmphasis(e, 'moderate') - } - break - case 'i': - case 'em': - case 'q': - case 'blockquote': - case 'cite': - case 'del': - case 'strike': - case 'sup': - case 'summary': - case 'caption': - case 'figcaption': - replaceWithEmphasis(e, 'moderate') - break - case 'b': - case 'strong': - case 'dt': - case 'dfn': - case 'u': - case 'mark': - case 'th': - case 'title': - case 'var': - replaceWithEmphasis(e, 'moderate') - break - case 'li': - replaceWithSentence(e) - break - default: { - const parent = ssml.createDocumentFragment() as unknown as Element - appendBookmarkElement(parent, e) - const text = (e as HTMLElement).innerText.trim() - const textElement = ssml.createTextNode(text) - parent.appendChild(textElement) - replaceElement(parent, e) - } - } - }) - prosodyElement.appendChild(htmlElement) - - return speakElement.outerHTML.replace(/ |\n/g, '') -} +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}) export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { @@ -389,8 +72,22 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( } const input = req.body as TextToSpeechInput try { - const { audioFileName, speechMarksFileName } = - await synthesizeTextToSpeech(input) + const audioFileName = `speech/${input.id}.mp3` + const audioFile = createGCSFile(input.bucket, audioFileName) + const writeStream = audioFile.createWriteStream({ + resumable: true, + }) + const { speechMarks } = await synthesizeTextToSpeech({ + ...input, + writeStream, + }) + // upload Speech Marks file to GCS + const speechMarksFileName = `speech/${input.id}.json` + await uploadToBucket( + speechMarksFileName, + Buffer.from(JSON.stringify(speechMarks)), + input.bucket + ) const updated = await updateSpeech( input.id, token, @@ -411,3 +108,42 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( res.send('OK') } ) + +export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + console.debug('Text to speech steaming request', req) + const token = req.query.token as string + if (!process.env.JWT_SECRET) { + console.error('JWT_SECRET not exists') + return res.status(500).send('JWT_SECRET not exists') + } + try { + jwt.verify(token, process.env.JWT_SECRET) + } catch (e) { + console.error(e) + return res.status(200).send('UNAUTHENTICATED') + } + + try { + const audioFileName = `./tmp/speech-${Date.now()}.mp3` + const writeStream = createWriteStream(audioFileName) + const input: TextToSpeechInput = { + id: req.query.id as string, + text: 'text', + bucket: req.query.bucket as string, + textType: 'ssml', + writeStream, + } + await synthesizeTextToSpeech(input) + + res.set({ + 'Content-Type': 'audio/mpeg', + 'Transfer-Encoding': 'chunked', + }) + writeStream.pipe(res) + } catch (e) { + console.error(e) + return res.status(500).send('Failed to synthesize') + } + } +) diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts new file mode 100644 index 000000000..77eb706f7 --- /dev/null +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -0,0 +1,163 @@ +import { + CancellationDetails, + CancellationReason, + ResultReason, + SpeechConfig, + SpeechSynthesisOutputFormat, + SpeechSynthesisResult, + SpeechSynthesizer, +} from 'microsoft-cognitiveservices-speech-sdk' +import { htmlToSsml, ssmlItemText } from './htmlToSsml' + +export interface TextToSpeechInput { + id: string + text: string + voice?: string + languageCode?: string + textType?: 'text' | 'ssml' + rate?: number + volume?: number + complimentaryVoice?: string + bucket: string + writeStream: NodeJS.WritableStream +} + +export interface TextToSpeechOutput { + speechMarks: SpeechMark[] +} + +export interface SpeechMark { + time: number + start?: number + length?: number + word: string + type: 'word' | 'bookmark' +} + +export const synthesizeTextToSpeech = async ( + input: TextToSpeechInput +): Promise => { + if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) { + throw new Error('Azure Speech Key or Region not set') + } + const writeStream = input.writeStream + const speechConfig = SpeechConfig.fromSubscription( + process.env.AZURE_SPEECH_KEY, + process.env.AZURE_SPEECH_REGION + ) + const textType = input.textType || 'text' + speechConfig.speechSynthesisOutputFormat = + SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 + + // Create the speech synthesizer. + const synthesizer = new SpeechSynthesizer(speechConfig) + const speechMarks: SpeechMark[] = [] + let timeOffset = 0 + const characterOffset = 0 + + synthesizer.synthesizing = function (s, e) { + // convert arrayBuffer to stream and write to gcs file + writeStream.write(Buffer.from(e.result.audioData)) + } + + // The event synthesis completed signals that the synthesis is completed. + synthesizer.synthesisCompleted = (s, e) => { + console.info( + `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${ + e.result.audioData.byteLength + }` + ) + } + + // The synthesis started event signals that the synthesis is started. + synthesizer.synthesisStarted = (s, e) => { + console.info('(synthesis started)') + } + + // The event signals that the service has stopped processing speech. + // This can happen when an error is encountered. + synthesizer.SynthesisCanceled = (s, e) => { + const cancellationDetails = CancellationDetails.fromResult(e.result) + let str = + '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason] + if (cancellationDetails.reason === CancellationReason.Error) { + str += ': ' + e.result.errorDetails + } + console.info(str) + } + + // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. + synthesizer.wordBoundary = (s, e) => { + speechMarks.push({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + start: characterOffset + e.textOffset, + length: e.wordLength, + type: 'word', + }) + } + + synthesizer.bookmarkReached = (s, e) => { + console.debug( + `(Bookmark reached), Audio offset: ${ + e.audioOffset / 10000 + }ms, bookmark text: ${e.text}` + ) + speechMarks.push({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + type: 'bookmark', + }) + } + + const speakSsmlAsyncPromise = ( + text: string + ): Promise => { + return new Promise((resolve, reject) => { + synthesizer.speakSsmlAsync( + text, + (result) => { + resolve(result) + }, + (error) => { + reject(error) + } + ) + }) + } + + if (textType === 'text') { + const ssmlItems = htmlToSsml(input.text, { + primaryVoice: input.voice || 'en-US-JennyNeural', + secondaryVoice: 'en-US-GuyNeural', + language: input.languageCode || 'en-US', + rate: '1', + }) + + for (const ssmlItem of Array.from(ssmlItems)) { + const ssml = ssmlItemText(ssmlItem) + console.debug(`synthesizing ${ssml}`) + const result = await speakSsmlAsyncPromise(ssml) + if (result.reason === ResultReason.Canceled) { + writeStream.end() + synthesizer.close() + throw new Error(result.errorDetails) + } + timeOffset = timeOffset + result.audioDuration + // characterOffset = characterOffset + htmlElement.innerText.length + } + } else { + const result = await speakSsmlAsyncPromise(input.text) + if (result.reason === ResultReason.Canceled) { + writeStream.end() + synthesizer.close() + throw new Error(result.errorDetails) + } + } + writeStream.end() + synthesizer.close() + + return { + speechMarks, + } +}