From 7baafc96bbe8ea8a4630cbcf451dd1a4c3e74da6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 31 Aug 2022 12:01:52 +0800 Subject: [PATCH 01/15] Create a cloud function to steam speech mp3 --- packages/text-to-speech/data/test.ssml | 0 packages/text-to-speech/src/index.ts | 388 ++++---------------- packages/text-to-speech/src/textToSpeech.ts | 163 ++++++++ 3 files changed, 225 insertions(+), 326 deletions(-) create mode 100644 packages/text-to-speech/data/test.ssml create mode 100644 packages/text-to-speech/src/textToSpeech.ts diff --git a/packages/text-to-speech/data/test.ssml b/packages/text-to-speech/data/test.ssml new file mode 100644 index 000000000..e69de29bb diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index a213d8b42..8c51640aa 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -4,49 +4,15 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import * as Sentry from '@sentry/serverless' -import { parseHTML } from 'linkedom' -import { File, Storage } from '@google-cloud/storage' -import { - CancellationDetails, - CancellationReason, - ResultReason, - SpeechConfig, - SpeechSynthesisOutputFormat, - SpeechSynthesisResult, - SpeechSynthesizer, -} from 'microsoft-cognitiveservices-speech-sdk' import axios from 'axios' import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import -import { htmlToSsml, ssmlItemText } from './htmlToSsml' +import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' +import { File, Storage } from '@google-cloud/storage' +import { createWriteStream } from 'fs' dotenv.config() -interface TextToSpeechInput { - id: string - text: string - voice?: string - languageCode?: string - textType?: 'text' | 'ssml' - rate?: number - volume?: number - complimentaryVoice?: string - bucket: string -} - -interface TextToSpeechOutput { - audioFileName: string - speechMarksFileName: string -} - -interface SpeechMark { - time: number - start?: number - length?: number - word: string - type: 'word' | 'bookmark' -} - const storage = new Storage() const uploadToBucket = async ( @@ -85,293 +51,10 @@ const updateSpeech = async ( return response.status === 200 } -const synthesizeTextToSpeech = async ( - input: TextToSpeechInput -): Promise => { - if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) { - throw new Error('Azure Speech Key or Region not set') - } - const audioFileName = `speech/${input.id}.mp3` - const audioFile = createGCSFile(input.bucket, audioFileName) - const writeStream = audioFile.createWriteStream({ - resumable: true, - }) - const speechConfig = SpeechConfig.fromSubscription( - process.env.AZURE_SPEECH_KEY, - process.env.AZURE_SPEECH_REGION - ) - const textType = input.textType || 'text' - if (textType === 'text') { - speechConfig.speechSynthesisLanguage = input.languageCode || 'en-US' - speechConfig.speechSynthesisVoiceName = input.voice || 'en-US-JennyNeural' - } - speechConfig.speechSynthesisOutputFormat = - SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 - - // Create the speech synthesizer. - const synthesizer = new SpeechSynthesizer(speechConfig) - const speechMarks: SpeechMark[] = [] - let timeOffset = 0 - let characterOffset = 0 - - synthesizer.synthesizing = function (s, e) { - // convert arrayBuffer to stream and write to gcs file - writeStream.write(Buffer.from(e.result.audioData)) - } - - // The event synthesis completed signals that the synthesis is completed. - synthesizer.synthesisCompleted = (s, e) => { - console.info( - `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${ - e.result.audioData.byteLength - }` - ) - } - - // The synthesis started event signals that the synthesis is started. - synthesizer.synthesisStarted = (s, e) => { - console.info('(synthesis started)') - } - - // The event signals that the service has stopped processing speech. - // This can happen when an error is encountered. - synthesizer.SynthesisCanceled = (s, e) => { - const cancellationDetails = CancellationDetails.fromResult(e.result) - let str = - '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason] - if (cancellationDetails.reason === CancellationReason.Error) { - str += ': ' + e.result.errorDetails - } - console.info(str) - } - - // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. - synthesizer.wordBoundary = (s, e) => { - speechMarks.push({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - start: characterOffset + e.textOffset, - length: e.wordLength, - type: 'word', - }) - } - - synthesizer.bookmarkReached = (s, e) => { - console.debug( - `(Bookmark reached), Audio offset: ${ - e.audioOffset / 10000 - }ms, bookmark text: ${e.text}` - ) - speechMarks.push({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - type: 'bookmark', - }) - } - - const speakTextAsyncPromise = ( - text: string - ): Promise => { - return new Promise((resolve, reject) => { - synthesizer.speakTextAsync( - text, - (result) => { - resolve(result) - }, - (error) => { - reject(error) - } - ) - }) - } - - const speakSsmlAsyncPromise = ( - text: string - ): Promise => { - return new Promise((resolve, reject) => { - synthesizer.speakSsmlAsync( - text, - (result) => { - resolve(result) - }, - (error) => { - reject(error) - } - ) - }) - } - - if (textType === 'text') { - // slice the text into chunks of 5,000 characters - let currentTextChunk = '' - const textChunks = input.text.split('\n') - for (let i = 0; i < textChunks.length; i++) { - currentTextChunk += textChunks[i] + '\n' - if (currentTextChunk.length < 5000 && i < textChunks.length - 1) { - continue - } - console.debug(`synthesizing ${currentTextChunk}`) - const result = await speakTextAsyncPromise(currentTextChunk) - timeOffset = timeOffset + result.audioDuration - characterOffset = characterOffset + currentTextChunk.length - currentTextChunk = '' - } - } else { - const ssmlItems = htmlToSsml(input.text, { - primaryVoice: input.voice || 'en-US-JennyNeural', - secondaryVoice: 'en-US-GuyNeural', - language: input.languageCode || 'en-US', - rate: '1', - }) - - for (const ssmlItem of Array.from(ssmlItems)) { - const ssml = ssmlItemText(ssmlItem) - console.debug(`synthesizing ${ssml}`) - const result = await speakSsmlAsyncPromise(ssml) - if (result.reason === ResultReason.Canceled) { - writeStream.end() - synthesizer.close() - throw new Error(result.errorDetails) - } - timeOffset = timeOffset + result.audioDuration - // characterOffset = characterOffset + htmlElement.innerText.length - } - } - writeStream.end() - synthesizer.close() - - console.debug(`audio file: ${audioFileName}`) - - // upload Speech Marks file to GCS - const speechMarksFileName = `speech/${input.id}.json` - await uploadToBucket( - speechMarksFileName, - Buffer.from(JSON.stringify(speechMarks)), - input.bucket - ) - - return { - audioFileName, - speechMarksFileName, - } -} - -const htmlElementToSsml = ({ - htmlElement, - language = 'en-US', - voice = 'en-US-JennyNeural', - rate = 1, - volume = 100, -}: { - htmlElement: Element - language?: string - voice?: string - rate?: number - volume?: number -}): string => { - const replaceElement = (newElement: Element, oldElement: Element) => { - const id = oldElement.getAttribute('data-omnivore-anchor-idx') - if (id) { - const e = htmlElement.querySelector(`[data-omnivore-anchor-idx="${id}"]`) - e?.parentNode?.replaceChild(newElement, e) - } - } - - const appendBookmarkElement = (parent: Element, element: Element) => { - const id = element.getAttribute('data-omnivore-anchor-idx') - if (id) { - const bookMark = ssml.createElement('bookmark') - bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`) - parent.appendChild(bookMark) - } - } - - const replaceWithEmphasis = (element: Element, level: string) => { - const parent = ssml.createDocumentFragment() as unknown as Element - appendBookmarkElement(parent, element) - const emphasisElement = ssml.createElement('emphasis') - emphasisElement.setAttribute('level', level) - emphasisElement.innerHTML = element.innerHTML.trim() - parent.appendChild(emphasisElement) - replaceElement(parent, element) - } - - const replaceWithSentence = (element: Element) => { - const parent = ssml.createDocumentFragment() as unknown as Element - appendBookmarkElement(parent, element) - const sentenceElement = ssml.createElement('s') - sentenceElement.innerHTML = element.innerHTML.trim() - parent.appendChild(sentenceElement) - replaceElement(parent, element) - } - - // create new ssml document - const ssml = parseHTML('').document - const speakElement = ssml.createElement('speak') - speakElement.setAttribute('version', '1.0') - speakElement.setAttribute('xmlns', 'http://www.w3.org/2001/10/synthesis') - speakElement.setAttribute('xml:lang', language) - const voiceElement = ssml.createElement('voice') - voiceElement.setAttribute('name', voice) - speakElement.appendChild(voiceElement) - const prosodyElement = ssml.createElement('prosody') - prosodyElement.setAttribute('rate', `${rate}`) - prosodyElement.setAttribute('volume', volume.toString()) - voiceElement.appendChild(prosodyElement) - // add each paragraph to the ssml document - appendBookmarkElement(prosodyElement, htmlElement) - // replace emphasis elements with ssml - htmlElement.querySelectorAll('*').forEach((e) => { - switch (e.tagName.toLowerCase()) { - case 's': - replaceWithEmphasis(e, 'moderate') - break - case 'sub': - if (e.getAttribute('alias') === null) { - replaceWithEmphasis(e, 'moderate') - } - break - case 'i': - case 'em': - case 'q': - case 'blockquote': - case 'cite': - case 'del': - case 'strike': - case 'sup': - case 'summary': - case 'caption': - case 'figcaption': - replaceWithEmphasis(e, 'moderate') - break - case 'b': - case 'strong': - case 'dt': - case 'dfn': - case 'u': - case 'mark': - case 'th': - case 'title': - case 'var': - replaceWithEmphasis(e, 'moderate') - break - case 'li': - replaceWithSentence(e) - break - default: { - const parent = ssml.createDocumentFragment() as unknown as Element - appendBookmarkElement(parent, e) - const text = (e as HTMLElement).innerText.trim() - const textElement = ssml.createTextNode(text) - parent.appendChild(textElement) - replaceElement(parent, e) - } - } - }) - prosodyElement.appendChild(htmlElement) - - return speakElement.outerHTML.replace(/ |\n/g, '') -} +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}) export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { @@ -389,8 +72,22 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( } const input = req.body as TextToSpeechInput try { - const { audioFileName, speechMarksFileName } = - await synthesizeTextToSpeech(input) + const audioFileName = `speech/${input.id}.mp3` + const audioFile = createGCSFile(input.bucket, audioFileName) + const writeStream = audioFile.createWriteStream({ + resumable: true, + }) + const { speechMarks } = await synthesizeTextToSpeech({ + ...input, + writeStream, + }) + // upload Speech Marks file to GCS + const speechMarksFileName = `speech/${input.id}.json` + await uploadToBucket( + speechMarksFileName, + Buffer.from(JSON.stringify(speechMarks)), + input.bucket + ) const updated = await updateSpeech( input.id, token, @@ -411,3 +108,42 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( res.send('OK') } ) + +export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + console.debug('Text to speech steaming request', req) + const token = req.query.token as string + if (!process.env.JWT_SECRET) { + console.error('JWT_SECRET not exists') + return res.status(500).send('JWT_SECRET not exists') + } + try { + jwt.verify(token, process.env.JWT_SECRET) + } catch (e) { + console.error(e) + return res.status(200).send('UNAUTHENTICATED') + } + + try { + const audioFileName = `./tmp/speech-${Date.now()}.mp3` + const writeStream = createWriteStream(audioFileName) + const input: TextToSpeechInput = { + id: req.query.id as string, + text: 'text', + bucket: req.query.bucket as string, + textType: 'ssml', + writeStream, + } + await synthesizeTextToSpeech(input) + + res.set({ + 'Content-Type': 'audio/mpeg', + 'Transfer-Encoding': 'chunked', + }) + writeStream.pipe(res) + } catch (e) { + console.error(e) + return res.status(500).send('Failed to synthesize') + } + } +) diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts new file mode 100644 index 000000000..77eb706f7 --- /dev/null +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -0,0 +1,163 @@ +import { + CancellationDetails, + CancellationReason, + ResultReason, + SpeechConfig, + SpeechSynthesisOutputFormat, + SpeechSynthesisResult, + SpeechSynthesizer, +} from 'microsoft-cognitiveservices-speech-sdk' +import { htmlToSsml, ssmlItemText } from './htmlToSsml' + +export interface TextToSpeechInput { + id: string + text: string + voice?: string + languageCode?: string + textType?: 'text' | 'ssml' + rate?: number + volume?: number + complimentaryVoice?: string + bucket: string + writeStream: NodeJS.WritableStream +} + +export interface TextToSpeechOutput { + speechMarks: SpeechMark[] +} + +export interface SpeechMark { + time: number + start?: number + length?: number + word: string + type: 'word' | 'bookmark' +} + +export const synthesizeTextToSpeech = async ( + input: TextToSpeechInput +): Promise => { + if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) { + throw new Error('Azure Speech Key or Region not set') + } + const writeStream = input.writeStream + const speechConfig = SpeechConfig.fromSubscription( + process.env.AZURE_SPEECH_KEY, + process.env.AZURE_SPEECH_REGION + ) + const textType = input.textType || 'text' + speechConfig.speechSynthesisOutputFormat = + SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 + + // Create the speech synthesizer. + const synthesizer = new SpeechSynthesizer(speechConfig) + const speechMarks: SpeechMark[] = [] + let timeOffset = 0 + const characterOffset = 0 + + synthesizer.synthesizing = function (s, e) { + // convert arrayBuffer to stream and write to gcs file + writeStream.write(Buffer.from(e.result.audioData)) + } + + // The event synthesis completed signals that the synthesis is completed. + synthesizer.synthesisCompleted = (s, e) => { + console.info( + `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${ + e.result.audioData.byteLength + }` + ) + } + + // The synthesis started event signals that the synthesis is started. + synthesizer.synthesisStarted = (s, e) => { + console.info('(synthesis started)') + } + + // The event signals that the service has stopped processing speech. + // This can happen when an error is encountered. + synthesizer.SynthesisCanceled = (s, e) => { + const cancellationDetails = CancellationDetails.fromResult(e.result) + let str = + '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason] + if (cancellationDetails.reason === CancellationReason.Error) { + str += ': ' + e.result.errorDetails + } + console.info(str) + } + + // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. + synthesizer.wordBoundary = (s, e) => { + speechMarks.push({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + start: characterOffset + e.textOffset, + length: e.wordLength, + type: 'word', + }) + } + + synthesizer.bookmarkReached = (s, e) => { + console.debug( + `(Bookmark reached), Audio offset: ${ + e.audioOffset / 10000 + }ms, bookmark text: ${e.text}` + ) + speechMarks.push({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + type: 'bookmark', + }) + } + + const speakSsmlAsyncPromise = ( + text: string + ): Promise => { + return new Promise((resolve, reject) => { + synthesizer.speakSsmlAsync( + text, + (result) => { + resolve(result) + }, + (error) => { + reject(error) + } + ) + }) + } + + if (textType === 'text') { + const ssmlItems = htmlToSsml(input.text, { + primaryVoice: input.voice || 'en-US-JennyNeural', + secondaryVoice: 'en-US-GuyNeural', + language: input.languageCode || 'en-US', + rate: '1', + }) + + for (const ssmlItem of Array.from(ssmlItems)) { + const ssml = ssmlItemText(ssmlItem) + console.debug(`synthesizing ${ssml}`) + const result = await speakSsmlAsyncPromise(ssml) + if (result.reason === ResultReason.Canceled) { + writeStream.end() + synthesizer.close() + throw new Error(result.errorDetails) + } + timeOffset = timeOffset + result.audioDuration + // characterOffset = characterOffset + htmlElement.innerText.length + } + } else { + const result = await speakSsmlAsyncPromise(input.text) + if (result.reason === ResultReason.Canceled) { + writeStream.end() + synthesizer.close() + throw new Error(result.errorDetails) + } + } + writeStream.end() + synthesizer.close() + + return { + speechMarks, + } +} From 0aa5447b9bf41b0857929cae9f07fbba1785c8b9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 31 Aug 2022 17:26:49 +0800 Subject: [PATCH 02/15] Read SSML file to stream --- packages/text-to-speech/data/ssml.xml | 8 ++ packages/text-to-speech/data/test.ssml | 0 packages/text-to-speech/package.json | 3 +- packages/text-to-speech/src/index.ts | 50 ++++++------ packages/text-to-speech/src/textToSpeech.ts | 87 +++++++++++---------- 5 files changed, 84 insertions(+), 64 deletions(-) create mode 100644 packages/text-to-speech/data/ssml.xml delete mode 100644 packages/text-to-speech/data/test.ssml diff --git a/packages/text-to-speech/data/ssml.xml b/packages/text-to-speech/data/ssml.xml new file mode 100644 index 000000000..2e8e27e77 --- /dev/null +++ b/packages/text-to-speech/data/ssml.xml @@ -0,0 +1,8 @@ + + + Good morning! + + + Good morning to you too Jenny! + + diff --git a/packages/text-to-speech/data/test.ssml b/packages/text-to-speech/data/test.ssml deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json index 6b21521bb..5416fd83e 100644 --- a/packages/text-to-speech/package.json +++ b/packages/text-to-speech/package.json @@ -14,7 +14,8 @@ "lint": "eslint src --ext ts,js,tsx,jsx", "compile": "tsc", "build": "tsc", - "start": "functions-framework --source=build/src/ --target=textToSpeechHandler", + "start": "functions-framework --target=textToSpeechHandler", + "start_streaming": "functions-framework --target=textToSpeechStreamingHandler", "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"", "gcloud-deploy": "gcloud functions deploy text-to-speech --gen2 --entry-point=textToSpeechHandler --trigger-http --allow-unauthenticated --region=us-west2 --runtime nodejs14", "deploy": "yarn build && yarn gcloud-deploy" diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 8c51640aa..0cff52eba 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -9,9 +9,14 @@ import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' import { File, Storage } from '@google-cloud/storage' -import { createWriteStream } from 'fs' +import { PassThrough } from 'stream' +import * as fs from 'fs' dotenv.config() +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}) const storage = new Storage() @@ -51,18 +56,13 @@ const updateSpeech = async ( return response.status === 200 } -Sentry.GCPFunction.init({ - dsn: process.env.SENTRY_DSN, - tracesSampleRate: 0, -}) - export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { console.debug('New text to speech request', req) const token = req.query.token as string if (!process.env.JWT_SECRET) { console.error('JWT_SECRET not exists') - return res.status(500).send('JWT_SECRET not exists') + return res.status(500).send({ errorCodes: 'JWT_SECRET_NOT_EXISTS' }) } try { jwt.verify(token, process.env.JWT_SECRET) @@ -71,25 +71,31 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( return res.status(200).send('UNAUTHENTICATED') } const input = req.body as TextToSpeechInput + const id = input.id + const bucket = input.bucket + if (!id || !bucket) { + return res.status(200).send('Invalid data') + } try { - const audioFileName = `speech/${input.id}.mp3` - const audioFile = createGCSFile(input.bucket, audioFileName) + const audioFileName = `speech/${id}.mp3` + const audioFile = createGCSFile(bucket, audioFileName) const writeStream = audioFile.createWriteStream({ resumable: true, }) const { speechMarks } = await synthesizeTextToSpeech({ ...input, + textType: 'html', writeStream, }) // upload Speech Marks file to GCS - const speechMarksFileName = `speech/${input.id}.json` + const speechMarksFileName = `speech/${id}.json` await uploadToBucket( speechMarksFileName, Buffer.from(JSON.stringify(speechMarks)), - input.bucket + bucket ) const updated = await updateSpeech( - input.id, + id, token, 'COMPLETED', audioFileName, @@ -97,12 +103,12 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( ) if (!updated) { - return res.status(500).send('Failed to update speech') + return res.status(500).send({ errorCodes: 'DB_ERROR' }) } } catch (e) { - console.error(e) - await updateSpeech(input.id, token, 'FAILED') - return res.status(500).send('Failed to synthesize') + console.error('Text to speech cloud function error', e) + await updateSpeech(id, token, 'FAILED') + return res.status(500).send({ errorCodes: 'SYNTHESIZER_ERROR' }) } res.send('OK') @@ -125,12 +131,10 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( } try { - const audioFileName = `./tmp/speech-${Date.now()}.mp3` - const writeStream = createWriteStream(audioFileName) + const ssml = fs.readFileSync('./data/ssml.xml', 'utf8') + const writeStream = new PassThrough() const input: TextToSpeechInput = { - id: req.query.id as string, - text: 'text', - bucket: req.query.bucket as string, + text: ssml, textType: 'ssml', writeStream, } @@ -142,8 +146,8 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( }) writeStream.pipe(res) } catch (e) { - console.error(e) - return res.status(500).send('Failed to synthesize') + console.error('Text to speech streaming error', e) + return res.status(500).send({ errorCodes: 'SYNTHESIZER_ERROR' }) } } ) diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 77eb706f7..e007e343d 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -10,15 +10,15 @@ import { import { htmlToSsml, ssmlItemText } from './htmlToSsml' export interface TextToSpeechInput { - id: string + id?: string text: string voice?: string languageCode?: string - textType?: 'text' | 'ssml' + textType?: 'html' | 'ssml' rate?: number volume?: number complimentaryVoice?: string - bucket: string + bucket?: string writeStream: NodeJS.WritableStream } @@ -45,7 +45,7 @@ export const synthesizeTextToSpeech = async ( process.env.AZURE_SPEECH_KEY, process.env.AZURE_SPEECH_REGION ) - const textType = input.textType || 'text' + const textType = input.textType || 'html' speechConfig.speechSynthesisOutputFormat = SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 @@ -53,12 +53,14 @@ export const synthesizeTextToSpeech = async ( const synthesizer = new SpeechSynthesizer(speechConfig) const speechMarks: SpeechMark[] = [] let timeOffset = 0 - const characterOffset = 0 - synthesizer.synthesizing = function (s, e) { - // convert arrayBuffer to stream and write to gcs file - writeStream.write(Buffer.from(e.result.audioData)) - } + // synthesizer.synthesizing = function (s, e) { + // // convert arrayBuffer to stream and write to stream + // console.debug( + // `(synthesizing): Audio length: ${e.result.audioData.byteLength}` + // ) + // writeStream.write(Buffer.from(e.result.audioData)) + // } // The event synthesis completed signals that the synthesis is completed. synthesizer.synthesisCompleted = (s, e) => { @@ -83,15 +85,20 @@ export const synthesizeTextToSpeech = async ( if (cancellationDetails.reason === CancellationReason.Error) { str += ': ' + e.result.errorDetails } - console.info(str) + console.error(str) } // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. synthesizer.wordBoundary = (s, e) => { + console.debug( + `(word boundary) Audio offset: ${e.audioOffset / 10000}ms, text: ${ + e.text + }` + ) speechMarks.push({ word: e.text, time: (timeOffset + e.audioOffset) / 10000, - start: characterOffset + e.textOffset, + start: e.textOffset, length: e.wordLength, type: 'word', }) @@ -99,7 +106,7 @@ export const synthesizeTextToSpeech = async ( synthesizer.bookmarkReached = (s, e) => { console.debug( - `(Bookmark reached), Audio offset: ${ + `(bookmark reached) Audio offset: ${ e.audioOffset / 10000 }ms, bookmark text: ${e.text}` ) @@ -111,12 +118,14 @@ export const synthesizeTextToSpeech = async ( } const speakSsmlAsyncPromise = ( - text: string + ssml: string, + writeStream: NodeJS.WritableStream ): Promise => { return new Promise((resolve, reject) => { synthesizer.speakSsmlAsync( - text, + ssml, (result) => { + writeStream.write(Buffer.from(result.audioData)) resolve(result) }, (error) => { @@ -126,36 +135,34 @@ export const synthesizeTextToSpeech = async ( }) } - if (textType === 'text') { - const ssmlItems = htmlToSsml(input.text, { - primaryVoice: input.voice || 'en-US-JennyNeural', - secondaryVoice: 'en-US-GuyNeural', - language: input.languageCode || 'en-US', - rate: '1', - }) + try { + if (textType === 'html') { + const ssmlItems = htmlToSsml(input.text, { + primaryVoice: input.voice || 'en-US-JennyNeural', + secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural', + language: input.languageCode || 'en-US', + rate: '1', + }) - for (const ssmlItem of Array.from(ssmlItems)) { - const ssml = ssmlItemText(ssmlItem) - console.debug(`synthesizing ${ssml}`) - const result = await speakSsmlAsyncPromise(ssml) - if (result.reason === ResultReason.Canceled) { - writeStream.end() - synthesizer.close() - throw new Error(result.errorDetails) + for (const ssmlItem of Array.from(ssmlItems)) { + const ssml = ssmlItemText(ssmlItem) + console.debug('start synthesizing', ssml) + const result = await speakSsmlAsyncPromise(ssml, writeStream) + timeOffset = timeOffset + result.audioDuration } - timeOffset = timeOffset + result.audioDuration - // characterOffset = characterOffset + htmlElement.innerText.length - } - } else { - const result = await speakSsmlAsyncPromise(input.text) - if (result.reason === ResultReason.Canceled) { - writeStream.end() - synthesizer.close() - throw new Error(result.errorDetails) + } else { + console.debug('start synthesizing', input.text) + await speakSsmlAsyncPromise(input.text, writeStream) } + } catch (error) { + console.error('synthesis error', error) + throw error + } finally { + console.debug('closing synthesizer') + writeStream.end() + synthesizer.close() + console.debug('synthesizer closed') } - writeStream.end() - synthesizer.close() return { speechMarks, From f7720ac156416f3f7dbf289b20eca390d5faa83e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 31 Aug 2022 17:43:34 +0800 Subject: [PATCH 03/15] Add debug logs --- packages/text-to-speech/src/index.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 0cff52eba..d9eab6993 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -58,7 +58,7 @@ const updateSpeech = async ( export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { - console.debug('New text to speech request', req) + console.info('Text to speech request received') const token = req.query.token as string if (!process.env.JWT_SECRET) { console.error('JWT_SECRET not exists') @@ -82,11 +82,15 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( const writeStream = audioFile.createWriteStream({ resumable: true, }) + const startTime = Date.now() const { speechMarks } = await synthesizeTextToSpeech({ ...input, textType: 'html', writeStream, }) + console.info( + `Synthesize text to speech completed in ${Date.now() - startTime} ms` + ) // upload Speech Marks file to GCS const speechMarksFileName = `speech/${id}.json` await uploadToBucket( @@ -103,15 +107,17 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( ) if (!updated) { + console.error('Failed to update speech') return res.status(500).send({ errorCodes: 'DB_ERROR' }) } + + console.info('Text to speech cloud function completed') + res.send('OK') } catch (e) { console.error('Text to speech cloud function error', e) await updateSpeech(id, token, 'FAILED') return res.status(500).send({ errorCodes: 'SYNTHESIZER_ERROR' }) } - - res.send('OK') } ) @@ -139,11 +145,14 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( writeStream, } await synthesizeTextToSpeech(input) + console.info('Synthesize text to speech completed') res.set({ 'Content-Type': 'audio/mpeg', 'Transfer-Encoding': 'chunked', }) + + console.info('Text to speech starts streaming') writeStream.pipe(res) } catch (e) { console.error('Text to speech streaming error', e) From be8b0e6eb7d6fb2ce0361f5830951fd01f2e6e39 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 31 Aug 2022 18:33:55 +0800 Subject: [PATCH 04/15] Use a large SSML to test --- packages/text-to-speech/data/ssml.xml | 88 +++++++++++++++++++-- packages/text-to-speech/src/index.ts | 5 +- packages/text-to-speech/src/textToSpeech.ts | 22 +++--- 3 files changed, 92 insertions(+), 23 deletions(-) diff --git a/packages/text-to-speech/data/ssml.xml b/packages/text-to-speech/data/ssml.xml index 2e8e27e77..aa97700d8 100644 --- a/packages/text-to-speech/data/ssml.xml +++ b/packages/text-to-speech/data/ssml.xml @@ -1,8 +1,80 @@ - - - Good morning! - - - Good morning to you too Jenny! - - + +

Back in 2010, during my first year of Business School, I helped give a presentation entitled “Twitter 101”:

+

My section was “The Twitter Value Proposition”, and after admitting that yes, you can find out what people are eating for lunch on Twitter, I stated “The truth is you can find anything you want on Twitter, and that’s a good thing.” The Twitter value proposition was that you could “See exactly what you need to see, in real-time, in one place, and nothing more”; I illustrated this by showing people how they could unfollow me:

+

The point was that Twitter required active management of your feed, but if you put in the effort, you could get something uniquely interesting to you that was incredibly valuable.

+

Most of the audience didn’t take me up on it.

+

Facebook Versus Instagram

+

If there is one axiom that governs the consumer Internet — consumer anything, really — it is that convenience matters more than anything. That was the problem with Twitter: it just wasn’t convenient for nearly enough people to figure out how to follow the right people. It was Facebook, which digitized offline relationships, that dominated the social media space.

+

Facebook’s social graph was the ultimate growth hack: from the moment you created an account Facebook worked assiduously to connect you with everyone you knew or wish you knew from high school, college, your hometown, workplace, you name an offline network and Facebook digitized it. Of course this meant that there were far too many updates and photos to keep track of, so Facebook ranked them, and presented them in a feed that you could scroll endlessly.

+

Users, famously, hated the News Feed when it was first launched: Facebook had protesters outside their doors in Palo Alto when it was introduced, and far more online; most were, ironically enough, organized on Facebook. CEO Mark Zuckerberg penned an apology:

+

+ We really messed this one up. When we launched News Feed and Mini-Feed we were trying to provide you with a stream of information about your social world. Instead, we did a bad job of explaining what the new features were and an even worse job of giving you control of them. I’d like to try to correct those errors now… +

+

The errors to be corrected were better controls over what might be shared; Facebook did not give the users what they claimed to want, which was abolishing the News Feed completely. That’s because the company correctly intuited a significant gap between its users stated preference — no News Feed — and their revealed preference, which was that they liked News Feed quite a bit. The next fifteen years would prove the company right.

+

It was hard to not think of that non-apology apology while watching Adam Mosseri’s Instagram update three weeks ago; Mosseri was clear that videos were going to be an ever great part of the Instagram experience, along with recommended posts. Zuckerberg reiterated the point on Facebook’s earnings call, noting that recommended posts in both Facebook and Instagram would continue to increase. A day later Mosseri told Casey Newton on Platformer that Instagram would scale back recommended posts, but was clear that the pullback was temporary:

+

+ “When you discover something in your feed that you didn’t follow before, there should be a high bar — it should just be great,” Mosseri said. “You should be delighted to see it. And I don’t think that’s happening enough right now. So I think we need to take a step back, in terms of the percentage of feed that are recommendations, get better at ranking and recommendations, and then — if and when we do — we can start to grow again.” (“I’m confident we will,” he added.) +

+

Michael Mignano calls this recommendation media in an article entitled The End of Social Media:

+

+ In recommendation media, content is not distributed to networks of connected people as the primary means of distribution. Instead, the main mechanism for the distribution of content is through opaque, platform-defined algorithms that favor maximum attention and engagement from consumers. The exact type of attention these recommendations seek is always defined by the platform and often tailored specifically to the user who is consuming content. For example, if the platform determines that someone loves movies, that person will likely see a lot of movie related content because that’s what captures that person’s attention best. This means platforms can also decide what consumers won’t see, such as problematic or polarizing content. + It’s ultimately up to the platform to decide what type of content gets recommended, not the social graph of the person producing the content. In contrast to social media, recommendation media is not a competition based on popularity; instead, it is a competition based on the absolute best content. Through this lens, it’s no wonder why Kylie Jenner opposes this change; her more than 360 million followers are simply worth less in a version of media dominated by algorithms and not followers. +

+

Sam Lessin, a former Facebook executive, traced this evolution from the analog days to what is next in a Twitter screenshot entitled “The Digital Media ‘Attention’ Food Chain in Progress”:

+

Lessin’s five steps:

+

+ The Pre-Internet ‘People Magazine’ Era + Content from ‘your friends’ kills People Magazine + Kardashians/Professional ‘friends’ kill real friends + Algorithmic everyone kills Kardashians + Next is pure-AI content which beats ‘algorithmic everyone’ +

+

This is a meta observation and, to make a cheap play on words, the first reason why it made sense for Facebook to change its name: Facebook the app is eternally stuck on Step 2 in terms of entertainment (the app has evolved to become much more of a utility, with a focus on groups, marketplace, etc.). It’s Instagram that is barreling forward. I wrote last summer about Instagram’s Evolution:

+

+ The reality, though, is that this is what Instagram is best at. When Mosseri said that Instagram was no longer a photo-sharing app — particularly a “square photo-sharing app” — he was not making a forward-looking pronouncement, but simply stating what has been true for many years now. More broadly, Instagram from the very beginning — including under former CEO Kevin Systrom — has been marked first and foremost by evolution. +

+

To put this in Lessin’s framework, Instagram started out as a utility for adding filters to photos put on other social networks, then it developed into a social network in its own right. What always made Instagram different than Facebook, though, is the fact that its content was default-public; this gave the space for the rise of brands, meme and highlight accounts, and the Instagram influencer. Sure, some number of people continued to use Instagram primarily as a social network, but Meta, more than anyone, had an understanding of how Instagram usage had evolved over time.

+

In other words, when Kylie Jenner posts a petition demanding that Meta “Make Instagram Instagram again”, the honest answer is that changing Instagram is the most Instagram-like behavior possible.

+

Three Trends

+

Still, it’s understandable why Instagram did back off, at least for now: the company is attempting to navigate three distinct trends, all at the same time.

+

The first trend is the shift towards ever more immersive mediums. Facebook, for example, started with text but exploded with the addition of photos. Instagram started with photos and expanded into video. Gaming was the first to make this progression, and is well into the 3D era. The next step is full immersion — virtual reality — and while the format has yet to penetrate the mainstream this progression in mediums is perhaps the most obvious reason to be bullish about the possibility.

+

The second trend is the increase in artificial intelligence. I’m using the term colloquially to refer to the overall trend of computers getting smarter and more useful, even if those smarts are a function of simple algorithms, machine learning, or, perhaps someday, something approaching general intelligence. To go back to Facebook, the original site didn’t have any smarts at all: it was just a collection of profile pages. Twitter came along and had the timeline, but the only smarts there was the ability to read a time stamp: all of the content was presented in chronological order. What made Facebook’s News Feed work was the application of ranking: from the very beginning the company tried to present users the content from their network that it thought you might be most interested in, mostly using simple signals and weights. Over time this ranking algorithm has evolved into a machine-learning driven model that is constantly iterating based on every click and linger, but on the limited set of content constrained by who you follow. Recommendations is the step beyond ranking: now the pool is not who you follow but all of the content on the entire network; it is a computation challenge that is many orders of magnitude beyond mere ranking (and AI-created content another massive step-change beyond that).

+

The third trend is the change in interaction models from user-directed to computer-controlled. The first version of Facebook relied on users clicking on links to visit different profiles; the News Feed changed the interaction model to scrolling. Stories reduced that to tapping, and Reels/TikTok is about swiping. YouTube has gone further than anyone here: Autoplay simply plays the next video without any interaction required at all.

+

One of the reasons Instagram got itself in trouble over the last few months is by introducing changes along all of these vectors at the same time. The company introduced more video into the feed (Trend 1), increased the percentage of recommended posts (Trend 2), and rolled out a new version of the app that was effectively a re-skinned TikTok to a limited set of users (Trend 3). It stands to reason that the company would have been better off doing one at a time.

+

That, though, would only be a temporary solution: it seems likely that all of these trends are inextricably intertwined.

+

Medium, Computing, and Interaction Models

+

Start with medium: text is easy, which is why it was the original medium of the Internet; effectively anyone can create it. The first implication is that there is far more text on the Internet than anything else; it also follows that the amount of high quality text is correspondingly high as well (a small fraction of a large number is still very large). The second implication has to do with AI: it is easier to process and glean insight from text. Text, meanwhile, takes focus and the application of an acquired skill for humans to interpret, not dissimilar to the deliberate movement of a mouse to interact with a link.

+

Photos used to be more difficult: digital cameras came along around the same time as the web, but even then you needed to have a dedicated device, move those photos to your computer, then upload them to a network. What is striking about the impact of smartphones is that not only did they make the device used to take pictures the same device used to upload and consume them, but they actually made it easier to take a picture than to write text. Still, it took time for AI to catch up: at first photos were ranked using the metadata surrounding them; only over the last few years has it become possible for services to understand what the photo actually is. The most reliable indicator of quality — beyond a like — remains the photo that you stop at while scrolling.

+

The ease of making a video followed a similar path to photos, but more extreme: making and uploading your own videos before the smartphone was even more difficult than photos; today the mechanics are just as easy, and it’s arguably even easier to make something interesting, given the amount of information conveyed by a video relative to photos, much less a text. Still, videos require more of a commitment than text or photos, because consuming them takes time; this is where the user interaction layer really matters. Lessin again, in another Twitter screenshot:

+

+ I saw someone recently complaining that Facebook was recommending to them…a very crass but probably pretty hilarious video. Their indignant response [was that] “the ranking must be broken.” Here is the thing: the ranking probably isn’t broken. He probably would love that video, but the fact that in order to engage with it he would have to go proactively click makes him feel bad. He doesn’t want to see himself as the type of person that clicks on things like that, even if he would enjoy it. + This is the brilliance of Tiktok and Facebook/Instagram’s challenge: TikTok’s interface eliminates the key problem of what people want to view themselves as wanting to follow/see versus what they actually want to see…it isn’t really about some big algorithm upgrade, it is about relesing emotional inner tension for people who show up to be entertained. +

+

This is the same tension between stated and revealed preference that Facebook encountered so many years ago, and its exactly why I fully expect the company to, after this pullback, continue to push forward with all three of the Instagram changes it is exploring.

+

Instagram’s Risk

+

Still, there is considerably more risk this time around: when Facebook pushed forward with the News Feed it was the young upstart moving aside incumbents like MySpace; it’s not as if its userbase was going to go backwards. This case is the opposite: Instagram is clearly aping TikTok, which is the young upstart in the space. It’s possible its users decide that if they must experience TikTok, they might as well go for the genuine thing.

+

This also highlights why TikTok is a much more serious challenge than Snapchat was: in that case Instagram’s network was the sword used to cut Snapchat off at the knees. I wrote in The Audacity of Copying Well:

+

+ For all of Snapchat’s explosive growth, Instagram is still more than double the size, with far more penetration across multiple demographics and international users. Rather than launch a “Stories” app without the network that is the most fundamental feature of any app built on sharing, Facebook is leveraging one of their most valuable assets: Instagram’s 500 million users…Instagram and Facebook are smart enough to know that Instagram Stories are not going to displace Snapchat’s place in its users lives. What Instagram Stories can do, though, is remove the motivation for the hundreds of millions of users on Instagram to even give Snapchat a shot. +

+

Instagram has no such power over TikTok, beyond inertia; in fact, the competitive situation is the opposite: if the goal is not to rank content from your network, but to recommend videos from the best creators anywhere, then it follows that TikTok is in the stronger relative position. Indeed, this is why Mosseri spent so much time talking about “small creators” with Newton:

+

+ I think one of the most important things is that we help new talent find an audience. I care a lot about large creators; I would like to do better than we have historically by smaller creators. I think we’ve done pretty well by large creators overall — I’m sure some people will disagree, but in general, that’s what the data suggests. I don’t think we’ve done nearly as well helping new talent break. And I think that’s super important. If we want to be a place where people push culture forward, to help realize the promise of the internet, which was to push power into the hands of more people, I think that we need to get better at that. +

+

There is the old Internet AMA question as to whether you would rather fight a horse-sized duck or 100 duck-sized horses. The analogy here is that in a world of ranking a horse-sized duck that everyone follows is valuable; in a world of recommendations 100 duck-sized horses are much more valuable, and Instagram is willing to sacrifice the former for the latter.

+

Meta’s Reward

+

The payoff, though, will not be “power” for these small creators: the implication of entertainment being dictated by recommendations and AI instead of reputation and ranking is that all of the power accrues to the platform doing the recommending. Indeed, this is where the potential reward comes in: this power isn’t only based on the sort of Aggregator dynamics underpinning dominant platforms today, but also absolutely massive amounts of investment in the computing necessary to power the AI that makes all of this possible.

+

In fact, you can make the case that if Meta survives the TikTok challenge, it will be on its way to the sort of moat enjoyed by the likes of Apple, Amazon, Google, and Microsoft, all of which have real world aspects to their differentiation. There is lots of talk about the $10 billion the company is spending on the Metaverse, but that is R&D; the more important number for this moat is the $30 billion this year in capital expditures, most of which is going to servers for AI. That AI is doing recommendations now, but Meta’s moat will only deepen if Lessin is right about a future where creators can be taken out of the equation entirely, in favor of artificially-generated content.

+

What is noteworty is that AI content will be an essential part of any sort of Metaverse future; I wrote earlier this year in DALL-E, the Metaverse, and Zero Marginal Content:

+

+ What is fascinating about DALL-E is that it points to a future where these three trends can be combined. DALL-E, at the end of the day, is ultimately a product of human-generated content, just like its GPT-3 cousin. The latter, of course, is about text, while DALL-E is about images. Notice, though, that progression from text to images; it follows that machine learning-generated video is next. This will likely take several years, of course; video is a much more difficult problem, and responsive 3D environments more difficult yet, but this is a path the industry has trod before: + Game developers pushed the limits on text, then images, then video, then 3D + Social media drives content creation costs to zero first on text, then images, then video + Machine learning models can now create text and images for zero marginal cost + In the very long run this points to a metaverse vision that is much less deterministic than your typical video game, yet much richer than what is generated on social media. Imagine environments that are not drawn by artists but rather created by AI: this not only increases the possibilities, but crucially, decreases the costs. +

+

These AI challenges, I would add, apply to monetization as well: one of the outcomes of Apple’s App Tracking Transparency changes is that advertising needs to shift from a deterministic model to a probabilistic one; the companies with the most data and the greatest amount of computing resources are going to make that shift more quickly and effectively, and I expect Meta to be top of the list.

+

None of this matters, though, without engagement. Instagram is following the medium trend to video, and Meta’s resources give it the long-term advantage in AI; the big question is which service users choose to interact with. To put it another way, Facebook’s next two decades are coming into sharper focus than ever; it is how well it navigates the TikTok minefield over the next two years that will determine if that long-term vision becomes a reality.

+

I wrote a follow-up to this Article in this Daily Update.

+

Post navigation

+ diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index d9eab6993..970864d0f 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -144,9 +144,6 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( textType: 'ssml', writeStream, } - await synthesizeTextToSpeech(input) - console.info('Synthesize text to speech completed') - res.set({ 'Content-Type': 'audio/mpeg', 'Transfer-Encoding': 'chunked', @@ -154,6 +151,8 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( console.info('Text to speech starts streaming') writeStream.pipe(res) + + await synthesizeTextToSpeech(input) } catch (e) { console.error('Text to speech streaming error', e) return res.status(500).send({ errorCodes: 'SYNTHESIZER_ERROR' }) diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index e007e343d..9fecc78c3 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -54,13 +54,13 @@ export const synthesizeTextToSpeech = async ( const speechMarks: SpeechMark[] = [] let timeOffset = 0 - // synthesizer.synthesizing = function (s, e) { - // // convert arrayBuffer to stream and write to stream - // console.debug( - // `(synthesizing): Audio length: ${e.result.audioData.byteLength}` - // ) - // writeStream.write(Buffer.from(e.result.audioData)) - // } + synthesizer.synthesizing = function (s, e) { + // convert arrayBuffer to stream and write to stream + console.debug( + `(synthesizing): Audio length: ${e.result.audioData.byteLength}` + ) + writeStream.write(Buffer.from(e.result.audioData)) + } // The event synthesis completed signals that the synthesis is completed. synthesizer.synthesisCompleted = (s, e) => { @@ -118,14 +118,12 @@ export const synthesizeTextToSpeech = async ( } const speakSsmlAsyncPromise = ( - ssml: string, - writeStream: NodeJS.WritableStream + ssml: string ): Promise => { return new Promise((resolve, reject) => { synthesizer.speakSsmlAsync( ssml, (result) => { - writeStream.write(Buffer.from(result.audioData)) resolve(result) }, (error) => { @@ -147,12 +145,12 @@ export const synthesizeTextToSpeech = async ( for (const ssmlItem of Array.from(ssmlItems)) { const ssml = ssmlItemText(ssmlItem) console.debug('start synthesizing', ssml) - const result = await speakSsmlAsyncPromise(ssml, writeStream) + const result = await speakSsmlAsyncPromise(ssml) timeOffset = timeOffset + result.audioDuration } } else { console.debug('start synthesizing', input.text) - await speakSsmlAsyncPromise(input.text, writeStream) + await speakSsmlAsyncPromise(input.text) } } catch (error) { console.error('synthesis error', error) From 3a40dd7266ce0bbad147ffa5f2ffd364d3ee8029 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 31 Aug 2022 18:45:23 +0800 Subject: [PATCH 05/15] Make test SSML shorter --- packages/text-to-speech/data/ssml.xml | 44 --------------------- packages/text-to-speech/src/textToSpeech.ts | 3 -- 2 files changed, 47 deletions(-) diff --git a/packages/text-to-speech/data/ssml.xml b/packages/text-to-speech/data/ssml.xml index aa97700d8..5492c0d5f 100644 --- a/packages/text-to-speech/data/ssml.xml +++ b/packages/text-to-speech/data/ssml.xml @@ -33,48 +33,4 @@

The reality, though, is that this is what Instagram is best at. When Mosseri said that Instagram was no longer a photo-sharing app — particularly a “square photo-sharing app” — he was not making a forward-looking pronouncement, but simply stating what has been true for many years now. More broadly, Instagram from the very beginning — including under former CEO Kevin Systrom — has been marked first and foremost by evolution.

-

To put this in Lessin’s framework, Instagram started out as a utility for adding filters to photos put on other social networks, then it developed into a social network in its own right. What always made Instagram different than Facebook, though, is the fact that its content was default-public; this gave the space for the rise of brands, meme and highlight accounts, and the Instagram influencer. Sure, some number of people continued to use Instagram primarily as a social network, but Meta, more than anyone, had an understanding of how Instagram usage had evolved over time.

-

In other words, when Kylie Jenner posts a petition demanding that Meta “Make Instagram Instagram again”, the honest answer is that changing Instagram is the most Instagram-like behavior possible.

-

Three Trends

-

Still, it’s understandable why Instagram did back off, at least for now: the company is attempting to navigate three distinct trends, all at the same time.

-

The first trend is the shift towards ever more immersive mediums. Facebook, for example, started with text but exploded with the addition of photos. Instagram started with photos and expanded into video. Gaming was the first to make this progression, and is well into the 3D era. The next step is full immersion — virtual reality — and while the format has yet to penetrate the mainstream this progression in mediums is perhaps the most obvious reason to be bullish about the possibility.

-

The second trend is the increase in artificial intelligence. I’m using the term colloquially to refer to the overall trend of computers getting smarter and more useful, even if those smarts are a function of simple algorithms, machine learning, or, perhaps someday, something approaching general intelligence. To go back to Facebook, the original site didn’t have any smarts at all: it was just a collection of profile pages. Twitter came along and had the timeline, but the only smarts there was the ability to read a time stamp: all of the content was presented in chronological order. What made Facebook’s News Feed work was the application of ranking: from the very beginning the company tried to present users the content from their network that it thought you might be most interested in, mostly using simple signals and weights. Over time this ranking algorithm has evolved into a machine-learning driven model that is constantly iterating based on every click and linger, but on the limited set of content constrained by who you follow. Recommendations is the step beyond ranking: now the pool is not who you follow but all of the content on the entire network; it is a computation challenge that is many orders of magnitude beyond mere ranking (and AI-created content another massive step-change beyond that).

-

The third trend is the change in interaction models from user-directed to computer-controlled. The first version of Facebook relied on users clicking on links to visit different profiles; the News Feed changed the interaction model to scrolling. Stories reduced that to tapping, and Reels/TikTok is about swiping. YouTube has gone further than anyone here: Autoplay simply plays the next video without any interaction required at all.

-

One of the reasons Instagram got itself in trouble over the last few months is by introducing changes along all of these vectors at the same time. The company introduced more video into the feed (Trend 1), increased the percentage of recommended posts (Trend 2), and rolled out a new version of the app that was effectively a re-skinned TikTok to a limited set of users (Trend 3). It stands to reason that the company would have been better off doing one at a time.

-

That, though, would only be a temporary solution: it seems likely that all of these trends are inextricably intertwined.

-

Medium, Computing, and Interaction Models

-

Start with medium: text is easy, which is why it was the original medium of the Internet; effectively anyone can create it. The first implication is that there is far more text on the Internet than anything else; it also follows that the amount of high quality text is correspondingly high as well (a small fraction of a large number is still very large). The second implication has to do with AI: it is easier to process and glean insight from text. Text, meanwhile, takes focus and the application of an acquired skill for humans to interpret, not dissimilar to the deliberate movement of a mouse to interact with a link.

-

Photos used to be more difficult: digital cameras came along around the same time as the web, but even then you needed to have a dedicated device, move those photos to your computer, then upload them to a network. What is striking about the impact of smartphones is that not only did they make the device used to take pictures the same device used to upload and consume them, but they actually made it easier to take a picture than to write text. Still, it took time for AI to catch up: at first photos were ranked using the metadata surrounding them; only over the last few years has it become possible for services to understand what the photo actually is. The most reliable indicator of quality — beyond a like — remains the photo that you stop at while scrolling.

-

The ease of making a video followed a similar path to photos, but more extreme: making and uploading your own videos before the smartphone was even more difficult than photos; today the mechanics are just as easy, and it’s arguably even easier to make something interesting, given the amount of information conveyed by a video relative to photos, much less a text. Still, videos require more of a commitment than text or photos, because consuming them takes time; this is where the user interaction layer really matters. Lessin again, in another Twitter screenshot:

-

- I saw someone recently complaining that Facebook was recommending to them…a very crass but probably pretty hilarious video. Their indignant response [was that] “the ranking must be broken.” Here is the thing: the ranking probably isn’t broken. He probably would love that video, but the fact that in order to engage with it he would have to go proactively click makes him feel bad. He doesn’t want to see himself as the type of person that clicks on things like that, even if he would enjoy it. - This is the brilliance of Tiktok and Facebook/Instagram’s challenge: TikTok’s interface eliminates the key problem of what people want to view themselves as wanting to follow/see versus what they actually want to see…it isn’t really about some big algorithm upgrade, it is about relesing emotional inner tension for people who show up to be entertained. -

-

This is the same tension between stated and revealed preference that Facebook encountered so many years ago, and its exactly why I fully expect the company to, after this pullback, continue to push forward with all three of the Instagram changes it is exploring.

-

Instagram’s Risk

-

Still, there is considerably more risk this time around: when Facebook pushed forward with the News Feed it was the young upstart moving aside incumbents like MySpace; it’s not as if its userbase was going to go backwards. This case is the opposite: Instagram is clearly aping TikTok, which is the young upstart in the space. It’s possible its users decide that if they must experience TikTok, they might as well go for the genuine thing.

-

This also highlights why TikTok is a much more serious challenge than Snapchat was: in that case Instagram’s network was the sword used to cut Snapchat off at the knees. I wrote in The Audacity of Copying Well:

-

- For all of Snapchat’s explosive growth, Instagram is still more than double the size, with far more penetration across multiple demographics and international users. Rather than launch a “Stories” app without the network that is the most fundamental feature of any app built on sharing, Facebook is leveraging one of their most valuable assets: Instagram’s 500 million users…Instagram and Facebook are smart enough to know that Instagram Stories are not going to displace Snapchat’s place in its users lives. What Instagram Stories can do, though, is remove the motivation for the hundreds of millions of users on Instagram to even give Snapchat a shot. -

-

Instagram has no such power over TikTok, beyond inertia; in fact, the competitive situation is the opposite: if the goal is not to rank content from your network, but to recommend videos from the best creators anywhere, then it follows that TikTok is in the stronger relative position. Indeed, this is why Mosseri spent so much time talking about “small creators” with Newton:

-

- I think one of the most important things is that we help new talent find an audience. I care a lot about large creators; I would like to do better than we have historically by smaller creators. I think we’ve done pretty well by large creators overall — I’m sure some people will disagree, but in general, that’s what the data suggests. I don’t think we’ve done nearly as well helping new talent break. And I think that’s super important. If we want to be a place where people push culture forward, to help realize the promise of the internet, which was to push power into the hands of more people, I think that we need to get better at that. -

-

There is the old Internet AMA question as to whether you would rather fight a horse-sized duck or 100 duck-sized horses. The analogy here is that in a world of ranking a horse-sized duck that everyone follows is valuable; in a world of recommendations 100 duck-sized horses are much more valuable, and Instagram is willing to sacrifice the former for the latter.

-

Meta’s Reward

-

The payoff, though, will not be “power” for these small creators: the implication of entertainment being dictated by recommendations and AI instead of reputation and ranking is that all of the power accrues to the platform doing the recommending. Indeed, this is where the potential reward comes in: this power isn’t only based on the sort of Aggregator dynamics underpinning dominant platforms today, but also absolutely massive amounts of investment in the computing necessary to power the AI that makes all of this possible.

-

In fact, you can make the case that if Meta survives the TikTok challenge, it will be on its way to the sort of moat enjoyed by the likes of Apple, Amazon, Google, and Microsoft, all of which have real world aspects to their differentiation. There is lots of talk about the $10 billion the company is spending on the Metaverse, but that is R&D; the more important number for this moat is the $30 billion this year in capital expditures, most of which is going to servers for AI. That AI is doing recommendations now, but Meta’s moat will only deepen if Lessin is right about a future where creators can be taken out of the equation entirely, in favor of artificially-generated content.

-

What is noteworty is that AI content will be an essential part of any sort of Metaverse future; I wrote earlier this year in DALL-E, the Metaverse, and Zero Marginal Content:

-

- What is fascinating about DALL-E is that it points to a future where these three trends can be combined. DALL-E, at the end of the day, is ultimately a product of human-generated content, just like its GPT-3 cousin. The latter, of course, is about text, while DALL-E is about images. Notice, though, that progression from text to images; it follows that machine learning-generated video is next. This will likely take several years, of course; video is a much more difficult problem, and responsive 3D environments more difficult yet, but this is a path the industry has trod before: - Game developers pushed the limits on text, then images, then video, then 3D - Social media drives content creation costs to zero first on text, then images, then video - Machine learning models can now create text and images for zero marginal cost - In the very long run this points to a metaverse vision that is much less deterministic than your typical video game, yet much richer than what is generated on social media. Imagine environments that are not drawn by artists but rather created by AI: this not only increases the possibilities, but crucially, decreases the costs. -

-

These AI challenges, I would add, apply to monetization as well: one of the outcomes of Apple’s App Tracking Transparency changes is that advertising needs to shift from a deterministic model to a probabilistic one; the companies with the most data and the greatest amount of computing resources are going to make that shift more quickly and effectively, and I expect Meta to be top of the list.

-

None of this matters, though, without engagement. Instagram is following the medium trend to video, and Meta’s resources give it the long-term advantage in AI; the big question is which service users choose to interact with. To put it another way, Facebook’s next two decades are coming into sharper focus than ever; it is how well it navigates the TikTok minefield over the next two years that will determine if that long-term vision becomes a reality.

-

I wrote a follow-up to this Article in this Daily Update.

-

Post navigation

diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 9fecc78c3..73cf49b28 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -56,9 +56,6 @@ export const synthesizeTextToSpeech = async ( synthesizer.synthesizing = function (s, e) { // convert arrayBuffer to stream and write to stream - console.debug( - `(synthesizing): Audio length: ${e.result.audioData.byteLength}` - ) writeStream.write(Buffer.from(e.result.audioData)) } From 7bb3d711c519df9ce87af8a15d302a3ad68fc913 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 1 Sep 2022 15:26:19 +0800 Subject: [PATCH 06/15] Pull token from Authorization header too --- packages/text-to-speech/src/index.ts | 19 +++++++++++-------- packages/text-to-speech/src/textToSpeech.ts | 4 ++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 970864d0f..fc0e630dd 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -81,12 +81,12 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( const audioFile = createGCSFile(bucket, audioFileName) const writeStream = audioFile.createWriteStream({ resumable: true, - }) + }) as NodeJS.WriteStream const startTime = Date.now() const { speechMarks } = await synthesizeTextToSpeech({ ...input, textType: 'html', - writeStream, + audioStream: writeStream, }) console.info( `Synthesize text to speech completed in ${Date.now() - startTime} ms` @@ -124,25 +124,28 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { console.debug('Text to speech steaming request', req) - const token = req.query.token as string if (!process.env.JWT_SECRET) { console.error('JWT_SECRET not exists') - return res.status(500).send('JWT_SECRET not exists') + return res.status(500).send({ errorCodes: 'JWT_SECRET_NOT_EXISTS' }) + } + const token = (req.query.token || req.headers.authorization) as string + if (!token) { + return res.status(200).send({ errorCode: 'UNAUTHORIZED' }) } try { jwt.verify(token, process.env.JWT_SECRET) } catch (e) { console.error(e) - return res.status(200).send('UNAUTHENTICATED') + return res.status(200).send({ errorCode: 'UNAUTHORIZED' }) } try { const ssml = fs.readFileSync('./data/ssml.xml', 'utf8') - const writeStream = new PassThrough() + const audioStream = new PassThrough() const input: TextToSpeechInput = { text: ssml, textType: 'ssml', - writeStream, + audioStream, } res.set({ 'Content-Type': 'audio/mpeg', @@ -150,7 +153,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( }) console.info('Text to speech starts streaming') - writeStream.pipe(res) + audioStream.pipe(res) await synthesizeTextToSpeech(input) } catch (e) { diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 73cf49b28..a79ee42c0 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -19,7 +19,7 @@ export interface TextToSpeechInput { volume?: number complimentaryVoice?: string bucket?: string - writeStream: NodeJS.WritableStream + audioStream: NodeJS.ReadWriteStream } export interface TextToSpeechOutput { @@ -40,7 +40,7 @@ export const synthesizeTextToSpeech = async ( if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) { throw new Error('Azure Speech Key or Region not set') } - const writeStream = input.writeStream + const writeStream = input.audioStream const speechConfig = SpeechConfig.fromSubscription( process.env.AZURE_SPEECH_KEY, process.env.AZURE_SPEECH_REGION From 3953a9098ec0b8f10aad2db87443556eedaa9617 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 1 Sep 2022 17:09:48 +0800 Subject: [PATCH 07/15] Send SSML items in the synthesis API --- packages/api/package.json | 1 + packages/api/src/routers/article_router.ts | 19 +++++++++++++++++-- packages/api/src/textToSpeech.d.ts | 18 ++++++++++++++++++ packages/text-to-speech/.npmignore | 1 + 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 packages/api/src/textToSpeech.d.ts create mode 100644 packages/text-to-speech/.npmignore diff --git a/packages/api/package.json b/packages/api/package.json index 28ba0df6c..7b275b385 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -19,6 +19,7 @@ "@google-cloud/storage": "^5.18.1", "@google-cloud/tasks": "^2.3.0", "@omnivore/readability": "1.0.0", + "@omnivore/text-to-speech-handler": "1.0.0", "@opentelemetry/api": "^1.0.1", "@opentelemetry/core": "^1.3.1", "@opentelemetry/exporter-jaeger": "^1.0.1", diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index e0f625aad..3302cdf11 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -21,6 +21,7 @@ import { getPageById, updatePage } from '../elastic/pages' import { generateDownloadSignedUrl } from '../utils/uploads' import { enqueueTextToSpeech } from '../utils/createTask' import { createPubSubClient } from '../datalayer/pubsub' +import { htmlToSsml } from '@omnivore/text-to-speech-handler' const logger = buildLogger('app.dispatch') @@ -81,7 +82,7 @@ export function articleRouter() { const priority = req.params.priority if ( !articleId || - !['mp3', 'speech-marks'].includes(outputFormat) || + !['mp3', 'speech-marks', 'ssml'].includes(outputFormat) || !['low', 'high'].includes(priority) ) { return res.status(400).send('Invalid data') @@ -91,7 +92,6 @@ export function articleRouter() { return res.status(401).send({ errorCode: 'UNAUTHORIZED' }) } const { uid } = jwt.decode(token) as Claims - logger.info(`Get article speech in ${outputFormat} format`, { params: req.params, labels: { @@ -100,6 +100,21 @@ export function articleRouter() { }, }) + if (outputFormat === 'ssml') { + const page = await getPageById(articleId) + if (!page) { + return res.status(404).send('Page not found') + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + const ssmlItems = htmlToSsml(page.content, { + primaryVoice: voice, + secondaryVoice: 'en-US-GuyNeural', + rate: '1', + language: page.language || 'en-US', + }) + return res.send({ ssmlItems }) + } + const existingSpeech = await getRepository(Speech).findOne({ where: { elasticPageId: articleId, diff --git a/packages/api/src/textToSpeech.d.ts b/packages/api/src/textToSpeech.d.ts new file mode 100644 index 000000000..194db1fbd --- /dev/null +++ b/packages/api/src/textToSpeech.d.ts @@ -0,0 +1,18 @@ +declare module '@omnivore/text-to-speech-handler' { + function htmlToSsml(html: string, options: SSMLOptions): SSMLItem[] + + interface SSMLOptions { + primaryVoice: string + secondaryVoice: string + rate: string + language: string + } + + interface SSMLItem { + open: string + close: string + textItems: string[] + } + + export { htmlToSsml } +} diff --git a/packages/text-to-speech/.npmignore b/packages/text-to-speech/.npmignore new file mode 100644 index 000000000..193378602 --- /dev/null +++ b/packages/text-to-speech/.npmignore @@ -0,0 +1 @@ +/test/ From ba147c0d99948240e8c6bf8d9adbecdbba3aa610 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 2 Sep 2022 12:22:59 +0800 Subject: [PATCH 08/15] Allow receiving an array of ssmlItems --- packages/text-to-speech/src/index.ts | 8 +++-- packages/text-to-speech/src/textToSpeech.ts | 36 +++++++++++---------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index fc0e630dd..d5329b11e 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -10,7 +10,7 @@ import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-d import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' import { File, Storage } from '@google-cloud/storage' import { PassThrough } from 'stream' -import * as fs from 'fs' +import { SSMLItem } from './htmlToSsml' dotenv.config() Sentry.GCPFunction.init({ @@ -140,12 +140,14 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( } try { - const ssml = fs.readFileSync('./data/ssml.xml', 'utf8') + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const ssmlItems = req.body.ssmlItems as SSMLItem[] const audioStream = new PassThrough() const input: TextToSpeechInput = { - text: ssml, + text: '', textType: 'ssml', audioStream, + ssmlItems, } res.set({ 'Content-Type': 'audio/mpeg', diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index a79ee42c0..2217fe1a1 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -7,7 +7,7 @@ import { SpeechSynthesisResult, SpeechSynthesizer, } from 'microsoft-cognitiveservices-speech-sdk' -import { htmlToSsml, ssmlItemText } from './htmlToSsml' +import { htmlToSsml, SSMLItem, ssmlItemText } from './htmlToSsml' export interface TextToSpeechInput { id?: string @@ -20,6 +20,7 @@ export interface TextToSpeechInput { complimentaryVoice?: string bucket?: string audioStream: NodeJS.ReadWriteStream + ssmlItems?: SSMLItem[] } export interface TextToSpeechOutput { @@ -131,23 +132,24 @@ export const synthesizeTextToSpeech = async ( } try { - if (textType === 'html') { - const ssmlItems = htmlToSsml(input.text, { - primaryVoice: input.voice || 'en-US-JennyNeural', - secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural', - language: input.languageCode || 'en-US', - rate: '1', - }) + const ssmlItems = + input.textType === 'ssml' + ? input.ssmlItems + : htmlToSsml(input.text, { + primaryVoice: input.voice || 'en-US-JennyNeural', + secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural', + language: input.languageCode || 'en-US', + rate: '1', + }) + if (!ssmlItems || ssmlItems.length === 0) { + throw new Error('No SSML items found') + } - for (const ssmlItem of Array.from(ssmlItems)) { - const ssml = ssmlItemText(ssmlItem) - console.debug('start synthesizing', ssml) - const result = await speakSsmlAsyncPromise(ssml) - timeOffset = timeOffset + result.audioDuration - } - } else { - console.debug('start synthesizing', input.text) - await speakSsmlAsyncPromise(input.text) + for (const ssmlItem of Array.from(ssmlItems)) { + const ssml = ssmlItemText(ssmlItem) + console.debug('start synthesizing', ssml) + const result = await speakSsmlAsyncPromise(ssml) + timeOffset = timeOffset + result.audioDuration } } catch (error) { console.error('synthesis error', error) From 71144d3c84f67236d55d10b993e85ceb796653bc Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 2 Sep 2022 13:19:44 +0800 Subject: [PATCH 09/15] Reduce SSML size --- packages/text-to-speech/src/htmlToSsml.ts | 8 +++----- packages/text-to-speech/src/index.ts | 8 +++++++- packages/text-to-speech/src/textToSpeech.ts | 1 - 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index b90142f63..ed125114d 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -91,7 +91,7 @@ function emitTextNode( if (ssmlElement) { emit(textItems, `<${ssmlElement}>`) } - emit(textItems, `${cleanedText}`) + emit(textItems, `${cleanedText.replace(/\s+/g, ' ')}`) if (ssmlElement) { emit(textItems, ``) } @@ -123,7 +123,7 @@ function emitElement( ) { const cleanedText = cleanTextNode(child) if (idx && cleanedText.length > 1) { - // Make sure its more than just a space + // Make sure it's more than just a space emit(textItems, ``) } emitTextNode(textItems, cleanedText, child) @@ -158,9 +158,7 @@ const startSsml = (element: Element, options: SSMLOptions): string => { element.nodeName === 'BLOCKQUOTE' ? options.secondaryVoice : options.primaryVoice - return ` - - ` + return `` } const endSsml = (): string => { diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index d5329b11e..55a9eabc6 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -10,7 +10,7 @@ import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-d import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' import { File, Storage } from '@google-cloud/storage' import { PassThrough } from 'stream' -import { SSMLItem } from './htmlToSsml' +import { htmlToSsml, SSMLItem } from './htmlToSsml' dotenv.config() Sentry.GCPFunction.init({ @@ -164,3 +164,9 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( } } ) + +module.exports = { + htmlToSsml, + textToSpeechStreamingHandler, + textToSpeechHandler, +} diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 2217fe1a1..ed7799e2b 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -46,7 +46,6 @@ export const synthesizeTextToSpeech = async ( process.env.AZURE_SPEECH_KEY, process.env.AZURE_SPEECH_REGION ) - const textType = input.textType || 'html' speechConfig.speechSynthesisOutputFormat = SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 From 9826262ef4692ef6de3deb0d113e89f33888bf28 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 2 Sep 2022 13:51:20 +0800 Subject: [PATCH 10/15] Return an array of SSMLs instead of ssmlItems --- packages/api/src/textToSpeech.d.ts | 8 +--- packages/text-to-speech/src/htmlToSsml.ts | 9 ++++- packages/text-to-speech/src/index.ts | 7 +++- packages/text-to-speech/src/textToSpeech.ts | 39 +++++++++---------- .../text-to-speech/test/htmlToSsml.test.ts | 22 +++++------ 5 files changed, 44 insertions(+), 41 deletions(-) diff --git a/packages/api/src/textToSpeech.d.ts b/packages/api/src/textToSpeech.d.ts index 194db1fbd..5e438e05e 100644 --- a/packages/api/src/textToSpeech.d.ts +++ b/packages/api/src/textToSpeech.d.ts @@ -1,5 +1,5 @@ declare module '@omnivore/text-to-speech-handler' { - function htmlToSsml(html: string, options: SSMLOptions): SSMLItem[] + function htmlToSsml(html: string, options: SSMLOptions): string[] interface SSMLOptions { primaryVoice: string @@ -8,11 +8,5 @@ declare module '@omnivore/text-to-speech-handler' { language: string } - interface SSMLItem { - open: string - close: string - textItems: string[] - } - export { htmlToSsml } } diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index ed125114d..e65755cfc 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -179,7 +179,10 @@ export const ssmlItemText = (item: SSMLItem): string => { return [item.open, ...item.textItems, item.close].join('') } -export const htmlToSsml = (html: string, options: SSMLOptions): SSMLItem[] => { +export const htmlToSsmlItems = ( + html: string, + options: SSMLOptions +): SSMLItem[] => { console.log('creating ssml with options', options) const dom = parseHTML(html) @@ -218,3 +221,7 @@ export const htmlToSsml = (html: string, options: SSMLOptions): SSMLItem[] => { return items } + +export const htmlToSsml = (html: string, options: SSMLOptions): string[] => { + return htmlToSsmlItems(html, options).map(ssmlItemText) +} diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 55a9eabc6..5006a2666 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -10,7 +10,7 @@ import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-d import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' import { File, Storage } from '@google-cloud/storage' import { PassThrough } from 'stream' -import { htmlToSsml, SSMLItem } from './htmlToSsml' +import { htmlToSsml } from './htmlToSsml' dotenv.config() Sentry.GCPFunction.init({ @@ -141,7 +141,10 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( try { // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - const ssmlItems = req.body.ssmlItems as SSMLItem[] + const ssmlItems = req.body.ssmlItems as string[] + if (!ssmlItems || ssmlItems.length === 0) { + return res.status(200).send({ errorCode: 'INVALID_DATA' }) + } const audioStream = new PassThrough() const input: TextToSpeechInput = { text: '', diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index ed7799e2b..0fdfc43ed 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -7,7 +7,7 @@ import { SpeechSynthesisResult, SpeechSynthesizer, } from 'microsoft-cognitiveservices-speech-sdk' -import { htmlToSsml, SSMLItem, ssmlItemText } from './htmlToSsml' +import { htmlToSsmlItems, ssmlItemText } from './htmlToSsml' export interface TextToSpeechInput { id?: string @@ -20,7 +20,7 @@ export interface TextToSpeechInput { complimentaryVoice?: string bucket?: string audioStream: NodeJS.ReadWriteStream - ssmlItems?: SSMLItem[] + ssmlItems?: string[] } export interface TextToSpeechOutput { @@ -41,6 +41,7 @@ export const synthesizeTextToSpeech = async ( if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) { throw new Error('Azure Speech Key or Region not set') } + const textType = input.textType || 'html' const writeStream = input.audioStream const speechConfig = SpeechConfig.fromSubscription( process.env.AZURE_SPEECH_KEY, @@ -131,24 +132,22 @@ export const synthesizeTextToSpeech = async ( } try { - const ssmlItems = - input.textType === 'ssml' - ? input.ssmlItems - : htmlToSsml(input.text, { - primaryVoice: input.voice || 'en-US-JennyNeural', - secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural', - language: input.languageCode || 'en-US', - rate: '1', - }) - if (!ssmlItems || ssmlItems.length === 0) { - throw new Error('No SSML items found') - } - - for (const ssmlItem of Array.from(ssmlItems)) { - const ssml = ssmlItemText(ssmlItem) - console.debug('start synthesizing', ssml) - const result = await speakSsmlAsyncPromise(ssml) - timeOffset = timeOffset + result.audioDuration + if (textType === 'html') { + const ssmlItems = htmlToSsmlItems(input.text, { + primaryVoice: input.voice || 'en-US-JennyNeural', + secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural', + language: input.languageCode || 'en-US', + rate: '1', + }) + for (const ssmlItem of ssmlItems) { + const ssml = ssmlItemText(ssmlItem) + const result = await speakSsmlAsyncPromise(ssml) + timeOffset = timeOffset + result.audioDuration + } + } else { + for (const ssmlItem of input.ssmlItems || []) { + await speakSsmlAsyncPromise(ssmlItem) + } } } catch (error) { console.error('synthesis error', error) diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index 46bd0243c..80fd3fbd3 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -1,8 +1,8 @@ import 'mocha' import { expect } from 'chai' -import { htmlToSsml } from '../src/htmlToSsml' +import { htmlToSsmlItems } from '../src/htmlToSsml' -describe('htmlToSsml', () => { +describe('htmlToSsmlItems', () => { const TEST_OPTIONS = { primaryVoice: 'test-primary', secondaryVoice: 'test-secondary', @@ -12,7 +12,7 @@ describe('htmlToSsml', () => { describe('a simple html file', () => { xit('should convert Html to SSML', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -28,7 +28,7 @@ describe('htmlToSsml', () => { }) describe('a file with nested elements', () => { xit('should collapse spans into the parent paragraph', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -57,7 +57,7 @@ describe('htmlToSsml', () => { ) }) xit('should extract child paragraphs to the top level', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -77,7 +77,7 @@ describe('htmlToSsml', () => { ) }) xit('should hoist paragraphs in spans to the top level', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -95,7 +95,7 @@ describe('htmlToSsml', () => { expect(text).to.equal(`TBD`.trim()) }) xit('should hoist lists to the top level', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -113,7 +113,7 @@ describe('htmlToSsml', () => { expect(text).to.equal(`TBD`.trim()) }) xit('should hoist headers to the top level', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -131,7 +131,7 @@ describe('htmlToSsml', () => { expect(text).to.equal(`TBD`.trim()) }) xit('should hoist blockquotes to the top level', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -151,7 +151,7 @@ describe('htmlToSsml', () => { }) describe('a file with blockquotes', () => { xit('should convert Html to SSML with complimentary voices', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -193,7 +193,7 @@ describe('htmlToSsml', () => { // continue // } // const html = fs.readFileSync(readablePath, { encoding: 'utf-8' }) - // const ssmlItems = htmlToSsml(html, TEST_OPTIONS) + // const ssmlItems = htmlToSsmlItems(html, TEST_OPTIONS) // console.log('SSML ITEMS', ssmlItems) // } // }) From b9eee9e4c56795a3bd7fed6c1644049563635594 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 2 Sep 2022 15:56:23 +0800 Subject: [PATCH 11/15] Fix docker build --- packages/api/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/api/Dockerfile b/packages/api/Dockerfile index ff72db699..c33bac1a4 100644 --- a/packages/api/Dockerfile +++ b/packages/api/Dockerfile @@ -13,11 +13,13 @@ COPY .eslintrc . COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json COPY /packages/api/package.json ./packages/api/package.json +COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json RUN yarn install --pure-lockfile ADD /packages/readabilityjs ./packages/readabilityjs ADD /packages/api ./packages/api +ADD /packages/text-to-speech ./packages/text-to-speech RUN yarn RUN yarn workspace @omnivore/api build @@ -41,6 +43,7 @@ COPY --from=builder /app/packages/api/package.json /app/packages/api/package.jso COPY --from=builder /app/packages/api/node_modules /app/packages/api/node_modules COPY --from=builder /app/node_modules /app/node_modules COPY --from=builder /app/package.json /app/package.json +COPY --from=builder /app/packages/text-to-speech/ /app/packages/text-to-speech/ EXPOSE 8080 CMD ["yarn", "workspace", "@omnivore/api", "start"] From 7fa092d95e6b8184b9e8e3c899d10d143750e107 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 5 Sep 2022 17:16:27 +0800 Subject: [PATCH 12/15] Write speech marks to file stream --- packages/text-to-speech/src/index.ts | 21 ++++++----- packages/text-to-speech/src/textToSpeech.ts | 41 +++++++++++++-------- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 5006a2666..306666e9c 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -79,25 +79,24 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( try { const audioFileName = `speech/${id}.mp3` const audioFile = createGCSFile(bucket, audioFileName) - const writeStream = audioFile.createWriteStream({ + const audioStream = audioFile.createWriteStream({ + resumable: true, + }) as NodeJS.WriteStream + const speechMarksFileName = `speech/${id}.json` + const speechMarksFile = createGCSFile(bucket, speechMarksFileName) + const speechMarksStream = speechMarksFile.createWriteStream({ resumable: true, }) as NodeJS.WriteStream const startTime = Date.now() - const { speechMarks } = await synthesizeTextToSpeech({ + await synthesizeTextToSpeech({ ...input, textType: 'html', - audioStream: writeStream, + audioStream, + speechMarksStream, }) console.info( `Synthesize text to speech completed in ${Date.now() - startTime} ms` ) - // upload Speech Marks file to GCS - const speechMarksFileName = `speech/${id}.json` - await uploadToBucket( - speechMarksFileName, - Buffer.from(JSON.stringify(speechMarks)), - bucket - ) const updated = await updateSpeech( id, token, @@ -146,11 +145,13 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( return res.status(200).send({ errorCode: 'INVALID_DATA' }) } const audioStream = new PassThrough() + const speechMarksStream = new PassThrough() const input: TextToSpeechInput = { text: '', textType: 'ssml', audioStream, ssmlItems, + speechMarksStream, } res.set({ 'Content-Type': 'audio/mpeg', diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 0fdfc43ed..18c7480b3 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -21,6 +21,7 @@ export interface TextToSpeechInput { bucket?: string audioStream: NodeJS.ReadWriteStream ssmlItems?: string[] + speechMarksStream: NodeJS.ReadWriteStream } export interface TextToSpeechOutput { @@ -42,7 +43,8 @@ export const synthesizeTextToSpeech = async ( throw new Error('Azure Speech Key or Region not set') } const textType = input.textType || 'html' - const writeStream = input.audioStream + const audioStream = input.audioStream + const speechMarksStream = input.speechMarksStream const speechConfig = SpeechConfig.fromSubscription( process.env.AZURE_SPEECH_KEY, process.env.AZURE_SPEECH_REGION @@ -57,7 +59,7 @@ export const synthesizeTextToSpeech = async ( synthesizer.synthesizing = function (s, e) { // convert arrayBuffer to stream and write to stream - writeStream.write(Buffer.from(e.result.audioData)) + audioStream.write(Buffer.from(e.result.audioData)) } // The event synthesis completed signals that the synthesis is completed. @@ -93,13 +95,17 @@ export const synthesizeTextToSpeech = async ( e.text }` ) - speechMarks.push({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - start: e.textOffset, - length: e.wordLength, - type: 'word', - }) + speechMarksStream.write( + Buffer.from( + JSON.stringify({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + start: e.textOffset, + length: e.wordLength, + type: 'word', + }) + ) + ) } synthesizer.bookmarkReached = (s, e) => { @@ -108,11 +114,15 @@ export const synthesizeTextToSpeech = async ( e.audioOffset / 10000 }ms, bookmark text: ${e.text}` ) - speechMarks.push({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - type: 'bookmark', - }) + speechMarksStream.write( + Buffer.from( + JSON.stringify({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + type: 'bookmark', + }) + ) + ) } const speakSsmlAsyncPromise = ( @@ -154,7 +164,8 @@ export const synthesizeTextToSpeech = async ( throw error } finally { console.debug('closing synthesizer') - writeStream.end() + audioStream.end() + speechMarksStream.end() synthesizer.close() console.debug('synthesizer closed') } From d00391b59349b3f45a8d78ca98956ee2de278d60 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 6 Sep 2022 10:03:36 +0800 Subject: [PATCH 13/15] Add wordCount to Page doc in elastic --- packages/api/src/elastic/types.ts | 1 + packages/db/elastic_migrations/index_settings.json | 3 +++ 2 files changed, 4 insertions(+) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index f25c33ae4..0f27998c6 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -216,6 +216,7 @@ export interface Page { language?: string readAt?: Date listenedAt?: Date + wordCount?: number } export interface SearchItem { diff --git a/packages/db/elastic_migrations/index_settings.json b/packages/db/elastic_migrations/index_settings.json index 71d4e64d1..3c30c357d 100644 --- a/packages/db/elastic_migrations/index_settings.json +++ b/packages/db/elastic_migrations/index_settings.json @@ -151,6 +151,9 @@ }, "listenedAt": { "type": "date" + }, + "wordCount": { + "type": "integer" } } } From 795ca47414b18acb8d1f125453fa171ea2c45606 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 6 Sep 2022 10:19:58 +0800 Subject: [PATCH 14/15] Save wordsCount when page is created --- packages/api/package.json | 3 +- packages/api/src/elastic/pages.ts | 2 + packages/api/src/elastic/types.ts | 3 +- .../db/elastic_migrations/index_settings.json | 2 +- yarn.lock | 81 +++++++++++++++++++ 5 files changed, 88 insertions(+), 3 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 7b275b385..5297027a2 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -86,7 +86,8 @@ "urlsafe-base64": "^1.0.0", "uuid": "^8.3.1", "voca": "^1.4.0", - "winston": "^3.3.3" + "winston": "^3.3.3", + "word-counting": "^1.1.4" }, "devDependencies": { "@babel/register": "^7.14.5", diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 97b00cdf0..a240a42e1 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -22,6 +22,7 @@ import { import { client, INDEX_ALIAS } from './index' import { EntityType } from '../datalayer/pubsub' import { ResponseError } from '@elastic/elasticsearch/lib/errors' +import wordsCounter from 'word-counting' const appendQuery = (body: SearchBody, query: string): void => { body.query.bool.should.push({ @@ -190,6 +191,7 @@ export const createPage = async ( ...page, updatedAt: new Date(), savedAt: new Date(), + wordsCount: wordsCounter(page.content, { isHtml: true }).wordsCount, }, refresh: ctx.refresh, }) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 0f27998c6..33d00b038 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -216,7 +216,7 @@ export interface Page { language?: string readAt?: Date listenedAt?: Date - wordCount?: number + wordsCount?: number } export interface SearchItem { @@ -246,6 +246,7 @@ export interface SearchItem { updatedAt?: Date labels?: Label[] highlights?: Highlight[] + wordsCount?: number } const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const diff --git a/packages/db/elastic_migrations/index_settings.json b/packages/db/elastic_migrations/index_settings.json index 3c30c357d..61ec732d4 100644 --- a/packages/db/elastic_migrations/index_settings.json +++ b/packages/db/elastic_migrations/index_settings.json @@ -152,7 +152,7 @@ "listenedAt": { "type": "date" }, - "wordCount": { + "wordsCount": { "type": "integer" } } diff --git a/yarn.lock b/yarn.lock index 09a7d55a3..c7a25aa2b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5097,6 +5097,14 @@ dset "^3.1.1" tiny-hashes "^1.0.1" +"@selderee/plugin-htmlparser2@^0.6.0": + version "0.6.0" + resolved "https://registry.yarnpkg.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.6.0.tgz#27e994afd1c2cb647ceb5406a185a5574188069d" + integrity sha512-J3jpy002TyBjd4N/p6s+s90eX42H2eRhK3SbsZuvTDv977/E8p2U3zikdiehyJja66do7FlxLomZLPlvl2/xaA== + dependencies: + domhandler "^4.2.0" + selderee "^0.6.0" + "@sendgrid/client@^7.6.0", "@sendgrid/client@^7.7.0": version "7.7.0" resolved "https://registry.yarnpkg.com/@sendgrid/client/-/client-7.7.0.tgz#f8f67abd604205a0d0b1af091b61517ef465fdbf" @@ -12274,6 +12282,11 @@ dir-glob@^3.0.1: dependencies: path-type "^4.0.0" +discontinuous-range@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/discontinuous-range/-/discontinuous-range-1.0.0.tgz#e38331f0844bba49b9a9cb71c771585aab1bc65a" + integrity sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ== + dlv@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/dlv/-/dlv-1.1.3.tgz#5c198a8a11453596e751494d49874bc7732f2e79" @@ -15211,6 +15224,18 @@ html-tags@^3.1.0: resolved "https://registry.yarnpkg.com/html-tags/-/html-tags-3.2.0.tgz#dbb3518d20b726524e4dd43de397eb0a95726961" integrity sha512-vy7ClnArOZwCnqZgvv+ddgHgJiAFXe3Ge9ML5/mBctVJoUoYPCdxVucOywjDARn6CVoh3dRSFdPHy2sX80L0Wg== +html-to-text@^8.1.0: + version "8.2.1" + resolved "https://registry.yarnpkg.com/html-to-text/-/html-to-text-8.2.1.tgz#4a75b8a1b646149bd71c50527adb568990bf459b" + integrity sha512-aN/3JvAk8qFsWVeE9InWAWueLXrbkoVZy0TkzaGhoRBC2gCFEeRLDDJN3/ijIGHohy6H+SZzUQWN/hcYtaPK8w== + dependencies: + "@selderee/plugin-htmlparser2" "^0.6.0" + deepmerge "^4.2.2" + he "^1.2.0" + htmlparser2 "^6.1.0" + minimist "^1.2.6" + selderee "^0.6.0" + html-void-elements@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/html-void-elements/-/html-void-elements-1.0.5.tgz#ce9159494e86d95e45795b166c2021c2cfca4483" @@ -18771,6 +18796,11 @@ module-details-from-path@^1.0.3: resolved "https://registry.yarnpkg.com/module-details-from-path/-/module-details-from-path-1.0.3.tgz#114c949673e2a8a35e9d35788527aa37b679da2b" integrity sha1-EUyUlnPiqKNenTV4hSeqN7Z52is= +moo@^0.5.0, moo@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/moo/-/moo-0.5.1.tgz#7aae7f384b9b09f620b6abf6f74ebbcd1b65dbc4" + integrity sha512-I1mnb5xn4fO80BH9BLcF0yLypy2UKl+Cb01Fu0hJRkJjlCRtxZMWkTdAtDd5ZqCOxtCkhmRwyI57vWT+1iZ67w== + move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92" @@ -18888,6 +18918,16 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= +nearley@^2.20.1: + version "2.20.1" + resolved "https://registry.yarnpkg.com/nearley/-/nearley-2.20.1.tgz#246cd33eff0d012faf197ff6774d7ac78acdd474" + integrity sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ== + dependencies: + commander "^2.19.0" + moo "^0.5.0" + railroad-diagrams "^1.0.0" + randexp "0.4.6" + negotiator@0.6.3: version "0.6.3" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" @@ -20139,6 +20179,14 @@ parse5@^5.1.1: resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178" integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug== +parseley@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/parseley/-/parseley-0.7.0.tgz#9949e3a0ed05c5072adb04f013c2810cf49171a8" + integrity sha512-xyOytsdDu077M3/46Am+2cGXEKM9U9QclBDv7fimY7e+BBlxh2JcBp2mgNsmkyA9uvgyTjVzDi7cP1v4hcFxbw== + dependencies: + moo "^0.5.1" + nearley "^2.20.1" + parseurl@^1.3.3, parseurl@~1.3.2, parseurl@~1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" @@ -21152,11 +21200,24 @@ quick-lru@^4.0.1: resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g== +railroad-diagrams@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/railroad-diagrams/-/railroad-diagrams-1.0.0.tgz#eb7e6267548ddedfb899c1b90e57374559cddb7e" + integrity sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A== + ramda@^0.21.0: version "0.21.0" resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.21.0.tgz#a001abedb3ff61077d4ff1d577d44de77e8d0a35" integrity sha1-oAGr7bP/YQd9T/HVd9RN536NCjU= +randexp@0.4.6: + version "0.4.6" + resolved "https://registry.yarnpkg.com/randexp/-/randexp-0.4.6.tgz#e986ad5e5e31dae13ddd6f7b3019aa7c87f60ca3" + integrity sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ== + dependencies: + discontinuous-range "1.0.0" + ret "~0.1.10" + randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5, randombytes@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" @@ -22329,6 +22390,13 @@ secure-json-parse@^2.3.1: resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.4.0.tgz#5aaeaaef85c7a417f76271a4f5b0cc3315ddca85" integrity sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg== +selderee@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.6.0.tgz#f3bee66cfebcb6f33df98e4a1df77388b42a96f7" + integrity sha512-ibqWGV5aChDvfVdqNYuaJP/HnVBhlRGSRrlbttmlMpHcLuTqqbMH36QkSs9GEgj5M88JDYLI8eyP94JaQ8xRlg== + dependencies: + parseley "^0.7.0" + select-hose@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" @@ -25360,6 +25428,19 @@ winston@^3.3.3: triple-beam "^1.3.0" winston-transport "^4.5.0" +word-counting@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/word-counting/-/word-counting-1.1.4.tgz#4d772df20bd86e2e8b00596c8e1ab2f355578ddd" + integrity sha512-SsAKEoa6FzQTV7fR27vDOHO9m2f7cnGhppP+e0c6JCn+pDg88kCMVfrt0Qr8XcbbW2o6HIum4yFX8WEnxZ5xLA== + dependencies: + html-to-text "^8.1.0" + word-regex "^0.1.2" + +word-regex@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/word-regex/-/word-regex-0.1.2.tgz#a3bc7f2d222ce4a93c246c3ef69458f61f511639" + integrity sha512-4jK/OibPeindR9o/sryObhVWNgD2LJCMJFWEME69p48sEYpE9axfyjHK+RqYcOeoEoqcqJEPE9iMdiiFpXHo0Q== + word-wrap@^1.2.3, word-wrap@~1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" From 3bcd77d87a75dc04f6a2c63bdcc939c0aaf2289d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 6 Sep 2022 10:25:40 +0800 Subject: [PATCH 15/15] Hardcode SSML --- packages/text-to-speech/src/index.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 306666e9c..ce2acb4b2 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -11,6 +11,7 @@ import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' import { File, Storage } from '@google-cloud/storage' import { PassThrough } from 'stream' import { htmlToSsml } from './htmlToSsml' +import * as fs from 'fs' dotenv.config() Sentry.GCPFunction.init({ @@ -140,17 +141,19 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( try { // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - const ssmlItems = req.body.ssmlItems as string[] - if (!ssmlItems || ssmlItems.length === 0) { - return res.status(200).send({ errorCode: 'INVALID_DATA' }) - } + // const ssmlItems = req.body.ssmlItems as string[] + // if (!ssmlItems || ssmlItems.length === 0) { + // return res.status(200).send({ errorCode: 'INVALID_DATA' }) + // } + // hardcoded for now + const ssml = fs.readFileSync('./data/ssml.xml', 'utf8') const audioStream = new PassThrough() const speechMarksStream = new PassThrough() const input: TextToSpeechInput = { text: '', textType: 'ssml', audioStream, - ssmlItems, + ssmlItems: [ssml], speechMarksStream, } res.set({