diff --git a/packages/api/Dockerfile b/packages/api/Dockerfile index ff72db699..c33bac1a4 100644 --- a/packages/api/Dockerfile +++ b/packages/api/Dockerfile @@ -13,11 +13,13 @@ COPY .eslintrc . COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json COPY /packages/api/package.json ./packages/api/package.json +COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json RUN yarn install --pure-lockfile ADD /packages/readabilityjs ./packages/readabilityjs ADD /packages/api ./packages/api +ADD /packages/text-to-speech ./packages/text-to-speech RUN yarn RUN yarn workspace @omnivore/api build @@ -41,6 +43,7 @@ COPY --from=builder /app/packages/api/package.json /app/packages/api/package.jso COPY --from=builder /app/packages/api/node_modules /app/packages/api/node_modules COPY --from=builder /app/node_modules /app/node_modules COPY --from=builder /app/package.json /app/package.json +COPY --from=builder /app/packages/text-to-speech/ /app/packages/text-to-speech/ EXPOSE 8080 CMD ["yarn", "workspace", "@omnivore/api", "start"] diff --git a/packages/api/package.json b/packages/api/package.json index 28ba0df6c..5297027a2 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -19,6 +19,7 @@ "@google-cloud/storage": "^5.18.1", "@google-cloud/tasks": "^2.3.0", "@omnivore/readability": "1.0.0", + "@omnivore/text-to-speech-handler": "1.0.0", "@opentelemetry/api": "^1.0.1", "@opentelemetry/core": "^1.3.1", "@opentelemetry/exporter-jaeger": "^1.0.1", @@ -85,7 +86,8 @@ "urlsafe-base64": "^1.0.0", "uuid": "^8.3.1", "voca": "^1.4.0", - "winston": "^3.3.3" + "winston": "^3.3.3", + "word-counting": "^1.1.4" }, "devDependencies": { "@babel/register": "^7.14.5", diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 97b00cdf0..a240a42e1 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -22,6 +22,7 @@ import { import { client, INDEX_ALIAS } from './index' import { EntityType } from '../datalayer/pubsub' import { ResponseError } from '@elastic/elasticsearch/lib/errors' +import wordsCounter from 'word-counting' const appendQuery = (body: SearchBody, query: string): void => { body.query.bool.should.push({ @@ -190,6 +191,7 @@ export const createPage = async ( ...page, updatedAt: new Date(), savedAt: new Date(), + wordsCount: wordsCounter(page.content, { isHtml: true }).wordsCount, }, refresh: ctx.refresh, }) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index f25c33ae4..33d00b038 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -216,6 +216,7 @@ export interface Page { language?: string readAt?: Date listenedAt?: Date + wordsCount?: number } export interface SearchItem { @@ -245,6 +246,7 @@ export interface SearchItem { updatedAt?: Date labels?: Label[] highlights?: Highlight[] + wordsCount?: number } const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index e0f625aad..3302cdf11 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -21,6 +21,7 @@ import { getPageById, updatePage } from '../elastic/pages' import { generateDownloadSignedUrl } from '../utils/uploads' import { enqueueTextToSpeech } from '../utils/createTask' import { createPubSubClient } from '../datalayer/pubsub' +import { htmlToSsml } from '@omnivore/text-to-speech-handler' const logger = buildLogger('app.dispatch') @@ -81,7 +82,7 @@ export function articleRouter() { const priority = req.params.priority if ( !articleId || - !['mp3', 'speech-marks'].includes(outputFormat) || + !['mp3', 'speech-marks', 'ssml'].includes(outputFormat) || !['low', 'high'].includes(priority) ) { return res.status(400).send('Invalid data') @@ -91,7 +92,6 @@ export function articleRouter() { return res.status(401).send({ errorCode: 'UNAUTHORIZED' }) } const { uid } = jwt.decode(token) as Claims - logger.info(`Get article speech in ${outputFormat} format`, { params: req.params, labels: { @@ -100,6 +100,21 @@ export function articleRouter() { }, }) + if (outputFormat === 'ssml') { + const page = await getPageById(articleId) + if (!page) { + return res.status(404).send('Page not found') + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + const ssmlItems = htmlToSsml(page.content, { + primaryVoice: voice, + secondaryVoice: 'en-US-GuyNeural', + rate: '1', + language: page.language || 'en-US', + }) + return res.send({ ssmlItems }) + } + const existingSpeech = await getRepository(Speech).findOne({ where: { elasticPageId: articleId, diff --git a/packages/api/src/textToSpeech.d.ts b/packages/api/src/textToSpeech.d.ts new file mode 100644 index 000000000..5e438e05e --- /dev/null +++ b/packages/api/src/textToSpeech.d.ts @@ -0,0 +1,12 @@ +declare module '@omnivore/text-to-speech-handler' { + function htmlToSsml(html: string, options: SSMLOptions): string[] + + interface SSMLOptions { + primaryVoice: string + secondaryVoice: string + rate: string + language: string + } + + export { htmlToSsml } +} diff --git a/packages/db/elastic_migrations/index_settings.json b/packages/db/elastic_migrations/index_settings.json index 71d4e64d1..61ec732d4 100644 --- a/packages/db/elastic_migrations/index_settings.json +++ b/packages/db/elastic_migrations/index_settings.json @@ -151,6 +151,9 @@ }, "listenedAt": { "type": "date" + }, + "wordsCount": { + "type": "integer" } } } diff --git a/packages/text-to-speech/.npmignore b/packages/text-to-speech/.npmignore new file mode 100644 index 000000000..193378602 --- /dev/null +++ b/packages/text-to-speech/.npmignore @@ -0,0 +1 @@ +/test/ diff --git a/packages/text-to-speech/data/ssml.xml b/packages/text-to-speech/data/ssml.xml new file mode 100644 index 000000000..5492c0d5f --- /dev/null +++ b/packages/text-to-speech/data/ssml.xml @@ -0,0 +1,36 @@ + +

Back in 2010, during my first year of Business School, I helped give a presentation entitled “Twitter 101”:

+

My section was “The Twitter Value Proposition”, and after admitting that yes, you can find out what people are eating for lunch on Twitter, I stated “The truth is you can find anything you want on Twitter, and that’s a good thing.” The Twitter value proposition was that you could “See exactly what you need to see, in real-time, in one place, and nothing more”; I illustrated this by showing people how they could unfollow me:

+

The point was that Twitter required active management of your feed, but if you put in the effort, you could get something uniquely interesting to you that was incredibly valuable.

+

Most of the audience didn’t take me up on it.

+

Facebook Versus Instagram

+

If there is one axiom that governs the consumer Internet — consumer anything, really — it is that convenience matters more than anything. That was the problem with Twitter: it just wasn’t convenient for nearly enough people to figure out how to follow the right people. It was Facebook, which digitized offline relationships, that dominated the social media space.

+

Facebook’s social graph was the ultimate growth hack: from the moment you created an account Facebook worked assiduously to connect you with everyone you knew or wish you knew from high school, college, your hometown, workplace, you name an offline network and Facebook digitized it. Of course this meant that there were far too many updates and photos to keep track of, so Facebook ranked them, and presented them in a feed that you could scroll endlessly.

+

Users, famously, hated the News Feed when it was first launched: Facebook had protesters outside their doors in Palo Alto when it was introduced, and far more online; most were, ironically enough, organized on Facebook. CEO Mark Zuckerberg penned an apology:

+

+ We really messed this one up. When we launched News Feed and Mini-Feed we were trying to provide you with a stream of information about your social world. Instead, we did a bad job of explaining what the new features were and an even worse job of giving you control of them. I’d like to try to correct those errors now… +

+

The errors to be corrected were better controls over what might be shared; Facebook did not give the users what they claimed to want, which was abolishing the News Feed completely. That’s because the company correctly intuited a significant gap between its users stated preference — no News Feed — and their revealed preference, which was that they liked News Feed quite a bit. The next fifteen years would prove the company right.

+

It was hard to not think of that non-apology apology while watching Adam Mosseri’s Instagram update three weeks ago; Mosseri was clear that videos were going to be an ever great part of the Instagram experience, along with recommended posts. Zuckerberg reiterated the point on Facebook’s earnings call, noting that recommended posts in both Facebook and Instagram would continue to increase. A day later Mosseri told Casey Newton on Platformer that Instagram would scale back recommended posts, but was clear that the pullback was temporary:

+

+ “When you discover something in your feed that you didn’t follow before, there should be a high bar — it should just be great,” Mosseri said. “You should be delighted to see it. And I don’t think that’s happening enough right now. So I think we need to take a step back, in terms of the percentage of feed that are recommendations, get better at ranking and recommendations, and then — if and when we do — we can start to grow again.” (“I’m confident we will,” he added.) +

+

Michael Mignano calls this recommendation media in an article entitled The End of Social Media:

+

+ In recommendation media, content is not distributed to networks of connected people as the primary means of distribution. Instead, the main mechanism for the distribution of content is through opaque, platform-defined algorithms that favor maximum attention and engagement from consumers. The exact type of attention these recommendations seek is always defined by the platform and often tailored specifically to the user who is consuming content. For example, if the platform determines that someone loves movies, that person will likely see a lot of movie related content because that’s what captures that person’s attention best. This means platforms can also decide what consumers won’t see, such as problematic or polarizing content. + It’s ultimately up to the platform to decide what type of content gets recommended, not the social graph of the person producing the content. In contrast to social media, recommendation media is not a competition based on popularity; instead, it is a competition based on the absolute best content. Through this lens, it’s no wonder why Kylie Jenner opposes this change; her more than 360 million followers are simply worth less in a version of media dominated by algorithms and not followers. +

+

Sam Lessin, a former Facebook executive, traced this evolution from the analog days to what is next in a Twitter screenshot entitled “The Digital Media ‘Attention’ Food Chain in Progress”:

+

Lessin’s five steps:

+

+ The Pre-Internet ‘People Magazine’ Era + Content from ‘your friends’ kills People Magazine + Kardashians/Professional ‘friends’ kill real friends + Algorithmic everyone kills Kardashians + Next is pure-AI content which beats ‘algorithmic everyone’ +

+

This is a meta observation and, to make a cheap play on words, the first reason why it made sense for Facebook to change its name: Facebook the app is eternally stuck on Step 2 in terms of entertainment (the app has evolved to become much more of a utility, with a focus on groups, marketplace, etc.). It’s Instagram that is barreling forward. I wrote last summer about Instagram’s Evolution:

+

+ The reality, though, is that this is what Instagram is best at. When Mosseri said that Instagram was no longer a photo-sharing app — particularly a “square photo-sharing app” — he was not making a forward-looking pronouncement, but simply stating what has been true for many years now. More broadly, Instagram from the very beginning — including under former CEO Kevin Systrom — has been marked first and foremost by evolution. +

+
diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json index 6b21521bb..5416fd83e 100644 --- a/packages/text-to-speech/package.json +++ b/packages/text-to-speech/package.json @@ -14,7 +14,8 @@ "lint": "eslint src --ext ts,js,tsx,jsx", "compile": "tsc", "build": "tsc", - "start": "functions-framework --source=build/src/ --target=textToSpeechHandler", + "start": "functions-framework --target=textToSpeechHandler", + "start_streaming": "functions-framework --target=textToSpeechStreamingHandler", "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"", "gcloud-deploy": "gcloud functions deploy text-to-speech --gen2 --entry-point=textToSpeechHandler --trigger-http --allow-unauthenticated --region=us-west2 --runtime nodejs14", "deploy": "yarn build && yarn gcloud-deploy" diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index b90142f63..e65755cfc 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -91,7 +91,7 @@ function emitTextNode( if (ssmlElement) { emit(textItems, `<${ssmlElement}>`) } - emit(textItems, `${cleanedText}`) + emit(textItems, `${cleanedText.replace(/\s+/g, ' ')}`) if (ssmlElement) { emit(textItems, ``) } @@ -123,7 +123,7 @@ function emitElement( ) { const cleanedText = cleanTextNode(child) if (idx && cleanedText.length > 1) { - // Make sure its more than just a space + // Make sure it's more than just a space emit(textItems, ``) } emitTextNode(textItems, cleanedText, child) @@ -158,9 +158,7 @@ const startSsml = (element: Element, options: SSMLOptions): string => { element.nodeName === 'BLOCKQUOTE' ? options.secondaryVoice : options.primaryVoice - return ` - - ` + return `` } const endSsml = (): string => { @@ -181,7 +179,10 @@ export const ssmlItemText = (item: SSMLItem): string => { return [item.open, ...item.textItems, item.close].join('') } -export const htmlToSsml = (html: string, options: SSMLOptions): SSMLItem[] => { +export const htmlToSsmlItems = ( + html: string, + options: SSMLOptions +): SSMLItem[] => { console.log('creating ssml with options', options) const dom = parseHTML(html) @@ -220,3 +221,7 @@ export const htmlToSsml = (html: string, options: SSMLOptions): SSMLItem[] => { return items } + +export const htmlToSsml = (html: string, options: SSMLOptions): string[] => { + return htmlToSsmlItems(html, options).map(ssmlItemText) +} diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index a213d8b42..ce2acb4b2 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -4,48 +4,20 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import * as Sentry from '@sentry/serverless' -import { parseHTML } from 'linkedom' -import { File, Storage } from '@google-cloud/storage' -import { - CancellationDetails, - CancellationReason, - ResultReason, - SpeechConfig, - SpeechSynthesisOutputFormat, - SpeechSynthesisResult, - SpeechSynthesizer, -} from 'microsoft-cognitiveservices-speech-sdk' import axios from 'axios' import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import -import { htmlToSsml, ssmlItemText } from './htmlToSsml' +import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' +import { File, Storage } from '@google-cloud/storage' +import { PassThrough } from 'stream' +import { htmlToSsml } from './htmlToSsml' +import * as fs from 'fs' dotenv.config() - -interface TextToSpeechInput { - id: string - text: string - voice?: string - languageCode?: string - textType?: 'text' | 'ssml' - rate?: number - volume?: number - complimentaryVoice?: string - bucket: string -} - -interface TextToSpeechOutput { - audioFileName: string - speechMarksFileName: string -} - -interface SpeechMark { - time: number - start?: number - length?: number - word: string - type: 'word' | 'bookmark' -} +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}) const storage = new Storage() @@ -85,301 +57,13 @@ const updateSpeech = async ( return response.status === 200 } -const synthesizeTextToSpeech = async ( - input: TextToSpeechInput -): Promise => { - if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) { - throw new Error('Azure Speech Key or Region not set') - } - const audioFileName = `speech/${input.id}.mp3` - const audioFile = createGCSFile(input.bucket, audioFileName) - const writeStream = audioFile.createWriteStream({ - resumable: true, - }) - const speechConfig = SpeechConfig.fromSubscription( - process.env.AZURE_SPEECH_KEY, - process.env.AZURE_SPEECH_REGION - ) - const textType = input.textType || 'text' - if (textType === 'text') { - speechConfig.speechSynthesisLanguage = input.languageCode || 'en-US' - speechConfig.speechSynthesisVoiceName = input.voice || 'en-US-JennyNeural' - } - speechConfig.speechSynthesisOutputFormat = - SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 - - // Create the speech synthesizer. - const synthesizer = new SpeechSynthesizer(speechConfig) - const speechMarks: SpeechMark[] = [] - let timeOffset = 0 - let characterOffset = 0 - - synthesizer.synthesizing = function (s, e) { - // convert arrayBuffer to stream and write to gcs file - writeStream.write(Buffer.from(e.result.audioData)) - } - - // The event synthesis completed signals that the synthesis is completed. - synthesizer.synthesisCompleted = (s, e) => { - console.info( - `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${ - e.result.audioData.byteLength - }` - ) - } - - // The synthesis started event signals that the synthesis is started. - synthesizer.synthesisStarted = (s, e) => { - console.info('(synthesis started)') - } - - // The event signals that the service has stopped processing speech. - // This can happen when an error is encountered. - synthesizer.SynthesisCanceled = (s, e) => { - const cancellationDetails = CancellationDetails.fromResult(e.result) - let str = - '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason] - if (cancellationDetails.reason === CancellationReason.Error) { - str += ': ' + e.result.errorDetails - } - console.info(str) - } - - // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. - synthesizer.wordBoundary = (s, e) => { - speechMarks.push({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - start: characterOffset + e.textOffset, - length: e.wordLength, - type: 'word', - }) - } - - synthesizer.bookmarkReached = (s, e) => { - console.debug( - `(Bookmark reached), Audio offset: ${ - e.audioOffset / 10000 - }ms, bookmark text: ${e.text}` - ) - speechMarks.push({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - type: 'bookmark', - }) - } - - const speakTextAsyncPromise = ( - text: string - ): Promise => { - return new Promise((resolve, reject) => { - synthesizer.speakTextAsync( - text, - (result) => { - resolve(result) - }, - (error) => { - reject(error) - } - ) - }) - } - - const speakSsmlAsyncPromise = ( - text: string - ): Promise => { - return new Promise((resolve, reject) => { - synthesizer.speakSsmlAsync( - text, - (result) => { - resolve(result) - }, - (error) => { - reject(error) - } - ) - }) - } - - if (textType === 'text') { - // slice the text into chunks of 5,000 characters - let currentTextChunk = '' - const textChunks = input.text.split('\n') - for (let i = 0; i < textChunks.length; i++) { - currentTextChunk += textChunks[i] + '\n' - if (currentTextChunk.length < 5000 && i < textChunks.length - 1) { - continue - } - console.debug(`synthesizing ${currentTextChunk}`) - const result = await speakTextAsyncPromise(currentTextChunk) - timeOffset = timeOffset + result.audioDuration - characterOffset = characterOffset + currentTextChunk.length - currentTextChunk = '' - } - } else { - const ssmlItems = htmlToSsml(input.text, { - primaryVoice: input.voice || 'en-US-JennyNeural', - secondaryVoice: 'en-US-GuyNeural', - language: input.languageCode || 'en-US', - rate: '1', - }) - - for (const ssmlItem of Array.from(ssmlItems)) { - const ssml = ssmlItemText(ssmlItem) - console.debug(`synthesizing ${ssml}`) - const result = await speakSsmlAsyncPromise(ssml) - if (result.reason === ResultReason.Canceled) { - writeStream.end() - synthesizer.close() - throw new Error(result.errorDetails) - } - timeOffset = timeOffset + result.audioDuration - // characterOffset = characterOffset + htmlElement.innerText.length - } - } - writeStream.end() - synthesizer.close() - - console.debug(`audio file: ${audioFileName}`) - - // upload Speech Marks file to GCS - const speechMarksFileName = `speech/${input.id}.json` - await uploadToBucket( - speechMarksFileName, - Buffer.from(JSON.stringify(speechMarks)), - input.bucket - ) - - return { - audioFileName, - speechMarksFileName, - } -} - -const htmlElementToSsml = ({ - htmlElement, - language = 'en-US', - voice = 'en-US-JennyNeural', - rate = 1, - volume = 100, -}: { - htmlElement: Element - language?: string - voice?: string - rate?: number - volume?: number -}): string => { - const replaceElement = (newElement: Element, oldElement: Element) => { - const id = oldElement.getAttribute('data-omnivore-anchor-idx') - if (id) { - const e = htmlElement.querySelector(`[data-omnivore-anchor-idx="${id}"]`) - e?.parentNode?.replaceChild(newElement, e) - } - } - - const appendBookmarkElement = (parent: Element, element: Element) => { - const id = element.getAttribute('data-omnivore-anchor-idx') - if (id) { - const bookMark = ssml.createElement('bookmark') - bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`) - parent.appendChild(bookMark) - } - } - - const replaceWithEmphasis = (element: Element, level: string) => { - const parent = ssml.createDocumentFragment() as unknown as Element - appendBookmarkElement(parent, element) - const emphasisElement = ssml.createElement('emphasis') - emphasisElement.setAttribute('level', level) - emphasisElement.innerHTML = element.innerHTML.trim() - parent.appendChild(emphasisElement) - replaceElement(parent, element) - } - - const replaceWithSentence = (element: Element) => { - const parent = ssml.createDocumentFragment() as unknown as Element - appendBookmarkElement(parent, element) - const sentenceElement = ssml.createElement('s') - sentenceElement.innerHTML = element.innerHTML.trim() - parent.appendChild(sentenceElement) - replaceElement(parent, element) - } - - // create new ssml document - const ssml = parseHTML('').document - const speakElement = ssml.createElement('speak') - speakElement.setAttribute('version', '1.0') - speakElement.setAttribute('xmlns', 'http://www.w3.org/2001/10/synthesis') - speakElement.setAttribute('xml:lang', language) - const voiceElement = ssml.createElement('voice') - voiceElement.setAttribute('name', voice) - speakElement.appendChild(voiceElement) - const prosodyElement = ssml.createElement('prosody') - prosodyElement.setAttribute('rate', `${rate}`) - prosodyElement.setAttribute('volume', volume.toString()) - voiceElement.appendChild(prosodyElement) - // add each paragraph to the ssml document - appendBookmarkElement(prosodyElement, htmlElement) - // replace emphasis elements with ssml - htmlElement.querySelectorAll('*').forEach((e) => { - switch (e.tagName.toLowerCase()) { - case 's': - replaceWithEmphasis(e, 'moderate') - break - case 'sub': - if (e.getAttribute('alias') === null) { - replaceWithEmphasis(e, 'moderate') - } - break - case 'i': - case 'em': - case 'q': - case 'blockquote': - case 'cite': - case 'del': - case 'strike': - case 'sup': - case 'summary': - case 'caption': - case 'figcaption': - replaceWithEmphasis(e, 'moderate') - break - case 'b': - case 'strong': - case 'dt': - case 'dfn': - case 'u': - case 'mark': - case 'th': - case 'title': - case 'var': - replaceWithEmphasis(e, 'moderate') - break - case 'li': - replaceWithSentence(e) - break - default: { - const parent = ssml.createDocumentFragment() as unknown as Element - appendBookmarkElement(parent, e) - const text = (e as HTMLElement).innerText.trim() - const textElement = ssml.createTextNode(text) - parent.appendChild(textElement) - replaceElement(parent, e) - } - } - }) - prosodyElement.appendChild(htmlElement) - - return speakElement.outerHTML.replace(/ |\n/g, '') -} - export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { - console.debug('New text to speech request', req) + console.info('Text to speech request received') const token = req.query.token as string if (!process.env.JWT_SECRET) { console.error('JWT_SECRET not exists') - return res.status(500).send('JWT_SECRET not exists') + return res.status(500).send({ errorCodes: 'JWT_SECRET_NOT_EXISTS' }) } try { jwt.verify(token, process.env.JWT_SECRET) @@ -388,11 +72,34 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( return res.status(200).send('UNAUTHENTICATED') } const input = req.body as TextToSpeechInput + const id = input.id + const bucket = input.bucket + if (!id || !bucket) { + return res.status(200).send('Invalid data') + } try { - const { audioFileName, speechMarksFileName } = - await synthesizeTextToSpeech(input) + const audioFileName = `speech/${id}.mp3` + const audioFile = createGCSFile(bucket, audioFileName) + const audioStream = audioFile.createWriteStream({ + resumable: true, + }) as NodeJS.WriteStream + const speechMarksFileName = `speech/${id}.json` + const speechMarksFile = createGCSFile(bucket, speechMarksFileName) + const speechMarksStream = speechMarksFile.createWriteStream({ + resumable: true, + }) as NodeJS.WriteStream + const startTime = Date.now() + await synthesizeTextToSpeech({ + ...input, + textType: 'html', + audioStream, + speechMarksStream, + }) + console.info( + `Synthesize text to speech completed in ${Date.now() - startTime} ms` + ) const updated = await updateSpeech( - input.id, + id, token, 'COMPLETED', audioFileName, @@ -400,14 +107,73 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( ) if (!updated) { - return res.status(500).send('Failed to update speech') + console.error('Failed to update speech') + return res.status(500).send({ errorCodes: 'DB_ERROR' }) } - } catch (e) { - console.error(e) - await updateSpeech(input.id, token, 'FAILED') - return res.status(500).send('Failed to synthesize') - } - res.send('OK') + console.info('Text to speech cloud function completed') + res.send('OK') + } catch (e) { + console.error('Text to speech cloud function error', e) + await updateSpeech(id, token, 'FAILED') + return res.status(500).send({ errorCodes: 'SYNTHESIZER_ERROR' }) + } } ) + +export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + console.debug('Text to speech steaming request', req) + if (!process.env.JWT_SECRET) { + console.error('JWT_SECRET not exists') + return res.status(500).send({ errorCodes: 'JWT_SECRET_NOT_EXISTS' }) + } + const token = (req.query.token || req.headers.authorization) as string + if (!token) { + return res.status(200).send({ errorCode: 'UNAUTHORIZED' }) + } + try { + jwt.verify(token, process.env.JWT_SECRET) + } catch (e) { + console.error(e) + return res.status(200).send({ errorCode: 'UNAUTHORIZED' }) + } + + try { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + // const ssmlItems = req.body.ssmlItems as string[] + // if (!ssmlItems || ssmlItems.length === 0) { + // return res.status(200).send({ errorCode: 'INVALID_DATA' }) + // } + // hardcoded for now + const ssml = fs.readFileSync('./data/ssml.xml', 'utf8') + const audioStream = new PassThrough() + const speechMarksStream = new PassThrough() + const input: TextToSpeechInput = { + text: '', + textType: 'ssml', + audioStream, + ssmlItems: [ssml], + speechMarksStream, + } + res.set({ + 'Content-Type': 'audio/mpeg', + 'Transfer-Encoding': 'chunked', + }) + + console.info('Text to speech starts streaming') + audioStream.pipe(res) + + await synthesizeTextToSpeech(input) + } catch (e) { + console.error('Text to speech streaming error', e) + return res.status(500).send({ errorCodes: 'SYNTHESIZER_ERROR' }) + } + } +) + +module.exports = { + htmlToSsml, + textToSpeechStreamingHandler, + textToSpeechHandler, +} diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts new file mode 100644 index 000000000..18c7480b3 --- /dev/null +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -0,0 +1,176 @@ +import { + CancellationDetails, + CancellationReason, + ResultReason, + SpeechConfig, + SpeechSynthesisOutputFormat, + SpeechSynthesisResult, + SpeechSynthesizer, +} from 'microsoft-cognitiveservices-speech-sdk' +import { htmlToSsmlItems, ssmlItemText } from './htmlToSsml' + +export interface TextToSpeechInput { + id?: string + text: string + voice?: string + languageCode?: string + textType?: 'html' | 'ssml' + rate?: number + volume?: number + complimentaryVoice?: string + bucket?: string + audioStream: NodeJS.ReadWriteStream + ssmlItems?: string[] + speechMarksStream: NodeJS.ReadWriteStream +} + +export interface TextToSpeechOutput { + speechMarks: SpeechMark[] +} + +export interface SpeechMark { + time: number + start?: number + length?: number + word: string + type: 'word' | 'bookmark' +} + +export const synthesizeTextToSpeech = async ( + input: TextToSpeechInput +): Promise => { + if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) { + throw new Error('Azure Speech Key or Region not set') + } + const textType = input.textType || 'html' + const audioStream = input.audioStream + const speechMarksStream = input.speechMarksStream + const speechConfig = SpeechConfig.fromSubscription( + process.env.AZURE_SPEECH_KEY, + process.env.AZURE_SPEECH_REGION + ) + speechConfig.speechSynthesisOutputFormat = + SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 + + // Create the speech synthesizer. + const synthesizer = new SpeechSynthesizer(speechConfig) + const speechMarks: SpeechMark[] = [] + let timeOffset = 0 + + synthesizer.synthesizing = function (s, e) { + // convert arrayBuffer to stream and write to stream + audioStream.write(Buffer.from(e.result.audioData)) + } + + // The event synthesis completed signals that the synthesis is completed. + synthesizer.synthesisCompleted = (s, e) => { + console.info( + `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${ + e.result.audioData.byteLength + }` + ) + } + + // The synthesis started event signals that the synthesis is started. + synthesizer.synthesisStarted = (s, e) => { + console.info('(synthesis started)') + } + + // The event signals that the service has stopped processing speech. + // This can happen when an error is encountered. + synthesizer.SynthesisCanceled = (s, e) => { + const cancellationDetails = CancellationDetails.fromResult(e.result) + let str = + '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason] + if (cancellationDetails.reason === CancellationReason.Error) { + str += ': ' + e.result.errorDetails + } + console.error(str) + } + + // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. + synthesizer.wordBoundary = (s, e) => { + console.debug( + `(word boundary) Audio offset: ${e.audioOffset / 10000}ms, text: ${ + e.text + }` + ) + speechMarksStream.write( + Buffer.from( + JSON.stringify({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + start: e.textOffset, + length: e.wordLength, + type: 'word', + }) + ) + ) + } + + synthesizer.bookmarkReached = (s, e) => { + console.debug( + `(bookmark reached) Audio offset: ${ + e.audioOffset / 10000 + }ms, bookmark text: ${e.text}` + ) + speechMarksStream.write( + Buffer.from( + JSON.stringify({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + type: 'bookmark', + }) + ) + ) + } + + const speakSsmlAsyncPromise = ( + ssml: string + ): Promise => { + return new Promise((resolve, reject) => { + synthesizer.speakSsmlAsync( + ssml, + (result) => { + resolve(result) + }, + (error) => { + reject(error) + } + ) + }) + } + + try { + if (textType === 'html') { + const ssmlItems = htmlToSsmlItems(input.text, { + primaryVoice: input.voice || 'en-US-JennyNeural', + secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural', + language: input.languageCode || 'en-US', + rate: '1', + }) + for (const ssmlItem of ssmlItems) { + const ssml = ssmlItemText(ssmlItem) + const result = await speakSsmlAsyncPromise(ssml) + timeOffset = timeOffset + result.audioDuration + } + } else { + for (const ssmlItem of input.ssmlItems || []) { + await speakSsmlAsyncPromise(ssmlItem) + } + } + } catch (error) { + console.error('synthesis error', error) + throw error + } finally { + console.debug('closing synthesizer') + audioStream.end() + speechMarksStream.end() + synthesizer.close() + console.debug('synthesizer closed') + } + + return { + speechMarks, + } +} diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index 46bd0243c..80fd3fbd3 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -1,8 +1,8 @@ import 'mocha' import { expect } from 'chai' -import { htmlToSsml } from '../src/htmlToSsml' +import { htmlToSsmlItems } from '../src/htmlToSsml' -describe('htmlToSsml', () => { +describe('htmlToSsmlItems', () => { const TEST_OPTIONS = { primaryVoice: 'test-primary', secondaryVoice: 'test-secondary', @@ -12,7 +12,7 @@ describe('htmlToSsml', () => { describe('a simple html file', () => { xit('should convert Html to SSML', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -28,7 +28,7 @@ describe('htmlToSsml', () => { }) describe('a file with nested elements', () => { xit('should collapse spans into the parent paragraph', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -57,7 +57,7 @@ describe('htmlToSsml', () => { ) }) xit('should extract child paragraphs to the top level', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -77,7 +77,7 @@ describe('htmlToSsml', () => { ) }) xit('should hoist paragraphs in spans to the top level', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -95,7 +95,7 @@ describe('htmlToSsml', () => { expect(text).to.equal(`TBD`.trim()) }) xit('should hoist lists to the top level', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -113,7 +113,7 @@ describe('htmlToSsml', () => { expect(text).to.equal(`TBD`.trim()) }) xit('should hoist headers to the top level', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -131,7 +131,7 @@ describe('htmlToSsml', () => { expect(text).to.equal(`TBD`.trim()) }) xit('should hoist blockquotes to the top level', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -151,7 +151,7 @@ describe('htmlToSsml', () => { }) describe('a file with blockquotes', () => { xit('should convert Html to SSML with complimentary voices', () => { - const ssml = htmlToSsml( + const ssml = htmlToSsmlItems( `
@@ -193,7 +193,7 @@ describe('htmlToSsml', () => { // continue // } // const html = fs.readFileSync(readablePath, { encoding: 'utf-8' }) - // const ssmlItems = htmlToSsml(html, TEST_OPTIONS) + // const ssmlItems = htmlToSsmlItems(html, TEST_OPTIONS) // console.log('SSML ITEMS', ssmlItems) // } // }) diff --git a/yarn.lock b/yarn.lock index 09a7d55a3..c7a25aa2b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5097,6 +5097,14 @@ dset "^3.1.1" tiny-hashes "^1.0.1" +"@selderee/plugin-htmlparser2@^0.6.0": + version "0.6.0" + resolved "https://registry.yarnpkg.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.6.0.tgz#27e994afd1c2cb647ceb5406a185a5574188069d" + integrity sha512-J3jpy002TyBjd4N/p6s+s90eX42H2eRhK3SbsZuvTDv977/E8p2U3zikdiehyJja66do7FlxLomZLPlvl2/xaA== + dependencies: + domhandler "^4.2.0" + selderee "^0.6.0" + "@sendgrid/client@^7.6.0", "@sendgrid/client@^7.7.0": version "7.7.0" resolved "https://registry.yarnpkg.com/@sendgrid/client/-/client-7.7.0.tgz#f8f67abd604205a0d0b1af091b61517ef465fdbf" @@ -12274,6 +12282,11 @@ dir-glob@^3.0.1: dependencies: path-type "^4.0.0" +discontinuous-range@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/discontinuous-range/-/discontinuous-range-1.0.0.tgz#e38331f0844bba49b9a9cb71c771585aab1bc65a" + integrity sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ== + dlv@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/dlv/-/dlv-1.1.3.tgz#5c198a8a11453596e751494d49874bc7732f2e79" @@ -15211,6 +15224,18 @@ html-tags@^3.1.0: resolved "https://registry.yarnpkg.com/html-tags/-/html-tags-3.2.0.tgz#dbb3518d20b726524e4dd43de397eb0a95726961" integrity sha512-vy7ClnArOZwCnqZgvv+ddgHgJiAFXe3Ge9ML5/mBctVJoUoYPCdxVucOywjDARn6CVoh3dRSFdPHy2sX80L0Wg== +html-to-text@^8.1.0: + version "8.2.1" + resolved "https://registry.yarnpkg.com/html-to-text/-/html-to-text-8.2.1.tgz#4a75b8a1b646149bd71c50527adb568990bf459b" + integrity sha512-aN/3JvAk8qFsWVeE9InWAWueLXrbkoVZy0TkzaGhoRBC2gCFEeRLDDJN3/ijIGHohy6H+SZzUQWN/hcYtaPK8w== + dependencies: + "@selderee/plugin-htmlparser2" "^0.6.0" + deepmerge "^4.2.2" + he "^1.2.0" + htmlparser2 "^6.1.0" + minimist "^1.2.6" + selderee "^0.6.0" + html-void-elements@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/html-void-elements/-/html-void-elements-1.0.5.tgz#ce9159494e86d95e45795b166c2021c2cfca4483" @@ -18771,6 +18796,11 @@ module-details-from-path@^1.0.3: resolved "https://registry.yarnpkg.com/module-details-from-path/-/module-details-from-path-1.0.3.tgz#114c949673e2a8a35e9d35788527aa37b679da2b" integrity sha1-EUyUlnPiqKNenTV4hSeqN7Z52is= +moo@^0.5.0, moo@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/moo/-/moo-0.5.1.tgz#7aae7f384b9b09f620b6abf6f74ebbcd1b65dbc4" + integrity sha512-I1mnb5xn4fO80BH9BLcF0yLypy2UKl+Cb01Fu0hJRkJjlCRtxZMWkTdAtDd5ZqCOxtCkhmRwyI57vWT+1iZ67w== + move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92" @@ -18888,6 +18918,16 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= +nearley@^2.20.1: + version "2.20.1" + resolved "https://registry.yarnpkg.com/nearley/-/nearley-2.20.1.tgz#246cd33eff0d012faf197ff6774d7ac78acdd474" + integrity sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ== + dependencies: + commander "^2.19.0" + moo "^0.5.0" + railroad-diagrams "^1.0.0" + randexp "0.4.6" + negotiator@0.6.3: version "0.6.3" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" @@ -20139,6 +20179,14 @@ parse5@^5.1.1: resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178" integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug== +parseley@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/parseley/-/parseley-0.7.0.tgz#9949e3a0ed05c5072adb04f013c2810cf49171a8" + integrity sha512-xyOytsdDu077M3/46Am+2cGXEKM9U9QclBDv7fimY7e+BBlxh2JcBp2mgNsmkyA9uvgyTjVzDi7cP1v4hcFxbw== + dependencies: + moo "^0.5.1" + nearley "^2.20.1" + parseurl@^1.3.3, parseurl@~1.3.2, parseurl@~1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" @@ -21152,11 +21200,24 @@ quick-lru@^4.0.1: resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g== +railroad-diagrams@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/railroad-diagrams/-/railroad-diagrams-1.0.0.tgz#eb7e6267548ddedfb899c1b90e57374559cddb7e" + integrity sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A== + ramda@^0.21.0: version "0.21.0" resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.21.0.tgz#a001abedb3ff61077d4ff1d577d44de77e8d0a35" integrity sha1-oAGr7bP/YQd9T/HVd9RN536NCjU= +randexp@0.4.6: + version "0.4.6" + resolved "https://registry.yarnpkg.com/randexp/-/randexp-0.4.6.tgz#e986ad5e5e31dae13ddd6f7b3019aa7c87f60ca3" + integrity sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ== + dependencies: + discontinuous-range "1.0.0" + ret "~0.1.10" + randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5, randombytes@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" @@ -22329,6 +22390,13 @@ secure-json-parse@^2.3.1: resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.4.0.tgz#5aaeaaef85c7a417f76271a4f5b0cc3315ddca85" integrity sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg== +selderee@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.6.0.tgz#f3bee66cfebcb6f33df98e4a1df77388b42a96f7" + integrity sha512-ibqWGV5aChDvfVdqNYuaJP/HnVBhlRGSRrlbttmlMpHcLuTqqbMH36QkSs9GEgj5M88JDYLI8eyP94JaQ8xRlg== + dependencies: + parseley "^0.7.0" + select-hose@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" @@ -25360,6 +25428,19 @@ winston@^3.3.3: triple-beam "^1.3.0" winston-transport "^4.5.0" +word-counting@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/word-counting/-/word-counting-1.1.4.tgz#4d772df20bd86e2e8b00596c8e1ab2f355578ddd" + integrity sha512-SsAKEoa6FzQTV7fR27vDOHO9m2f7cnGhppP+e0c6JCn+pDg88kCMVfrt0Qr8XcbbW2o6HIum4yFX8WEnxZ5xLA== + dependencies: + html-to-text "^8.1.0" + word-regex "^0.1.2" + +word-regex@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/word-regex/-/word-regex-0.1.2.tgz#a3bc7f2d222ce4a93c246c3ef69458f61f511639" + integrity sha512-4jK/OibPeindR9o/sryObhVWNgD2LJCMJFWEME69p48sEYpE9axfyjHK+RqYcOeoEoqcqJEPE9iMdiiFpXHo0Q== + word-wrap@^1.2.3, word-wrap@~1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"