diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index 817049114..117149de4 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -44,6 +44,7 @@ export type SSMLOptions = { const DEFAULT_LANGUAGE = 'en-US' const DEFAULT_VOICE = 'en-US-JennyNeural' +const DEFAULT_SECONDARY_VOICE = 'en-US-GuyNeural' const DEFAULT_RATE = '1.0' const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ @@ -186,13 +187,11 @@ function emitElement( export const startSsml = (options: SSMLOptions, element?: Element): string => { const voice = element?.nodeName === 'BLOCKQUOTE' - ? options.secondaryVoice - : options.primaryVoice + ? options.secondaryVoice ?? DEFAULT_SECONDARY_VOICE + : options.primaryVoice ?? DEFAULT_VOICE return `` + }">` } export const endSsml = (): string => { diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index bcc124a39..24ea470a0 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -7,9 +7,15 @@ import * as Sentry from '@sentry/serverless' import axios from 'axios' import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import -import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' +import { + SpeechMark, + synthesizeTextToSpeech, + TextToSpeechInput, +} from './textToSpeech' import { File, Storage } from '@google-cloud/storage' -import { htmlToSpeechFile } from './htmlToSsml' +import { endSsml, htmlToSpeechFile, startSsml } from './htmlToSsml' +import crypto from 'crypto' +import { createRedisClient } from './redis' interface UtteranceInput { voice?: string @@ -29,6 +35,11 @@ interface HTMLInput { bucket: string } +interface CacheResult { + audioDataString: string + speechMarks: SpeechMark[] +} + dotenv.config() Sentry.GCPFunction.init({ dsn: process.env.SENTRY_DSN, @@ -160,17 +171,56 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( try { const utteranceInput = req.body as UtteranceInput + const ssmlOptions = { + primaryVoice: utteranceInput.voice, + secondaryVoice: utteranceInput.voice, + language: utteranceInput.language, + rate: utteranceInput.rate, + } + // for utterance, assemble the ssml and pass it through + const ssml = `${startSsml(ssmlOptions)}${utteranceInput.text}${endSsml()}` + // hash ssml to get the cache key + const cacheKey = crypto.createHash('md5').update(ssml).digest('hex') + const redisClient = await createRedisClient() + // find audio data in cache + const cacheResult = await redisClient.get(cacheKey) + if (cacheResult) { + console.log('Cache hit') + const { audioDataString, speechMarks }: CacheResult = + JSON.parse(cacheResult) + res.send({ + idx: utteranceInput.idx, + audioData: audioDataString, + speechMarks, + }) + return + } + console.log('Cache miss') + // synthesize text to speech if cache miss const input: TextToSpeechInput = { ...utteranceInput, - textType: 'utterance', + textType: 'ssml', + text: ssml, } const { audioData, speechMarks } = await synthesizeTextToSpeech(input) if (!audioData) { return res.status(500).send({ errorCode: 'SYNTHESIZER_ERROR' }) } + const audioDataString = audioData.toString('hex') + // save audio data to cache for 1 hour + await redisClient.set( + cacheKey, + JSON.stringify({ audioDataString, speechMarks }), + { + EX: 3600, // in seconds + NX: true, + } + ) + console.log('Cache saved') + res.send({ idx: utteranceInput.idx, - audioData: audioData.toString('hex'), + audioData: audioDataString, speechMarks, }) } catch (e) { diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 27508175e..042e53454 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -7,13 +7,13 @@ import { SpeechSynthesisResult, SpeechSynthesizer, } from 'microsoft-cognitiveservices-speech-sdk' -import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml' +import { htmlToSsmlItems, ssmlItemText } from './htmlToSsml' export interface TextToSpeechInput { text: string voice?: string language?: string - textType?: 'html' | 'utterance' + textType?: 'html' | 'ssml' rate?: string secondaryVoice?: string audioStream?: NodeJS.ReadWriteStream @@ -51,7 +51,7 @@ export const synthesizeTextToSpeech = async ( const synthesizer = new SpeechSynthesizer(speechConfig) const speechMarks: SpeechMark[] = [] let timeOffset = 0 - let wordOffset = 0 + const wordOffset = 0 synthesizer.synthesizing = function (s, e) { // convert arrayBuffer to stream and write to stream @@ -137,11 +137,7 @@ export const synthesizeTextToSpeech = async ( speechMarks, } } - // for utterance, just assemble the ssml and pass it through - const start = startSsml(ssmlOptions) - wordOffset = -start.length - const ssml = `${start}${input.text}${endSsml()}` - const result = await speakSsmlAsyncPromise(ssml) + const result = await speakSsmlAsyncPromise(input.text) if (result.reason === ResultReason.Canceled) { throw new Error(result.errorDetails) }