Get audio data from Redis cache

packages/text-to-speech/src/htmlToSsml.ts

@@ -44,6 +44,7 @@ export type SSMLOptions = {
 
 const DEFAULT_LANGUAGE = 'en-US'
 const DEFAULT_VOICE = 'en-US-JennyNeural'
+const DEFAULT_SECONDARY_VOICE = 'en-US-GuyNeural'
 const DEFAULT_RATE = '1.0'
 
 const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [
@@ -186,13 +187,11 @@ function emitElement(
 export const startSsml = (options: SSMLOptions, element?: Element): string => {
   const voice =
     element?.nodeName === 'BLOCKQUOTE'
-      ? options.secondaryVoice
-      : options.primaryVoice
+      ? options.secondaryVoice ?? DEFAULT_SECONDARY_VOICE
+      : options.primaryVoice ?? DEFAULT_VOICE
   return `<speak xmlns="http://www.w3.org/2001/10/synthesis" version="1.0" xml:lang="${
     options.language || DEFAULT_LANGUAGE
-  }"><voice name="${voice || DEFAULT_VOICE}"><prosody rate="${
-    options.rate || DEFAULT_RATE
-  }">`
+  }"><voice name="${voice}"><prosody rate="${options.rate || DEFAULT_RATE}">`
 }
 
 export const endSsml = (): string => {

packages/text-to-speech/src/index.ts

@@ -7,9 +7,15 @@ import * as Sentry from '@sentry/serverless'
 import axios from 'axios'
 import * as jwt from 'jsonwebtoken'
 import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
-import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech'
+import {
+  SpeechMark,
+  synthesizeTextToSpeech,
+  TextToSpeechInput,
+} from './textToSpeech'
 import { File, Storage } from '@google-cloud/storage'
-import { htmlToSpeechFile } from './htmlToSsml'
+import { endSsml, htmlToSpeechFile, startSsml } from './htmlToSsml'
+import crypto from 'crypto'
+import { createRedisClient } from './redis'
 
 interface UtteranceInput {
   voice?: string
@@ -29,6 +35,11 @@ interface HTMLInput {
   bucket: string
 }
 
+interface CacheResult {
+  audioDataString: string
+  speechMarks: SpeechMark[]
+}
+
 dotenv.config()
 Sentry.GCPFunction.init({
   dsn: process.env.SENTRY_DSN,
@@ -160,17 +171,56 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
 
     try {
       const utteranceInput = req.body as UtteranceInput
+      const ssmlOptions = {
+        primaryVoice: utteranceInput.voice,
+        secondaryVoice: utteranceInput.voice,
+        language: utteranceInput.language,
+        rate: utteranceInput.rate,
+      }
+      // for utterance, assemble the ssml and pass it through
+      const ssml = `${startSsml(ssmlOptions)}${utteranceInput.text}${endSsml()}`
+      // hash ssml to get the cache key
+      const cacheKey = crypto.createHash('md5').update(ssml).digest('hex')
+      const redisClient = await createRedisClient()
+      // find audio data in cache
+      const cacheResult = await redisClient.get(cacheKey)
+      if (cacheResult) {
+        console.log('Cache hit')
+        const { audioDataString, speechMarks }: CacheResult =
+          JSON.parse(cacheResult)
+        res.send({
+          idx: utteranceInput.idx,
+          audioData: audioDataString,
+          speechMarks,
+        })
+        return
+      }
+      console.log('Cache miss')
+      // synthesize text to speech if cache miss
       const input: TextToSpeechInput = {
         ...utteranceInput,
-        textType: 'utterance',
+        textType: 'ssml',
+        text: ssml,
       }
       const { audioData, speechMarks } = await synthesizeTextToSpeech(input)
       if (!audioData) {
         return res.status(500).send({ errorCode: 'SYNTHESIZER_ERROR' })
       }
+      const audioDataString = audioData.toString('hex')
+      // save audio data to cache for 1 hour
+      await redisClient.set(
+        cacheKey,
+        JSON.stringify({ audioDataString, speechMarks }),
+        {
+          EX: 3600, // in seconds
+          NX: true,
+        }
+      )
+      console.log('Cache saved')
+
       res.send({
         idx: utteranceInput.idx,
-        audioData: audioData.toString('hex'),
+        audioData: audioDataString,
         speechMarks,
       })
     } catch (e) {

packages/text-to-speech/src/textToSpeech.ts

@@ -7,13 +7,13 @@ import {
   SpeechSynthesisResult,
   SpeechSynthesizer,
 } from 'microsoft-cognitiveservices-speech-sdk'
-import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml'
+import { htmlToSsmlItems, ssmlItemText } from './htmlToSsml'
 
 export interface TextToSpeechInput {
   text: string
   voice?: string
   language?: string
-  textType?: 'html' | 'utterance'
+  textType?: 'html' | 'ssml'
   rate?: string
   secondaryVoice?: string
   audioStream?: NodeJS.ReadWriteStream
@@ -51,7 +51,7 @@ export const synthesizeTextToSpeech = async (
   const synthesizer = new SpeechSynthesizer(speechConfig)
   const speechMarks: SpeechMark[] = []
   let timeOffset = 0
-  let wordOffset = 0
+  const wordOffset = 0
 
   synthesizer.synthesizing = function (s, e) {
     // convert arrayBuffer to stream and write to stream
@@ -137,11 +137,7 @@ export const synthesizeTextToSpeech = async (
         speechMarks,
       }
     }
-    // for utterance, just assemble the ssml and pass it through
-    const start = startSsml(ssmlOptions)
-    wordOffset = -start.length
-    const ssml = `${start}${input.text}${endSsml()}`
-    const result = await speakSsmlAsyncPromise(ssml)
+    const result = await speakSsmlAsyncPromise(input.text)
     if (result.reason === ResultReason.Canceled) {
       throw new Error(result.errorDetails)
     }