Add a realistic voice api provider

packages/text-to-speech/src/index.ts

@@ -7,7 +7,7 @@ import * as Sentry from '@sentry/serverless'
 import axios from 'axios'
 import * as jwt from 'jsonwebtoken'
 import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
-import { AzureTextToSpeech } from './azure'
+import { AzureTextToSpeech } from './azureTextToSpeech'
 import { File, Storage } from '@google-cloud/storage'
 import { endSsml, htmlToSpeechFile, startSsml } from './htmlToSsml'
 import crypto from 'crypto'
@@ -18,6 +18,7 @@ import {
   TextToSpeechOutput,
 } from './textToSpeech'
 import { createClient } from 'redis'
+import { RealisticTextToSpeech } from './realisticTextToSpeech'
 
 // explicitly create the return type of RedisClient
 type RedisClient = ReturnType<typeof createClient>
@@ -55,7 +56,10 @@ Sentry.GCPFunction.init({
 const MAX_CHARACTER_COUNT = 50000
 const storage = new Storage()
 
-const textToSpeechHandlers = [new AzureTextToSpeech()]
+const textToSpeechHandlers = [
+  new AzureTextToSpeech(),
+  new RealisticTextToSpeech(),
+]
 
 const synthesizeTextToSpeech = async (
   input: TextToSpeechInput
@@ -107,9 +111,9 @@ const updateSpeech = async (
 
 const getCharacterCountFromRedis = async (
   redisClient: RedisClient,
-  token: string
+  uid: string
 ): Promise<number> => {
-  const wordCount = await redisClient.get(`tts:charCount:${token}`)
+  const wordCount = await redisClient.get(`tts:charCount:${uid}`)
   return wordCount ? parseInt(wordCount) : 0
 }
 
@@ -118,10 +122,10 @@ const getCharacterCountFromRedis = async (
 // expires after 1 day
 const updateCharacterCountInRedis = async (
   redisClient: RedisClient,
-  token: string,
+  uid: string,
   wordCount: number
 ): Promise<void> => {
-  await redisClient.set(`tts:charCount:${token}`, wordCount.toString(), {
+  await redisClient.set(`tts:charCount:${uid}`, wordCount.toString(), {
     EX: 3600 * 24, // in seconds
     NX: true,
   })
@@ -205,8 +209,15 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
     if (!token) {
       return res.status(401).send({ errorCode: 'INVALID_TOKEN' })
     }
+
+    let uid: string
     try {
       jwt.verify(token, process.env.JWT_SECRET)
+      const claim = jwt.decode(token) as { uid: string }
+      uid = claim.uid
+      if (!uid) {
+        throw new Error('uid not exists')
+      }
     } catch (e) {
       console.error('Authentication error:', e)
       return res.status(401).send({ errorCode: 'UNAUTHENTICATED' })
@@ -226,7 +237,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       // validate character count
       const characterCount =
-        (await getCharacterCountFromRedis(redisClient, token)) +
+        (await getCharacterCountFromRedis(redisClient, uid)) +
         utteranceInput.text.length
       if (characterCount > MAX_CHARACTER_COUNT) {
         return res.status(429).send('RATE_LIMITED')
@@ -278,7 +289,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
       console.log('Cache saved')
 
       // update character count
-      await updateCharacterCountInRedis(redisClient, token, characterCount)
+      await updateCharacterCountInRedis(redisClient, uid, characterCount)
 
       res.send({
         idx: utteranceInput.idx,

packages/text-to-speech/src/realisticTextToSpeech.ts

@@ -0,0 +1,85 @@
+import {
+  TextToSpeech,
+  TextToSpeechInput,
+  TextToSpeechOutput,
+} from './textToSpeech'
+import axios from 'axios'
+
+interface PlayHtConvertResponse {
+  message: string
+  payload: string[]
+}
+
+export class RealisticTextToSpeech implements TextToSpeech {
+  synthesizeTextToSpeech = async (
+    input: TextToSpeechInput
+  ): Promise<TextToSpeechOutput> => {
+    const apiEndpoint = process.env.REALISTIC_VOICE_API_ENDPOINT
+    const apiKey = process.env.REALISTIC_VOICE_API_KEY
+    const userId = process.env.REALISTIC_VOICE_USER_ID
+    if (!apiEndpoint || !apiKey || !userId) {
+      throw new Error('PlayHT API credentials not set')
+    }
+
+    const HEADERS = {
+      Authorization: apiKey,
+      'X-User-ID': userId,
+      'Content-Type': 'application/json',
+    }
+
+    const data = {
+      voice: input.voice,
+      content: [input.text],
+    }
+
+    // get the download url first
+    const response = await axios.post<PlayHtConvertResponse>(
+      apiEndpoint,
+      data,
+      {
+        headers: HEADERS,
+      }
+    )
+
+    if (response.data.payload.length === 0) {
+      throw new Error('No payload returned')
+    }
+
+    const downloadUrl = response.data.payload[0]
+
+    // polling the download url until the file is ready
+    // timeout after 5 minutes
+    const timeout = 5 * 60 * 1000
+    const startTime = Date.now()
+    let audioData: Buffer | undefined
+    while (!audioData) {
+      if (Date.now() - startTime > timeout) {
+        throw new Error('Timeout when polling the download url')
+      }
+
+      // download the audio file
+      try {
+        const downloadResponse = await axios.get(downloadUrl, {
+          responseType: 'arraybuffer',
+          headers: {
+            'Content-Type': 'audio/wav',
+          },
+        })
+        // convert the wav file to buffer
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+        audioData = Buffer.from(downloadResponse.data, 'binary')
+      } catch (e) {
+        // ignore error
+        console.debug('checking status of audio file', downloadUrl)
+      }
+    }
+
+    return {
+      audioData,
+    }
+  }
+
+  use(input: TextToSpeechInput): boolean {
+    return !!input.isUltraRealisticVoice
+  }
+}