From 718f6716bc7724026500fb4506c47b0e42b26cf5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 4 Nov 2022 13:50:49 +0800 Subject: [PATCH] Add a realistic voice api provider --- .../src/{azure.ts => azureTextToSpeech.ts} | 0 packages/text-to-speech/src/index.ts | 27 ++++-- .../src/realisticTextToSpeech.ts | 85 +++++++++++++++++++ 3 files changed, 104 insertions(+), 8 deletions(-) rename packages/text-to-speech/src/{azure.ts => azureTextToSpeech.ts} (100%) create mode 100644 packages/text-to-speech/src/realisticTextToSpeech.ts diff --git a/packages/text-to-speech/src/azure.ts b/packages/text-to-speech/src/azureTextToSpeech.ts similarity index 100% rename from packages/text-to-speech/src/azure.ts rename to packages/text-to-speech/src/azureTextToSpeech.ts diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index eb9e6c166..530b73a61 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -7,7 +7,7 @@ import * as Sentry from '@sentry/serverless' import axios from 'axios' import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import -import { AzureTextToSpeech } from './azure' +import { AzureTextToSpeech } from './azureTextToSpeech' import { File, Storage } from '@google-cloud/storage' import { endSsml, htmlToSpeechFile, startSsml } from './htmlToSsml' import crypto from 'crypto' @@ -18,6 +18,7 @@ import { TextToSpeechOutput, } from './textToSpeech' import { createClient } from 'redis' +import { RealisticTextToSpeech } from './realisticTextToSpeech' // explicitly create the return type of RedisClient type RedisClient = ReturnType @@ -55,7 +56,10 @@ Sentry.GCPFunction.init({ const MAX_CHARACTER_COUNT = 50000 const storage = new Storage() -const textToSpeechHandlers = [new AzureTextToSpeech()] +const textToSpeechHandlers = [ + new AzureTextToSpeech(), + new RealisticTextToSpeech(), +] const synthesizeTextToSpeech = async ( input: TextToSpeechInput @@ -107,9 +111,9 @@ const updateSpeech = async ( const getCharacterCountFromRedis = async ( redisClient: RedisClient, - token: string + uid: string ): Promise => { - const wordCount = await redisClient.get(`tts:charCount:${token}`) + const wordCount = await redisClient.get(`tts:charCount:${uid}`) return wordCount ? parseInt(wordCount) : 0 } @@ -118,10 +122,10 @@ const getCharacterCountFromRedis = async ( // expires after 1 day const updateCharacterCountInRedis = async ( redisClient: RedisClient, - token: string, + uid: string, wordCount: number ): Promise => { - await redisClient.set(`tts:charCount:${token}`, wordCount.toString(), { + await redisClient.set(`tts:charCount:${uid}`, wordCount.toString(), { EX: 3600 * 24, // in seconds NX: true, }) @@ -205,8 +209,15 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( if (!token) { return res.status(401).send({ errorCode: 'INVALID_TOKEN' }) } + + let uid: string try { jwt.verify(token, process.env.JWT_SECRET) + const claim = jwt.decode(token) as { uid: string } + uid = claim.uid + if (!uid) { + throw new Error('uid not exists') + } } catch (e) { console.error('Authentication error:', e) return res.status(401).send({ errorCode: 'UNAUTHENTICATED' }) @@ -226,7 +237,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( // validate character count const characterCount = - (await getCharacterCountFromRedis(redisClient, token)) + + (await getCharacterCountFromRedis(redisClient, uid)) + utteranceInput.text.length if (characterCount > MAX_CHARACTER_COUNT) { return res.status(429).send('RATE_LIMITED') @@ -278,7 +289,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( console.log('Cache saved') // update character count - await updateCharacterCountInRedis(redisClient, token, characterCount) + await updateCharacterCountInRedis(redisClient, uid, characterCount) res.send({ idx: utteranceInput.idx, diff --git a/packages/text-to-speech/src/realisticTextToSpeech.ts b/packages/text-to-speech/src/realisticTextToSpeech.ts new file mode 100644 index 000000000..1cb28decd --- /dev/null +++ b/packages/text-to-speech/src/realisticTextToSpeech.ts @@ -0,0 +1,85 @@ +import { + TextToSpeech, + TextToSpeechInput, + TextToSpeechOutput, +} from './textToSpeech' +import axios from 'axios' + +interface PlayHtConvertResponse { + message: string + payload: string[] +} + +export class RealisticTextToSpeech implements TextToSpeech { + synthesizeTextToSpeech = async ( + input: TextToSpeechInput + ): Promise => { + const apiEndpoint = process.env.REALISTIC_VOICE_API_ENDPOINT + const apiKey = process.env.REALISTIC_VOICE_API_KEY + const userId = process.env.REALISTIC_VOICE_USER_ID + if (!apiEndpoint || !apiKey || !userId) { + throw new Error('PlayHT API credentials not set') + } + + const HEADERS = { + Authorization: apiKey, + 'X-User-ID': userId, + 'Content-Type': 'application/json', + } + + const data = { + voice: input.voice, + content: [input.text], + } + + // get the download url first + const response = await axios.post( + apiEndpoint, + data, + { + headers: HEADERS, + } + ) + + if (response.data.payload.length === 0) { + throw new Error('No payload returned') + } + + const downloadUrl = response.data.payload[0] + + // polling the download url until the file is ready + // timeout after 5 minutes + const timeout = 5 * 60 * 1000 + const startTime = Date.now() + let audioData: Buffer | undefined + while (!audioData) { + if (Date.now() - startTime > timeout) { + throw new Error('Timeout when polling the download url') + } + + // download the audio file + try { + const downloadResponse = await axios.get(downloadUrl, { + responseType: 'arraybuffer', + headers: { + 'Content-Type': 'audio/wav', + }, + }) + // convert the wav file to buffer + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + audioData = Buffer.from(downloadResponse.data, 'binary') + } catch (e) { + // ignore error + console.debug('checking status of audio file', downloadUrl) + } + } + + return { + audioData, + } + } + + use(input: TextToSpeechInput): boolean { + return !!input.isUltraRealisticVoice + } +}