diff --git a/packages/api/src/routers/text_to_speech.ts b/packages/api/src/routers/text_to_speech.ts index 8a89a3202..171593853 100644 --- a/packages/api/src/routers/text_to_speech.ts +++ b/packages/api/src/routers/text_to_speech.ts @@ -4,7 +4,7 @@ import express from 'express' import cors from 'cors' import { corsConfig } from '../utils/corsConfig' -import { getRepository, setClaims } from '../entity/utils' +import { setClaims } from '../entity/utils' import { getPageById } from '../elastic/pages' import { Speech, SpeechState } from '../entity/speech' import { buildLogger } from '../utils/logger' @@ -13,7 +13,7 @@ import { shouldSynthesize } from '../services/speech' import { readPushSubscription } from '../datalayer/pubsub' import { AppDataSource } from '../server' import { enqueueTextToSpeech } from '../utils/createTask' -import { UserPersonalization } from '../entity/user_personalization' +import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler' const logger = buildLogger('app.dispatch') @@ -57,26 +57,29 @@ export function textToSpeechRouter() { // checks if this page needs to be synthesized automatically if (await shouldSynthesize(userId, page)) { logger.info('page needs to be synthesized') - const userPersonalization = await getRepository( - UserPersonalization - ).findOneBy({ user: { id: userId } }) - // initialize state - const speech = await getRepository(Speech).save({ - user: { id: userId }, - elasticPageId: id, - state: SpeechState.INITIALIZED, - voice: userPersonalization?.speechVoice || 'Harrison', + + const speechFile = htmlToSpeechFile({ + title: page.title, + content: page.content, + options: { + primaryVoice: 'Harrison', + secondaryVoice: 'Evelyn', + }, }) - // enqueue a task to convert text to speech - const taskName = await enqueueTextToSpeech({ - userId, - speechId: speech.id, - text: page.content, - voice: speech.voice, - priority: 'low', - isUltraRealisticVoice: true, - }) - logger.info('Start Text to speech task', { taskName }) + + for (const utterance of speechFile.utterances) { + // enqueue a task to convert text to speech + const taskName = await enqueueTextToSpeech({ + userId, + speechId: utterance.idx, + text: utterance.text, + voice: utterance.voice || 'Harrison', + priority: 'low', + isUltraRealisticVoice: true, + }) + logger.info('Start Text to speech task', { taskName }) + } + return res.status(202).send('Text to speech task started') } diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index a57a4a098..7bdb5f40f 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -40,7 +40,6 @@ interface HTMLInput { rate?: string complimentaryVoice?: string bucket: string - isUltraRealisticVoice?: boolean } interface CacheResult { @@ -83,7 +82,7 @@ const uploadToBucket = async ( await storage.bucket(bucket).file(filePath).save(data, options) } -const createGCSFile = (bucket: string, filename: string): File => { +export const createGCSFile = (bucket: string, filename: string): File => { return storage.bucket(bucket).file(filename) } @@ -168,11 +167,11 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( // synthesize text to speech const startTime = Date.now() // temporary solution to use realistic text to speech - input.isUltraRealisticVoice = true const { speechMarks } = await synthesizeTextToSpeech({ ...input, textType: 'html', audioStream, + key: id, }) console.info( `Synthesize text to speech completed in ${Date.now() - startTime} ms` @@ -284,6 +283,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( const input: TextToSpeechInput = { ...utteranceInput, textType: 'ssml', + key: cacheKey, } const { audioData, speechMarks } = await synthesizeTextToSpeech(input) if (!audioData) { diff --git a/packages/text-to-speech/src/realisticTextToSpeech.ts b/packages/text-to-speech/src/realisticTextToSpeech.ts index 4b5c5d4d2..19bde0f5a 100644 --- a/packages/text-to-speech/src/realisticTextToSpeech.ts +++ b/packages/text-to-speech/src/realisticTextToSpeech.ts @@ -7,7 +7,7 @@ import axios from 'axios' import ffmpegPath from '@ffmpeg-installer/ffmpeg' import ffmpeg from 'fluent-ffmpeg' import { PassThrough } from 'stream' -import { htmlToSpeechFile } from './htmlToSsml' +import { createGCSFile } from './index' ffmpeg.setFfmpegPath(ffmpegPath.path) @@ -16,24 +16,23 @@ interface PlayHtConvertResponse { payload: string[] } -const streamWavToMp3 = ( +const convertWavToMp3AndUpload = async ( inputStream: PassThrough, outputStream: PassThrough ) => { - ffmpeg(inputStream) - .inputFormat('wav') - .format('mp3') - .audioBitrate('32k') - .audioChannels(2) - .audioCodec('libmp3lame') - .on('error', (err) => { - throw err - }) - .on('end', () => { - console.debug('transcoding finished') - outputStream.end() - }) - .pipe(outputStream, { end: true }) + return new Promise((resolve, reject) => { + ffmpeg(inputStream) + .audioCodec('libmp3lame') + .format('mp3') + .on('error', (err) => { + reject(err) + }) + .on('end', () => { + console.debug('Finished processing') + resolve() + }) + .pipe(outputStream, { end: true }) + }) } export class RealisticTextToSpeech implements TextToSpeech { @@ -47,8 +46,18 @@ export class RealisticTextToSpeech implements TextToSpeech { throw new Error('PlayHT API credentials not set') } + const bucket = process.env.GCS_UPLOAD_BUCKET + if (!bucket) { + throw new Error('GCS_UPLOAD_BUCKET not set') + } + + // audio file to be saved in GCS + const audioFileName = `speech/${input.key}.mp3` + const audioFile = createGCSFile(bucket, audioFileName) + const outputStream = audioFile.createWriteStream({ + resumable: true, + }) as PassThrough const inputStream = new PassThrough() - const outputStream = input.audioStream as PassThrough const HEADERS = { Authorization: apiKey, @@ -56,19 +65,9 @@ export class RealisticTextToSpeech implements TextToSpeech { 'Content-Type': 'application/json', } - const speechFile = htmlToSpeechFile({ - title: '', - content: input.text, - options: { - primaryVoice: input.voice, - secondaryVoice: input.secondaryVoice, - language: input.language, - }, - }) - const content = speechFile.utterances.map((u) => u.text) const data = { voice: input.voice, - content, + content: [input.text], } // get the download url first @@ -116,7 +115,7 @@ export class RealisticTextToSpeech implements TextToSpeech { } // transcode the audio file to mp3 - streamWavToMp3(inputStream, outputStream) + await convertWavToMp3AndUpload(inputStream, outputStream) return { audioData, diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 3bc6f7b4b..a446b18b3 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -1,5 +1,6 @@ export interface TextToSpeechInput { text: string + key: string voice?: string language?: string textType?: 'html' | 'ssml'