Save steaming data in GCS

packages/api/src/routers/text_to_speech.ts

@@ -4,7 +4,7 @@
 import express from 'express'
 import cors from 'cors'
 import { corsConfig } from '../utils/corsConfig'
-import { getRepository, setClaims } from '../entity/utils'
+import { setClaims } from '../entity/utils'
 import { getPageById } from '../elastic/pages'
 import { Speech, SpeechState } from '../entity/speech'
 import { buildLogger } from '../utils/logger'
@@ -13,7 +13,7 @@ import { shouldSynthesize } from '../services/speech'
 import { readPushSubscription } from '../datalayer/pubsub'
 import { AppDataSource } from '../server'
 import { enqueueTextToSpeech } from '../utils/createTask'
-import { UserPersonalization } from '../entity/user_personalization'
+import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler'
 
 const logger = buildLogger('app.dispatch')
 
@@ -57,26 +57,29 @@ export function textToSpeechRouter() {
       // checks if this page needs to be synthesized automatically
       if (await shouldSynthesize(userId, page)) {
         logger.info('page needs to be synthesized')
-        const userPersonalization = await getRepository(
-          UserPersonalization
-        ).findOneBy({ user: { id: userId } })
-        // initialize state
-        const speech = await getRepository(Speech).save({
-          user: { id: userId },
-          elasticPageId: id,
-          state: SpeechState.INITIALIZED,
-          voice: userPersonalization?.speechVoice || 'Harrison',
+
+        const speechFile = htmlToSpeechFile({
+          title: page.title,
+          content: page.content,
+          options: {
+            primaryVoice: 'Harrison',
+            secondaryVoice: 'Evelyn',
+          },
         })
-        // enqueue a task to convert text to speech
-        const taskName = await enqueueTextToSpeech({
-          userId,
-          speechId: speech.id,
-          text: page.content,
-          voice: speech.voice,
-          priority: 'low',
-          isUltraRealisticVoice: true,
-        })
-        logger.info('Start Text to speech task', { taskName })
+
+        for (const utterance of speechFile.utterances) {
+          // enqueue a task to convert text to speech
+          const taskName = await enqueueTextToSpeech({
+            userId,
+            speechId: utterance.idx,
+            text: utterance.text,
+            voice: utterance.voice || 'Harrison',
+            priority: 'low',
+            isUltraRealisticVoice: true,
+          })
+          logger.info('Start Text to speech task', { taskName })
+        }
+
         return res.status(202).send('Text to speech task started')
       }
 

packages/text-to-speech/src/index.ts

@@ -40,7 +40,6 @@ interface HTMLInput {
   rate?: string
   complimentaryVoice?: string
   bucket: string
-  isUltraRealisticVoice?: boolean
 }
 
 interface CacheResult {
@@ -83,7 +82,7 @@ const uploadToBucket = async (
   await storage.bucket(bucket).file(filePath).save(data, options)
 }
 
-const createGCSFile = (bucket: string, filename: string): File => {
+export const createGCSFile = (bucket: string, filename: string): File => {
   return storage.bucket(bucket).file(filename)
 }
 
@@ -168,11 +167,11 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction(
       // synthesize text to speech
       const startTime = Date.now()
       // temporary solution to use realistic text to speech
-      input.isUltraRealisticVoice = true
       const { speechMarks } = await synthesizeTextToSpeech({
         ...input,
         textType: 'html',
         audioStream,
+        key: id,
       })
       console.info(
         `Synthesize text to speech completed in ${Date.now() - startTime} ms`
@@ -284,6 +283,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
       const input: TextToSpeechInput = {
         ...utteranceInput,
         textType: 'ssml',
+        key: cacheKey,
       }
       const { audioData, speechMarks } = await synthesizeTextToSpeech(input)
       if (!audioData) {

packages/text-to-speech/src/realisticTextToSpeech.ts

@@ -7,7 +7,7 @@ import axios from 'axios'
 import ffmpegPath from '@ffmpeg-installer/ffmpeg'
 import ffmpeg from 'fluent-ffmpeg'
 import { PassThrough } from 'stream'
-import { htmlToSpeechFile } from './htmlToSsml'
+import { createGCSFile } from './index'
 
 ffmpeg.setFfmpegPath(ffmpegPath.path)
 
@@ -16,24 +16,23 @@ interface PlayHtConvertResponse {
   payload: string[]
 }
 
-const streamWavToMp3 = (
+const convertWavToMp3AndUpload = async (
   inputStream: PassThrough,
   outputStream: PassThrough
 ) => {
-  ffmpeg(inputStream)
-    .inputFormat('wav')
-    .format('mp3')
-    .audioBitrate('32k')
-    .audioChannels(2)
-    .audioCodec('libmp3lame')
-    .on('error', (err) => {
-      throw err
-    })
-    .on('end', () => {
-      console.debug('transcoding finished')
-      outputStream.end()
-    })
-    .pipe(outputStream, { end: true })
+  return new Promise<void>((resolve, reject) => {
+    ffmpeg(inputStream)
+      .audioCodec('libmp3lame')
+      .format('mp3')
+      .on('error', (err) => {
+        reject(err)
+      })
+      .on('end', () => {
+        console.debug('Finished processing')
+        resolve()
+      })
+      .pipe(outputStream, { end: true })
+  })
 }
 
 export class RealisticTextToSpeech implements TextToSpeech {
@@ -47,8 +46,18 @@ export class RealisticTextToSpeech implements TextToSpeech {
       throw new Error('PlayHT API credentials not set')
     }
 
+    const bucket = process.env.GCS_UPLOAD_BUCKET
+    if (!bucket) {
+      throw new Error('GCS_UPLOAD_BUCKET not set')
+    }
+
+    // audio file to be saved in GCS
+    const audioFileName = `speech/${input.key}.mp3`
+    const audioFile = createGCSFile(bucket, audioFileName)
+    const outputStream = audioFile.createWriteStream({
+      resumable: true,
+    }) as PassThrough
     const inputStream = new PassThrough()
-    const outputStream = input.audioStream as PassThrough
 
     const HEADERS = {
       Authorization: apiKey,
@@ -56,19 +65,9 @@ export class RealisticTextToSpeech implements TextToSpeech {
       'Content-Type': 'application/json',
     }
 
-    const speechFile = htmlToSpeechFile({
-      title: '',
-      content: input.text,
-      options: {
-        primaryVoice: input.voice,
-        secondaryVoice: input.secondaryVoice,
-        language: input.language,
-      },
-    })
-    const content = speechFile.utterances.map((u) => u.text)
     const data = {
       voice: input.voice,
-      content,
+      content: [input.text],
     }
 
     // get the download url first
@@ -116,7 +115,7 @@ export class RealisticTextToSpeech implements TextToSpeech {
     }
 
     // transcode the audio file to mp3
-    streamWavToMp3(inputStream, outputStream)
+    await convertWavToMp3AndUpload(inputStream, outputStream)
 
     return {
       audioData,

packages/text-to-speech/src/textToSpeech.ts

@@ -1,5 +1,6 @@
 export interface TextToSpeechInput {
   text: string
+  key: string
   voice?: string
   language?: string
   textType?: 'html' | 'ssml'