diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json index e63b348b2..2c4e8591b 100644 --- a/packages/text-to-speech/package.json +++ b/packages/text-to-speech/package.json @@ -21,6 +21,7 @@ "deploy": "yarn build && yarn gcloud-deploy" }, "devDependencies": { + "@types/fluent-ffmpeg": "^2.1.20", "@types/html-to-text": "^8.1.1", "@types/natural": "^5.1.1", "@types/node": "^14.11.2", @@ -30,11 +31,13 @@ "mocha": "^10.0.0" }, "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.1.0", "@google-cloud/functions-framework": "3.1.2", "@google-cloud/storage": "^6.4.1", "@sentry/serverless": "^6.16.1", "axios": "^0.27.2", "dotenv": "^16.0.1", + "fluent-ffmpeg": "^2.1.2", "html-to-text": "^8.2.1", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.12", diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 530b73a61..21f7b07ba 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -40,6 +40,7 @@ interface HTMLInput { rate?: string complimentaryVoice?: string bucket: string + isUltraRealisticVoice?: boolean } interface CacheResult { @@ -134,11 +135,16 @@ const updateCharacterCountInRedis = async ( export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { console.info('Text to speech request body:', req.body) - const token = req.query.token as string if (!process.env.JWT_SECRET) { console.error('JWT_SECRET not exists') return res.status(500).send({ errorCodes: 'JWT_SECRET_NOT_EXISTS' }) } + + const token = (req.query.token || req.headers.authorization) as string + if (!token) { + return res.status(401).send({ errorCode: 'INVALID_TOKEN' }) + } + try { jwt.verify(token, process.env.JWT_SECRET) } catch (e) { @@ -169,13 +175,18 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( console.info( `Synthesize text to speech completed in ${Date.now() - startTime} ms` ) + // speech marks file to be saved in GCS - const speechMarksFileName = `speech/${id}.json` - await uploadToBucket( - speechMarksFileName, - Buffer.from(JSON.stringify(speechMarks)), - bucket - ) + let speechMarksFileName: string | undefined + if (speechMarks.length > 0) { + speechMarksFileName = `speech/${id}.json` + await uploadToBucket( + speechMarksFileName, + Buffer.from(JSON.stringify(speechMarks)), + bucket + ) + } + // update speech state const updated = await updateSpeech( id, diff --git a/packages/text-to-speech/src/realisticTextToSpeech.ts b/packages/text-to-speech/src/realisticTextToSpeech.ts index 329058611..295067d03 100644 --- a/packages/text-to-speech/src/realisticTextToSpeech.ts +++ b/packages/text-to-speech/src/realisticTextToSpeech.ts @@ -4,12 +4,28 @@ import { TextToSpeechOutput, } from './textToSpeech' import axios from 'axios' +import ffmpegPath from '@ffmpeg-installer/ffmpeg' +import ffmpeg from 'fluent-ffmpeg' +import { PassThrough } from 'stream' + +ffmpeg.setFfmpegPath(ffmpegPath.path) interface PlayHtConvertResponse { message: string payload: string[] } +const streamWavToMp3 = (inputStream: PassThrough, outputSteam: PassThrough) => { + ffmpeg(inputStream) + .on('error', (err) => { + throw err + }) + .on('end', () => { + outputSteam.end() + }) + .pipe(outputSteam, { end: true }) +} + export class RealisticTextToSpeech implements TextToSpeech { synthesizeTextToSpeech = async ( input: TextToSpeechInput @@ -21,6 +37,9 @@ export class RealisticTextToSpeech implements TextToSpeech { throw new Error('PlayHT API credentials not set') } + const inputStream = new PassThrough() + const outputStream = input.audioStream + const HEADERS = { Authorization: apiKey, 'X-User-ID': userId, @@ -48,11 +67,10 @@ export class RealisticTextToSpeech implements TextToSpeech { const downloadUrl = response.data.payload[0] // polling the download url until the file is ready - // timeout after 5 minutes - const timeout = 5 * 60 * 1000 + // timeout after 1 hour + const timeout = 60 * 60 * 1000 const startTime = Date.now() - let audioData: Buffer | undefined - while (!audioData) { + while (true) { if (Date.now() - startTime > timeout) { throw new Error('Timeout when polling the download url') } @@ -65,17 +83,22 @@ export class RealisticTextToSpeech implements TextToSpeech { 'Content-Type': 'audio/wav', }, }) - // convert the wav file to buffer - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - audioData = Buffer.from(downloadResponse.data, 'binary') + + // write the audio file to the input stream + inputStream.end(downloadResponse.data) + break } catch (e) { // ignore error console.debug('checking status of audio file', downloadUrl) } } + // transcode the audio file to mp3 + if (outputStream) { + streamWavToMp3(inputStream, outputStream as PassThrough) + } + return { - audioData, speechMarks: [], } } diff --git a/yarn.lock b/yarn.lock index 27452d1b1..b1b9a214d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2286,6 +2286,60 @@ lodash.isundefined "^3.0.1" lodash.uniq "^4.5.0" +"@ffmpeg-installer/darwin-arm64@4.1.5": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz#b7b5c262dd96d1aea4807514e1cdcf6e11f82743" + integrity sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA== + +"@ffmpeg-installer/darwin-x64@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz#48e1706c690e628148482bfb64acb67472089aaa" + integrity sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw== + +"@ffmpeg-installer/ffmpeg@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.1.0.tgz#87fdb9e7d180e8d78f7903f9441e36f978938a90" + integrity sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg== + optionalDependencies: + "@ffmpeg-installer/darwin-arm64" "4.1.5" + "@ffmpeg-installer/darwin-x64" "4.1.0" + "@ffmpeg-installer/linux-arm" "4.1.3" + "@ffmpeg-installer/linux-arm64" "4.1.4" + "@ffmpeg-installer/linux-ia32" "4.1.0" + "@ffmpeg-installer/linux-x64" "4.1.0" + "@ffmpeg-installer/win32-ia32" "4.1.0" + "@ffmpeg-installer/win32-x64" "4.1.0" + +"@ffmpeg-installer/linux-arm64@4.1.4": + version "4.1.4" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz#7219f3f901bb67f7926cb060b56b6974a6cad29f" + integrity sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg== + +"@ffmpeg-installer/linux-arm@4.1.3": + version "4.1.3" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz#c554f105ed5f10475ec25d7bec94926ce18db4c1" + integrity sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg== + +"@ffmpeg-installer/linux-ia32@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz#adad70b0d0d9d8d813983d6e683c5a338a75e442" + integrity sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ== + +"@ffmpeg-installer/linux-x64@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz#b4a5d89c4e12e6d9306dbcdc573df716ec1c4323" + integrity sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A== + +"@ffmpeg-installer/win32-ia32@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz#6eac4fb691b64c02e7a116c1e2d167f3e9b40638" + integrity sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw== + +"@ffmpeg-installer/win32-x64@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz#17e8699b5798d4c60e36e2d6326a8ebe5e95a2c5" + integrity sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg== + "@firebase/app-types@0.7.0": version "0.7.0" resolved "https://registry.yarnpkg.com/@firebase/app-types/-/app-types-0.7.0.tgz#c9e16d1b8bed1a991840b8d2a725fb58d0b5899f" @@ -7702,6 +7756,13 @@ resolved "https://registry.yarnpkg.com/@types/fined/-/fined-1.1.3.tgz#83f03e8f0a8d3673dfcafb18fce3571f6250e1bc" integrity sha512-CWYnSRnun3CGbt6taXeVo2lCbuaj4mchVJ4UF/BdU5TSuIn3AmS13pGMwCsBUoehGbhZrBrpNJZSZI5EVilXww== +"@types/fluent-ffmpeg@^2.1.20": + version "2.1.20" + resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz#3b5f42fc8263761d58284fa46ee6759a64ce54ac" + integrity sha512-B+OvhCdJ3LgEq2PhvWNOiB/EfwnXLElfMCgc4Z1K5zXgSfo9I6uGKwR/lqmNPFQuebNnes7re3gqkV77SyypLg== + dependencies: + "@types/node" "*" + "@types/glob@*": version "7.2.0" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb" @@ -9590,6 +9651,11 @@ async-retry@^1.2.1, async-retry@^1.3.3: dependencies: retry "0.13.1" +async@>=0.2.9: + version "3.2.4" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" + integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== + async@^2.6.2: version "2.6.4" resolved "https://registry.yarnpkg.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221" @@ -14018,6 +14084,14 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.2.tgz#64bfed5cb68fe3ca78b3eb214ad97b63bedce561" integrity sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA== +fluent-ffmpeg@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz#c952de2240f812ebda0aa8006d7776ee2acf7d74" + integrity sha512-IZTB4kq5GK0DPp7sGQ0q/BWurGHffRtQQwVkiqDgeO6wYJLLV5ZhgNOQ65loZxxuPMKZKZcICCUnaGtlxBiR0Q== + dependencies: + async ">=0.2.9" + which "^1.1.1" + flush-write-stream@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8" @@ -25709,7 +25783,7 @@ which@2.0.2, which@^2.0.1, which@^2.0.2: dependencies: isexe "^2.0.0" -which@^1.2.14, which@^1.2.9, which@^1.3.1: +which@^1.1.1, which@^1.2.14, which@^1.2.9, which@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==