From 9566c17db31cf98104ce5003d2defd9ea39ecfd8 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 16 Feb 2023 10:56:24 +0800 Subject: [PATCH 1/7] Update API for ultra realistic voices --- .../Sources/Models/DataModels/FeedItem.swift | 4 + .../src/realisticTextToSpeech.ts | 113 +++--------------- 2 files changed, 19 insertions(+), 98 deletions(-) diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift index e07f2edf9..8615eb371 100644 --- a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift +++ b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift @@ -69,6 +69,10 @@ public extension LinkedItem { (labels?.count ?? 0) > 0 } + var isUnread: Bool { + readAt == nil + } + var isRead: Bool { readingProgress >= 0.98 } diff --git a/packages/text-to-speech/src/realisticTextToSpeech.ts b/packages/text-to-speech/src/realisticTextToSpeech.ts index 94a0a9c7a..a93f40e3a 100644 --- a/packages/text-to-speech/src/realisticTextToSpeech.ts +++ b/packages/text-to-speech/src/realisticTextToSpeech.ts @@ -4,126 +4,43 @@ import { TextToSpeechOutput, } from './textToSpeech' import axios from 'axios' -import ffmpegPath from '@ffmpeg-installer/ffmpeg' -import ffmpeg from 'fluent-ffmpeg' -import { PassThrough } from 'stream' - -ffmpeg.setFfmpegPath(ffmpegPath.path) - -interface PlayHtConvertResponse { - message: string - payload: string[] -} - -const convertWavToMp3AndUpload = async ( - inputStream: PassThrough, - outputStream: PassThrough -) => { - return new Promise((resolve, reject) => { - ffmpeg(inputStream) - .audioCodec('libmp3lame') - .format('mp3') - .on('error', (err) => { - reject(err) - }) - .on('end', () => { - console.debug('Finished processing') - resolve() - }) - .pipe(outputStream, { end: true }) - }) -} export class RealisticTextToSpeech implements TextToSpeech { synthesizeTextToSpeech = async ( input: TextToSpeechInput ): Promise => { - const apiEndpoint = process.env.REALISTIC_VOICE_API_ENDPOINT + const voiceId = process.env.REALISTIC_VOICE_ID const apiKey = process.env.REALISTIC_VOICE_API_KEY - const userId = process.env.REALISTIC_VOICE_USER_ID - if (!apiEndpoint || !apiKey || !userId) { - throw new Error('PlayHT API credentials not set') + const apiEndpoint = process.env.REALISTIC_API_ENDPOINT + + if (!apiEndpoint || !apiKey || !voiceId) { + throw new Error('API credentials not set') } - const inputStream = new PassThrough() - const HEADERS = { - Authorization: apiKey, - 'X-User-ID': userId, + 'xi-api-key': apiKey, + voice_id: voiceId, 'Content-Type': 'application/json', } - const data = { - voice: input.voice, - content: [input.text], - } - - // get the download url first - const response = await axios.post( - apiEndpoint, - data, + const requestUrl = `${apiEndpoint}/${voiceId}` + const response = await axios.post( + requestUrl, + { + text: input.text, + }, { headers: HEADERS, } ) - if (response.data.payload.length === 0) { + if (response.data.length === 0) { throw new Error('No payload returned') } - const downloadUrl = response.data.payload[0] - - // polling the download url until the file is ready - // timeout after 1 hour - const timeout = 60 * 60 * 1000 - const startTime = Date.now() - let isReady = false - while (!isReady) { - if (Date.now() - startTime > timeout) { - throw new Error('Timeout when polling the download url') - } - - // download the audio file - try { - const downloadResponse = await axios.get(downloadUrl, { - responseType: 'arraybuffer', - headers: { - 'Content-Type': 'audio/wav', - }, - }) - - // write the audio file to the input stream - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - inputStream.end(Buffer.from(downloadResponse.data, 'binary')) - isReady = true - } catch (e) { - // ignore error - console.debug('checking status of audio file', downloadUrl) - } - } - - const outputStream = new PassThrough() - // transcode the audio file to mp3 - await convertWavToMp3AndUpload(inputStream, outputStream) - - // convert the buffer stream to a buffer - const audioData = await new Promise((resolve, reject) => { - const chunks: Buffer[] = [] - outputStream.on('data', (chunk) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - chunks.push(chunk) - }) - outputStream.on('end', () => { - resolve(Buffer.concat(chunks)) - }) - outputStream.on('error', (err) => { - reject(err) - }) - }) - return { - audioData, speechMarks: [], + audioData: response.data, } } From 7177279951ce7c5393630db1cbe91b9fd3b4f481 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 16 Feb 2023 11:23:28 +0800 Subject: [PATCH 2/7] Fix envar name --- packages/text-to-speech/src/realisticTextToSpeech.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/text-to-speech/src/realisticTextToSpeech.ts b/packages/text-to-speech/src/realisticTextToSpeech.ts index a93f40e3a..9b76942f2 100644 --- a/packages/text-to-speech/src/realisticTextToSpeech.ts +++ b/packages/text-to-speech/src/realisticTextToSpeech.ts @@ -11,7 +11,7 @@ export class RealisticTextToSpeech implements TextToSpeech { ): Promise => { const voiceId = process.env.REALISTIC_VOICE_ID const apiKey = process.env.REALISTIC_VOICE_API_KEY - const apiEndpoint = process.env.REALISTIC_API_ENDPOINT + const apiEndpoint = process.env.REALISTIC_VOICE_API_ENDPOINT if (!apiEndpoint || !apiKey || !voiceId) { throw new Error('API credentials not set') From 9d5cb8464845e05d57b647f52dd1690e96db588e Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 16 Feb 2023 13:19:24 +0800 Subject: [PATCH 3/7] Return arraybuffer from realistic voice API --- packages/text-to-speech/src/realisticTextToSpeech.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/text-to-speech/src/realisticTextToSpeech.ts b/packages/text-to-speech/src/realisticTextToSpeech.ts index 9b76942f2..fa3e99159 100644 --- a/packages/text-to-speech/src/realisticTextToSpeech.ts +++ b/packages/text-to-speech/src/realisticTextToSpeech.ts @@ -23,7 +23,7 @@ export class RealisticTextToSpeech implements TextToSpeech { 'Content-Type': 'application/json', } - const requestUrl = `${apiEndpoint}/${voiceId}` + const requestUrl = `${apiEndpoint}${voiceId}` const response = await axios.post( requestUrl, { @@ -31,6 +31,7 @@ export class RealisticTextToSpeech implements TextToSpeech { }, { headers: HEADERS, + responseType: 'arraybuffer', } ) From cd9c2e50b55519926229bfef0f7a1a52d4d1a923 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 16 Feb 2023 15:43:12 +0800 Subject: [PATCH 4/7] Use a voice mapping --- .../src/realisticTextToSpeech.ts | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/packages/text-to-speech/src/realisticTextToSpeech.ts b/packages/text-to-speech/src/realisticTextToSpeech.ts index fa3e99159..c19c293a6 100644 --- a/packages/text-to-speech/src/realisticTextToSpeech.ts +++ b/packages/text-to-speech/src/realisticTextToSpeech.ts @@ -5,11 +5,41 @@ import { } from './textToSpeech' import axios from 'axios' +const getRealisticVoiceId = (name: string | undefined) => { + const voiceList = [ + { + voiceId: '21m00Tcm4TlvDq8ikWAM', + name: 'Rachel', + }, + { + voiceId: 'EXAVITQu4vr4xnSDxMaL', + name: 'Bella', + }, + { + voiceId: 'MF3mGyEYCl7XYWbV9V6O', + name: 'Elli', + }, + { + voiceId: 'TxGEqnHWrfWFTfGW9XjX', + name: 'Josh', + }, + { + voiceId: 'VR6AewLTigWG4xSOukaG', + name: 'Arnold', + }, + { + voiceId: 'pNInz6obpgDQGcFmaJgB', + name: 'Adam', + }, + ] + return voiceList.find((voice) => voice.name === name)?.voiceId +} + export class RealisticTextToSpeech implements TextToSpeech { synthesizeTextToSpeech = async ( input: TextToSpeechInput ): Promise => { - const voiceId = process.env.REALISTIC_VOICE_ID + const voiceId = getRealisticVoiceId(input.voice) const apiKey = process.env.REALISTIC_VOICE_API_KEY const apiEndpoint = process.env.REALISTIC_VOICE_API_ENDPOINT From f25b9f09a71f45358ed6dc33c89ebb651f9e2de2 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 16 Feb 2023 16:23:03 +0800 Subject: [PATCH 5/7] Add some logging --- packages/text-to-speech/src/realisticTextToSpeech.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/text-to-speech/src/realisticTextToSpeech.ts b/packages/text-to-speech/src/realisticTextToSpeech.ts index c19c293a6..051921327 100644 --- a/packages/text-to-speech/src/realisticTextToSpeech.ts +++ b/packages/text-to-speech/src/realisticTextToSpeech.ts @@ -54,6 +54,8 @@ export class RealisticTextToSpeech implements TextToSpeech { } const requestUrl = `${apiEndpoint}${voiceId}` + console.log('using voice id', voiceId, requestUrl) + const response = await axios.post( requestUrl, { From f06ebe4e258db33e555f3167458772535567015d Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 16 Feb 2023 16:44:08 +0800 Subject: [PATCH 6/7] Revert change to swift file --- apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift index 8615eb371..e07f2edf9 100644 --- a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift +++ b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift @@ -69,10 +69,6 @@ public extension LinkedItem { (labels?.count ?? 0) > 0 } - var isUnread: Bool { - readAt == nil - } - var isRead: Bool { readingProgress >= 0.98 } From 3dde59c4a6002973d081aca48dcdf96309eed5e9 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 16 Feb 2023 16:45:37 +0800 Subject: [PATCH 7/7] Clean up logging of API errors --- packages/text-to-speech/src/realisticTextToSpeech.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/text-to-speech/src/realisticTextToSpeech.ts b/packages/text-to-speech/src/realisticTextToSpeech.ts index 051921327..9a14f36cd 100644 --- a/packages/text-to-speech/src/realisticTextToSpeech.ts +++ b/packages/text-to-speech/src/realisticTextToSpeech.ts @@ -54,8 +54,6 @@ export class RealisticTextToSpeech implements TextToSpeech { } const requestUrl = `${apiEndpoint}${voiceId}` - console.log('using voice id', voiceId, requestUrl) - const response = await axios.post( requestUrl, { @@ -68,6 +66,7 @@ export class RealisticTextToSpeech implements TextToSpeech { ) if (response.data.length === 0) { + console.log('No payload returned: ', response) throw new Error('No payload returned') }