From e976c9e169be8dfabda87886bb3f99587f3a1844 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Aug 2022 11:47:01 +0800 Subject: [PATCH 1/5] Add text-to-speech queue name and location to env --- packages/api/src/util.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index ea61b3ba2..53d2daa83 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -65,6 +65,8 @@ interface BackendEnv { reminderTaskHanderUrl: string integrationTaskHandlerUrl: string textToSpeechTaskHandlerUrl: string + textToSpeechName: string + textToSpeechLocation: string } fileUpload: { gcsUploadBucket: string @@ -233,6 +235,8 @@ export function getEnv(): BackendEnv { reminderTaskHanderUrl: parse('REMINDER_TASK_HANDLER_URL'), integrationTaskHandlerUrl: parse('INTEGRATION_TASK_HANDLER_URL'), textToSpeechTaskHandlerUrl: parse('TEXT_TO_SPEECH_TASK_HANDLER_URL'), + textToSpeechName: parse('TEXT_TO_SPEECH_QUEUE_NAME'), + textToSpeechLocation: parse('TEXT_TO_SPEECH_QUEUE_LOCATION'), } const imageProxy = { url: parse('IMAGE_PROXY_URL'), From f903a92245b75f34568827dfad77b6b5326b7d6c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Aug 2022 11:47:42 +0800 Subject: [PATCH 2/5] Use text-to-speech queue --- packages/api/src/utils/createTask.ts | 35 +++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index ddc21be2b..c4de5d58b 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -328,14 +328,27 @@ export const enqueueSyncWithIntegration = async ( return createdTasks[0].name } -export const enqueueTextToSpeech = async ( - userId: string, - speechId: string, - text: string, - textType: 'text' | 'ssml', - voice: string, - bucket: string -): Promise => { +export const enqueueTextToSpeech = async ({ + userId, + text, + speechId, + voice, + priority, + textType = 'ssml', + bucket = env.fileUpload.gcsUploadBucket, + queue = env.queue.textToSpeechName, + location = env.queue.textToSpeechLocation, +}: { + userId: string + speechId: string + text: string + voice: string + priority: 'low' | 'high' + bucket?: string + textType?: 'text' | 'ssml' + queue?: string + location?: string +}): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { id: speechId, @@ -344,6 +357,9 @@ export const enqueueTextToSpeech = async ( bucket, textType, } + if (priority === 'low') { + queue = `${queue}-low` + } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore const token = await signToken({ uid: userId }, env.server.jwtSecret, { @@ -364,6 +380,9 @@ export const enqueueTextToSpeech = async ( project: GOOGLE_CLOUD_PROJECT, payload, taskHandlerUrl, + queue, + location, + priority, }) if (!createdTasks || !createdTasks[0].name) { From 4af340f30e93fd0f26f3414b9eb086a4c13e40e8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Aug 2022 11:49:10 +0800 Subject: [PATCH 3/5] Add the priority to request param and use the correct queue accordingly --- packages/api/src/routers/article_router.ts | 24 +++++++++++++--------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 68e804071..487b32c77 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -73,13 +73,18 @@ export function articleRouter() { }) router.get( - '/:id/:outputFormat/:voice?', + '/:id/:outputFormat/:priority/:voice?', cors(corsConfig), async (req, res) => { const articleId = req.params.id const outputFormat = req.params.outputFormat const voice = req.params.voice - if (!articleId || !['mp3', 'speech-marks'].includes(outputFormat)) { + const priority = req.params.priority + if ( + !articleId || + !['mp3', 'speech-marks'].includes(outputFormat) || + !['low', 'high'].includes(priority) + ) { return res.status(400).send('Invalid data') } const token = req.cookies?.auth || req.headers?.authorization @@ -153,14 +158,13 @@ export function articleRouter() { voice: voice || userPersonalization?.speechVoice || 'en-US-JennyNeural', }) // enqueue a task to convert text to speech - const taskName = await enqueueTextToSpeech( - uid, - speech.id, - page.content, - 'ssml', - speech.voice, - env.fileUpload.gcsUploadBucket - ) + const taskName = await enqueueTextToSpeech({ + userId: uid, + speechId: speech.id, + text: page.content, + voice: speech.voice, + priority: priority as 'low' | 'high', + }) logger.info('Start Text to speech task', { taskName }) res.status(202).send('Text to speech task started') } From 2e4cb7f4b08c5475f757ea7810796ad6ce8b79ad Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Aug 2022 12:01:17 +0800 Subject: [PATCH 4/5] Fix using wrong taskHandlerUrl for low priority task --- packages/api/src/utils/createTask.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index c4de5d58b..962b796ee 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -45,11 +45,7 @@ const createHttpTaskWithToken = async ({ ] > => { // Construct the fully qualified queue name. - if (priority === 'low') { - queue = `${queue}-low` - // use GCF url for low priority tasks - taskHandlerUrl = env.queue.contentFetchGCFUrl - } + priority === 'low' && (queue = `${queue}-low`) const parent = client.queuePath(project, location, queue) console.log(`Task creation options: `, { @@ -240,10 +236,17 @@ export const enqueueParseRequest = async ( return '' } + // use GCF url for low priority tasks + const taskHandlerUrl = + priority === 'low' + ? env.queue.contentFetchGCFUrl + : env.queue.contentFetchUrl + const createdTasks = await createHttpTaskWithToken({ project: GOOGLE_CLOUD_PROJECT, payload, priority, + taskHandlerUrl, }) if (!createdTasks || !createdTasks[0].name) { logger.error(`Unable to get the name of the task`, { @@ -357,9 +360,6 @@ export const enqueueTextToSpeech = async ({ bucket, textType, } - if (priority === 'low') { - queue = `${queue}-low` - } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore const token = await signToken({ uid: userId }, env.server.jwtSecret, { From a19ec8efea5cb41abaa1427fe9c736f87b3cb3a8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Aug 2022 13:20:39 +0800 Subject: [PATCH 5/5] Hardcode queue name --- packages/api/src/util.ts | 13 +++++++++---- packages/api/src/utils/createTask.ts | 9 ++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 53d2daa83..0ca9ad8b8 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -65,8 +65,6 @@ interface BackendEnv { reminderTaskHanderUrl: string integrationTaskHandlerUrl: string textToSpeechTaskHandlerUrl: string - textToSpeechName: string - textToSpeechLocation: string } fileUpload: { gcsUploadBucket: string @@ -96,6 +94,9 @@ interface BackendEnv { speechKey: string speechRegion: string } + gcp: { + location: string + } } /*** @@ -150,6 +151,7 @@ const nullableEnvVars = [ 'TEXT_TO_SPEECH_TASK_HANDLER_URL', 'AZURE_SPEECH_KEY', 'AZURE_SPEECH_REGION', + 'GCP_LOCATION', ] // Allow some vars to be null/empty /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */ @@ -235,8 +237,6 @@ export function getEnv(): BackendEnv { reminderTaskHanderUrl: parse('REMINDER_TASK_HANDLER_URL'), integrationTaskHandlerUrl: parse('INTEGRATION_TASK_HANDLER_URL'), textToSpeechTaskHandlerUrl: parse('TEXT_TO_SPEECH_TASK_HANDLER_URL'), - textToSpeechName: parse('TEXT_TO_SPEECH_QUEUE_NAME'), - textToSpeechLocation: parse('TEXT_TO_SPEECH_QUEUE_LOCATION'), } const imageProxy = { url: parse('IMAGE_PROXY_URL'), @@ -277,6 +277,10 @@ export function getEnv(): BackendEnv { speechRegion: parse('AZURE_SPEECH_REGION'), } + const gcp = { + location: parse('GCP_LOCATION'), + } + return { pg, client, @@ -296,6 +300,7 @@ export function getEnv(): BackendEnv { sendgrid, readwise, azure, + gcp, } } diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 962b796ee..c8102fa02 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -207,13 +207,15 @@ export const deleteTask = async ( * @param userId - Id of the user authorized * @param saveRequestId - Id of the article_saving_request table record * @param priority - Priority of the task + * @param queue - Queue name * @returns Name of the task created */ export const enqueueParseRequest = async ( url: string, userId: string, saveRequestId: string, - priority: 'low' | 'high' = 'high' + priority: 'low' | 'high' = 'high', + queue = env.queue.name ): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { @@ -247,6 +249,7 @@ export const enqueueParseRequest = async ( payload, priority, taskHandlerUrl, + queue, }) if (!createdTasks || !createdTasks[0].name) { logger.error(`Unable to get the name of the task`, { @@ -339,8 +342,8 @@ export const enqueueTextToSpeech = async ({ priority, textType = 'ssml', bucket = env.fileUpload.gcsUploadBucket, - queue = env.queue.textToSpeechName, - location = env.queue.textToSpeechLocation, + queue = 'omnivore-demo-text-to-speech-queue', + location = env.gcp.location, }: { userId: string speechId: string