From 2298635e560ec19b3a4d9ceafc5c04e855e0e12e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 22 Aug 2022 16:09:56 +0800 Subject: [PATCH 1/8] Add function to set speech to failed state --- packages/api/src/services/speech.ts | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 packages/api/src/services/speech.ts diff --git a/packages/api/src/services/speech.ts b/packages/api/src/services/speech.ts new file mode 100644 index 000000000..01d86c768 --- /dev/null +++ b/packages/api/src/services/speech.ts @@ -0,0 +1,9 @@ +import { getRepository } from '../entity/utils' +import { Speech, SpeechState } from '../entity/speech' + +export const setSpeechFailure = async (id: string) => { + // update state + await getRepository(Speech).update(id, { + state: SpeechState.FAILED, + }) +} From 4852777f6673383e9e8ad92b878cf52cd140a340 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 22 Aug 2022 16:10:49 +0800 Subject: [PATCH 2/8] Throw error if synthesizing was cancelled --- packages/api/src/utils/textToSpeech.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index edfde939e..de0ee2714 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -128,7 +128,6 @@ export const synthesizeTextToSpeech = async ( resolve(result) }, (error) => { - synthesizer.close() reject(error) } ) @@ -145,7 +144,6 @@ export const synthesizeTextToSpeech = async ( resolve(result) }, (error) => { - synthesizer.close() reject(error) } ) @@ -183,6 +181,10 @@ export const synthesizeTextToSpeech = async ( ) logger.debug(`synthesizing ${ssml}`) const result = await speakSsmlAsyncPromise(ssml) + if (result.reason === ResultReason.Canceled) { + synthesizer.close() + throw new Error(result.errorDetails) + } timeOffset = timeOffset + result.audioDuration // characterOffset = characterOffset + htmlElement.innerText.length } From 476e43a1f8935324a1c96ee74041e70afb1c00f7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 22 Aug 2022 16:11:49 +0800 Subject: [PATCH 3/8] Set speech state to FAILED if encountered error during transcription --- packages/api/src/routers/svc/speech.ts | 70 +++++++++++--------------- 1 file changed, 28 insertions(+), 42 deletions(-) diff --git a/packages/api/src/routers/svc/speech.ts b/packages/api/src/routers/svc/speech.ts index b6f5136ab..6c188d4fb 100644 --- a/packages/api/src/routers/svc/speech.ts +++ b/packages/api/src/routers/svc/speech.ts @@ -5,9 +5,9 @@ import { getRepository } from '../../entity/utils' import { getPageById } from '../../elastic/pages' import { synthesizeTextToSpeech } from '../../utils/textToSpeech' import { Speech, SpeechState } from '../../entity/speech' -import { UserPersonalization } from '../../entity/user_personalization' import { buildLogger } from '../../utils/logger' import { getClaimsByToken } from '../../utils/auth' +import { setSpeechFailure } from '../../services/speech' const logger = buildLogger('app.dispatch') @@ -21,68 +21,57 @@ export function speechServiceRouter() { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment body: req.body, }) + const { userId, speechId } = req.body as { + userId: string + speechId: string + } + if (!userId || !speechId) { + speechId && (await setSpeechFailure(speechId)) + return res.status(200).send('Invalid data') + } + const token = req.query.token as string try { if (!(await getClaimsByToken(token))) { logger.info('Unauthorized request', { token }) + await setSpeechFailure(speechId) return res.status(200).send('UNAUTHORIZED') } } catch (error) { logger.error('Unauthorized request', { token, error }) + await setSpeechFailure(speechId) return res.status(200).send('UNAUTHORIZED') } - const { userId, pageId } = req.body as { - userId: string - pageId: string - } - - if (!userId || !pageId) { - return res.status(200).send('Invalid data') - } - - const userPersonalization = await getRepository( - UserPersonalization - ).findOneBy({ - user: { id: userId }, - }) - if (!userPersonalization) { - return res.status(200).send('User Personalization not found') - } - - const page = await getPageById(pageId) - if (!page) { - return res.status(200).send('Page not found') - } - // const text = parseHTML(page.content).document.documentElement.innerText - // if (!text) { - // return res.status(200).send('Page has no text') - // } logger.info(`Create article speech`, { body: { userId, - pageId, + speechId, }, labels: { source: 'CreateArticleSpeech', }, }) - - // initialize state - const speech = await getRepository(Speech).save({ - user: { id: userId }, - elasticPageId: pageId, - state: SpeechState.INITIALIZED, - voice: userPersonalization.speechVoice, + const speech = await getRepository(Speech).findOneBy({ + id: speechId, }) + if (!speech) { + return res.status(200).send('Speech not found') + } + + const page = await getPageById(speech.elasticPageId) + if (!page) { + await setSpeechFailure(speech.id) + return res.status(200).send('Page not found') + } try { const startTime = Date.now() const speechOutput = await synthesizeTextToSpeech({ - id: pageId, + id: speech.id, text: page.content, languageCode: page.language, - voice: userPersonalization.speechVoice, + voice: speech.voice, textType: 'ssml', }) logger.info('Created speech', { @@ -91,7 +80,7 @@ export function speechServiceRouter() { duration: Date.now() - startTime, }) - // update state + // set state to completed await getRepository(Speech).update(speech.id, { audioFileName: speechOutput.audioFileName, speechMarksFileName: speechOutput.speechMarksFileName, @@ -101,10 +90,7 @@ export function speechServiceRouter() { res.status(200).send('OK') } catch (error) { logger.error(`Error creating article speech`, { error }) - // update state - await getRepository(Speech).update(speech.id, { - state: SpeechState.FAILED, - }) + await setSpeechFailure(speech.id) res.status(500).send('Error creating article speech') } }) From efce6dad94e7404d2f4f49f21c42729ff4a7ff81 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 22 Aug 2022 16:12:27 +0800 Subject: [PATCH 4/8] Stop queueing transcription after calling createArticle API --- packages/api/src/resolvers/article/index.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index f3251e6ff..77ebd901e 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -94,7 +94,6 @@ import { updatePage, } from '../../elastic/pages' import { searchHighlights } from '../../elastic/highlights' -import { enqueueTextToSpeech } from '../../utils/createTask' export type PartialArticle = Omit< Article, @@ -372,11 +371,6 @@ export const createArticleResolver = authorized< } articleToSave.id = newPageId } - - // enqueue a task to convert text to speech - const taskName = await enqueueTextToSpeech(uid, articleToSave.id) - log.info('Text to speech task name', { taskName }) - log.info( 'page created in elastic', articleToSave.id, From 463f423cab00ce61691cb64a079335873b4be534 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 22 Aug 2022 16:13:34 +0800 Subject: [PATCH 5/8] Enqueue TTS task in the get speech of article API if speech not found --- packages/api/src/routers/article_router.ts | 84 +++++++--------------- packages/api/src/utils/createTask.ts | 4 +- 2 files changed, 28 insertions(+), 60 deletions(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 31fbc2eac..b472e2099 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -18,9 +18,9 @@ import { Claims } from '../resolvers/types' import { getRepository } from '../entity/utils' import { Speech, SpeechState } from '../entity/speech' import { getPageById } from '../elastic/pages' -import { synthesizeTextToSpeech } from '../utils/textToSpeech' -import { UserPersonalization } from '../entity/user_personalization' import { generateDownloadSignedUrl } from '../utils/uploads' +import { enqueueTextToSpeech } from '../utils/createTask' +import { UserPersonalization } from '../entity/user_personalization' const logger = buildLogger('app.dispatch') @@ -72,12 +72,13 @@ export function articleRouter() { }) router.get( - '/:id/:outputFormat', + '/:id/:outputFormat/:voice?', cors(corsConfig), async (req, res) => { - const id = req.params.id + const articleId = req.params.id const outputFormat = req.params.outputFormat - if (!id || !['mp3', 'speech-marks'].includes(outputFormat)) { + const voice = req.params.voice + if (!articleId || !['mp3', 'speech-marks'].includes(outputFormat)) { return res.status(400).send('Invalid data') } const token = req.cookies?.auth || req.headers?.authorization @@ -94,8 +95,14 @@ export function articleRouter() { }, }) - const existingSpeech = await getRepository(Speech).findOneBy({ - elasticPageId: id, + const existingSpeech = await getRepository(Speech).findOne({ + where: { + elasticPageId: articleId, + voice, + }, + order: { + createdAt: 'DESC', + }, }) if (existingSpeech?.state === SpeechState.COMPLETED) { logger.info('Found existing completed speech', { @@ -107,69 +114,30 @@ export function articleRouter() { if (existingSpeech?.state === SpeechState.INITIALIZED) { logger.info('Found existing in progress speech') // retry later - return res.status(429).send('Speech is in progress') + return res.status(202).send('Speech is in progress') } - logger.debug('Text to speech request', { articleId: id }) + logger.info('Text to speech request', { articleId }) + const page = await getPageById(articleId) + if (!page) { + return res.status(404).send('Page not found') + } const userPersonalization = await getRepository( UserPersonalization ).findOneBy({ user: { id: uid }, }) - if (!userPersonalization) { - return res.status(404).send('User Personalization not found') - } - - const page = await getPageById(id) - if (!page) { - return res.status(404).send('Page not found') - } - - // const text = parseHTML(page.content).document.documentElement.innerText - // if (!text) { - // return res.status(404).send('Page has no text') - // } - // initialize state const speech = await getRepository(Speech).save({ user: { id: uid }, - elasticPageId: id, + elasticPageId: articleId, state: SpeechState.INITIALIZED, - voice: userPersonalization.speechVoice, + voice: voice || userPersonalization?.speechVoice, }) - try { - const startTime = Date.now() - const speechOutput = await synthesizeTextToSpeech({ - id, - text: page.content, - languageCode: page.language, - voice: userPersonalization.speechVoice, - textType: 'ssml', - }) - logger.info('Created speech', { - audioFileName: speechOutput.audioFileName, - speechMarksFileName: speechOutput.speechMarksFileName, - duration: Date.now() - startTime, - }) - - // update state - await getRepository(Speech).update(speech.id, { - state: SpeechState.COMPLETED, - audioFileName: speechOutput.audioFileName, - speechMarksFileName: speechOutput.speechMarksFileName, - }) - speech.audioFileName = speechOutput.audioFileName - speech.speechMarksFileName = speechOutput.speechMarksFileName - - res.redirect(await redirectUrl(speech, outputFormat)) - } catch (error) { - logger.error('Text to speech error', { error }) - // update state - await getRepository(Speech).update(speech.id, { - state: SpeechState.FAILED, - }) - res.status(500).send('Text to speech error') - } + // enqueue a task to convert text to speech + const taskName = await enqueueTextToSpeech(uid, speech.id) + logger.info('Start Text to speech task', { taskName }) + res.status(202).send('Text to speech task started') } ) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 3f6ecfbce..fc6a858cb 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -330,12 +330,12 @@ export const enqueueSyncWithIntegration = async ( export const enqueueTextToSpeech = async ( userId: string, - pageId: string + speechId: string ): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { userId, - pageId, + speechId, } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore From cfdc2362974ae07e12fc987a838c0bd61da46008 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 22 Aug 2022 16:24:39 +0800 Subject: [PATCH 6/8] Validate token and authenticate user before setting Speech state to FAILED --- packages/api/src/routers/svc/speech.ts | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/packages/api/src/routers/svc/speech.ts b/packages/api/src/routers/svc/speech.ts index 6c188d4fb..87ddfb981 100644 --- a/packages/api/src/routers/svc/speech.ts +++ b/packages/api/src/routers/svc/speech.ts @@ -21,28 +21,25 @@ export function speechServiceRouter() { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment body: req.body, }) + const token = req.query.token as string + try { + if (!(await getClaimsByToken(token))) { + logger.info('Unauthorized request', { token }) + return res.status(200).send('UNAUTHORIZED') + } + } catch (error) { + logger.error('Unauthorized request', { token, error }) + return res.status(200).send('UNAUTHORIZED') + } + const { userId, speechId } = req.body as { userId: string speechId: string } if (!userId || !speechId) { - speechId && (await setSpeechFailure(speechId)) return res.status(200).send('Invalid data') } - const token = req.query.token as string - try { - if (!(await getClaimsByToken(token))) { - logger.info('Unauthorized request', { token }) - await setSpeechFailure(speechId) - return res.status(200).send('UNAUTHORIZED') - } - } catch (error) { - logger.error('Unauthorized request', { token, error }) - await setSpeechFailure(speechId) - return res.status(200).send('UNAUTHORIZED') - } - logger.info(`Create article speech`, { body: { userId, @@ -54,6 +51,7 @@ export function speechServiceRouter() { }) const speech = await getRepository(Speech).findOneBy({ id: speechId, + user: { id: userId }, }) if (!speech) { return res.status(200).send('Speech not found') From 5cfe0a14e9586ff6529b0226290f9c686673828b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 22 Aug 2022 16:44:05 +0800 Subject: [PATCH 7/8] Authorize user to get or create speech in the API --- packages/api/src/routers/article_router.ts | 32 ++++++++++++++-------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index b472e2099..cd798ae05 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -103,18 +103,28 @@ export function articleRouter() { order: { createdAt: 'DESC', }, + relations: ['user'], }) - if (existingSpeech?.state === SpeechState.COMPLETED) { - logger.info('Found existing completed speech', { - audioUrl: existingSpeech.audioFileName, - speechMarksUrl: existingSpeech.speechMarksFileName, - }) - return res.redirect(await redirectUrl(existingSpeech, outputFormat)) - } - if (existingSpeech?.state === SpeechState.INITIALIZED) { - logger.info('Found existing in progress speech') - // retry later - return res.status(202).send('Speech is in progress') + if (existingSpeech) { + if (existingSpeech.user.id !== uid) { + logger.info('User is not allowed to access speech of the article', { + userId: uid, + articleId, + }) + return res.status(401).send({ errorCode: 'UNAUTHORIZED' }) + } + if (existingSpeech.state === SpeechState.COMPLETED) { + logger.info('Found existing completed speech', { + audioUrl: existingSpeech.audioFileName, + speechMarksUrl: existingSpeech.speechMarksFileName, + }) + return res.redirect(await redirectUrl(existingSpeech, outputFormat)) + } + if (existingSpeech.state === SpeechState.INITIALIZED) { + logger.info('Found existing in progress speech') + // retry later + return res.status(202).send('Speech is in progress') + } } logger.info('Text to speech request', { articleId }) From 45c1ba2d2556009e6728b8977663ca274885f8b3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 22 Aug 2022 16:45:10 +0800 Subject: [PATCH 8/8] Update log --- packages/api/src/routers/article_router.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index cd798ae05..43f8373fc 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -127,7 +127,7 @@ export function articleRouter() { } } - logger.info('Text to speech request', { articleId }) + logger.info('Create Text to speech task', { articleId }) const page = await getPageById(articleId) if (!page) { return res.status(404).send('Page not found')