From bca0f5ed594eac369b708080b09dff3a21385270 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 11 Aug 2022 16:58:59 +0800 Subject: [PATCH 01/28] Create Amazon Polly text to speech integration --- packages/api/package.json | 1 + packages/api/src/utils/textToSpeech.ts | 96 ++++++++++++++++++++++++++ yarn.lock | 66 +++++++++++++++++- 3 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 packages/api/src/utils/textToSpeech.ts diff --git a/packages/api/package.json b/packages/api/package.json index a51bece89..688c2ef87 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -40,6 +40,7 @@ "analytics-node": "^6.0.0", "apollo-datasource": "^3.3.1", "apollo-server-express": "^3.6.3", + "aws-sdk": "^2.1192.0", "axios": "^0.27.2", "bcryptjs": "^2.4.3", "cookie": "^0.5.0", diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts new file mode 100644 index 000000000..c348b9c4b --- /dev/null +++ b/packages/api/src/utils/textToSpeech.ts @@ -0,0 +1,96 @@ +import * as AWS from 'aws-sdk' +import { buildLogger } from './logger' +import { SynthesizeSpeechInput } from 'aws-sdk/clients/polly' +import { + generateUploadFilePathName, + generateUploadSignedUrl, + getFilePublicUrl, + uploadToSignedUrl, +} from './uploads' + +export interface TextToSpeechInput { + id: string + title: string + text: string + voice?: string + textType?: 'text' | 'ssml' + engine?: 'standard' | 'neural' +} + +export interface TextToSpeechOutput { + audioUrl: string + speechMarks: string +} + +const logger = buildLogger('app.dispatch') + +// create a new AWS Polly client +const client = new AWS.Polly() + +export const createAudio = async ( + input: TextToSpeechInput +): Promise => { + const { text, voice, textType, engine } = input + const params: SynthesizeSpeechInput = { + OutputFormat: 'mp3', + Text: text, + TextType: textType || 'text', + VoiceId: voice || 'Joanna', + Engine: engine || 'neural', + } + try { + const data = await client.synthesizeSpeech(params).promise() + return data.AudioStream as Buffer + } catch (error) { + logger.error('Unable to create audio file', { error }) + throw error + } +} + +export const createSpeechMarks = async ( + input: TextToSpeechInput +): Promise => { + const { text, voice, textType, engine } = input + const params: SynthesizeSpeechInput = { + OutputFormat: 'json', + Text: text, + TextType: textType || 'text', + VoiceId: voice || 'Joanna', + Engine: engine || 'neural', + SpeechMarkTypes: ['sentence', 'word'], + } + try { + const data = await client.synthesizeSpeech(params).promise() + return data.AudioStream as string + } catch (error) { + logger.error('Unable to create speech marks', { error }) + throw error + } +} + +export const createAudioWithSpeechMarks = async ( + input: TextToSpeechInput +): Promise => { + try { + const audio = await createAudio(input) + // upload audio to google cloud storage + logger.info('generating upload url...') + const filePathName = generateUploadFilePathName(input.id, input.title) + const contentType = 'audio/mpeg' + const uploadUrl = await generateUploadSignedUrl(filePathName, contentType) + + logger.info('start uploading...', { uploadUrl }) + await uploadToSignedUrl(uploadUrl, audio, contentType) + + // get public url for audio file + const publicUrl = getFilePublicUrl(filePathName) + const speechMarks = await createSpeechMarks(input) + return { + audioUrl: publicUrl, + speechMarks, + } + } catch (error) { + logger.error('Unable to create audio with speech marks', { error }) + throw error + } +} diff --git a/yarn.lock b/yarn.lock index f5c4a4d79..e706dd28c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9484,6 +9484,22 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== +aws-sdk@^2.1192.0: + version "2.1192.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.1192.0.tgz#13fe38ec8dae3232f17d52b370e69daa9c7a0e9c" + integrity sha512-6uzrlG1Ow3qcOnL0+et+DBTGhYgJzgNydVvos1Eg01vPc/ZhxR7roZ3epZQcPmOR0thQuzzckTq7FBO6wzZA2w== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.16.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + util "^0.12.4" + uuid "8.0.0" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -10132,7 +10148,7 @@ buffer-xor@^1.0.3: resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= -buffer@^4.3.0: +buffer@4.9.2, buffer@^4.3.0: version "4.9.2" resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== @@ -13160,6 +13176,11 @@ eventid@^2.0.0: dependencies: uuid "^8.0.0" +events@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + integrity sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw== + events@^3.0.0, events@^3.2.0: version "3.3.0" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" @@ -15336,6 +15357,11 @@ icss-utils@^5.0.0, icss-utils@^5.1.0: resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae" integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA== +ieee754@1.1.13: + version "1.1.13" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" + integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== + ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -16813,6 +16839,11 @@ jest@^27.4.5: import-local "^3.0.2" jest-cli "^27.5.1" +jmespath@0.16.0: + version "0.16.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076" + integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw== + join-component@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/join-component/-/join-component-1.1.0.tgz#b8417b750661a392bee2c2537c68b2a9d4977cd5" @@ -22066,6 +22097,11 @@ sanitize-html@^2.3.2: parse-srcset "^1.0.2" postcss "^8.0.2" +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== + sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -24416,6 +24452,14 @@ url-template@^2.0.8: resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21" integrity sha1-/FZaPMy/93MMd19WQflVV5FDnyE= +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + integrity sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ== + dependencies: + punycode "1.3.2" + querystring "0.2.0" + url@^0.11.0: version "0.11.0" resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" @@ -24530,12 +24574,17 @@ uuid-browser@^3.1.0: resolved "https://registry.yarnpkg.com/uuid-browser/-/uuid-browser-3.1.0.tgz#0f05a40aef74f9e5951e20efbf44b11871e56410" integrity sha1-DwWkCu90+eWVHiDvv0SxGHHlZBA= +uuid@8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c" + integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw== + uuid@^3.2.1, uuid@^3.3.2, uuid@^3.3.3: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== -uuid@^8.0.0, uuid@^8.3.1, uuid@^8.3.2: +uuid@^8.0.0, uuid@^8.3.2: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== @@ -25300,6 +25349,14 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + xml2js@^0.4.23: version "0.4.23" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" @@ -25313,6 +25370,11 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From dd332f5ee6a75dd3a8714608542278dc31d68e00 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 11 Aug 2022 17:37:35 +0800 Subject: [PATCH 02/28] Add tests --- packages/api/src/utils/textToSpeech.ts | 4 ++-- packages/api/test/utils/textToSpeech.test.ts | 23 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 packages/api/test/utils/textToSpeech.test.ts diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index c348b9c4b..d9cf0af7e 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -61,7 +61,7 @@ export const createSpeechMarks = async ( } try { const data = await client.synthesizeSpeech(params).promise() - return data.AudioStream as string + return (data.AudioStream as Buffer).toString() } catch (error) { logger.error('Unable to create speech marks', { error }) throw error @@ -90,7 +90,7 @@ export const createAudioWithSpeechMarks = async ( speechMarks, } } catch (error) { - logger.error('Unable to create audio with speech marks', { error }) + logger.error('Unable to create audio with speech marks', error) throw error } } diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts new file mode 100644 index 000000000..62a3e53c2 --- /dev/null +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -0,0 +1,23 @@ +import 'mocha' +import { + createAudioWithSpeechMarks, + TextToSpeechInput, +} from '../../src/utils/textToSpeech' +import { expect } from 'chai' +import { generateFakeUuid } from '../util' + +describe('textToSpeech', () => { + describe('createAudioWithSpeechMarks', () => { + it('should create an audio file with speech marks', async () => { + const input: TextToSpeechInput = { + id: generateFakeUuid(), + title: 'Hello World', + text: 'Hello World', + engine: 'standard', + } + const output = await createAudioWithSpeechMarks(input) + expect(output.audioUrl).to.be.a('string') + expect(output.speechMarks).to.be.a('string') + }) + }) +}) From 0419472c2e7aca79501d8d803ae875254c70fc5d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 11 Aug 2022 19:14:22 +0800 Subject: [PATCH 03/28] Upload audio file with public access right --- packages/api/src/routers/svc/upload.ts | 48 +++++++++++--------------- packages/api/src/utils/textToSpeech.ts | 24 ++++++------- packages/api/src/utils/uploads.ts | 24 +++++-------- 3 files changed, 38 insertions(+), 58 deletions(-) diff --git a/packages/api/src/routers/svc/upload.ts b/packages/api/src/routers/svc/upload.ts index ebc935f86..c2378225d 100644 --- a/packages/api/src/routers/svc/upload.ts +++ b/packages/api/src/routers/svc/upload.ts @@ -3,60 +3,52 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' import { readPushSubscription } from '../../datalayer/pubsub' -import { generateUploadSignedUrl, uploadToSignedUrl } from '../../utils/uploads' +import { uploadToBucket } from '../../utils/uploads' import { v4 as uuidv4 } from 'uuid' import { env } from '../../env' import { DateTime } from 'luxon' +import { buildLogger } from '../../utils/logger' + +const logger = buildLogger('app.dispatch') export function uploadServiceRouter() { const router = express.Router() router.post('/:folder', async (req, res) => { - console.log('upload data to folder', req.params.folder) + logger.info('upload data to folder', req.params.folder) const { message: msgStr, expired } = readPushSubscription(req) if (!msgStr) { - res.status(400).send('Bad Request') - return + return res.status(400).send('Bad Request') } if (expired) { - console.log('discarding expired message') - res.status(200).send('Expired') - return + logger.info('discarding expired message') + return res.status(200).send('Expired') } try { const data: { userId: string; type: string } = JSON.parse(msgStr) if (!data.userId || !data.type) { - console.log('No userId or type found in message') - res.status(400).send('Bad Request') - return + logger.info('No userId or type found in message') + return res.status(400).send('Bad Request') } - const contentType = 'application/json' - const bucketName = env.fileUpload.gcsUploadPrivateBucket + const filePath = `${req.params.folder}/${data.type}/${ + data.userId + }/${DateTime.now().toFormat('yyyy-LL-dd')}/${uuidv4()}.json` - console.log('generate upload url') - - const uploadUrl = await generateUploadSignedUrl( - `${req.params.folder}/${data.type}/${ - data.userId - }/${DateTime.now().toFormat('yyyy-LL-dd')}/${uuidv4()}.json`, - contentType, - bucketName - ) - - console.log('start uploading', uploadUrl) - - await uploadToSignedUrl( - uploadUrl, + logger.info('uploading data to', filePath) + await uploadToBucket( + filePath, Buffer.from(msgStr, 'utf8'), - contentType + { contentType: 'application/json' }, + env.fileUpload.gcsUploadPrivateBucket ) + res.status(200).send('OK') } catch (err) { - console.log('upload page data failed', err) + logger.error('upload page data failed', err) res.status(500).send(err) } }) diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index d9cf0af7e..34e2e860b 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -1,16 +1,10 @@ import * as AWS from 'aws-sdk' import { buildLogger } from './logger' import { SynthesizeSpeechInput } from 'aws-sdk/clients/polly' -import { - generateUploadFilePathName, - generateUploadSignedUrl, - getFilePublicUrl, - uploadToSignedUrl, -} from './uploads' +import { getFilePublicUrl, uploadToBucket } from './uploads' export interface TextToSpeechInput { id: string - title: string text: string voice?: string textType?: 'text' | 'ssml' @@ -74,16 +68,18 @@ export const createAudioWithSpeechMarks = async ( try { const audio = await createAudio(input) // upload audio to google cloud storage - logger.info('generating upload url...') - const filePathName = generateUploadFilePathName(input.id, input.title) - const contentType = 'audio/mpeg' - const uploadUrl = await generateUploadSignedUrl(filePathName, contentType) + const filePath = `speech/${input.id}.mp3` - logger.info('start uploading...', { uploadUrl }) - await uploadToSignedUrl(uploadUrl, audio, contentType) + logger.info('start uploading...', { filePath }) + await uploadToBucket(filePath, audio, { + contentType: 'audio/mpeg', + public: true, + }) // get public url for audio file - const publicUrl = getFilePublicUrl(filePathName) + const publicUrl = getFilePublicUrl(filePath) + logger.info('upload complete', { publicUrl }) + const speechMarks = await createSpeechMarks(input) return { audioUrl: publicUrl, diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 6c4492479..351799a34 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -2,7 +2,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { env } from '../env' import { GetSignedUrlConfig, Storage } from '@google-cloud/storage' -import axios from 'axios' /* On GAE/Prod, we shall rely on default app engine service account credentials. * Two changes needed: 1) add default service account to our uploads GCS Bucket @@ -102,21 +101,14 @@ export const generateUploadFilePathName = ( return `u/${id}/${fileName}` } -export const uploadToSignedUrl = async ( - uploadUrl: string, +export const uploadToBucket = async ( + filePath: string, data: Buffer, - contentType: string + options?: { contentType?: string; public?: boolean }, + selectedBucket?: string ): Promise => { - // if (env.dev.isLocal) { - // return - // } - - await axios.put(uploadUrl, data, { - headers: { - 'Content-Type': contentType, - }, - maxBodyLength: 1000000000, - maxContentLength: 100000000, - timeout: 30000, - }) + await storage + .bucket(selectedBucket || bucketName) + .file(filePath) + .save(data, options) } From 4b42d013cafd0f597a02c4ac571138bb8568f84d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 12 Aug 2022 11:49:11 +0800 Subject: [PATCH 04/28] Convert text to OGG file --- packages/api/src/utils/textToSpeech.ts | 8 ++++---- packages/api/test/utils/textToSpeech.test.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index 34e2e860b..5d3f40eca 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -26,7 +26,7 @@ export const createAudio = async ( ): Promise => { const { text, voice, textType, engine } = input const params: SynthesizeSpeechInput = { - OutputFormat: 'mp3', + OutputFormat: 'ogg_vorbis', Text: text, TextType: textType || 'text', VoiceId: voice || 'Joanna', @@ -51,7 +51,7 @@ export const createSpeechMarks = async ( TextType: textType || 'text', VoiceId: voice || 'Joanna', Engine: engine || 'neural', - SpeechMarkTypes: ['sentence', 'word'], + SpeechMarkTypes: ['sentence'], } try { const data = await client.synthesizeSpeech(params).promise() @@ -68,11 +68,11 @@ export const createAudioWithSpeechMarks = async ( try { const audio = await createAudio(input) // upload audio to google cloud storage - const filePath = `speech/${input.id}.mp3` + const filePath = `speech/${input.id}.ogg` logger.info('start uploading...', { filePath }) await uploadToBucket(filePath, audio, { - contentType: 'audio/mpeg', + contentType: 'audio/ogg', public: true, }) diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts index 62a3e53c2..0dc698c33 100644 --- a/packages/api/test/utils/textToSpeech.test.ts +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -11,9 +11,9 @@ describe('textToSpeech', () => { it('should create an audio file with speech marks', async () => { const input: TextToSpeechInput = { id: generateFakeUuid(), - title: 'Hello World', - text: 'Hello World', + text: 'The rumor mill suggests that Google may be looking to kill off its game streaming platform, Stadia, for good before the end of the year.', engine: 'standard', + textType: 'ssml', } const output = await createAudioWithSpeechMarks(input) expect(output.audioUrl).to.be.a('string') From 94f9dd9e6e55779ca2c744bb7f2b6534f9a37181 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 12 Aug 2022 17:53:41 +0800 Subject: [PATCH 05/28] Enqueue text to speech tasks --- packages/api/src/entity/speech.ts | 38 +++++++++++++ .../api/src/entity/user_personalization.ts | 53 +++++++++++++++++++ packages/api/src/util.ts | 3 ++ packages/api/src/utils/createTask.ts | 31 +++++++++++ packages/api/src/utils/textToSpeech.ts | 11 ++-- packages/db/migrations/0093.do.speech.sql | 28 ++++++++++ packages/db/migrations/0093.undo.speech.sql | 14 +++++ 7 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 packages/api/src/entity/speech.ts create mode 100644 packages/api/src/entity/user_personalization.ts create mode 100755 packages/db/migrations/0093.do.speech.sql create mode 100755 packages/db/migrations/0093.undo.speech.sql diff --git a/packages/api/src/entity/speech.ts b/packages/api/src/entity/speech.ts new file mode 100644 index 000000000..27e189d6b --- /dev/null +++ b/packages/api/src/entity/speech.ts @@ -0,0 +1,38 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' +import { User } from './user' + +@Entity({ name: 'speech' }) +export class Speech { + @PrimaryGeneratedColumn('uuid') + id!: string + + @ManyToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'user_id' }) + user!: User + + @Column('text') + elasticPageId!: string + + @Column('text') + audioUrl!: string + + @Column('text') + speechMarks!: string + + @Column('text') + voiceId!: string + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + createdAt!: Date + + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + updatedAt!: Date +} diff --git a/packages/api/src/entity/user_personalization.ts b/packages/api/src/entity/user_personalization.ts new file mode 100644 index 000000000..8044ef1a7 --- /dev/null +++ b/packages/api/src/entity/user_personalization.ts @@ -0,0 +1,53 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + OneToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' +import { User } from './user' + +@Entity({ name: 'user_personalization' }) +export class UserPersonalization { + @PrimaryGeneratedColumn('uuid') + id!: string + + @OneToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'user_id' }) + user!: User + + @Column('text', { nullable: true }) + fontFamily?: string + + @Column('integer', { nullable: true }) + fontSize?: number + + @Column('text', { nullable: true }) + margin?: number + + @Column('text', { nullable: true }) + theme?: string + + @Column('text', { nullable: true }) + libraryLayoutType?: string + + @Column('text', { nullable: true }) + librarySortOrder?: string + + @Column('text', { nullable: true }) + speechVoice?: string + + @Column('integer', { nullable: true }) + speechRate?: number + + @Column('integer', { nullable: true }) + speechVolume?: number + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + createdAt!: Date + + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + updatedAt!: Date +} diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index f43a9a8b2..3cd8f7edb 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -63,6 +63,7 @@ interface BackendEnv { contentFetchGCFUrl: string reminderTaskHanderUrl: string integrationTaskHandlerUrl: string + textToSpeechTaskHandlerUrl: string } fileUpload: { gcsUploadBucket: string @@ -138,6 +139,7 @@ const nullableEnvVars = [ 'SENDGRID_INSTALLATION_TEMPLATE_ID', 'READWISE_API_URL', 'INTEGRATION_TASK_HANDLER_URL', + 'TEXT_TO_SPEECH_TASK_HANDLER_URL', ] // Allow some vars to be null/empty /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */ @@ -221,6 +223,7 @@ export function getEnv(): BackendEnv { contentFetchGCFUrl: parse('CONTENT_FETCH_GCF_URL'), reminderTaskHanderUrl: parse('REMINDER_TASK_HANDLER_URL'), integrationTaskHandlerUrl: parse('INTEGRATION_TASK_HANDLER_URL'), + textToSpeechTaskHandlerUrl: parse('TEXT_TO_SPEECH_TASK_HANDLER_URL'), } const imageProxy = { url: parse('IMAGE_PROXY_URL'), diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 3dec00231..c69d5a986 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -325,4 +325,35 @@ export const enqueueSyncWithIntegration = async ( return createdTasks[0].name } +export const enqueueTextToSpeech = async ( + userId: string, + pageId: string +): Promise => { + const { GOOGLE_CLOUD_PROJECT } = process.env + const payload = { + userId, + pageId, + } + + // If there is no Google Cloud Project Id exposed, it means that we are in local environment + if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { + return nanoid() + } + + const createdTasks = await createHttpTaskWithToken({ + project: GOOGLE_CLOUD_PROJECT, + payload, + taskHandlerUrl: env.queue.textToSpeechTaskHandlerUrl, + }) + + if (!createdTasks || !createdTasks[0].name) { + logger.error(`Unable to get the name of the task`, { + payload, + createdTasks, + }) + throw new CreateTaskError(`Unable to get the name of the task`) + } + return createdTasks[0].name +} + export default createHttpTaskWithToken diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index 5d3f40eca..7f855cab7 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -1,7 +1,7 @@ import * as AWS from 'aws-sdk' import { buildLogger } from './logger' -import { SynthesizeSpeechInput } from 'aws-sdk/clients/polly' import { getFilePublicUrl, uploadToBucket } from './uploads' +import { SynthesizeSpeechInput } from 'aws-sdk/clients/polly' export interface TextToSpeechInput { id: string @@ -9,6 +9,7 @@ export interface TextToSpeechInput { voice?: string textType?: 'text' | 'ssml' engine?: 'standard' | 'neural' + languageCode?: string } export interface TextToSpeechOutput { @@ -24,13 +25,14 @@ const client = new AWS.Polly() export const createAudio = async ( input: TextToSpeechInput ): Promise => { - const { text, voice, textType, engine } = input + const { text, voice, textType, engine, languageCode } = input const params: SynthesizeSpeechInput = { OutputFormat: 'ogg_vorbis', Text: text, TextType: textType || 'text', VoiceId: voice || 'Joanna', Engine: engine || 'neural', + LanguageCode: languageCode || 'en-US', } try { const data = await client.synthesizeSpeech(params).promise() @@ -44,14 +46,15 @@ export const createAudio = async ( export const createSpeechMarks = async ( input: TextToSpeechInput ): Promise => { - const { text, voice, textType, engine } = input + const { text, voice, textType, engine, languageCode } = input const params: SynthesizeSpeechInput = { OutputFormat: 'json', Text: text, TextType: textType || 'text', VoiceId: voice || 'Joanna', Engine: engine || 'neural', - SpeechMarkTypes: ['sentence'], + SpeechMarkTypes: ['word'], + LanguageCode: languageCode || 'en-US', } try { const data = await client.synthesizeSpeech(params).promise() diff --git a/packages/db/migrations/0093.do.speech.sql b/packages/db/migrations/0093.do.speech.sql new file mode 100755 index 000000000..39a50aadf --- /dev/null +++ b/packages/db/migrations/0093.do.speech.sql @@ -0,0 +1,28 @@ +-- Type: DO +-- Name: speech +-- Description: Add speech table containing text to speech audio_url and speech_marks + +BEGIN; + +CREATE TABLE omnivore.speech ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + user_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE, + elastic_page_id TEXT NOT NULL, + voice text, + audio_url text NOT NULL, + speech_marks text NOT NULL, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp +); + +CREATE TRIGGER speech_modtime BEFORE UPDATE ON omnivore.speech FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +-- No permission to delete on the speech table, only superuser can delete. +GRANT SELECT, INSERT, UPDATE ON omnivore.speech TO omnivore_user; + +ALTER TABLE omnivore.user_personalization + ADD COLUMN speech_voice TEXT, + ADD COLUMN speech_rate INTEGER, + ADD COLUMN speech_volume INTEGER; + +COMMIT; diff --git a/packages/db/migrations/0093.undo.speech.sql b/packages/db/migrations/0093.undo.speech.sql new file mode 100755 index 000000000..9457f5bd8 --- /dev/null +++ b/packages/db/migrations/0093.undo.speech.sql @@ -0,0 +1,14 @@ +-- Type: UNDO +-- Name: speech +-- Description: Add speech table containing text to speech audio_url and speech_marks + +BEGIN; + +DROP TABLE IF EXISTS omnivore.speech; + +ALTER TABLE omnivore.user_personalization + DROP COLUMN IF EXISTS speech_voice, + DROP COLUMN IF EXISTS speech_rate, + DROP COLUMN IF EXISTS speech_volume; + +COMMIT; From 77570aa5ab7630e56aa1231334a8daed2e2938ca Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 15 Aug 2022 19:04:05 +0800 Subject: [PATCH 06/28] Synthesize text to speech with azure API --- packages/api/package.json | 1 + packages/api/src/entity/speech.ts | 2 +- packages/api/src/entity/user.ts | 7 + packages/api/src/routers/svc/textToSpeech.ts | 57 +++++ packages/api/src/util.ts | 12 + packages/api/src/utils/textToSpeech.ts | 253 ++++++++++++++----- packages/api/src/utils/uploads.ts | 6 +- packages/api/test/utils/textToSpeech.test.ts | 75 +++++- yarn.lock | 136 +++++++++- 9 files changed, 468 insertions(+), 81 deletions(-) create mode 100644 packages/api/src/routers/svc/textToSpeech.ts diff --git a/packages/api/package.json b/packages/api/package.json index 688c2ef87..6a1fef691 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -68,6 +68,7 @@ "knex-stringcase": "^1.4.2", "linkedom": "^0.14.9", "luxon": "^2.3.1", + "microsoft-cognitiveservices-speech-sdk": "^1.22.0", "nanoid": "^3.1.25", "nodemailer": "^6.7.3", "normalize-url": "^6.1.0", diff --git a/packages/api/src/entity/speech.ts b/packages/api/src/entity/speech.ts index 27e189d6b..0f61d7dd7 100644 --- a/packages/api/src/entity/speech.ts +++ b/packages/api/src/entity/speech.ts @@ -28,7 +28,7 @@ export class Speech { speechMarks!: string @Column('text') - voiceId!: string + voice!: string @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) createdAt!: Date diff --git a/packages/api/src/entity/user.ts b/packages/api/src/entity/user.ts index 58c4a6975..b03c36e11 100644 --- a/packages/api/src/entity/user.ts +++ b/packages/api/src/entity/user.ts @@ -12,6 +12,7 @@ import { NewsletterEmail } from './newsletter_email' import { Profile } from './profile' import { Label } from './label' import { Subscription } from './subscription' +import { UserPersonalization } from './user_personalization' @Entity() export class User { @@ -53,4 +54,10 @@ export class User { @Column({ type: 'enum', enum: StatusType }) status!: StatusType + + @OneToOne( + () => UserPersonalization, + (userPersonalization) => userPersonalization.user + ) + userPersonalization!: UserPersonalization } diff --git a/packages/api/src/routers/svc/textToSpeech.ts b/packages/api/src/routers/svc/textToSpeech.ts new file mode 100644 index 000000000..42ec982e4 --- /dev/null +++ b/packages/api/src/routers/svc/textToSpeech.ts @@ -0,0 +1,57 @@ +import express from 'express' +import cors from 'cors' +import { corsConfig } from '../../utils/corsConfig' +import { getRepository } from '../../entity/utils' +import { User } from '../../entity/user' +import { getPageById } from '../../elastic/pages' +import { htmlToSsml, synthesizeTextToSpeech } from '../../utils/textToSpeech' +import { Speech } from '../../entity/speech' + +export function textToSpeechServiceRouter() { + const router = express.Router() + + router.options('/', cors({ ...corsConfig, maxAge: 600 })) + router.post('/', async (req, res) => { + const { userId, pageId } = req.body as { + userId: string + pageId: string + } + + if (!userId || !pageId) { + return res.status(400).send({ errorCode: 'BAD_DATA' }) + } + + const user = await getRepository(User).findOne({ + where: { id: userId }, + relations: ['user_personalization'], + }) + if (!user) { + return res.status(400).send({ errorCode: 'BAD_DATA' }) + } + + const page = await getPageById(pageId) + if (!page) { + return res.status(400).send({ errorCode: 'BAD_DATA' }) + } + + const html = page.content + const language = page.language + const voice = user.userPersonalization.speechVoice || 'en-US_AllisonVoice' + const rate = user.userPersonalization.speechRate || 100 + const volume = user.userPersonalization.speechVolume || 100 + const ssml = htmlToSsml(html, language, voice, rate, volume) + + const audioAndSpeechMarks = await synthesizeTextToSpeech({ + id: pageId, + text: ssml, + }) + + await getRepository(Speech).save({ + elasticPageId: pageId, + audioUrl: audioAndSpeechMarks.audioUrl, + speechMarks: JSON.stringify(audioAndSpeechMarks.speechMarks), + id: pageId, + user, + }) + }) +} diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 3cd8f7edb..1ca344d02 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -89,6 +89,10 @@ interface BackendEnv { readwise: { apiUrl: string } + azure: { + speechKey: string + speechRegion: string + } } /*** @@ -140,6 +144,8 @@ const nullableEnvVars = [ 'READWISE_API_URL', 'INTEGRATION_TASK_HANDLER_URL', 'TEXT_TO_SPEECH_TASK_HANDLER_URL', + 'AZURE_SPEECH_KEY', + 'AZURE_SPEECH_REGION', ] // Allow some vars to be null/empty /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */ @@ -259,6 +265,11 @@ export function getEnv(): BackendEnv { apiUrl: parse('READWISE_API_URL'), } + const azure = { + speechKey: parse('AZURE_SPEECH_KEY'), + speechRegion: parse('AZURE_SPEECH_REGION'), + } + return { pg, client, @@ -277,6 +288,7 @@ export function getEnv(): BackendEnv { sender, sendgrid, readwise, + azure, } } diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index 7f855cab7..9a1a3ad1a 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -1,7 +1,7 @@ -import * as AWS from 'aws-sdk' import { buildLogger } from './logger' -import { getFilePublicUrl, uploadToBucket } from './uploads' -import { SynthesizeSpeechInput } from 'aws-sdk/clients/polly' +import { createGCSFile, getFilePublicUrl } from './uploads' +import * as sdk from 'microsoft-cognitiveservices-speech-sdk' +import { env } from '../env' export interface TextToSpeechInput { id: string @@ -14,82 +14,197 @@ export interface TextToSpeechInput { export interface TextToSpeechOutput { audioUrl: string - speechMarks: string + speechMarks: SpeechMark[] +} + +export interface SpeechMark { + time: number + start: number + length: number + word: string } const logger = buildLogger('app.dispatch') -// create a new AWS Polly client -const client = new AWS.Polly() +// // create a new AWS Polly client +// const client = new AWS.Polly() -export const createAudio = async ( - input: TextToSpeechInput -): Promise => { - const { text, voice, textType, engine, languageCode } = input - const params: SynthesizeSpeechInput = { - OutputFormat: 'ogg_vorbis', - Text: text, - TextType: textType || 'text', - VoiceId: voice || 'Joanna', - Engine: engine || 'neural', - LanguageCode: languageCode || 'en-US', - } - try { - const data = await client.synthesizeSpeech(params).promise() - return data.AudioStream as Buffer - } catch (error) { - logger.error('Unable to create audio file', { error }) - throw error - } -} - -export const createSpeechMarks = async ( - input: TextToSpeechInput -): Promise => { - const { text, voice, textType, engine, languageCode } = input - const params: SynthesizeSpeechInput = { - OutputFormat: 'json', - Text: text, - TextType: textType || 'text', - VoiceId: voice || 'Joanna', - Engine: engine || 'neural', - SpeechMarkTypes: ['word'], - LanguageCode: languageCode || 'en-US', - } - try { - const data = await client.synthesizeSpeech(params).promise() - return (data.AudioStream as Buffer).toString() - } catch (error) { - logger.error('Unable to create speech marks', { error }) - throw error - } -} - -export const createAudioWithSpeechMarks = async ( +export const synthesizeTextToSpeech = async ( input: TextToSpeechInput ): Promise => { - try { - const audio = await createAudio(input) - // upload audio to google cloud storage - const filePath = `speech/${input.id}.ogg` + const audioFile = `speech/${input.id}.mp3` + const gcsFile = createGCSFile(audioFile) + const writeStream = gcsFile.createWriteStream({ + public: true, + resumable: true, + }) + const speechConfig = sdk.SpeechConfig.fromSubscription( + env.azure.speechKey, + env.azure.speechRegion + ) + speechConfig.speechSynthesisLanguage = input.languageCode || 'en-US' + speechConfig.speechSynthesisVoiceName = input.voice || 'en-US-JennyNeural' + speechConfig.speechSynthesisOutputFormat = + sdk.SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 - logger.info('start uploading...', { filePath }) - await uploadToBucket(filePath, audio, { - contentType: 'audio/ogg', - public: true, - }) + // Create the speech synthesizer. + const synthesizer = new sdk.SpeechSynthesizer(speechConfig) + const speechMarks: SpeechMark[] = [] - // get public url for audio file - const publicUrl = getFilePublicUrl(filePath) - logger.info('upload complete', { publicUrl }) + synthesizer.synthesizing = function (s, e) { + logger.debug(`synthesizing ${e.result.audioData.byteLength} bytes`) + // convert arrayBuffer to stream and write to gcs file + writeStream.write(Buffer.from(e.result.audioData)) + } - const speechMarks = await createSpeechMarks(input) - return { - audioUrl: publicUrl, - speechMarks, + // The event synthesis completed signals that the synthesis is completed. + synthesizer.synthesisCompleted = function (s, e) { + logger.info( + '(synthesized) Reason: ' + + sdk.ResultReason[e.result.reason] + + ' Audio length: ' + + e.result.audioData.byteLength + ) + } + + // The synthesis started event signals that the synthesis is started. + synthesizer.synthesisStarted = function (s, e) { + logger.info('(synthesis started)') + } + + // The event signals that the service has stopped processing speech. + // This can happen when an error is encountered. + synthesizer.SynthesisCanceled = function (s, e) { + const cancellationDetails = sdk.CancellationDetails.fromResult(e.result) + let str = + '(cancel) Reason: ' + sdk.CancellationReason[cancellationDetails.reason] + if (cancellationDetails.reason === sdk.CancellationReason.Error) { + str += ': ' + e.result.errorDetails } - } catch (error) { - logger.error('Unable to create audio with speech marks', error) - throw error + logger.info(str) + } + + synthesizer.wordBoundary = function (s, e) { + speechMarks.push({ + word: e.text, + time: e.audioOffset, + start: e.textOffset, + length: e.wordLength, + }) + } + + const speakTextAsyncPromise = ( + text: string + ): Promise => { + return new Promise((resolve, reject) => { + synthesizer.speakTextAsync( + text, + (result) => { + resolve(result) + }, + (error) => { + synthesizer.close() + reject(error) + } + ) + }) + } + // slice the text into chunks of 1,000 characters + const textChunks = input.text.match(/.{1,1000}/g) || [] + for (const textChunk of textChunks) { + console.debug(`synthesizing ${textChunk}`) + await speakTextAsyncPromise(textChunk) + } + writeStream.end() + synthesizer.close() + + logger.debug(`audio file: ${audioFile}`) + logger.debug(`speechMarks: ${speechMarks}`) + + return { + audioUrl: getFilePublicUrl(audioFile), + speechMarks, } } + +// export const createAudio = async ( +// input: TextToSpeechInput +// ): Promise => { +// const { text, voice, textType, engine, languageCode } = input +// const params: SynthesizeSpeechInput = { +// OutputFormat: 'ogg_vorbis', +// Text: text, +// TextType: textType || 'text', +// VoiceId: voice || 'Joanna', +// Engine: engine || 'neural', +// LanguageCode: languageCode || 'en-US', +// } +// try { +// const data = await client.synthesizeSpeech(params).promise() +// return data.AudioStream as Buffer +// } catch (error) { +// logger.error('Unable to create audio file', { error }) +// throw error +// } +// } + +// export const createSpeechMarks = async ( +// input: TextToSpeechInput +// ): Promise => { +// const { text, voice, textType, engine, languageCode } = input +// const params: SynthesizeSpeechInput = { +// OutputFormat: 'json', +// Text: text, +// TextType: textType || 'text', +// VoiceId: voice || 'Joanna', +// Engine: engine || 'neural', +// SpeechMarkTypes: ['word'], +// LanguageCode: languageCode || 'en-US', +// } +// try { +// const data = await client.synthesizeSpeech(params).promise() +// return (data.AudioStream as Buffer).toString() +// } catch (error) { +// logger.error('Unable to create speech marks', { error }) +// throw error +// } +// } +// +// export const createAudioWithSpeechMarks = async ( +// input: TextToSpeechInput +// ): Promise => { +// try { +// const audio = await createAudio(input) +// // upload audio to google cloud storage +// const filePath = `speech/${input.id}.ogg` +// +// logger.info('start uploading...', { filePath }) +// await uploadToBucket(filePath, audio, { +// contentType: 'audio/ogg', +// public: true, +// }) +// +// // get public url for audio file +// const publicUrl = getFilePublicUrl(filePath) +// logger.info('upload complete', { publicUrl }) +// +// const speechMarks = await createSpeechMarks(input) +// return { +// audioUrl: publicUrl, +// speechMarks, +// } +// } catch (error) { +// logger.error('Unable to create audio with speech marks', error) +// throw error +// } +// } + +export const htmlToSsml = ( + html: string, + language = 'en-US', + voice = 'en-US-JennyNeural', + rate = 100, + volume = 100 +): string => { + return `${html}` +} diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 351799a34..8e631f3e1 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { env } from '../env' -import { GetSignedUrlConfig, Storage } from '@google-cloud/storage' +import { File, GetSignedUrlConfig, Storage } from '@google-cloud/storage' /* On GAE/Prod, we shall rely on default app engine service account credentials. * Two changes needed: 1) add default service account to our uploads GCS Bucket @@ -112,3 +112,7 @@ export const uploadToBucket = async ( .file(filePath) .save(data, options) } + +export const createGCSFile = (filename: string): File => { + return storage.bucket(bucketName).file(filename) +} diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts index 0dc698c33..4909192a5 100644 --- a/packages/api/test/utils/textToSpeech.test.ts +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -1,6 +1,6 @@ import 'mocha' import { - createAudioWithSpeechMarks, + synthesizeTextToSpeech, TextToSpeechInput, } from '../../src/utils/textToSpeech' import { expect } from 'chai' @@ -11,13 +11,76 @@ describe('textToSpeech', () => { it('should create an audio file with speech marks', async () => { const input: TextToSpeechInput = { id: generateFakeUuid(), - text: 'The rumor mill suggests that Google may be looking to kill off its game streaming platform, Stadia, for good before the end of the year.', - engine: 'standard', - textType: 'ssml', + text: + 'MIT spinout Quaise Energy is working to create geothermal wells made from the deepest holes in the world.\n' + + '\n' + + 'Publication Date:\n' + + '\n' + + 'June 28, 2022\n' + + '\n' + + "A graphic depicting the heat at the earth's core\n" + + 'Caption:\n' + + '\n' + + 'Quaise Energy wants to repurpose coal and gas plants into deep geothermal wells by using X-rays to melt rock.\n' + + '\n' + + 'Credits:\n' + + '\n' + + 'Image: Collage by MIT News with images courtesy of Quaise Energy\n' + + '\n' + + 'There’s an abandoned coal power plant in upstate New York that most people regard as a useless relic. But MIT’s Paul Woskov sees things differently.\n' + + '\n' + + 'Woskov, a research engineer in MIT’s Plasma Science and Fusion Center, notes the plant’s power turbine is still intact and the transmission lines still run to the grid. Using an approach he’s been working on for the last 14 years, he’s hoping it will be back online, completely carbon-free, within the decade.\n' + + 'In fact, Quaise Energy, the company commercializing Woskov’s work, believes if it can retrofit one power plant, the same process will work on virtually every coal and gas power plant in the world.\n' + + '\n' + + 'Quaise is hoping to accomplish those lofty goals by tapping into the energy source below our feet. The company plans to vaporize enough rock to create the world’s deepest holes and harvest geothermal energy at a scale that could satisfy human energy consumption for millions of years. They haven’t yet solved all the related engineering challenges, but Quaise’s founders have set an ambitious timeline to begin harvesting energy from a pilot well by 2026.\n' + + '\n' + + 'The plan would be easier to dismiss as unrealistic if it were based on a new and unproven technology. But Quaise’s drilling systems center around a microwave-emitting device called a gyrotron that has been used in research and manufacturing for decades.\n' + + '\n' + + '“This will happen quickly once we solve the immediate engineering problems of transmitting a clean beam and having it operate at a high energy density without breakdown,” explains Woskov, who is not formally affiliated with Quaise but serves as an advisor. “It’ll go fast because the underlying technology, gyrotrons, are commercially available. You could place an order with a company and have a system delivered right now — granted, these beam sources have never been used 24/7, but they are engineered to be operational for long time periods. In five or six years, I think we’ll have a plant running if we solve these engineering problems. I’m very optimistic.”\n' + + 'Woskov and many other researchers have been using gyrotrons to heat material in nuclear fusion experiments for decades. It wasn’t until 2008, however, after the MIT Energy Initiative (MITEI) published a request for proposals on new geothermal drilling technologies, that Woskov thought of using gyrotrons for a new application.\n' + + '\n' + + '“[Gyrotrons] haven’t been well-publicized in the general science community, but those of us in fusion research understood they were very powerful beam sources — like lasers, but in a different frequency range,” Woskov says. “I thought, why not direct these high-power beams, instead of into fusion plasma, down into rock and vaporize the hole?”\n' + + '\n' + + 'As power from other renewable energy sources has exploded in recent decades, geothermal energy has plateaued, mainly because geothermal plants only exist in places where natural conditions allow for energy extraction at relatively shallow depths of up to 400 feet beneath the Earth’s surface. At a certain point, conventional drilling becomes impractical because deeper crust is both hotter and harder, which wears down mechanical drill bits.\n' + + '\n' + + 'Woskov’s idea to use gyrotron beams to vaporize rock sent him on a research journey that has never really stopped. With some funding from MITEI, he began running tests, quickly filling his office with small rock formations he’d blasted with millimeter waves from a small gyrotron in MIT’s Plasma Science and Fusion Center.\n' + + '\n' + + 'Paul Woskov with blasted rock samples\n' + + 'Woskov displaying samples in his lab in 2016.\n' + + '\n' + + 'Photo: Paul Rivenberg\n' + + '\n' + + 'Around 2018, Woskov’s rocks got the attention of Carlos Araque ’01, SM ’02, who had spent his career in the oil and gas industry and was the technical director of MIT’s investment fund The Engine at the time.\n' + + '\n' + + 'That year, Araque and Matt Houde, who’d been working with geothermal company AltaRock Energy, founded Quaise. Quaise was soon given a grant by the Department of Energy to scale up Woskov’s experiments using a larger gyrotron.\n' + + '\n' + + 'With the larger machine, the team hopes to vaporize a hole 10 times the depth of Woskov’s lab experiments. That is expected to be accomplished by the end of this year. After that, the team will vaporize a hole 10 times the depth of the previous one — what Houde calls a 100-to-1 hole.\n' + + '“That’s something [the DOE] is particularly interested in, because they want to address the challenges posed by material removal over those greater lengths — in other words, can we show we’re fully flushing out the rock vapors?” Houde explains. “We believe the 100-to-1 test also gives us the confidence to go out and mobilize a prototype gyrotron drilling rig in the field for the first field demonstrations.”\n' + + '\n' + + 'Tests on the 100-to-1 hole are expected to be completed sometime next year. Quaise is also hoping to begin vaporizing rock in field tests late next year. The short timeline reflects the progress Woskov has already made in his lab.\n' + + '\n' + + "Although more engineering research is needed, ultimately, the team expects to be able to drill and operate these geothermal wells safely. “We believe, because of Paul’s work at MIT over the past decade, that most if not all of the core physics questions have been answered and addressed,” Houde says. “It’s really engineering challenges we have to answer, which doesn’t mean they’re easy to solve, but we’re not working against the laws of physics, to which there is no answer. It's more a matter of overcoming some of the more technical and cost considerations to making this work at a large scale.”\n" + + '\n' + + 'The company plans to begin harvesting energy from pilot geothermal wells that reach rock temperatures at up to 500 C by 2026. From there, the team hopes to begin repurposing coal and natural gas plants using its system.\n' + + '\n' + + '“We believe, if we can drill down to 20 kilometers, we can access these super-hot temperatures in greater than 90 percent of locations across the globe,” Houde says.\n' + + '\n' + + 'Quaise’s work with the DOE is addressing what it sees as the biggest remaining questions about drilling holes of unprecedented depth and pressure, such as material removal and determining the best casing to keep the hole stable and open. For the latter problem of well stability, Houde believes additional computer modeling is needed and expects to complete that modeling by the end of 2024.\n' + + '\n' + + 'By drilling the holes at existing power plants, Quaise will be able to move faster than if it had to get permits to build new plants and transmission lines. And by making their millimeter-wave drilling equipment compatible with the existing global fleet of drilling rigs, it will also allow the company to tap into the oil and gas industry’s global workforce.\n' + + '\n' + + '“At these high temperatures [we’re accessing], we’re producing steam very close to, if not exceeding, the temperature that today’s coal and gas-fired power plants operate at,” Houde says. “So, we can go to existing power plants and say, ‘We can replace 95 to 100 percent of your coal use by developing a geothermal field and producing steam from the Earth, at the same temperature you’re burning coal to run your turbine, directly replacing carbon emissions.”\n' + + '\n' + + 'Transforming the world’s energy systems in such a short timeframe is something the founders see as critical to help avoid the most catastrophic global warming scenarios.\n' + + '\n' + + '“There have been tremendous gains in renewables over the last decade, but the big picture today is we’re not going nearly fast enough to hit the milestones we need for limiting the worst impacts of climate change,” Houde says. “[Deep geothermal] is a power resource that can scale anywhere and has the ability to tap into a large workforce in the energy industry to readily repackage their skills for a totally carbon free energy source.”\n' + + '\n' + + 'Related Topics\n' + + 'Related Articles', } - const output = await createAudioWithSpeechMarks(input) + const output = await synthesizeTextToSpeech(input) expect(output.audioUrl).to.be.a('string') - expect(output.speechMarks).to.be.a('string') + expect(output.speechMarks).to.be.a('array') }) }) }) diff --git a/yarn.lock b/yarn.lock index e706dd28c..c73f437a3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8739,6 +8739,11 @@ addressparser@^1.0.1: resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746" integrity sha512-aQX7AISOMM7HFE0iZ3+YnD07oIeJqWGVnJ+ZIKaBZAk03ftmVYVqsGas/rbXKR21n4D/hKCSHypvcyOkds/xzg== +agent-base@5: + version "5.1.1" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-5.1.1.tgz#e8fb3f242959db44d63be665db7a8e739537a32c" + integrity sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g== + agent-base@6: version "6.0.1" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.1.tgz#808007e4e5867decb0ab6ab2f928fbdb5a596db4" @@ -8746,7 +8751,7 @@ agent-base@6: dependencies: debug "4" -agent-base@^6.0.2: +agent-base@^6.0.1, agent-base@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== @@ -9360,7 +9365,21 @@ asap@^2.0.0, asap@~2.0.3: resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= -asn1.js@^5.2.0: +asn1.js-rfc2560@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/asn1.js-rfc2560/-/asn1.js-rfc2560-5.0.1.tgz#cff99b903e714756b29503ad49de01c72f131e60" + integrity sha512-1PrVg6kuBziDN3PGFmRk3QrjpKvP9h/Hv5yMrFZvC1kpzP6dQRzf5BpKstANqHBkaOUmTpakJWhicTATOA/SbA== + dependencies: + asn1.js-rfc5280 "^3.0.0" + +asn1.js-rfc5280@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/asn1.js-rfc5280/-/asn1.js-rfc5280-3.0.0.tgz#94e60498d5d4984b842d1a825485837574ccc902" + integrity sha512-Y2LZPOWeZ6qehv698ZgOGGCZXBQShObWnGthTrIFlIQjuV1gg2B8QOhWFRExq/MR1VnPpIIe7P9vX2vElxv+Pg== + dependencies: + asn1.js "^5.0.0" + +asn1.js@^5.0.0, asn1.js@^5.2.0: version "5.4.1" resolved "https://registry.yarnpkg.com/asn1.js/-/asn1.js-5.4.1.tgz#11a980b84ebb91781ce35b0fdc2ee294e3783f07" integrity sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA== @@ -9422,6 +9441,19 @@ astral-regex@^2.0.0: resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== +async-disk-cache@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/async-disk-cache/-/async-disk-cache-2.1.0.tgz#e0f37b187ed8c41a5991518a9556d206ae2843a2" + integrity sha512-iH+boep2xivfD9wMaZWkywYIURSmsL96d6MoqrC94BnGSvXE4Quf8hnJiHGFYhw/nLeIa1XyRaf4vvcvkwAefg== + dependencies: + debug "^4.1.1" + heimdalljs "^0.2.3" + istextorbinary "^2.5.1" + mkdirp "^0.5.0" + rimraf "^3.0.0" + rsvp "^4.8.5" + username-sync "^1.0.2" + async-each@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.3.tgz#b727dbf87d7651602f06f4d4ac387f47d91b0cbf" @@ -9845,6 +9877,15 @@ benchmark@^2.1.4: lodash "^4.17.4" platform "^1.3.3" +bent@^7.3.12: + version "7.3.12" + resolved "https://registry.yarnpkg.com/bent/-/bent-7.3.12.tgz#e0a2775d4425e7674c64b78b242af4f49da6b035" + integrity sha512-T3yrKnVGB63zRuoco/7Ybl7BwwGZR0lceoVG5XmQyMIH9s19SV5m+a8qam4if0zQuAmOQTyPTPmsQBdAorGK3w== + dependencies: + bytesish "^0.4.1" + caseless "~0.12.0" + is-stream "^2.0.0" + better-opn@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/better-opn/-/better-opn-2.1.1.tgz#94a55b4695dc79288f31d7d0e5f658320759f7c6" @@ -9872,6 +9913,11 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== +binaryextensions@^2.1.2: + version "2.3.0" + resolved "https://registry.yarnpkg.com/binaryextensions/-/binaryextensions-2.3.0.tgz#1d269cbf7e6243ea886aa41453c3651ccbe13c22" + integrity sha512-nAihlQsYGyc5Bwq6+EsubvANYGExeJKHDO3RjnvwU042fawQTQfM3Kxn7IHUXQOz4bzfwsGYYHGSvXyW4zOGLg== + bindings@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" @@ -10218,6 +10264,11 @@ bytes@3.1.1: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.1.tgz#3f018291cb4cbad9accb6e6970bca9c8889e879a" integrity sha512-dWe4nWO/ruEOY7HkUJ5gFt1DCFV9zPRoJr8pV0/ASQermOZjtq8jMjOprC0Kd10GLN+l7xaUPvxzJFWtxGu8Fg== +bytesish@^0.4.1: + version "0.4.4" + resolved "https://registry.yarnpkg.com/bytesish/-/bytesish-0.4.4.tgz#f3b535a0f1153747427aee27256748cff92347e6" + integrity sha512-i4uu6M4zuMUiyfZN4RU2+i9+peJh//pXhd9x1oSe1LBkZ3LEbCoygu8W0bXTukU1Jme2txKuotpCZRaC3FLxcQ== + c8@^7.6.0: version "7.11.0" resolved "https://registry.yarnpkg.com/c8/-/c8-7.11.0.tgz#b3ab4e9e03295a102c47ce11d4ef6d735d9a9ac9" @@ -12488,6 +12539,14 @@ ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: dependencies: safe-buffer "^5.0.1" +editions@^2.2.0: + version "2.3.1" + resolved "https://registry.yarnpkg.com/editions/-/editions-2.3.1.tgz#3bc9962f1978e801312fbd0aebfed63b49bfe698" + integrity sha512-ptGvkwTvGdGfC0hfhKg0MT+TRLRKGtUiWGBInxOm5pz7ssADezahjCUaYuZ8Dr+C05FW0AECIIPt4WBxVINEhA== + dependencies: + errlop "^2.0.0" + semver "^6.3.0" + editorconfig@^0.15.3: version "0.15.3" resolved "https://registry.yarnpkg.com/editorconfig/-/editorconfig-0.15.3.tgz#bef84c4e75fb8dcb0ce5cee8efd51c15999befc5" @@ -12664,6 +12723,11 @@ err-code@^2.0.2: resolved "https://registry.yarnpkg.com/err-code/-/err-code-2.0.3.tgz#23c2f3b756ffdfc608d30e27c9a941024807e7f9" integrity sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA== +errlop@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/errlop/-/errlop-2.2.0.tgz#1ff383f8f917ae328bebb802d6ca69666a42d21b" + integrity sha512-e64Qj9+4aZzjzzFpZC7p5kmm/ccCrbLhAJplhsDXQFs87XTsXwOpH4s1Io2s90Tau/8r2j9f4l/thhDevRjzxw== + errno@^0.1.3, errno@~0.1.7: version "0.1.8" resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.8.tgz#8bb3e9c7d463be4976ff888f76b4809ebc2e811f" @@ -14961,6 +15025,13 @@ header-case@^2.0.4: capital-case "^1.0.4" tslib "^2.0.3" +heimdalljs@^0.2.3: + version "0.2.6" + resolved "https://registry.yarnpkg.com/heimdalljs/-/heimdalljs-0.2.6.tgz#b0eebabc412813aeb9542f9cc622cb58dbdcd9fe" + integrity sha512-o9bd30+5vLBvBtzCPwwGqpry2+n0Hi6H1+qwt6y+0kwRHGGF8TFIhJPmnuM0xO97zaKrDZMwO/V56fAnn8m/tA== + dependencies: + rsvp "~3.2.1" + hexer@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/hexer/-/hexer-1.5.0.tgz#b86ce808598e8a9d1892c571f3cedd86fc9f0653" @@ -15309,6 +15380,14 @@ https-proxy-agent@5.0.1, https-proxy-agent@^5.0.0: agent-base "6" debug "4" +https-proxy-agent@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz#702b71fb5520a132a66de1f67541d9e62154d82b" + integrity sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg== + dependencies: + agent-base "5" + debug "4" + human-signals@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" @@ -16335,6 +16414,15 @@ istanbul-reports@^3.0.2, istanbul-reports@^3.1.3: html-escaper "^2.0.0" istanbul-lib-report "^3.0.0" +istextorbinary@^2.5.1: + version "2.6.0" + resolved "https://registry.yarnpkg.com/istextorbinary/-/istextorbinary-2.6.0.tgz#60776315fb0fa3999add276c02c69557b9ca28ab" + integrity sha512-+XRlFseT8B3L9KyjxxLjfXSLMuErKDsd8DBNrsaxoViABMEZlOSCstwmw0qpoFX3+U6yWU1yhLudAe6/lETGGA== + dependencies: + binaryextensions "^2.1.2" + editions "^2.2.0" + textextensions "^2.5.0" + iterall@^1.2.1: version "1.3.0" resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.3.0.tgz#afcb08492e2915cbd8a0884eb93a8c94d0d72fea" @@ -18255,6 +18343,21 @@ micromatch@^4.0.0, micromatch@^4.0.2, micromatch@^4.0.4: braces "^3.0.1" picomatch "^2.2.3" +microsoft-cognitiveservices-speech-sdk@^1.22.0: + version "1.22.0" + resolved "https://registry.yarnpkg.com/microsoft-cognitiveservices-speech-sdk/-/microsoft-cognitiveservices-speech-sdk-1.22.0.tgz#4c6f82147cbb364c5fa7478c7de691af781d6594" + integrity sha512-C1YV5jui3SD02DlmAlN+i7BKdBevETIbGxmkpFy/19yefja14Y7zOR/Hh0qb+ixuU49tPXWmTf2cL+FZE4YD6Q== + dependencies: + agent-base "^6.0.1" + asn1.js-rfc2560 "^5.0.1" + asn1.js-rfc5280 "^3.0.0" + async-disk-cache "^2.1.0" + bent "^7.3.12" + https-proxy-agent "^4.0.0" + simple-lru-cache "0.0.2" + uuid "^8.3.0" + ws "^7.5.6" + microtime@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/microtime/-/microtime-3.0.0.tgz#d140914bde88aa89b4f9fd2a18620b435af0f39b" @@ -21994,11 +22097,16 @@ ripemd160@^2.0.0, ripemd160@^2.0.1: hash-base "^3.0.0" inherits "^2.0.1" -rsvp@^4.8.4: +rsvp@^4.8.4, rsvp@^4.8.5: version "4.8.5" resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-4.8.5.tgz#c8f155311d167f68f21e168df71ec5b083113734" integrity sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA== +rsvp@~3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-3.2.1.tgz#07cb4a5df25add9e826ebc67dcc9fd89db27d84a" + integrity sha512-Rf4YVNYpKjZ6ASAmibcwTNciQ5Co5Ztq6iZPEykHpkoflnD/K5ryE/rHehFsTm4NJj8nKDhbi3eKBWGogmNnkg== + run-async@^2.4.0: version "2.4.1" resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455" @@ -22423,6 +22531,11 @@ signedsource@^1.0.0: resolved "https://registry.yarnpkg.com/signedsource/-/signedsource-1.0.0.tgz#1ddace4981798f93bd833973803d80d52e93ad6a" integrity sha1-HdrOSYF5j5O9gzlzgD2A1S6TrWo= +simple-lru-cache@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz#d59cc3a193c1a5d0320f84ee732f6e4713e511dd" + integrity sha512-uEv/AFO0ADI7d99OHDmh1QfYzQk/izT1vCmu/riQfh7qjBVUUgRT87E5s5h7CxWCA/+YoZerykpEthzVrW3LIw== + simple-swizzle@^0.2.2: version "0.2.2" resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a" @@ -23561,6 +23674,11 @@ text-table@^0.2.0: resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= +textextensions@^2.5.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/textextensions/-/textextensions-2.6.0.tgz#d7e4ab13fe54e32e08873be40d51b74229b00fc4" + integrity sha512-49WtAWS+tcsy93dRt6P0P3AMD2m5PvXRhuEA0kaXos5ZLlujtYmpmFsB+QvWUSxE1ZsstmYXfQ7L40+EcQgpAQ== + thenify-all@^1.0.0: version "1.6.0" resolved "https://registry.yarnpkg.com/thenify-all/-/thenify-all-1.6.0.tgz#1a1918d402d8fc3f98fbf234db0bcc8cc10e9726" @@ -24513,6 +24631,11 @@ user-home@^1.1.1: resolved "https://registry.yarnpkg.com/user-home/-/user-home-1.1.1.tgz#2b5be23a32b63a7c9deb8d0f28d485724a3df190" integrity sha1-K1viOjK2Onyd640PKNSFcko98ZA= +username-sync@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/username-sync/-/username-sync-1.0.3.tgz#ae41c5c8a4c8c2ecc1443a7d0742742bd7e36732" + integrity sha512-m/7/FSqjJNAzF2La448c/aEom0gJy7HY7Y509h6l0ePvEkFictAGptwWaj1msWJ38JbfEDOUoE8kqFee9EHKdA== + util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" @@ -24584,7 +24707,7 @@ uuid@^3.2.1, uuid@^3.3.2, uuid@^3.3.3: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== -uuid@^8.0.0, uuid@^8.3.2: +uuid@^8.0.0, uuid@^8.3.0, uuid@^8.3.1, uuid@^8.3.2: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== @@ -25339,6 +25462,11 @@ ws@8.8.1, ws@^8.2.3, ws@^8.3.0, ws@^8.4.2: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.7.tgz#9e0ac77ee50af70d58326ecff7e85eb3fa375e67" integrity sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A== +ws@^7.5.6: + version "7.5.9" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" + integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== + xdg-basedir@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13" From e086a14ac22f02635828819e66efd15a3de91b7b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 15 Aug 2022 21:58:31 +0800 Subject: [PATCH 07/28] Convert time to ms --- packages/api/src/utils/textToSpeech.ts | 17 ++++++++++------- packages/api/test/utils/textToSpeech.test.ts | 1 + 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index 9a1a3ad1a..b3de66131 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -50,9 +50,10 @@ export const synthesizeTextToSpeech = async ( // Create the speech synthesizer. const synthesizer = new sdk.SpeechSynthesizer(speechConfig) const speechMarks: SpeechMark[] = [] + let timeOffset = 0 + let characterOffset = 0 synthesizer.synthesizing = function (s, e) { - logger.debug(`synthesizing ${e.result.audioData.byteLength} bytes`) // convert arrayBuffer to stream and write to gcs file writeStream.write(Buffer.from(e.result.audioData)) } @@ -84,11 +85,12 @@ export const synthesizeTextToSpeech = async ( logger.info(str) } + // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. synthesizer.wordBoundary = function (s, e) { speechMarks.push({ word: e.text, - time: e.audioOffset, - start: e.textOffset, + time: (timeOffset + e.audioOffset) / 10000, + start: characterOffset + e.textOffset, length: e.wordLength, }) } @@ -110,16 +112,17 @@ export const synthesizeTextToSpeech = async ( }) } // slice the text into chunks of 1,000 characters - const textChunks = input.text.match(/.{1,1000}/g) || [] + const textChunks = input.text.match(/(.|[\r\n]){1,1000}/g) || [] for (const textChunk of textChunks) { - console.debug(`synthesizing ${textChunk}`) - await speakTextAsyncPromise(textChunk) + logger.debug(`synthesizing ${textChunk}`) + const result = await speakTextAsyncPromise(textChunk) + timeOffset = timeOffset + result.audioDuration + characterOffset = characterOffset + textChunk.length } writeStream.end() synthesizer.close() logger.debug(`audio file: ${audioFile}`) - logger.debug(`speechMarks: ${speechMarks}`) return { audioUrl: getFilePublicUrl(audioFile), diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts index 4909192a5..fbc685a4e 100644 --- a/packages/api/test/utils/textToSpeech.test.ts +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -81,6 +81,7 @@ describe('textToSpeech', () => { const output = await synthesizeTextToSpeech(input) expect(output.audioUrl).to.be.a('string') expect(output.speechMarks).to.be.a('array') + console.log(output.speechMarks) }) }) }) From 355e33b6489067b9473c155d04cb8b660963cc55 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 15 Aug 2022 22:24:16 +0800 Subject: [PATCH 08/28] Test Mandarin --- packages/api/src/routers/svc/textToSpeech.ts | 27 ++++--- packages/api/src/utils/createTask.ts | 4 +- packages/api/src/utils/textToSpeech.ts | 52 ++++++++------ packages/api/test/utils/textToSpeech.test.ts | 75 +++----------------- 4 files changed, 54 insertions(+), 104 deletions(-) diff --git a/packages/api/src/routers/svc/textToSpeech.ts b/packages/api/src/routers/svc/textToSpeech.ts index 42ec982e4..56b8e2a82 100644 --- a/packages/api/src/routers/svc/textToSpeech.ts +++ b/packages/api/src/routers/svc/textToSpeech.ts @@ -4,7 +4,7 @@ import { corsConfig } from '../../utils/corsConfig' import { getRepository } from '../../entity/utils' import { User } from '../../entity/user' import { getPageById } from '../../elastic/pages' -import { htmlToSsml, synthesizeTextToSpeech } from '../../utils/textToSpeech' +import { synthesizeTextToSpeech } from '../../utils/textToSpeech' import { Speech } from '../../entity/speech' export function textToSpeechServiceRouter() { @@ -12,13 +12,14 @@ export function textToSpeechServiceRouter() { router.options('/', cors({ ...corsConfig, maxAge: 600 })) router.post('/', async (req, res) => { - const { userId, pageId } = req.body as { + const { userId, pageId, text } = req.body as { userId: string pageId: string + text: string } - if (!userId || !pageId) { - return res.status(400).send({ errorCode: 'BAD_DATA' }) + if (!userId || !pageId || !text) { + return res.status(200).send('Invalid data') } const user = await getRepository(User).findOne({ @@ -26,32 +27,28 @@ export function textToSpeechServiceRouter() { relations: ['user_personalization'], }) if (!user) { - return res.status(400).send({ errorCode: 'BAD_DATA' }) + return res.status(200).send('User not found') } const page = await getPageById(pageId) if (!page) { - return res.status(400).send({ errorCode: 'BAD_DATA' }) + return res.status(200).send('Page not found') } - const html = page.content - const language = page.language - const voice = user.userPersonalization.speechVoice || 'en-US_AllisonVoice' - const rate = user.userPersonalization.speechRate || 100 - const volume = user.userPersonalization.speechVolume || 100 - const ssml = htmlToSsml(html, language, voice, rate, volume) - const audioAndSpeechMarks = await synthesizeTextToSpeech({ id: pageId, - text: ssml, + text, + languageCode: page.language, + voice: user.userPersonalization.speechVoice, }) await getRepository(Speech).save({ elasticPageId: pageId, audioUrl: audioAndSpeechMarks.audioUrl, speechMarks: JSON.stringify(audioAndSpeechMarks.speechMarks), - id: pageId, user, }) + + res.status(200).send('OK') }) } diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index c69d5a986..41174d891 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -327,12 +327,14 @@ export const enqueueSyncWithIntegration = async ( export const enqueueTextToSpeech = async ( userId: string, - pageId: string + pageId: string, + text: string ): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { userId, pageId, + text, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index b3de66131..a4daa9743 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -1,6 +1,14 @@ import { buildLogger } from './logger' import { createGCSFile, getFilePublicUrl } from './uploads' -import * as sdk from 'microsoft-cognitiveservices-speech-sdk' +import { + CancellationDetails, + CancellationReason, + ResultReason, + SpeechConfig, + SpeechSynthesisOutputFormat, + SpeechSynthesisResult, + SpeechSynthesizer, +} from 'microsoft-cognitiveservices-speech-sdk' import { env } from '../env' export interface TextToSpeechInput { @@ -38,17 +46,17 @@ export const synthesizeTextToSpeech = async ( public: true, resumable: true, }) - const speechConfig = sdk.SpeechConfig.fromSubscription( + const speechConfig = SpeechConfig.fromSubscription( env.azure.speechKey, env.azure.speechRegion ) speechConfig.speechSynthesisLanguage = input.languageCode || 'en-US' speechConfig.speechSynthesisVoiceName = input.voice || 'en-US-JennyNeural' speechConfig.speechSynthesisOutputFormat = - sdk.SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 + SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 // Create the speech synthesizer. - const synthesizer = new sdk.SpeechSynthesizer(speechConfig) + const synthesizer = new SpeechSynthesizer(speechConfig) const speechMarks: SpeechMark[] = [] let timeOffset = 0 let characterOffset = 0 @@ -59,34 +67,34 @@ export const synthesizeTextToSpeech = async ( } // The event synthesis completed signals that the synthesis is completed. - synthesizer.synthesisCompleted = function (s, e) { + synthesizer.synthesisCompleted = (s, e) => { logger.info( '(synthesized) Reason: ' + - sdk.ResultReason[e.result.reason] + + ResultReason[e.result.reason] + ' Audio length: ' + e.result.audioData.byteLength ) } // The synthesis started event signals that the synthesis is started. - synthesizer.synthesisStarted = function (s, e) { + synthesizer.synthesisStarted = (s, e) => { logger.info('(synthesis started)') } // The event signals that the service has stopped processing speech. // This can happen when an error is encountered. - synthesizer.SynthesisCanceled = function (s, e) { - const cancellationDetails = sdk.CancellationDetails.fromResult(e.result) + synthesizer.SynthesisCanceled = (s, e) => { + const cancellationDetails = CancellationDetails.fromResult(e.result) let str = - '(cancel) Reason: ' + sdk.CancellationReason[cancellationDetails.reason] - if (cancellationDetails.reason === sdk.CancellationReason.Error) { + '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason] + if (cancellationDetails.reason === CancellationReason.Error) { str += ': ' + e.result.errorDetails } logger.info(str) } // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. - synthesizer.wordBoundary = function (s, e) { + synthesizer.wordBoundary = (s, e) => { speechMarks.push({ word: e.text, time: (timeOffset + e.audioOffset) / 10000, @@ -97,7 +105,7 @@ export const synthesizeTextToSpeech = async ( const speakTextAsyncPromise = ( text: string - ): Promise => { + ): Promise => { return new Promise((resolve, reject) => { synthesizer.speakTextAsync( text, @@ -202,12 +210,12 @@ export const synthesizeTextToSpeech = async ( // } // } -export const htmlToSsml = ( - html: string, - language = 'en-US', - voice = 'en-US-JennyNeural', - rate = 100, - volume = 100 -): string => { - return `${html}` -} +// export const htmlToSsml = ( +// html: string, +// language = 'en-US', +// voice = 'en-US-JennyNeural', +// rate = 100, +// volume = 100 +// ): string => { +// return `${html}` +// } diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts index fbc685a4e..48a65abe0 100644 --- a/packages/api/test/utils/textToSpeech.test.ts +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -12,76 +12,19 @@ describe('textToSpeech', () => { const input: TextToSpeechInput = { id: generateFakeUuid(), text: - 'MIT spinout Quaise Energy is working to create geothermal wells made from the deepest holes in the world.\n' + - '\n' + - 'Publication Date:\n' + - '\n' + - 'June 28, 2022\n' + - '\n' + - "A graphic depicting the heat at the earth's core\n" + - 'Caption:\n' + - '\n' + - 'Quaise Energy wants to repurpose coal and gas plants into deep geothermal wells by using X-rays to melt rock.\n' + - '\n' + - 'Credits:\n' + - '\n' + - 'Image: Collage by MIT News with images courtesy of Quaise Energy\n' + - '\n' + - 'There’s an abandoned coal power plant in upstate New York that most people regard as a useless relic. But MIT’s Paul Woskov sees things differently.\n' + - '\n' + - 'Woskov, a research engineer in MIT’s Plasma Science and Fusion Center, notes the plant’s power turbine is still intact and the transmission lines still run to the grid. Using an approach he’s been working on for the last 14 years, he’s hoping it will be back online, completely carbon-free, within the decade.\n' + - 'In fact, Quaise Energy, the company commercializing Woskov’s work, believes if it can retrofit one power plant, the same process will work on virtually every coal and gas power plant in the world.\n' + - '\n' + - 'Quaise is hoping to accomplish those lofty goals by tapping into the energy source below our feet. The company plans to vaporize enough rock to create the world’s deepest holes and harvest geothermal energy at a scale that could satisfy human energy consumption for millions of years. They haven’t yet solved all the related engineering challenges, but Quaise’s founders have set an ambitious timeline to begin harvesting energy from a pilot well by 2026.\n' + - '\n' + - 'The plan would be easier to dismiss as unrealistic if it were based on a new and unproven technology. But Quaise’s drilling systems center around a microwave-emitting device called a gyrotron that has been used in research and manufacturing for decades.\n' + - '\n' + - '“This will happen quickly once we solve the immediate engineering problems of transmitting a clean beam and having it operate at a high energy density without breakdown,” explains Woskov, who is not formally affiliated with Quaise but serves as an advisor. “It’ll go fast because the underlying technology, gyrotrons, are commercially available. You could place an order with a company and have a system delivered right now — granted, these beam sources have never been used 24/7, but they are engineered to be operational for long time periods. In five or six years, I think we’ll have a plant running if we solve these engineering problems. I’m very optimistic.”\n' + - 'Woskov and many other researchers have been using gyrotrons to heat material in nuclear fusion experiments for decades. It wasn’t until 2008, however, after the MIT Energy Initiative (MITEI) published a request for proposals on new geothermal drilling technologies, that Woskov thought of using gyrotrons for a new application.\n' + - '\n' + - '“[Gyrotrons] haven’t been well-publicized in the general science community, but those of us in fusion research understood they were very powerful beam sources — like lasers, but in a different frequency range,” Woskov says. “I thought, why not direct these high-power beams, instead of into fusion plasma, down into rock and vaporize the hole?”\n' + - '\n' + - 'As power from other renewable energy sources has exploded in recent decades, geothermal energy has plateaued, mainly because geothermal plants only exist in places where natural conditions allow for energy extraction at relatively shallow depths of up to 400 feet beneath the Earth’s surface. At a certain point, conventional drilling becomes impractical because deeper crust is both hotter and harder, which wears down mechanical drill bits.\n' + - '\n' + - 'Woskov’s idea to use gyrotron beams to vaporize rock sent him on a research journey that has never really stopped. With some funding from MITEI, he began running tests, quickly filling his office with small rock formations he’d blasted with millimeter waves from a small gyrotron in MIT’s Plasma Science and Fusion Center.\n' + - '\n' + - 'Paul Woskov with blasted rock samples\n' + - 'Woskov displaying samples in his lab in 2016.\n' + - '\n' + - 'Photo: Paul Rivenberg\n' + - '\n' + - 'Around 2018, Woskov’s rocks got the attention of Carlos Araque ’01, SM ’02, who had spent his career in the oil and gas industry and was the technical director of MIT’s investment fund The Engine at the time.\n' + - '\n' + - 'That year, Araque and Matt Houde, who’d been working with geothermal company AltaRock Energy, founded Quaise. Quaise was soon given a grant by the Department of Energy to scale up Woskov’s experiments using a larger gyrotron.\n' + - '\n' + - 'With the larger machine, the team hopes to vaporize a hole 10 times the depth of Woskov’s lab experiments. That is expected to be accomplished by the end of this year. After that, the team will vaporize a hole 10 times the depth of the previous one — what Houde calls a 100-to-1 hole.\n' + - '“That’s something [the DOE] is particularly interested in, because they want to address the challenges posed by material removal over those greater lengths — in other words, can we show we’re fully flushing out the rock vapors?” Houde explains. “We believe the 100-to-1 test also gives us the confidence to go out and mobilize a prototype gyrotron drilling rig in the field for the first field demonstrations.”\n' + - '\n' + - 'Tests on the 100-to-1 hole are expected to be completed sometime next year. Quaise is also hoping to begin vaporizing rock in field tests late next year. The short timeline reflects the progress Woskov has already made in his lab.\n' + - '\n' + - "Although more engineering research is needed, ultimately, the team expects to be able to drill and operate these geothermal wells safely. “We believe, because of Paul’s work at MIT over the past decade, that most if not all of the core physics questions have been answered and addressed,” Houde says. “It’s really engineering challenges we have to answer, which doesn’t mean they’re easy to solve, but we’re not working against the laws of physics, to which there is no answer. It's more a matter of overcoming some of the more technical and cost considerations to making this work at a large scale.”\n" + - '\n' + - 'The company plans to begin harvesting energy from pilot geothermal wells that reach rock temperatures at up to 500 C by 2026. From there, the team hopes to begin repurposing coal and natural gas plants using its system.\n' + - '\n' + - '“We believe, if we can drill down to 20 kilometers, we can access these super-hot temperatures in greater than 90 percent of locations across the globe,” Houde says.\n' + - '\n' + - 'Quaise’s work with the DOE is addressing what it sees as the biggest remaining questions about drilling holes of unprecedented depth and pressure, such as material removal and determining the best casing to keep the hole stable and open. For the latter problem of well stability, Houde believes additional computer modeling is needed and expects to complete that modeling by the end of 2024.\n' + - '\n' + - 'By drilling the holes at existing power plants, Quaise will be able to move faster than if it had to get permits to build new plants and transmission lines. And by making their millimeter-wave drilling equipment compatible with the existing global fleet of drilling rigs, it will also allow the company to tap into the oil and gas industry’s global workforce.\n' + - '\n' + - '“At these high temperatures [we’re accessing], we’re producing steam very close to, if not exceeding, the temperature that today’s coal and gas-fired power plants operate at,” Houde says. “So, we can go to existing power plants and say, ‘We can replace 95 to 100 percent of your coal use by developing a geothermal field and producing steam from the Earth, at the same temperature you’re burning coal to run your turbine, directly replacing carbon emissions.”\n' + - '\n' + - 'Transforming the world’s energy systems in such a short timeframe is something the founders see as critical to help avoid the most catastrophic global warming scenarios.\n' + - '\n' + - '“There have been tremendous gains in renewables over the last decade, but the big picture today is we’re not going nearly fast enough to hit the milestones we need for limiting the worst impacts of climate change,” Houde says. “[Deep geothermal] is a power resource that can scale anywhere and has the ability to tap into a large workforce in the energy industry to readily repackage their skills for a totally carbon free energy source.”\n' + - '\n' + - 'Related Topics\n' + - 'Related Articles', + '《太阁立志传5 DX》清洲会议触发教程\n' + + '玩家要亲历清洲会议事件，需要位于织田家。\n' + + '清洲会议需要完成以下条件才能触发：\n' + + '本能寺发生之后，织田信长和织田信忠死亡。\n' + + '羽柴秀吉、柴田胜家、织田信雄、织田信孝为大名。\n' + + '清洲城必须为信雄的直辖城，或者清洲城主为信雄一方。\n' + + '前两个条件都很容易达成，主要是要保证清洲城主为信雄这一条件比较难办，需要玩家控制城主封地。', + languageCode: 'zh-CN', + voice: 'zh-CN-XiaochenNeural', } const output = await synthesizeTextToSpeech(input) expect(output.audioUrl).to.be.a('string') expect(output.speechMarks).to.be.a('array') - console.log(output.speechMarks) }) }) }) From def8f28138bc61ad2eb0fe2c39e80337ae0a2f59 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 16 Aug 2022 22:12:32 +0800 Subject: [PATCH 09/28] Add Azure Speech Service dependencies --- packages/api/package.json | 1 - yarn.lock | 64 +-------------------------------------- 2 files changed, 1 insertion(+), 64 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 6a1fef691..28ba0df6c 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -40,7 +40,6 @@ "analytics-node": "^6.0.0", "apollo-datasource": "^3.3.1", "apollo-server-express": "^3.6.3", - "aws-sdk": "^2.1192.0", "axios": "^0.27.2", "bcryptjs": "^2.4.3", "cookie": "^0.5.0", diff --git a/yarn.lock b/yarn.lock index c73f437a3..14046d3e7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9516,22 +9516,6 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== -aws-sdk@^2.1192.0: - version "2.1192.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.1192.0.tgz#13fe38ec8dae3232f17d52b370e69daa9c7a0e9c" - integrity sha512-6uzrlG1Ow3qcOnL0+et+DBTGhYgJzgNydVvos1Eg01vPc/ZhxR7roZ3epZQcPmOR0thQuzzckTq7FBO6wzZA2w== - dependencies: - buffer "4.9.2" - events "1.1.1" - ieee754 "1.1.13" - jmespath "0.16.0" - querystring "0.2.0" - sax "1.2.1" - url "0.10.3" - util "^0.12.4" - uuid "8.0.0" - xml2js "0.4.19" - aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -10194,7 +10178,7 @@ buffer-xor@^1.0.3: resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= -buffer@4.9.2, buffer@^4.3.0: +buffer@^4.3.0: version "4.9.2" resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== @@ -13240,11 +13224,6 @@ eventid@^2.0.0: dependencies: uuid "^8.0.0" -events@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" - integrity sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw== - events@^3.0.0, events@^3.2.0: version "3.3.0" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" @@ -15436,11 +15415,6 @@ icss-utils@^5.0.0, icss-utils@^5.1.0: resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae" integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA== -ieee754@1.1.13: - version "1.1.13" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" - integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== - ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -16927,11 +16901,6 @@ jest@^27.4.5: import-local "^3.0.2" jest-cli "^27.5.1" -jmespath@0.16.0: - version "0.16.0" - resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076" - integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw== - join-component@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/join-component/-/join-component-1.1.0.tgz#b8417b750661a392bee2c2537c68b2a9d4977cd5" @@ -22205,11 +22174,6 @@ sanitize-html@^2.3.2: parse-srcset "^1.0.2" postcss "^8.0.2" -sax@1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" - integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== - sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -24570,14 +24534,6 @@ url-template@^2.0.8: resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21" integrity sha1-/FZaPMy/93MMd19WQflVV5FDnyE= -url@0.10.3: - version "0.10.3" - resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" - integrity sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ== - dependencies: - punycode "1.3.2" - querystring "0.2.0" - url@^0.11.0: version "0.11.0" resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" @@ -24697,11 +24653,6 @@ uuid-browser@^3.1.0: resolved "https://registry.yarnpkg.com/uuid-browser/-/uuid-browser-3.1.0.tgz#0f05a40aef74f9e5951e20efbf44b11871e56410" integrity sha1-DwWkCu90+eWVHiDvv0SxGHHlZBA= -uuid@8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c" - integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw== - uuid@^3.2.1, uuid@^3.3.2, uuid@^3.3.3: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" @@ -25477,14 +25428,6 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - xml2js@^0.4.23: version "0.4.23" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" @@ -25498,11 +25441,6 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== - xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From 447e413605c76c50a080fcea8b425c28f9a6b2c5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 16 Aug 2022 22:15:44 +0800 Subject: [PATCH 10/28] Add function to parse HTML to SSML --- packages/api/src/utils/createTask.ts | 4 +- packages/api/src/utils/parser.ts | 42 ++++++ packages/api/src/utils/textToSpeech.ts | 145 +++++++------------ packages/api/test/utils/textToSpeech.test.ts | 13 +- 4 files changed, 101 insertions(+), 103 deletions(-) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 41174d891..c69d5a986 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -327,14 +327,12 @@ export const enqueueSyncWithIntegration = async ( export const enqueueTextToSpeech = async ( userId: string, - pageId: string, - text: string + pageId: string ): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { userId, pageId, - text, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 8900fc5d3..2e9997e58 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -276,6 +276,48 @@ export const parsePreparedContent = async ( }) article.content = article.dom.outerHTML } + + const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ + 'omnivore-highlight-id', + 'data-twitter-tweet-id', + 'data-instagram-id', + ] + + // Get the top level element? + const pageNode = article.dom.firstElementChild as HTMLElement + console.log('pageNode: ', pageNode) + const nodesToVisitStack: [HTMLElement] = [pageNode] + const visitedNodeList = [] + + while (nodesToVisitStack.length > 0) { + const currentNode = nodesToVisitStack.pop() + console.log('currentNode: ', currentNode?.nodeType) + if ( + currentNode?.nodeType !== 1 || + // Avoiding dynamic elements from being counted as anchor-allowed elements + ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES.some((attrib) => + currentNode.hasAttribute(attrib) + ) + ) { + continue + } + visitedNodeList.push(currentNode) + ;[].slice + .call(currentNode.childNodes) + .reverse() + .forEach(function (node) { + nodesToVisitStack.push(node) + }) + } + + visitedNodeList.shift() + visitedNodeList.forEach((node, index) => { + // start from index 1, index 0 reserved for anchor unknown. + node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString()) + }) + + console.log('article content:', article.dom.outerHTML) + article.content = article.dom.outerHTML } const newWindow = parseHTML('') diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index a4daa9743..53aefe881 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -10,13 +10,12 @@ import { SpeechSynthesizer, } from 'microsoft-cognitiveservices-speech-sdk' import { env } from '../env' +import { parseHTML } from 'linkedom' export interface TextToSpeechInput { id: string text: string voice?: string - textType?: 'text' | 'ssml' - engine?: 'standard' | 'neural' languageCode?: string } @@ -27,16 +26,14 @@ export interface TextToSpeechOutput { export interface SpeechMark { time: number - start: number - length: number + start?: number + length?: number word: string + type: 'word' | 'bookmark' } const logger = buildLogger('app.dispatch') -// // create a new AWS Polly client -// const client = new AWS.Polly() - export const synthesizeTextToSpeech = async ( input: TextToSpeechInput ): Promise => { @@ -69,10 +66,9 @@ export const synthesizeTextToSpeech = async ( // The event synthesis completed signals that the synthesis is completed. synthesizer.synthesisCompleted = (s, e) => { logger.info( - '(synthesized) Reason: ' + - ResultReason[e.result.reason] + - ' Audio length: ' + + `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${ e.result.audioData.byteLength + }` ) } @@ -100,6 +96,20 @@ export const synthesizeTextToSpeech = async ( time: (timeOffset + e.audioOffset) / 10000, start: characterOffset + e.textOffset, length: e.wordLength, + type: 'word', + }) + } + + synthesizer.bookmarkReached = (s, e) => { + logger.info( + `(Bookmark reached), Audio offset: ${ + e.audioOffset / 10000 + }ms, bookmark text: ${e.text}` + ) + speechMarks.push({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + type: 'bookmark', }) } @@ -138,84 +148,39 @@ export const synthesizeTextToSpeech = async ( } } -// export const createAudio = async ( -// input: TextToSpeechInput -// ): Promise => { -// const { text, voice, textType, engine, languageCode } = input -// const params: SynthesizeSpeechInput = { -// OutputFormat: 'ogg_vorbis', -// Text: text, -// TextType: textType || 'text', -// VoiceId: voice || 'Joanna', -// Engine: engine || 'neural', -// LanguageCode: languageCode || 'en-US', -// } -// try { -// const data = await client.synthesizeSpeech(params).promise() -// return data.AudioStream as Buffer -// } catch (error) { -// logger.error('Unable to create audio file', { error }) -// throw error -// } -// } +export const htmlToSsml = ( + html: string, + language = 'en-US', + voice = 'en-US-JennyNeural', + rate = 100, + volume = 100 +): string => { + const document = parseHTML(html).document + const paragraphs = document.querySelectorAll('p') + // create new ssml document + const ssml = parseHTML('').document + const speakElement = ssml.createElement('speak') + speakElement.setAttribute('version', '1.0') + speakElement.setAttribute('xmlns', 'http://www.w3.org/2001/10/synthesis') + speakElement.setAttribute('xml:lang', language) + const voiceElement = ssml.createElement('voice') + voiceElement.setAttribute('name', voice) + speakElement.appendChild(voiceElement) + const prosodyElement = ssml.createElement('prosody') + prosodyElement.setAttribute('rate', `${rate}%`) + prosodyElement.setAttribute('volume', volume.toString()) + voiceElement.appendChild(prosodyElement) + // add each paragraph to the ssml document + paragraphs.forEach((p) => { + const id = p.getAttribute('data-omnivore-anchor-idx') + if (id) { + const text = p.innerText + const bookMark = ssml.createElement('bookmark') + bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`) + bookMark.innerText = text + prosodyElement.appendChild(bookMark) + } + }) -// export const createSpeechMarks = async ( -// input: TextToSpeechInput -// ): Promise => { -// const { text, voice, textType, engine, languageCode } = input -// const params: SynthesizeSpeechInput = { -// OutputFormat: 'json', -// Text: text, -// TextType: textType || 'text', -// VoiceId: voice || 'Joanna', -// Engine: engine || 'neural', -// SpeechMarkTypes: ['word'], -// LanguageCode: languageCode || 'en-US', -// } -// try { -// const data = await client.synthesizeSpeech(params).promise() -// return (data.AudioStream as Buffer).toString() -// } catch (error) { -// logger.error('Unable to create speech marks', { error }) -// throw error -// } -// } -// -// export const createAudioWithSpeechMarks = async ( -// input: TextToSpeechInput -// ): Promise => { -// try { -// const audio = await createAudio(input) -// // upload audio to google cloud storage -// const filePath = `speech/${input.id}.ogg` -// -// logger.info('start uploading...', { filePath }) -// await uploadToBucket(filePath, audio, { -// contentType: 'audio/ogg', -// public: true, -// }) -// -// // get public url for audio file -// const publicUrl = getFilePublicUrl(filePath) -// logger.info('upload complete', { publicUrl }) -// -// const speechMarks = await createSpeechMarks(input) -// return { -// audioUrl: publicUrl, -// speechMarks, -// } -// } catch (error) { -// logger.error('Unable to create audio with speech marks', error) -// throw error -// } -// } - -// export const htmlToSsml = ( -// html: string, -// language = 'en-US', -// voice = 'en-US-JennyNeural', -// rate = 100, -// volume = 100 -// ): string => { -// return `${html}` -// } + return speakElement.outerHTML +} diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts index 48a65abe0..89d8f5edf 100644 --- a/packages/api/test/utils/textToSpeech.test.ts +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -11,16 +11,9 @@ describe('textToSpeech', () => { it('should create an audio file with speech marks', async () => { const input: TextToSpeechInput = { id: generateFakeUuid(), - text: - '《太阁立志传5 DX》清洲会议触发教程\n' + - '玩家要亲历清洲会议事件，需要位于织田家。\n' + - '清洲会议需要完成以下条件才能触发：\n' + - '本能寺发生之后，织田信长和织田信忠死亡。\n' + - '羽柴秀吉、柴田胜家、织田信雄、织田信孝为大名。\n' + - '清洲城必须为信雄的直辖城，或者清洲城主为信雄一方。\n' + - '前两个条件都很容易达成，主要是要保证清洲城主为信雄这一条件比较难办，需要玩家控制城主封地。', - languageCode: 'zh-CN', - voice: 'zh-CN-XiaochenNeural', + text: 'Marry had a little lamb', + languageCode: 'en-US', + voice: 'en-US-JennyNeural', } const output = await synthesizeTextToSpeech(input) expect(output.audioUrl).to.be.a('string') From c74f2bc3fdc85baa9c9737ecaeb5ea47d7dcb16b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 16 Aug 2022 22:16:43 +0800 Subject: [PATCH 11/28] Enqueue text to speech task after creating article --- packages/api/src/resolvers/article/index.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index bd9e26ee5..f3251e6ff 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -94,6 +94,7 @@ import { updatePage, } from '../../elastic/pages' import { searchHighlights } from '../../elastic/highlights' +import { enqueueTextToSpeech } from '../../utils/createTask' export type PartialArticle = Omit< Article, @@ -372,6 +373,10 @@ export const createArticleResolver = authorized< articleToSave.id = newPageId } + // enqueue a task to convert text to speech + const taskName = await enqueueTextToSpeech(uid, articleToSave.id) + log.info('Text to speech task name', { taskName }) + log.info( 'page created in elastic', articleToSave.id, From 301746d49400c5a72f961939c2aa38a6fbfffcc3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 16 Aug 2022 22:17:30 +0800 Subject: [PATCH 12/28] Add /:id/mp3 router to redirect request to speech MP3 URL --- packages/api/src/routers/article_router.ts | 46 +++++++++++++++++-- .../svc/{textToSpeech.ts => speech.ts} | 22 ++++++--- packages/api/src/server.ts | 2 + 3 files changed, 60 insertions(+), 10 deletions(-) rename packages/api/src/routers/svc/{textToSpeech.ts => speech.ts} (69%) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 5774483d6..836980cac 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -3,15 +3,20 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' -import { CreateArticleErrorCode } from './../generated/graphql' -import { isSiteBlockedForParse } from './../utils/blocked' +import { CreateArticleErrorCode } from '../generated/graphql' +import { isSiteBlockedForParse } from '../utils/blocked' import cors from 'cors' -import { buildLogger } from './../utils/logger' +import { buildLogger } from '../utils/logger' import { corsConfig } from '../utils/corsConfig' import { createPageSaveRequest } from '../services/create_page_save_request' import { initModels } from '../server' import { kx } from '../datalayer/knex_config' import { getClaimsByToken } from '../utils/auth' +import * as jwt from 'jsonwebtoken' +import { env } from '../env' +import { Claims } from '../resolvers/types' +import { getRepository } from '../entity/utils' +import { Speech } from '../entity/speech' const logger = buildLogger('app.dispatch') @@ -61,5 +66,40 @@ export function articleRouter() { articleSavingRequestId: result.id, }) }) + + router.get( + '/:id/mp3', + cors(corsConfig), + async (req, res) => { + const id = req.params.id + const token = req.cookies?.auth || req.headers?.authorization + if (!token || !jwt.verify(token, env.server.jwtSecret)) { + return res.status(401).send({ errorCode: 'UNAUTHORIZED' }) + } + const { uid } = jwt.decode(token) as Claims + + logger.info('Get article speech in mp3 format', { + params: req.params, + labels: { + userId: uid, + source: 'GetArticleSpeechMp3', + articleId: id, + }, + }) + + const speech = await getRepository(Speech).findOneBy({ + elasticPageId: id, + user: { id: uid }, + }) + + if (!speech) { + return res.status(404).send({ errorCode: 'NOT_FOUND' }) + } + + logger.info('Found speech mp3', { audioUrl: speech.audioUrl }) + res.redirect(speech.audioUrl) + } + ) + return router } diff --git a/packages/api/src/routers/svc/textToSpeech.ts b/packages/api/src/routers/svc/speech.ts similarity index 69% rename from packages/api/src/routers/svc/textToSpeech.ts rename to packages/api/src/routers/svc/speech.ts index 56b8e2a82..958750fc9 100644 --- a/packages/api/src/routers/svc/textToSpeech.ts +++ b/packages/api/src/routers/svc/speech.ts @@ -6,19 +6,20 @@ import { User } from '../../entity/user' import { getPageById } from '../../elastic/pages' import { synthesizeTextToSpeech } from '../../utils/textToSpeech' import { Speech } from '../../entity/speech' +import { parseHTML } from 'linkedom' -export function textToSpeechServiceRouter() { +export function speechServiceRouter() { const router = express.Router() router.options('/', cors({ ...corsConfig, maxAge: 600 })) + // eslint-disable-next-line @typescript-eslint/no-misused-promises router.post('/', async (req, res) => { - const { userId, pageId, text } = req.body as { + const { userId, pageId } = req.body as { userId: string pageId: string - text: string } - if (!userId || !pageId || !text) { + if (!userId || !pageId) { return res.status(200).send('Invalid data') } @@ -35,7 +36,12 @@ export function textToSpeechServiceRouter() { return res.status(200).send('Page not found') } - const audioAndSpeechMarks = await synthesizeTextToSpeech({ + const text = parseHTML(page.content).document.textContent + if (!text) { + return res.status(200).send('Page has no text') + } + + const speech = await synthesizeTextToSpeech({ id: pageId, text, languageCode: page.language, @@ -44,11 +50,13 @@ export function textToSpeechServiceRouter() { await getRepository(Speech).save({ elasticPageId: pageId, - audioUrl: audioAndSpeechMarks.audioUrl, - speechMarks: JSON.stringify(audioAndSpeechMarks.speechMarks), + audioUrl: speech.audioUrl, + speechMarks: JSON.stringify(speech.speechMarks), user, }) res.status(200).send('OK') }) + + return router } diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 619a4fda5..1945b3e2c 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -45,6 +45,7 @@ import { uploadServiceRouter } from './routers/svc/upload' import rateLimit from 'express-rate-limit' import { webhooksServiceRouter } from './routers/svc/webhooks' import { integrationsServiceRouter } from './routers/svc/integrations' +import { speechServiceRouter } from './routers/svc/speech' const PORT = process.env.PORT || 4000 @@ -119,6 +120,7 @@ export const createApp = (): { app.use('/svc/pubsub/integrations', integrationsServiceRouter()) app.use('/svc/reminders', remindersServiceRouter()) app.use('/svc/pdf-attachments', pdfAttachmentsRouter()) + app.use('/svc/speech', speechServiceRouter()) if (env.dev.isLocal) { app.use('/local/debug', localDebugRouter()) From b0bf4fc5cee1278207c223cc85acdb20bc3e1208 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 17 Aug 2022 10:51:39 +0800 Subject: [PATCH 13/28] Create speech while requesting --- packages/api/src/routers/article_router.ts | 39 +++++++++++++++++++--- packages/api/src/routers/svc/speech.ts | 2 +- packages/api/src/server.ts | 2 +- packages/api/src/utils/createTask.ts | 10 +++++- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 836980cac..89625429f 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -17,6 +17,10 @@ import { env } from '../env' import { Claims } from '../resolvers/types' import { getRepository } from '../entity/utils' import { Speech } from '../entity/speech' +import { getPageById } from '../elastic/pages' +import { parseHTML } from 'linkedom' +import { synthesizeTextToSpeech } from '../utils/textToSpeech' +import { UserPersonalization } from '../entity/user_personalization' const logger = buildLogger('app.dispatch') @@ -87,15 +91,40 @@ export function articleRouter() { }, }) - const speech = await getRepository(Speech).findOneBy({ - elasticPageId: id, + logger.debug('Text to speech request', { articleId: id }) + const userPersonalization = await getRepository( + UserPersonalization + ).findOneBy({ user: { id: uid }, }) - - if (!speech) { - return res.status(404).send({ errorCode: 'NOT_FOUND' }) + if (!userPersonalization) { + return res.status(200).send('userPersonalization not found') } + const page = await getPageById(id) + if (!page) { + return res.status(200).send('Page not found') + } + + const text = parseHTML(page.content).document.documentElement.textContent + if (!text) { + return res.status(200).send('Page has no text') + } + + const speech = await synthesizeTextToSpeech({ + id, + text, + languageCode: page.language, + voice: userPersonalization.speechVoice, + }) + + await getRepository(Speech).save({ + elasticPageId: id, + audioUrl: speech.audioUrl, + speechMarks: JSON.stringify(speech.speechMarks), + user: { id: uid }, + }) + logger.info('Found speech mp3', { audioUrl: speech.audioUrl }) res.redirect(speech.audioUrl) } diff --git a/packages/api/src/routers/svc/speech.ts b/packages/api/src/routers/svc/speech.ts index 958750fc9..71c978e5a 100644 --- a/packages/api/src/routers/svc/speech.ts +++ b/packages/api/src/routers/svc/speech.ts @@ -36,7 +36,7 @@ export function speechServiceRouter() { return res.status(200).send('Page not found') } - const text = parseHTML(page.content).document.textContent + const text = parseHTML(page.content).document.documentElement.textContent if (!text) { return res.status(200).send('Page has no text') } diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 1945b3e2c..7c9273f92 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -120,7 +120,7 @@ export const createApp = (): { app.use('/svc/pubsub/integrations', integrationsServiceRouter()) app.use('/svc/reminders', remindersServiceRouter()) app.use('/svc/pdf-attachments', pdfAttachmentsRouter()) - app.use('/svc/speech', speechServiceRouter()) + app.use('/svc/text-to-speech', speechServiceRouter()) if (env.dev.isLocal) { app.use('/local/debug', localDebugRouter()) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index c69d5a986..ae9e230b3 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -337,7 +337,15 @@ export const enqueueTextToSpeech = async ( // If there is no Google Cloud Project Id exposed, it means that we are in local environment if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { - return nanoid() + // Calling the handler function directly. + setTimeout(() => { + axios + .post(env.queue.textToSpeechTaskHandlerUrl, payload) + .catch((error) => { + logger.error(error) + }) + }, 0) + return '' } const createdTasks = await createHttpTaskWithToken({ From c482ad320b2b3bbf3b43eb32a85223738e991d36 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 17 Aug 2022 11:03:11 +0800 Subject: [PATCH 14/28] Add duration in logging --- packages/api/src/routers/article_router.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 89625429f..d9baf737c 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -82,6 +82,7 @@ export function articleRouter() { } const { uid } = jwt.decode(token) as Claims + const startTime = Date.now() logger.info('Get article speech in mp3 format', { params: req.params, labels: { @@ -125,7 +126,10 @@ export function articleRouter() { user: { id: uid }, }) - logger.info('Found speech mp3', { audioUrl: speech.audioUrl }) + logger.info('Found speech mp3', { + audioUrl: speech.audioUrl, + duration: Date.now() - startTime, + }) res.redirect(speech.audioUrl) } ) From 02ba753375ac559745f34585ce3aed3c560da708 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 17 Aug 2022 11:24:53 +0800 Subject: [PATCH 15/28] Upload speech marks file to gcs too --- packages/api/src/entity/speech.ts | 2 +- packages/api/src/routers/article_router.ts | 41 +++++++++++++++++----- packages/api/src/utils/textToSpeech.ts | 16 +++++++-- packages/db/migrations/0093.do.speech.sql | 2 +- 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/packages/api/src/entity/speech.ts b/packages/api/src/entity/speech.ts index 0f61d7dd7..ac64f7e71 100644 --- a/packages/api/src/entity/speech.ts +++ b/packages/api/src/entity/speech.ts @@ -25,7 +25,7 @@ export class Speech { audioUrl!: string @Column('text') - speechMarks!: string + speechMarksUrl!: string @Column('text') voice!: string diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index d9baf737c..85ccf9b0a 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -72,10 +72,11 @@ export function articleRouter() { }) router.get( - '/:id/mp3', + '/:id/:outputFormat', cors(corsConfig), async (req, res) => { const id = req.params.id + const outputFormat = req.params.outputFormat const token = req.cookies?.auth || req.headers?.authorization if (!token || !jwt.verify(token, env.server.jwtSecret)) { return res.status(401).send({ errorCode: 'UNAUTHORIZED' }) @@ -83,15 +84,27 @@ export function articleRouter() { const { uid } = jwt.decode(token) as Claims const startTime = Date.now() - logger.info('Get article speech in mp3 format', { + logger.info(`Get article speech in ${outputFormat} format`, { params: req.params, labels: { userId: uid, source: 'GetArticleSpeechMp3', articleId: id, + outputFormat, }, }) + const existingSpeech = await getRepository(Speech).findOneBy({ + elasticPageId: id, + }) + if (existingSpeech) { + logger.info('Found existing speech', { + audioUrl: existingSpeech.audioUrl, + speechMarksUrl: existingSpeech.speechMarksUrl, + }) + return res.redirect(redirectUrl(existingSpeech, outputFormat)) + } + logger.debug('Text to speech request', { articleId: id }) const userPersonalization = await getRepository( UserPersonalization @@ -112,27 +125,39 @@ export function articleRouter() { return res.status(200).send('Page has no text') } - const speech = await synthesizeTextToSpeech({ + const speechOutput = await synthesizeTextToSpeech({ id, text, languageCode: page.language, voice: userPersonalization.speechVoice, }) - await getRepository(Speech).save({ + const speech = await getRepository(Speech).save({ elasticPageId: id, - audioUrl: speech.audioUrl, - speechMarks: JSON.stringify(speech.speechMarks), + audioUrl: speechOutput.audioUrl, + speechMarksUrl: speechOutput.speechMarksUrl, user: { id: uid }, }) - logger.info('Found speech mp3', { + logger.info('Created speech', { audioUrl: speech.audioUrl, + speechMarksUrl: speech.speechMarksUrl, duration: Date.now() - startTime, }) - res.redirect(speech.audioUrl) + res.redirect(redirectUrl(speech, outputFormat)) } ) return router } + +const redirectUrl = (speech: Speech, outputFormat: string) => { + switch (outputFormat) { + case 'mp3': + return speech.audioUrl + case 'json': + return speech.speechMarksUrl + default: + return speech.audioUrl + } +} diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index 53aefe881..7f1a1d45c 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -1,5 +1,5 @@ import { buildLogger } from './logger' -import { createGCSFile, getFilePublicUrl } from './uploads' +import { createGCSFile, getFilePublicUrl, uploadToBucket } from './uploads' import { CancellationDetails, CancellationReason, @@ -21,7 +21,7 @@ export interface TextToSpeechInput { export interface TextToSpeechOutput { audioUrl: string - speechMarks: SpeechMark[] + speechMarksUrl: string } export interface SpeechMark { @@ -142,9 +142,19 @@ export const synthesizeTextToSpeech = async ( logger.debug(`audio file: ${audioFile}`) + // upload Speech Marks file to GCS + const speechMarksFile = `speech/${input.id}.json` + await uploadToBucket( + speechMarksFile, + Buffer.from(JSON.stringify(speechMarks)), + { + public: true, + } + ) + return { audioUrl: getFilePublicUrl(audioFile), - speechMarks, + speechMarksUrl: getFilePublicUrl(speechMarksFile), } } diff --git a/packages/db/migrations/0093.do.speech.sql b/packages/db/migrations/0093.do.speech.sql index 39a50aadf..fcda74145 100755 --- a/packages/db/migrations/0093.do.speech.sql +++ b/packages/db/migrations/0093.do.speech.sql @@ -10,7 +10,7 @@ CREATE TABLE omnivore.speech ( elastic_page_id TEXT NOT NULL, voice text, audio_url text NOT NULL, - speech_marks text NOT NULL, + speech_marks_url text NOT NULL, created_at timestamptz NOT NULL DEFAULT current_timestamp, updated_at timestamptz NOT NULL DEFAULT current_timestamp ); From c9d26acb25cf77edf88120c7333925f73cc860af Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 17 Aug 2022 11:27:09 +0800 Subject: [PATCH 16/28] Fix typo --- packages/api/src/routers/svc/speech.ts | 2 +- packages/api/test/utils/textToSpeech.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/routers/svc/speech.ts b/packages/api/src/routers/svc/speech.ts index 71c978e5a..d31807110 100644 --- a/packages/api/src/routers/svc/speech.ts +++ b/packages/api/src/routers/svc/speech.ts @@ -51,7 +51,7 @@ export function speechServiceRouter() { await getRepository(Speech).save({ elasticPageId: pageId, audioUrl: speech.audioUrl, - speechMarks: JSON.stringify(speech.speechMarks), + speechMarks: speech.speechMarksUrl, user, }) diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts index 89d8f5edf..f60a8a8c0 100644 --- a/packages/api/test/utils/textToSpeech.test.ts +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -17,7 +17,7 @@ describe('textToSpeech', () => { } const output = await synthesizeTextToSpeech(input) expect(output.audioUrl).to.be.a('string') - expect(output.speechMarks).to.be.a('array') + expect(output.speechMarksUrl).to.be.a('string') }) }) }) From 83c1930378900908f62af485de6b656b3a96405e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 17 Aug 2022 12:25:23 +0800 Subject: [PATCH 17/28] Split text into chunks of 5000 characters in paragraph --- packages/api/src/routers/article_router.ts | 2 +- packages/api/src/utils/textToSpeech.ts | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 85ccf9b0a..e9f851286 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -120,7 +120,7 @@ export function articleRouter() { return res.status(200).send('Page not found') } - const text = parseHTML(page.content).document.documentElement.textContent + const text = parseHTML(page.content).document.documentElement.innerText if (!text) { return res.status(200).send('Page has no text') } diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index 7f1a1d45c..b8249c14e 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -129,13 +129,19 @@ export const synthesizeTextToSpeech = async ( ) }) } - // slice the text into chunks of 1,000 characters - const textChunks = input.text.match(/(.|[\r\n]){1,1000}/g) || [] + // slice the text into chunks of 5,000 characters + let currentTextChunk = '' + const textChunks = input.text.split('\n') for (const textChunk of textChunks) { - logger.debug(`synthesizing ${textChunk}`) - const result = await speakTextAsyncPromise(textChunk) + currentTextChunk += textChunk + '\n' + if (currentTextChunk.length < 5000) { + continue + } + logger.debug(`synthesizing ${currentTextChunk}`) + const result = await speakTextAsyncPromise(currentTextChunk) timeOffset = timeOffset + result.audioDuration - characterOffset = characterOffset + textChunk.length + characterOffset = characterOffset + currentTextChunk.length + currentTextChunk = '' } writeStream.end() synthesizer.close() From 19b13d164d3ba8b0a8876b2d25fd4d02904e4f3b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 17 Aug 2022 15:05:58 +0800 Subject: [PATCH 18/28] Fix a bug not getting user personalization --- packages/api/src/routers/article_router.ts | 55 ++++++++++--------- packages/api/src/routers/svc/speech.ts | 63 +++++++++++++++------- 2 files changed, 73 insertions(+), 45 deletions(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index e9f851286..ab40d3365 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -83,14 +83,11 @@ export function articleRouter() { } const { uid } = jwt.decode(token) as Claims - const startTime = Date.now() logger.info(`Get article speech in ${outputFormat} format`, { params: req.params, labels: { userId: uid, - source: 'GetArticleSpeechMp3', - articleId: id, - outputFormat, + source: `GetArticleSpeech-${outputFormat}`, }, }) @@ -112,39 +109,45 @@ export function articleRouter() { user: { id: uid }, }) if (!userPersonalization) { - return res.status(200).send('userPersonalization not found') + return res.status(404).send('User Personalization not found') } const page = await getPageById(id) if (!page) { - return res.status(200).send('Page not found') + return res.status(404).send('Page not found') } const text = parseHTML(page.content).document.documentElement.innerText if (!text) { - return res.status(200).send('Page has no text') + return res.status(404).send('Page has no text') } - const speechOutput = await synthesizeTextToSpeech({ - id, - text, - languageCode: page.language, - voice: userPersonalization.speechVoice, - }) + try { + const startTime = Date.now() + const speechOutput = await synthesizeTextToSpeech({ + id, + text, + languageCode: page.language, + voice: userPersonalization.speechVoice, + }) + logger.info('Created speech', { + audioUrl: speechOutput.audioUrl, + speechMarksUrl: speechOutput.speechMarksUrl, + duration: Date.now() - startTime, + }) - const speech = await getRepository(Speech).save({ - elasticPageId: id, - audioUrl: speechOutput.audioUrl, - speechMarksUrl: speechOutput.speechMarksUrl, - user: { id: uid }, - }) + const speech = await getRepository(Speech).save({ + elasticPageId: id, + audioUrl: speechOutput.audioUrl, + speechMarksUrl: speechOutput.speechMarksUrl, + user: { id: uid }, + }) - logger.info('Created speech', { - audioUrl: speech.audioUrl, - speechMarksUrl: speech.speechMarksUrl, - duration: Date.now() - startTime, - }) - res.redirect(redirectUrl(speech, outputFormat)) + res.redirect(redirectUrl(speech, outputFormat)) + } catch (error) { + logger.error('Text to speech error', { error }) + res.status(500).send('Text to speech error') + } } ) @@ -155,7 +158,7 @@ const redirectUrl = (speech: Speech, outputFormat: string) => { switch (outputFormat) { case 'mp3': return speech.audioUrl - case 'json': + case 'speech-marks': return speech.speechMarksUrl default: return speech.audioUrl diff --git a/packages/api/src/routers/svc/speech.ts b/packages/api/src/routers/svc/speech.ts index d31807110..4899cecda 100644 --- a/packages/api/src/routers/svc/speech.ts +++ b/packages/api/src/routers/svc/speech.ts @@ -2,11 +2,14 @@ import express from 'express' import cors from 'cors' import { corsConfig } from '../../utils/corsConfig' import { getRepository } from '../../entity/utils' -import { User } from '../../entity/user' import { getPageById } from '../../elastic/pages' import { synthesizeTextToSpeech } from '../../utils/textToSpeech' import { Speech } from '../../entity/speech' import { parseHTML } from 'linkedom' +import { UserPersonalization } from '../../entity/user_personalization' +import { buildLogger } from '../../utils/logger' + +const logger = buildLogger('app.dispatch') export function speechServiceRouter() { const router = express.Router() @@ -23,12 +26,13 @@ export function speechServiceRouter() { return res.status(200).send('Invalid data') } - const user = await getRepository(User).findOne({ - where: { id: userId }, - relations: ['user_personalization'], + const userPersonalization = await getRepository( + UserPersonalization + ).findOneBy({ + user: { id: userId }, }) - if (!user) { - return res.status(200).send('User not found') + if (!userPersonalization) { + return res.status(200).send('User Personalization not found') } const page = await getPageById(pageId) @@ -36,26 +40,47 @@ export function speechServiceRouter() { return res.status(200).send('Page not found') } - const text = parseHTML(page.content).document.documentElement.textContent + const text = parseHTML(page.content).document.documentElement.innerText if (!text) { return res.status(200).send('Page has no text') } - const speech = await synthesizeTextToSpeech({ - id: pageId, - text, - languageCode: page.language, - voice: user.userPersonalization.speechVoice, + logger.info(`Create article speech`, { + body: { + userId, + pageId, + }, + labels: { + source: 'CreateArticleSpeech', + }, }) - await getRepository(Speech).save({ - elasticPageId: pageId, - audioUrl: speech.audioUrl, - speechMarks: speech.speechMarksUrl, - user, - }) + try { + const startTime = Date.now() + const speechOutput = await synthesizeTextToSpeech({ + id: pageId, + text, + languageCode: page.language, + voice: userPersonalization.speechVoice, + }) + logger.info('Created speech', { + audioUrl: speechOutput.audioUrl, + speechMarksUrl: speechOutput.speechMarksUrl, + duration: Date.now() - startTime, + }) - res.status(200).send('OK') + await getRepository(Speech).save({ + elasticPageId: pageId, + audioUrl: speechOutput.audioUrl, + speechMarksUrl: speechOutput.speechMarksUrl, + user: { id: userId }, + }) + + res.status(200).send('OK') + } catch (error) { + logger.error(`Error creating article speech`, { error }) + res.status(500).send('Error creating article speech') + } }) return router From dee94f7c932505fdae54d60c76c1612208d07fab Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 17 Aug 2022 16:03:30 +0800 Subject: [PATCH 19/28] Fix a bug for converting text to speech for articles with less than 5000 characters --- packages/api/src/utils/parser.ts | 3 --- packages/api/src/utils/textToSpeech.ts | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 2e9997e58..a46aed2bc 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -285,13 +285,11 @@ export const parsePreparedContent = async ( // Get the top level element? const pageNode = article.dom.firstElementChild as HTMLElement - console.log('pageNode: ', pageNode) const nodesToVisitStack: [HTMLElement] = [pageNode] const visitedNodeList = [] while (nodesToVisitStack.length > 0) { const currentNode = nodesToVisitStack.pop() - console.log('currentNode: ', currentNode?.nodeType) if ( currentNode?.nodeType !== 1 || // Avoiding dynamic elements from being counted as anchor-allowed elements @@ -316,7 +314,6 @@ export const parsePreparedContent = async ( node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString()) }) - console.log('article content:', article.dom.outerHTML) article.content = article.dom.outerHTML } diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index b8249c14e..ae2101240 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -132,9 +132,9 @@ export const synthesizeTextToSpeech = async ( // slice the text into chunks of 5,000 characters let currentTextChunk = '' const textChunks = input.text.split('\n') - for (const textChunk of textChunks) { - currentTextChunk += textChunk + '\n' - if (currentTextChunk.length < 5000) { + for (let i = 0; i < textChunks.length; i++) { + currentTextChunk += textChunks[i] + '\n' + if (currentTextChunk.length < 5000 && i < textChunks.length - 1) { continue } logger.debug(`synthesizing ${currentTextChunk}`) From c79651202d4a124268ebb88bb63929b9ddf55d43 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 18 Aug 2022 11:49:11 +0800 Subject: [PATCH 20/28] Add synthesize SSML to speech and test --- packages/api/src/utils/textToSpeech.ts | 102 +++++++++++++------ packages/api/test/utils/textToSpeech.test.ts | 17 +++- 2 files changed, 88 insertions(+), 31 deletions(-) diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index ae2101240..5d7eeb795 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -17,6 +17,9 @@ export interface TextToSpeechInput { text: string voice?: string languageCode?: string + textType?: 'text' | 'ssml' + rate?: number + volume?: number } export interface TextToSpeechOutput { @@ -47,8 +50,11 @@ export const synthesizeTextToSpeech = async ( env.azure.speechKey, env.azure.speechRegion ) - speechConfig.speechSynthesisLanguage = input.languageCode || 'en-US' - speechConfig.speechSynthesisVoiceName = input.voice || 'en-US-JennyNeural' + const textType = input.textType || 'text' + if (textType === 'text') { + speechConfig.speechSynthesisLanguage = input.languageCode || 'en-US' + speechConfig.speechSynthesisVoiceName = input.voice || 'en-US-JennyNeural' + } speechConfig.speechSynthesisOutputFormat = SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 @@ -129,19 +135,59 @@ export const synthesizeTextToSpeech = async ( ) }) } - // slice the text into chunks of 5,000 characters - let currentTextChunk = '' - const textChunks = input.text.split('\n') - for (let i = 0; i < textChunks.length; i++) { - currentTextChunk += textChunks[i] + '\n' - if (currentTextChunk.length < 5000 && i < textChunks.length - 1) { - continue + + const speakSsmlAsyncPromise = ( + text: string + ): Promise => { + return new Promise((resolve, reject) => { + synthesizer.speakSsmlAsync( + text, + (result) => { + resolve(result) + }, + (error) => { + synthesizer.close() + reject(error) + } + ) + }) + } + + if (textType === 'text') { + // slice the text into chunks of 5,000 characters + let currentTextChunk = '' + const textChunks = input.text.split('\n') + for (let i = 0; i < textChunks.length; i++) { + currentTextChunk += textChunks[i] + '\n' + if (currentTextChunk.length < 5000 && i < textChunks.length - 1) { + continue + } + logger.debug(`synthesizing ${currentTextChunk}`) + const result = await speakTextAsyncPromise(currentTextChunk) + timeOffset = timeOffset + result.audioDuration + characterOffset = characterOffset + currentTextChunk.length + currentTextChunk = '' + } + } else { + const document = parseHTML(input.text).document + const elements = document.querySelectorAll('h1, h2, h3, p, li') + // convert html elements to the ssml document + for (const e of Array.from(elements)) { + const htmlElement = e as HTMLElement + if (htmlElement.innerText) { + const result = await speakSsmlAsyncPromise( + htmlElementToSsml( + htmlElement, + input.languageCode, + input.voice, + input.rate, + input.volume + ) + ) + timeOffset = timeOffset + result.audioDuration + characterOffset = characterOffset + htmlElement.innerText.length + } } - logger.debug(`synthesizing ${currentTextChunk}`) - const result = await speakTextAsyncPromise(currentTextChunk) - timeOffset = timeOffset + result.audioDuration - characterOffset = characterOffset + currentTextChunk.length - currentTextChunk = '' } writeStream.end() synthesizer.close() @@ -164,15 +210,13 @@ export const synthesizeTextToSpeech = async ( } } -export const htmlToSsml = ( - html: string, +export const htmlElementToSsml = ( + htmlElement: HTMLElement, language = 'en-US', voice = 'en-US-JennyNeural', - rate = 100, + rate = 1, volume = 100 ): string => { - const document = parseHTML(html).document - const paragraphs = document.querySelectorAll('p') // create new ssml document const ssml = parseHTML('').document const speakElement = ssml.createElement('speak') @@ -183,20 +227,18 @@ export const htmlToSsml = ( voiceElement.setAttribute('name', voice) speakElement.appendChild(voiceElement) const prosodyElement = ssml.createElement('prosody') - prosodyElement.setAttribute('rate', `${rate}%`) + prosodyElement.setAttribute('rate', `${rate}`) prosodyElement.setAttribute('volume', volume.toString()) voiceElement.appendChild(prosodyElement) // add each paragraph to the ssml document - paragraphs.forEach((p) => { - const id = p.getAttribute('data-omnivore-anchor-idx') - if (id) { - const text = p.innerText - const bookMark = ssml.createElement('bookmark') - bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`) - bookMark.innerText = text - prosodyElement.appendChild(bookMark) - } - }) + const id = htmlElement.getAttribute('data-omnivore-anchor-idx') + if (id) { + const text = htmlElement.innerText + const bookMark = ssml.createElement('bookmark') + bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`) + prosodyElement.appendChild(bookMark) + prosodyElement.appendChild(ssml.createTextNode(text)) + } return speakElement.outerHTML } diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts index f60a8a8c0..2ae0c76ce 100644 --- a/packages/api/test/utils/textToSpeech.test.ts +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -1,23 +1,38 @@ import 'mocha' import { + htmlElementToSsml, synthesizeTextToSpeech, TextToSpeechInput, } from '../../src/utils/textToSpeech' import { expect } from 'chai' import { generateFakeUuid } from '../util' +import { parseHTML } from 'linkedom' describe('textToSpeech', () => { - describe('createAudioWithSpeechMarks', () => { + describe('synthesizeTextToSpeech', () => { it('should create an audio file with speech marks', async () => { const input: TextToSpeechInput = { id: generateFakeUuid(), text: 'Marry had a little lamb', languageCode: 'en-US', voice: 'en-US-JennyNeural', + textType: 'text', } const output = await synthesizeTextToSpeech(input) expect(output.audioUrl).to.be.a('string') expect(output.speechMarksUrl).to.be.a('string') }) }) + + describe('htmlElementToSsml', () => { + it('should convert Html Element to SSML', async () => { + const htmlElement = parseHTML( + `

Marry had a little lamb

` + ).document.documentElement + const ssml = htmlElementToSsml(htmlElement) + expect(ssml).to.equal( + `Marry had a little lamb` + ) + }) + }) }) From a4a8fa9241999a1524e617fd84ac40fe9199aa8d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 18 Aug 2022 16:32:50 +0800 Subject: [PATCH 21/28] Replace emphasis elements in HTML --- packages/api/src/routers/article_router.ts | 12 +-- packages/api/src/routers/svc/speech.ts | 14 ++- packages/api/src/utils/textToSpeech.ts | 97 +++++++++++++++---- .../api/test/utils/data/text-to-speech.html | 1 + packages/api/test/utils/textToSpeech.test.ts | 12 ++- 5 files changed, 100 insertions(+), 36 deletions(-) create mode 100644 packages/api/test/utils/data/text-to-speech.html diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index ab40d3365..cbc740894 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -18,7 +18,6 @@ import { Claims } from '../resolvers/types' import { getRepository } from '../entity/utils' import { Speech } from '../entity/speech' import { getPageById } from '../elastic/pages' -import { parseHTML } from 'linkedom' import { synthesizeTextToSpeech } from '../utils/textToSpeech' import { UserPersonalization } from '../entity/user_personalization' @@ -117,18 +116,19 @@ export function articleRouter() { return res.status(404).send('Page not found') } - const text = parseHTML(page.content).document.documentElement.innerText - if (!text) { - return res.status(404).send('Page has no text') - } + // const text = parseHTML(page.content).document.documentElement.innerText + // if (!text) { + // return res.status(404).send('Page has no text') + // } try { const startTime = Date.now() const speechOutput = await synthesizeTextToSpeech({ id, - text, + text: page.content, languageCode: page.language, voice: userPersonalization.speechVoice, + textType: 'ssml', }) logger.info('Created speech', { audioUrl: speechOutput.audioUrl, diff --git a/packages/api/src/routers/svc/speech.ts b/packages/api/src/routers/svc/speech.ts index 4899cecda..4aea9b2dc 100644 --- a/packages/api/src/routers/svc/speech.ts +++ b/packages/api/src/routers/svc/speech.ts @@ -5,7 +5,6 @@ import { getRepository } from '../../entity/utils' import { getPageById } from '../../elastic/pages' import { synthesizeTextToSpeech } from '../../utils/textToSpeech' import { Speech } from '../../entity/speech' -import { parseHTML } from 'linkedom' import { UserPersonalization } from '../../entity/user_personalization' import { buildLogger } from '../../utils/logger' @@ -39,12 +38,10 @@ export function speechServiceRouter() { if (!page) { return res.status(200).send('Page not found') } - - const text = parseHTML(page.content).document.documentElement.innerText - if (!text) { - return res.status(200).send('Page has no text') - } - + // const text = parseHTML(page.content).document.documentElement.innerText + // if (!text) { + // return res.status(200).send('Page has no text') + // } logger.info(`Create article speech`, { body: { userId, @@ -59,9 +56,10 @@ export function speechServiceRouter() { const startTime = Date.now() const speechOutput = await synthesizeTextToSpeech({ id: pageId, - text, + text: page.content, languageCode: page.language, voice: userPersonalization.speechVoice, + textType: 'ssml', }) logger.info('Created speech', { audioUrl: speechOutput.audioUrl, diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index 5d7eeb795..30352b133 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -107,7 +107,7 @@ export const synthesizeTextToSpeech = async ( } synthesizer.bookmarkReached = (s, e) => { - logger.info( + logger.debug( `(Bookmark reached), Audio offset: ${ e.audioOffset / 10000 }ms, bookmark text: ${e.text}` @@ -170,20 +170,20 @@ export const synthesizeTextToSpeech = async ( } } else { const document = parseHTML(input.text).document - const elements = document.querySelectorAll('h1, h2, h3, p, li') + const elements = document.querySelectorAll('h1, h2, h3, p, ul, ol') // convert html elements to the ssml document for (const e of Array.from(elements)) { const htmlElement = e as HTMLElement if (htmlElement.innerText) { - const result = await speakSsmlAsyncPromise( - htmlElementToSsml( - htmlElement, - input.languageCode, - input.voice, - input.rate, - input.volume - ) + const ssml = htmlElementToSsml( + e, + input.languageCode, + input.voice, + input.rate, + input.volume ) + logger.debug(`synthesizing ${ssml}`) + const result = await speakSsmlAsyncPromise(ssml) timeOffset = timeOffset + result.audioDuration characterOffset = characterOffset + htmlElement.innerText.length } @@ -211,12 +211,32 @@ export const synthesizeTextToSpeech = async ( } export const htmlElementToSsml = ( - htmlElement: HTMLElement, + htmlElement: Element, language = 'en-US', voice = 'en-US-JennyNeural', rate = 1, volume = 100 ): string => { + const appendBookmarkElement = (parent: Element, element: Element) => { + const id = element.getAttribute('data-omnivore-anchor-idx') + if (id) { + const bookMark = ssml.createElement('bookmark') + bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`) + parent.appendChild(bookMark) + } + } + + const replaceEmphasisElement = (element: Element, level: string) => { + logger.debug(`replaceEmphasisElement: ${element.innerHTML}`) + const parent = ssml.createDocumentFragment() as unknown as Element + appendBookmarkElement(parent, element) + const emphasisElement = ssml.createElement('emphasis') + emphasisElement.setAttribute('level', level) + emphasisElement.innerHTML = element.innerHTML.trim() + parent.appendChild(emphasisElement) + element?.parentNode?.replaceChild(parent, element) + } + // create new ssml document const ssml = parseHTML('').document const speakElement = ssml.createElement('speak') @@ -231,14 +251,53 @@ export const htmlElementToSsml = ( prosodyElement.setAttribute('volume', volume.toString()) voiceElement.appendChild(prosodyElement) // add each paragraph to the ssml document - const id = htmlElement.getAttribute('data-omnivore-anchor-idx') - if (id) { - const text = htmlElement.innerText - const bookMark = ssml.createElement('bookmark') - bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`) - prosodyElement.appendChild(bookMark) - prosodyElement.appendChild(ssml.createTextNode(text)) - } + appendBookmarkElement(prosodyElement, htmlElement) + // add text to the ssml document + htmlElement.querySelectorAll('*').forEach((e) => { + switch (e.tagName.toLowerCase()) { + case 's': + replaceEmphasisElement(e, 'reduced') + break + case 'sub': + if (e.getAttribute('alias') === null) { + replaceEmphasisElement(e, 'reduced') + } + break + case 'i': + case 'em': + case 'q': + case 'blockquote': + case 'cite': + case 'del': + case 'strike': + case 'sup': + case 'summary': + case 'caption': + case 'figcaption': + replaceEmphasisElement(e, 'reduced') + break + case 'b': + case 'strong': + case 'dt': + case 'dfn': + case 'u': + case 'li': + case 'mark': + case 'th': + case 'title': + case 'var': + replaceEmphasisElement(e, 'moderate') + break + default: { + const text = (e as HTMLElement).innerText.trim() + if (text) { + const textElement = ssml.createTextNode(text) + e.parentNode?.replaceChild(textElement, e) + } + } + } + }) + prosodyElement.appendChild(htmlElement) return speakElement.outerHTML } diff --git a/packages/api/test/utils/data/text-to-speech.html b/packages/api/test/utils/data/text-to-speech.html new file mode 100644 index 000000000..65245fe69 --- /dev/null +++ b/packages/api/test/utils/data/text-to-speech.html @@ -0,0 +1 @@ +

An Instinct for Dragons is a book by University of Central Florida anthropologist, David E. Jones, in which he seeks to explain the universality of dragon images in the folklore of human societies. In the introduction, Jones conducts a survey of dragon myths from cultures around the world and argues that certain aspects of dragons or dragon-like mythical creatures are found very widely. He claims that even the Inuit have a reptilian dragon-like monster, even though (living in a frigid environment unsuited for cold-blooded animals) they had never seen an actual reptile.

Jones then argues against the common hypothesis that dragon myths might be motivated by primitive discoveries of dinosaur fossils (he argues that there are widespread traits of dragons in folklore which are not observable from fossils), and claims that the common traits of dragons seem to be an amalgam of the principal predators of our ancestral hominids, which he names as the raptors, great cats (especially leopards) and pythons.

The hypothesis to which Jones conforms is that over millions of years of evolution, members of a species will evolve an instinctive fear of their predators, and he proposes ways in which these fearful images may be merged in artistic or cultural expression to create the dragon image and, perhaps, other kinds of hybrid monster.

Finally he suggests sociological reasons for why such images may be perceived differently at different stages of a culture to try to explain why Chinese dragons are considered basically good and representative of government, but the great majority (although not all) European dragons are evil and often represent chaos.

Reception

Jones' theory was opposed in an article by Paul Jordan-Smith in the Spring 2002 issue of Western Folklore and by other authors. Jordan-Smith criticized the lack of evidence given to prove why dragon myths could not have been passed from culture to culture. He also notes that it cannot be demonstrated that the fears of ancestral hominids are coded into the human brain. He concludes his review by writing "One is tempted to say, as Dorothy Parker once did, that this is a book not to be tossed aside lightly but thrown violently. But no, it is not worth spending even that much energy on."[1]

D. Ogden writes that Jones' ideas "might offer pause for thought given the universality of dragon-slaying narratives". He adds, though, that the compound cat, snake, raptor creature imagined by Jones is mostly the Western stereotype based on mediaeval imagery, and that Jones has sought out similar images in a way that lacks rigor. In particular, Ogden notes that the dragons of Graeco-Roman myth do not fit with Jones's prototype, typically lacking one or more of the hybrid components (with the exception of Typhon, who, however, combines many more animals than Jones's three).[2]

References

  1. ^ Jordan-Smith, Paul (2002). "Review: An Instinct for Dragons". Western Folklore. JSTOR .
  2. ^ Ogden, Daniel (2013). Drakon: Dragon Myth and Serpent Cult in the Greek and Roman Worlds. Oxford University Press. pp. 24–25.
diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts index 2ae0c76ce..4d9608415 100644 --- a/packages/api/test/utils/textToSpeech.test.ts +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -7,16 +7,22 @@ import { import { expect } from 'chai' import { generateFakeUuid } from '../util' import { parseHTML } from 'linkedom' +import fs from 'fs' describe('textToSpeech', () => { + const load = (path: string): string => { + return fs.readFileSync(path, 'utf8') + } + describe('synthesizeTextToSpeech', () => { it('should create an audio file with speech marks', async () => { + const html = load('./test/utils/data/text-to-speech.html') const input: TextToSpeechInput = { id: generateFakeUuid(), - text: 'Marry had a little lamb', + text: html, languageCode: 'en-US', voice: 'en-US-JennyNeural', - textType: 'text', + textType: 'ssml', } const output = await synthesizeTextToSpeech(input) expect(output.audioUrl).to.be.a('string') @@ -31,7 +37,7 @@ describe('textToSpeech', () => { ).document.documentElement const ssml = htmlElementToSsml(htmlElement) expect(ssml).to.equal( - `Marry had a little lamb` + `

Marry had a little lamb

` ) }) }) From 29061cf7b1f178bea24495f5aa152269318e8b77 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 18 Aug 2022 17:05:42 +0800 Subject: [PATCH 22/28] Remove   in SSML --- packages/api/src/utils/textToSpeech.ts | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index 30352b133..418affe2f 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -177,15 +177,15 @@ export const synthesizeTextToSpeech = async ( if (htmlElement.innerText) { const ssml = htmlElementToSsml( e, - input.languageCode, - input.voice, - input.rate, - input.volume + input.languageCode || 'en-US', + input.voice || 'en-US-JennyNeural', + input.rate || 1, + input.volume || 100 ) logger.debug(`synthesizing ${ssml}`) const result = await speakSsmlAsyncPromise(ssml) timeOffset = timeOffset + result.audioDuration - characterOffset = characterOffset + htmlElement.innerText.length + // characterOffset = characterOffset + htmlElement.innerText.length } } } @@ -227,7 +227,6 @@ export const htmlElementToSsml = ( } const replaceEmphasisElement = (element: Element, level: string) => { - logger.debug(`replaceEmphasisElement: ${element.innerHTML}`) const parent = ssml.createDocumentFragment() as unknown as Element appendBookmarkElement(parent, element) const emphasisElement = ssml.createElement('emphasis') @@ -252,7 +251,7 @@ export const htmlElementToSsml = ( voiceElement.appendChild(prosodyElement) // add each paragraph to the ssml document appendBookmarkElement(prosodyElement, htmlElement) - // add text to the ssml document + // replace emphasis elements with ssml htmlElement.querySelectorAll('*').forEach((e) => { switch (e.tagName.toLowerCase()) { case 's': @@ -290,14 +289,12 @@ export const htmlElementToSsml = ( break default: { const text = (e as HTMLElement).innerText.trim() - if (text) { - const textElement = ssml.createTextNode(text) - e.parentNode?.replaceChild(textElement, e) - } + const textElement = ssml.createTextNode(text) + e.parentNode?.replaceChild(textElement, e) } } }) prosodyElement.appendChild(htmlElement) - return speakElement.outerHTML + return speakElement.outerHTML.replace(/ /g, '') } From 4cf5b934eb14a42c50d1ea9cdd251aff6bbe528c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 18 Aug 2022 17:15:42 +0800 Subject: [PATCH 23/28] Add validation to request params --- packages/api/src/routers/article_router.ts | 3 +++ packages/api/test/utils/textToSpeech.test.ts | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index cbc740894..6219af45a 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -76,6 +76,9 @@ export function articleRouter() { async (req, res) => { const id = req.params.id const outputFormat = req.params.outputFormat + if (!id || !['mp3', 'speech-marks'].includes(outputFormat)) { + return res.status(400).send('Invalid data') + } const token = req.cookies?.auth || req.headers?.authorization if (!token || !jwt.verify(token, env.server.jwtSecret)) { return res.status(401).send({ errorCode: 'UNAUTHORIZED' }) diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts index 4d9608415..357f68445 100644 --- a/packages/api/test/utils/textToSpeech.test.ts +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -15,7 +15,7 @@ describe('textToSpeech', () => { } describe('synthesizeTextToSpeech', () => { - it('should create an audio file with speech marks', async () => { + xit('should create an audio file with speech marks', async () => { const html = load('./test/utils/data/text-to-speech.html') const input: TextToSpeechInput = { id: generateFakeUuid(), From 249a5cda4a4aaa001be9f37db51c30aef56a1677 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 18 Aug 2022 17:39:12 +0800 Subject: [PATCH 24/28] Add bucket, audioFileName, speechMarksFileName, state to speech table --- packages/api/src/entity/speech.ts | 14 ++++++++++++-- .../migrations/0094.do.add_state_to_speech.sql | 17 +++++++++++++++++ .../0094.undo.add_state_to_speech.sql | 17 +++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) create mode 100755 packages/db/migrations/0094.do.add_state_to_speech.sql create mode 100755 packages/db/migrations/0094.undo.add_state_to_speech.sql diff --git a/packages/api/src/entity/speech.ts b/packages/api/src/entity/speech.ts index ac64f7e71..d35fac57f 100644 --- a/packages/api/src/entity/speech.ts +++ b/packages/api/src/entity/speech.ts @@ -9,6 +9,13 @@ import { } from 'typeorm' import { User } from './user' +export enum SpeechState { + INITIALIZED = 'INITIALIZED', + COMPLETED = 'COMPLETED', + FAILED = 'FAILED', + CANCELLED = 'CANCELLED', +} + @Entity({ name: 'speech' }) export class Speech { @PrimaryGeneratedColumn('uuid') @@ -22,14 +29,17 @@ export class Speech { elasticPageId!: string @Column('text') - audioUrl!: string + audioFileName!: string @Column('text') - speechMarksUrl!: string + speechMarksFileName!: string @Column('text') voice!: string + @Column('enum', { enum: SpeechState }) + state!: SpeechState + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) createdAt!: Date diff --git a/packages/db/migrations/0094.do.add_state_to_speech.sql b/packages/db/migrations/0094.do.add_state_to_speech.sql new file mode 100755 index 000000000..878dbbba7 --- /dev/null +++ b/packages/db/migrations/0094.do.add_state_to_speech.sql @@ -0,0 +1,17 @@ +-- Type: DO +-- Name: add_state_to_speech +-- Description: Add state field to speech table + +BEGIN; + +CREATE TYPE speech_state_type AS ENUM ('INITIALIZED', 'COMPLETED', 'FAILED', 'CANCELLED'); + +ALTER TABLE omnivore.speech + DROP COLUMN audio_url, + DROP COLUMN speech_marks_url, + ADD COLUMN bucket VARCHAR(255) NOT NULL DEFAULT '', + ADD COLUMN audio_file_name VARCHAR(255) NOT NULL DEFAULT '', + ADD COLUMN speech_marks_file_name VARCHAR(255) NOT NULL DEFAULT '', + ADD COLUMN state speech_state_type NOT NULL DEFAULT 'INITIALIZED'; + +COMMIT; diff --git a/packages/db/migrations/0094.undo.add_state_to_speech.sql b/packages/db/migrations/0094.undo.add_state_to_speech.sql new file mode 100755 index 000000000..727277958 --- /dev/null +++ b/packages/db/migrations/0094.undo.add_state_to_speech.sql @@ -0,0 +1,17 @@ +-- Type: UNDO +-- Name: add_state_to_speech +-- Description: Add state field to speech table + +BEGIN; + +DROP TYPE IF EXISTS speech_state_type; + +ALTER TABLE omnivore.speech + DROP COLUMN bucket, + DROP COLUMN audio_file_name, + DROP COLUMN speech_marks_file_name, + DROP COLUMN state speech_state_type, + ADD COLUMN audio_url text NOT NULL, + ADD COLUMN speech_marks_url text NOT NULL; + +COMMIT; From 401633eda6e84822f285808a87738b761164c070 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 18 Aug 2022 18:03:33 +0800 Subject: [PATCH 25/28] Return signed url --- packages/api/src/entity/speech.ts | 6 +- packages/api/src/routers/article_router.ts | 55 +++++++++++++------ packages/api/src/routers/svc/speech.ts | 28 +++++++--- packages/api/src/utils/textToSpeech.ts | 28 ++++------ .../0094.do.add_state_to_speech.sql | 1 - .../0094.undo.add_state_to_speech.sql | 6 +- 6 files changed, 75 insertions(+), 49 deletions(-) diff --git a/packages/api/src/entity/speech.ts b/packages/api/src/entity/speech.ts index d35fac57f..38d5ccd86 100644 --- a/packages/api/src/entity/speech.ts +++ b/packages/api/src/entity/speech.ts @@ -28,16 +28,16 @@ export class Speech { @Column('text') elasticPageId!: string - @Column('text') + @Column('text', { default: '' }) audioFileName!: string - @Column('text') + @Column('text', { default: '' }) speechMarksFileName!: string @Column('text') voice!: string - @Column('enum', { enum: SpeechState }) + @Column('enum', { enum: SpeechState, default: SpeechState.INITIALIZED }) state!: SpeechState @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 6219af45a..25731076a 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -16,10 +16,11 @@ import * as jwt from 'jsonwebtoken' import { env } from '../env' import { Claims } from '../resolvers/types' import { getRepository } from '../entity/utils' -import { Speech } from '../entity/speech' +import { Speech, SpeechState } from '../entity/speech' import { getPageById } from '../elastic/pages' import { synthesizeTextToSpeech } from '../utils/textToSpeech' import { UserPersonalization } from '../entity/user_personalization' +import { generateDownloadSignedUrl } from '../utils/uploads' const logger = buildLogger('app.dispatch') @@ -96,12 +97,17 @@ export function articleRouter() { const existingSpeech = await getRepository(Speech).findOneBy({ elasticPageId: id, }) - if (existingSpeech) { - logger.info('Found existing speech', { - audioUrl: existingSpeech.audioUrl, - speechMarksUrl: existingSpeech.speechMarksUrl, + if (existingSpeech?.state === SpeechState.COMPLETED) { + logger.info('Found existing completed speech', { + audioUrl: existingSpeech.audioFileName, + speechMarksUrl: existingSpeech.speechMarksFileName, }) - return res.redirect(redirectUrl(existingSpeech, outputFormat)) + return res.redirect(await redirectUrl(existingSpeech, outputFormat)) + } + if (existingSpeech?.state === SpeechState.INITIALIZED) { + logger.info('Found existing in progress speech') + // retry later + return res.status(429).send('Speech is in progress') } logger.debug('Text to speech request', { articleId: id }) @@ -124,6 +130,13 @@ export function articleRouter() { // return res.status(404).send('Page has no text') // } + // initialize state + const speech = await getRepository(Speech).save({ + user: { id: uid }, + elasticPageId: id, + state: SpeechState.INITIALIZED, + voice: userPersonalization.speechVoice, + }) try { const startTime = Date.now() const speechOutput = await synthesizeTextToSpeech({ @@ -134,21 +147,27 @@ export function articleRouter() { textType: 'ssml', }) logger.info('Created speech', { - audioUrl: speechOutput.audioUrl, - speechMarksUrl: speechOutput.speechMarksUrl, + audioFileName: speechOutput.audioFileName, + speechMarksFileName: speechOutput.speechMarksFileName, duration: Date.now() - startTime, }) - const speech = await getRepository(Speech).save({ - elasticPageId: id, - audioUrl: speechOutput.audioUrl, - speechMarksUrl: speechOutput.speechMarksUrl, - user: { id: uid }, + // update state + await getRepository(Speech).update(speech.id, { + state: SpeechState.COMPLETED, + audioFileName: speech.audioFileName, + speechMarksFileName: speech.speechMarksFileName, }) + speech.audioFileName = speechOutput.audioFileName + speech.speechMarksFileName = speechOutput.speechMarksFileName - res.redirect(redirectUrl(speech, outputFormat)) + res.redirect(await redirectUrl(speech, outputFormat)) } catch (error) { logger.error('Text to speech error', { error }) + // update state + await getRepository(Speech).update(speech.id, { + state: SpeechState.FAILED, + }) res.status(500).send('Text to speech error') } } @@ -157,13 +176,13 @@ export function articleRouter() { return router } -const redirectUrl = (speech: Speech, outputFormat: string) => { +const redirectUrl = async (speech: Speech, outputFormat: string) => { switch (outputFormat) { case 'mp3': - return speech.audioUrl + return generateDownloadSignedUrl(speech.audioFileName) case 'speech-marks': - return speech.speechMarksUrl + return generateDownloadSignedUrl(speech.speechMarksFileName) default: - return speech.audioUrl + return generateDownloadSignedUrl(speech.audioFileName) } } diff --git a/packages/api/src/routers/svc/speech.ts b/packages/api/src/routers/svc/speech.ts index 4aea9b2dc..6c9e40d56 100644 --- a/packages/api/src/routers/svc/speech.ts +++ b/packages/api/src/routers/svc/speech.ts @@ -4,7 +4,7 @@ import { corsConfig } from '../../utils/corsConfig' import { getRepository } from '../../entity/utils' import { getPageById } from '../../elastic/pages' import { synthesizeTextToSpeech } from '../../utils/textToSpeech' -import { Speech } from '../../entity/speech' +import { Speech, SpeechState } from '../../entity/speech' import { UserPersonalization } from '../../entity/user_personalization' import { buildLogger } from '../../utils/logger' @@ -52,6 +52,14 @@ export function speechServiceRouter() { }, }) + // initialize state + const speech = await getRepository(Speech).save({ + user: { id: userId }, + elasticPageId: pageId, + state: SpeechState.INITIALIZED, + voice: userPersonalization.speechVoice, + }) + try { const startTime = Date.now() const speechOutput = await synthesizeTextToSpeech({ @@ -62,21 +70,25 @@ export function speechServiceRouter() { textType: 'ssml', }) logger.info('Created speech', { - audioUrl: speechOutput.audioUrl, - speechMarksUrl: speechOutput.speechMarksUrl, + audioFileName: speechOutput.audioFileName, + speechMarksFileName: speechOutput.speechMarksFileName, duration: Date.now() - startTime, }) - await getRepository(Speech).save({ - elasticPageId: pageId, - audioUrl: speechOutput.audioUrl, - speechMarksUrl: speechOutput.speechMarksUrl, - user: { id: userId }, + // update state + await getRepository(Speech).update(speech.id, { + audioFileName: speechOutput.audioFileName, + speechMarksFileName: speechOutput.speechMarksFileName, + state: SpeechState.COMPLETED, }) res.status(200).send('OK') } catch (error) { logger.error(`Error creating article speech`, { error }) + // update state + await getRepository(Speech).update(speech.id, { + state: SpeechState.FAILED, + }) res.status(500).send('Error creating article speech') } }) diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index 418affe2f..42a057b3b 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -1,5 +1,5 @@ import { buildLogger } from './logger' -import { createGCSFile, getFilePublicUrl, uploadToBucket } from './uploads' +import { createGCSFile, uploadToBucket } from './uploads' import { CancellationDetails, CancellationReason, @@ -23,8 +23,8 @@ export interface TextToSpeechInput { } export interface TextToSpeechOutput { - audioUrl: string - speechMarksUrl: string + audioFileName: string + speechMarksFileName: string } export interface SpeechMark { @@ -40,10 +40,9 @@ const logger = buildLogger('app.dispatch') export const synthesizeTextToSpeech = async ( input: TextToSpeechInput ): Promise => { - const audioFile = `speech/${input.id}.mp3` - const gcsFile = createGCSFile(audioFile) - const writeStream = gcsFile.createWriteStream({ - public: true, + const audioFileName = `speech/${input.id}.mp3` + const audioFile = createGCSFile(audioFileName) + const writeStream = audioFile.createWriteStream({ resumable: true, }) const speechConfig = SpeechConfig.fromSubscription( @@ -192,21 +191,18 @@ export const synthesizeTextToSpeech = async ( writeStream.end() synthesizer.close() - logger.debug(`audio file: ${audioFile}`) + logger.debug(`audio file: ${audioFileName}`) // upload Speech Marks file to GCS - const speechMarksFile = `speech/${input.id}.json` + const speechMarksFileName = `speech/${input.id}.json` await uploadToBucket( - speechMarksFile, - Buffer.from(JSON.stringify(speechMarks)), - { - public: true, - } + speechMarksFileName, + Buffer.from(JSON.stringify(speechMarks)) ) return { - audioUrl: getFilePublicUrl(audioFile), - speechMarksUrl: getFilePublicUrl(speechMarksFile), + audioFileName, + speechMarksFileName, } } diff --git a/packages/db/migrations/0094.do.add_state_to_speech.sql b/packages/db/migrations/0094.do.add_state_to_speech.sql index 878dbbba7..edd6d2eea 100755 --- a/packages/db/migrations/0094.do.add_state_to_speech.sql +++ b/packages/db/migrations/0094.do.add_state_to_speech.sql @@ -9,7 +9,6 @@ CREATE TYPE speech_state_type AS ENUM ('INITIALIZED', 'COMPLETED', 'FAILED', 'CA ALTER TABLE omnivore.speech DROP COLUMN audio_url, DROP COLUMN speech_marks_url, - ADD COLUMN bucket VARCHAR(255) NOT NULL DEFAULT '', ADD COLUMN audio_file_name VARCHAR(255) NOT NULL DEFAULT '', ADD COLUMN speech_marks_file_name VARCHAR(255) NOT NULL DEFAULT '', ADD COLUMN state speech_state_type NOT NULL DEFAULT 'INITIALIZED'; diff --git a/packages/db/migrations/0094.undo.add_state_to_speech.sql b/packages/db/migrations/0094.undo.add_state_to_speech.sql index 727277958..ff865b40a 100755 --- a/packages/db/migrations/0094.undo.add_state_to_speech.sql +++ b/packages/db/migrations/0094.undo.add_state_to_speech.sql @@ -4,14 +4,14 @@ BEGIN; -DROP TYPE IF EXISTS speech_state_type; - ALTER TABLE omnivore.speech DROP COLUMN bucket, DROP COLUMN audio_file_name, DROP COLUMN speech_marks_file_name, - DROP COLUMN state speech_state_type, + DROP COLUMN state, ADD COLUMN audio_url text NOT NULL, ADD COLUMN speech_marks_url text NOT NULL; +DROP TYPE IF EXISTS speech_state_type CASCADE; + COMMIT; From 576318fcbe82cc2d6ac9868b44e94a9645deaa0a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 18 Aug 2022 19:16:40 +0800 Subject: [PATCH 26/28] Add authentication to speech svc endpoint --- packages/api/src/routers/svc/speech.ts | 11 +++++++++++ packages/api/src/utils/createTask.ts | 21 +++++++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/api/src/routers/svc/speech.ts b/packages/api/src/routers/svc/speech.ts index 6c9e40d56..2798fe953 100644 --- a/packages/api/src/routers/svc/speech.ts +++ b/packages/api/src/routers/svc/speech.ts @@ -7,6 +7,7 @@ import { synthesizeTextToSpeech } from '../../utils/textToSpeech' import { Speech, SpeechState } from '../../entity/speech' import { UserPersonalization } from '../../entity/user_personalization' import { buildLogger } from '../../utils/logger' +import { getClaimsByToken } from '../../utils/auth' const logger = buildLogger('app.dispatch') @@ -16,6 +17,16 @@ export function speechServiceRouter() { router.options('/', cors({ ...corsConfig, maxAge: 600 })) // eslint-disable-next-line @typescript-eslint/no-misused-promises router.post('/', async (req, res) => { + logger.info('Speech svc request', { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + body: req.body, + }) + const token = req.query.token as string + if (!(await getClaimsByToken(token))) { + logger.info('Unauthorized request', { token }) + return res.status(200).send('UNAUTHORIZED') + } + const { userId, pageId } = req.body as { userId: string pageId: string diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index ae9e230b3..3f6ecfbce 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -9,9 +9,12 @@ import { buildLogger } from './logger' import { nanoid } from 'nanoid' import { google } from '@google-cloud/tasks/build/protos/protos' import { IntegrationType } from '../entity/integration' +import { promisify } from 'util' +import * as jwt from 'jsonwebtoken' import View = google.cloud.tasks.v2.Task.View const logger = buildLogger('app.dispatch') +const signToken = promisify(jwt.sign) // Instantiates a client. const client = new CloudTasksClient() @@ -334,24 +337,26 @@ export const enqueueTextToSpeech = async ( userId, pageId, } - + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const token = await signToken({ uid: userId }, env.server.jwtSecret, { + expiresIn: '1h', + }) + const taskHandlerUrl = `${env.queue.textToSpeechTaskHandlerUrl}?token=${token}` // If there is no Google Cloud Project Id exposed, it means that we are in local environment if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { // Calling the handler function directly. setTimeout(() => { - axios - .post(env.queue.textToSpeechTaskHandlerUrl, payload) - .catch((error) => { - logger.error(error) - }) + axios.post(taskHandlerUrl, payload).catch((error) => { + logger.error(error) + }) }, 0) return '' } - const createdTasks = await createHttpTaskWithToken({ project: GOOGLE_CLOUD_PROJECT, payload, - taskHandlerUrl: env.queue.textToSpeechTaskHandlerUrl, + taskHandlerUrl, }) if (!createdTasks || !createdTasks[0].name) { From a1171c795271394f3c91e4f68b6a018813eecf3b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 18 Aug 2022 19:19:59 +0800 Subject: [PATCH 27/28] Catch authentication error --- packages/api/src/routers/svc/speech.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/api/src/routers/svc/speech.ts b/packages/api/src/routers/svc/speech.ts index 2798fe953..b6f5136ab 100644 --- a/packages/api/src/routers/svc/speech.ts +++ b/packages/api/src/routers/svc/speech.ts @@ -22,8 +22,13 @@ export function speechServiceRouter() { body: req.body, }) const token = req.query.token as string - if (!(await getClaimsByToken(token))) { - logger.info('Unauthorized request', { token }) + try { + if (!(await getClaimsByToken(token))) { + logger.info('Unauthorized request', { token }) + return res.status(200).send('UNAUTHORIZED') + } + } catch (error) { + logger.error('Unauthorized request', { token, error }) return res.status(200).send('UNAUTHORIZED') } From 530464f54da8e6c02c68b626debbc5281fe30da7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 18 Aug 2022 21:29:35 +0800 Subject: [PATCH 28/28] Fix tests --- packages/api/test/utils/textToSpeech.test.ts | 4 ++-- yarn.lock | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts index 357f68445..2276b2ac4 100644 --- a/packages/api/test/utils/textToSpeech.test.ts +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -25,8 +25,8 @@ describe('textToSpeech', () => { textType: 'ssml', } const output = await synthesizeTextToSpeech(input) - expect(output.audioUrl).to.be.a('string') - expect(output.speechMarksUrl).to.be.a('string') + expect(output.audioFileName).to.be.a('string') + expect(output.speechMarksFileName).to.be.a('string') }) }) diff --git a/yarn.lock b/yarn.lock index 14046d3e7..075a3d86b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18598,7 +18598,7 @@ mkdirp-infer-owner@^2.0.0: infer-owner "^1.0.4" mkdirp "^1.0.3" -mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.5: +mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.5: version "0.5.6" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==