From bca0f5ed594eac369b708080b09dff3a21385270 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 11 Aug 2022 16:58:59 +0800 Subject: [PATCH] Create Amazon Polly text to speech integration --- packages/api/package.json | 1 + packages/api/src/utils/textToSpeech.ts | 96 ++++++++++++++++++++++++++ yarn.lock | 66 +++++++++++++++++- 3 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 packages/api/src/utils/textToSpeech.ts diff --git a/packages/api/package.json b/packages/api/package.json index a51bece89..688c2ef87 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -40,6 +40,7 @@ "analytics-node": "^6.0.0", "apollo-datasource": "^3.3.1", "apollo-server-express": "^3.6.3", + "aws-sdk": "^2.1192.0", "axios": "^0.27.2", "bcryptjs": "^2.4.3", "cookie": "^0.5.0", diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts new file mode 100644 index 000000000..c348b9c4b --- /dev/null +++ b/packages/api/src/utils/textToSpeech.ts @@ -0,0 +1,96 @@ +import * as AWS from 'aws-sdk' +import { buildLogger } from './logger' +import { SynthesizeSpeechInput } from 'aws-sdk/clients/polly' +import { + generateUploadFilePathName, + generateUploadSignedUrl, + getFilePublicUrl, + uploadToSignedUrl, +} from './uploads' + +export interface TextToSpeechInput { + id: string + title: string + text: string + voice?: string + textType?: 'text' | 'ssml' + engine?: 'standard' | 'neural' +} + +export interface TextToSpeechOutput { + audioUrl: string + speechMarks: string +} + +const logger = buildLogger('app.dispatch') + +// create a new AWS Polly client +const client = new AWS.Polly() + +export const createAudio = async ( + input: TextToSpeechInput +): Promise => { + const { text, voice, textType, engine } = input + const params: SynthesizeSpeechInput = { + OutputFormat: 'mp3', + Text: text, + TextType: textType || 'text', + VoiceId: voice || 'Joanna', + Engine: engine || 'neural', + } + try { + const data = await client.synthesizeSpeech(params).promise() + return data.AudioStream as Buffer + } catch (error) { + logger.error('Unable to create audio file', { error }) + throw error + } +} + +export const createSpeechMarks = async ( + input: TextToSpeechInput +): Promise => { + const { text, voice, textType, engine } = input + const params: SynthesizeSpeechInput = { + OutputFormat: 'json', + Text: text, + TextType: textType || 'text', + VoiceId: voice || 'Joanna', + Engine: engine || 'neural', + SpeechMarkTypes: ['sentence', 'word'], + } + try { + const data = await client.synthesizeSpeech(params).promise() + return data.AudioStream as string + } catch (error) { + logger.error('Unable to create speech marks', { error }) + throw error + } +} + +export const createAudioWithSpeechMarks = async ( + input: TextToSpeechInput +): Promise => { + try { + const audio = await createAudio(input) + // upload audio to google cloud storage + logger.info('generating upload url...') + const filePathName = generateUploadFilePathName(input.id, input.title) + const contentType = 'audio/mpeg' + const uploadUrl = await generateUploadSignedUrl(filePathName, contentType) + + logger.info('start uploading...', { uploadUrl }) + await uploadToSignedUrl(uploadUrl, audio, contentType) + + // get public url for audio file + const publicUrl = getFilePublicUrl(filePathName) + const speechMarks = await createSpeechMarks(input) + return { + audioUrl: publicUrl, + speechMarks, + } + } catch (error) { + logger.error('Unable to create audio with speech marks', { error }) + throw error + } +} diff --git a/yarn.lock b/yarn.lock index f5c4a4d79..e706dd28c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9484,6 +9484,22 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== +aws-sdk@^2.1192.0: + version "2.1192.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.1192.0.tgz#13fe38ec8dae3232f17d52b370e69daa9c7a0e9c" + integrity sha512-6uzrlG1Ow3qcOnL0+et+DBTGhYgJzgNydVvos1Eg01vPc/ZhxR7roZ3epZQcPmOR0thQuzzckTq7FBO6wzZA2w== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.16.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + util "^0.12.4" + uuid "8.0.0" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -10132,7 +10148,7 @@ buffer-xor@^1.0.3: resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= -buffer@^4.3.0: +buffer@4.9.2, buffer@^4.3.0: version "4.9.2" resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== @@ -13160,6 +13176,11 @@ eventid@^2.0.0: dependencies: uuid "^8.0.0" +events@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + integrity sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw== + events@^3.0.0, events@^3.2.0: version "3.3.0" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" @@ -15336,6 +15357,11 @@ icss-utils@^5.0.0, icss-utils@^5.1.0: resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae" integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA== +ieee754@1.1.13: + version "1.1.13" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" + integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== + ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -16813,6 +16839,11 @@ jest@^27.4.5: import-local "^3.0.2" jest-cli "^27.5.1" +jmespath@0.16.0: + version "0.16.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076" + integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw== + join-component@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/join-component/-/join-component-1.1.0.tgz#b8417b750661a392bee2c2537c68b2a9d4977cd5" @@ -22066,6 +22097,11 @@ sanitize-html@^2.3.2: parse-srcset "^1.0.2" postcss "^8.0.2" +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== + sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -24416,6 +24452,14 @@ url-template@^2.0.8: resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21" integrity sha1-/FZaPMy/93MMd19WQflVV5FDnyE= +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + integrity sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ== + dependencies: + punycode "1.3.2" + querystring "0.2.0" + url@^0.11.0: version "0.11.0" resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" @@ -24530,12 +24574,17 @@ uuid-browser@^3.1.0: resolved "https://registry.yarnpkg.com/uuid-browser/-/uuid-browser-3.1.0.tgz#0f05a40aef74f9e5951e20efbf44b11871e56410" integrity sha1-DwWkCu90+eWVHiDvv0SxGHHlZBA= +uuid@8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c" + integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw== + uuid@^3.2.1, uuid@^3.3.2, uuid@^3.3.3: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== -uuid@^8.0.0, uuid@^8.3.1, uuid@^8.3.2: +uuid@^8.0.0, uuid@^8.3.2: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== @@ -25300,6 +25349,14 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + xml2js@^0.4.23: version "0.4.23" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" @@ -25313,6 +25370,11 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"