diff --git a/packages/api/package.json b/packages/api/package.json index a51bece89..28ba0df6c 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -67,6 +67,7 @@ "knex-stringcase": "^1.4.2", "linkedom": "^0.14.9", "luxon": "^2.3.1", + "microsoft-cognitiveservices-speech-sdk": "^1.22.0", "nanoid": "^3.1.25", "nodemailer": "^6.7.3", "normalize-url": "^6.1.0", diff --git a/packages/api/src/entity/speech.ts b/packages/api/src/entity/speech.ts new file mode 100644 index 000000000..38d5ccd86 --- /dev/null +++ b/packages/api/src/entity/speech.ts @@ -0,0 +1,48 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' +import { User } from './user' + +export enum SpeechState { + INITIALIZED = 'INITIALIZED', + COMPLETED = 'COMPLETED', + FAILED = 'FAILED', + CANCELLED = 'CANCELLED', +} + +@Entity({ name: 'speech' }) +export class Speech { + @PrimaryGeneratedColumn('uuid') + id!: string + + @ManyToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'user_id' }) + user!: User + + @Column('text') + elasticPageId!: string + + @Column('text', { default: '' }) + audioFileName!: string + + @Column('text', { default: '' }) + speechMarksFileName!: string + + @Column('text') + voice!: string + + @Column('enum', { enum: SpeechState, default: SpeechState.INITIALIZED }) + state!: SpeechState + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + createdAt!: Date + + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + updatedAt!: Date +} diff --git a/packages/api/src/entity/user.ts b/packages/api/src/entity/user.ts index 58c4a6975..b03c36e11 100644 --- a/packages/api/src/entity/user.ts +++ b/packages/api/src/entity/user.ts @@ -12,6 +12,7 @@ import { NewsletterEmail } from './newsletter_email' import { Profile } from './profile' import { Label } from './label' import { Subscription } from './subscription' +import { UserPersonalization } from './user_personalization' @Entity() export class User { @@ -53,4 +54,10 @@ export class User { @Column({ type: 'enum', enum: StatusType }) status!: StatusType + + @OneToOne( + () => UserPersonalization, + (userPersonalization) => userPersonalization.user + ) + userPersonalization!: UserPersonalization } diff --git a/packages/api/src/entity/user_personalization.ts b/packages/api/src/entity/user_personalization.ts new file mode 100644 index 000000000..8044ef1a7 --- /dev/null +++ b/packages/api/src/entity/user_personalization.ts @@ -0,0 +1,53 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + OneToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' +import { User } from './user' + +@Entity({ name: 'user_personalization' }) +export class UserPersonalization { + @PrimaryGeneratedColumn('uuid') + id!: string + + @OneToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'user_id' }) + user!: User + + @Column('text', { nullable: true }) + fontFamily?: string + + @Column('integer', { nullable: true }) + fontSize?: number + + @Column('text', { nullable: true }) + margin?: number + + @Column('text', { nullable: true }) + theme?: string + + @Column('text', { nullable: true }) + libraryLayoutType?: string + + @Column('text', { nullable: true }) + librarySortOrder?: string + + @Column('text', { nullable: true }) + speechVoice?: string + + @Column('integer', { nullable: true }) + speechRate?: number + + @Column('integer', { nullable: true }) + speechVolume?: number + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + createdAt!: Date + + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + updatedAt!: Date +} diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index bd9e26ee5..f3251e6ff 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -94,6 +94,7 @@ import { updatePage, } from '../../elastic/pages' import { searchHighlights } from '../../elastic/highlights' +import { enqueueTextToSpeech } from '../../utils/createTask' export type PartialArticle = Omit< Article, @@ -372,6 +373,10 @@ export const createArticleResolver = authorized< articleToSave.id = newPageId } + // enqueue a task to convert text to speech + const taskName = await enqueueTextToSpeech(uid, articleToSave.id) + log.info('Text to speech task name', { taskName }) + log.info( 'page created in elastic', articleToSave.id, diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 5774483d6..25731076a 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -3,15 +3,24 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' -import { CreateArticleErrorCode } from './../generated/graphql' -import { isSiteBlockedForParse } from './../utils/blocked' +import { CreateArticleErrorCode } from '../generated/graphql' +import { isSiteBlockedForParse } from '../utils/blocked' import cors from 'cors' -import { buildLogger } from './../utils/logger' +import { buildLogger } from '../utils/logger' import { corsConfig } from '../utils/corsConfig' import { createPageSaveRequest } from '../services/create_page_save_request' import { initModels } from '../server' import { kx } from '../datalayer/knex_config' import { getClaimsByToken } from '../utils/auth' +import * as jwt from 'jsonwebtoken' +import { env } from '../env' +import { Claims } from '../resolvers/types' +import { getRepository } from '../entity/utils' +import { Speech, SpeechState } from '../entity/speech' +import { getPageById } from '../elastic/pages' +import { synthesizeTextToSpeech } from '../utils/textToSpeech' +import { UserPersonalization } from '../entity/user_personalization' +import { generateDownloadSignedUrl } from '../utils/uploads' const logger = buildLogger('app.dispatch') @@ -61,5 +70,119 @@ export function articleRouter() { articleSavingRequestId: result.id, }) }) + + router.get( + '/:id/:outputFormat', + cors(corsConfig), + async (req, res) => { + const id = req.params.id + const outputFormat = req.params.outputFormat + if (!id || !['mp3', 'speech-marks'].includes(outputFormat)) { + return res.status(400).send('Invalid data') + } + const token = req.cookies?.auth || req.headers?.authorization + if (!token || !jwt.verify(token, env.server.jwtSecret)) { + return res.status(401).send({ errorCode: 'UNAUTHORIZED' }) + } + const { uid } = jwt.decode(token) as Claims + + logger.info(`Get article speech in ${outputFormat} format`, { + params: req.params, + labels: { + userId: uid, + source: `GetArticleSpeech-${outputFormat}`, + }, + }) + + const existingSpeech = await getRepository(Speech).findOneBy({ + elasticPageId: id, + }) + if (existingSpeech?.state === SpeechState.COMPLETED) { + logger.info('Found existing completed speech', { + audioUrl: existingSpeech.audioFileName, + speechMarksUrl: existingSpeech.speechMarksFileName, + }) + return res.redirect(await redirectUrl(existingSpeech, outputFormat)) + } + if (existingSpeech?.state === SpeechState.INITIALIZED) { + logger.info('Found existing in progress speech') + // retry later + return res.status(429).send('Speech is in progress') + } + + logger.debug('Text to speech request', { articleId: id }) + const userPersonalization = await getRepository( + UserPersonalization + ).findOneBy({ + user: { id: uid }, + }) + if (!userPersonalization) { + return res.status(404).send('User Personalization not found') + } + + const page = await getPageById(id) + if (!page) { + return res.status(404).send('Page not found') + } + + // const text = parseHTML(page.content).document.documentElement.innerText + // if (!text) { + // return res.status(404).send('Page has no text') + // } + + // initialize state + const speech = await getRepository(Speech).save({ + user: { id: uid }, + elasticPageId: id, + state: SpeechState.INITIALIZED, + voice: userPersonalization.speechVoice, + }) + try { + const startTime = Date.now() + const speechOutput = await synthesizeTextToSpeech({ + id, + text: page.content, + languageCode: page.language, + voice: userPersonalization.speechVoice, + textType: 'ssml', + }) + logger.info('Created speech', { + audioFileName: speechOutput.audioFileName, + speechMarksFileName: speechOutput.speechMarksFileName, + duration: Date.now() - startTime, + }) + + // update state + await getRepository(Speech).update(speech.id, { + state: SpeechState.COMPLETED, + audioFileName: speech.audioFileName, + speechMarksFileName: speech.speechMarksFileName, + }) + speech.audioFileName = speechOutput.audioFileName + speech.speechMarksFileName = speechOutput.speechMarksFileName + + res.redirect(await redirectUrl(speech, outputFormat)) + } catch (error) { + logger.error('Text to speech error', { error }) + // update state + await getRepository(Speech).update(speech.id, { + state: SpeechState.FAILED, + }) + res.status(500).send('Text to speech error') + } + } + ) + return router } + +const redirectUrl = async (speech: Speech, outputFormat: string) => { + switch (outputFormat) { + case 'mp3': + return generateDownloadSignedUrl(speech.audioFileName) + case 'speech-marks': + return generateDownloadSignedUrl(speech.speechMarksFileName) + default: + return generateDownloadSignedUrl(speech.audioFileName) + } +} diff --git a/packages/api/src/routers/svc/speech.ts b/packages/api/src/routers/svc/speech.ts new file mode 100644 index 000000000..b6f5136ab --- /dev/null +++ b/packages/api/src/routers/svc/speech.ts @@ -0,0 +1,113 @@ +import express from 'express' +import cors from 'cors' +import { corsConfig } from '../../utils/corsConfig' +import { getRepository } from '../../entity/utils' +import { getPageById } from '../../elastic/pages' +import { synthesizeTextToSpeech } from '../../utils/textToSpeech' +import { Speech, SpeechState } from '../../entity/speech' +import { UserPersonalization } from '../../entity/user_personalization' +import { buildLogger } from '../../utils/logger' +import { getClaimsByToken } from '../../utils/auth' + +const logger = buildLogger('app.dispatch') + +export function speechServiceRouter() { + const router = express.Router() + + router.options('/', cors({ ...corsConfig, maxAge: 600 })) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + router.post('/', async (req, res) => { + logger.info('Speech svc request', { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + body: req.body, + }) + const token = req.query.token as string + try { + if (!(await getClaimsByToken(token))) { + logger.info('Unauthorized request', { token }) + return res.status(200).send('UNAUTHORIZED') + } + } catch (error) { + logger.error('Unauthorized request', { token, error }) + return res.status(200).send('UNAUTHORIZED') + } + + const { userId, pageId } = req.body as { + userId: string + pageId: string + } + + if (!userId || !pageId) { + return res.status(200).send('Invalid data') + } + + const userPersonalization = await getRepository( + UserPersonalization + ).findOneBy({ + user: { id: userId }, + }) + if (!userPersonalization) { + return res.status(200).send('User Personalization not found') + } + + const page = await getPageById(pageId) + if (!page) { + return res.status(200).send('Page not found') + } + // const text = parseHTML(page.content).document.documentElement.innerText + // if (!text) { + // return res.status(200).send('Page has no text') + // } + logger.info(`Create article speech`, { + body: { + userId, + pageId, + }, + labels: { + source: 'CreateArticleSpeech', + }, + }) + + // initialize state + const speech = await getRepository(Speech).save({ + user: { id: userId }, + elasticPageId: pageId, + state: SpeechState.INITIALIZED, + voice: userPersonalization.speechVoice, + }) + + try { + const startTime = Date.now() + const speechOutput = await synthesizeTextToSpeech({ + id: pageId, + text: page.content, + languageCode: page.language, + voice: userPersonalization.speechVoice, + textType: 'ssml', + }) + logger.info('Created speech', { + audioFileName: speechOutput.audioFileName, + speechMarksFileName: speechOutput.speechMarksFileName, + duration: Date.now() - startTime, + }) + + // update state + await getRepository(Speech).update(speech.id, { + audioFileName: speechOutput.audioFileName, + speechMarksFileName: speechOutput.speechMarksFileName, + state: SpeechState.COMPLETED, + }) + + res.status(200).send('OK') + } catch (error) { + logger.error(`Error creating article speech`, { error }) + // update state + await getRepository(Speech).update(speech.id, { + state: SpeechState.FAILED, + }) + res.status(500).send('Error creating article speech') + } + }) + + return router +} diff --git a/packages/api/src/routers/svc/upload.ts b/packages/api/src/routers/svc/upload.ts index ebc935f86..c2378225d 100644 --- a/packages/api/src/routers/svc/upload.ts +++ b/packages/api/src/routers/svc/upload.ts @@ -3,60 +3,52 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' import { readPushSubscription } from '../../datalayer/pubsub' -import { generateUploadSignedUrl, uploadToSignedUrl } from '../../utils/uploads' +import { uploadToBucket } from '../../utils/uploads' import { v4 as uuidv4 } from 'uuid' import { env } from '../../env' import { DateTime } from 'luxon' +import { buildLogger } from '../../utils/logger' + +const logger = buildLogger('app.dispatch') export function uploadServiceRouter() { const router = express.Router() router.post('/:folder', async (req, res) => { - console.log('upload data to folder', req.params.folder) + logger.info('upload data to folder', req.params.folder) const { message: msgStr, expired } = readPushSubscription(req) if (!msgStr) { - res.status(400).send('Bad Request') - return + return res.status(400).send('Bad Request') } if (expired) { - console.log('discarding expired message') - res.status(200).send('Expired') - return + logger.info('discarding expired message') + return res.status(200).send('Expired') } try { const data: { userId: string; type: string } = JSON.parse(msgStr) if (!data.userId || !data.type) { - console.log('No userId or type found in message') - res.status(400).send('Bad Request') - return + logger.info('No userId or type found in message') + return res.status(400).send('Bad Request') } - const contentType = 'application/json' - const bucketName = env.fileUpload.gcsUploadPrivateBucket + const filePath = `${req.params.folder}/${data.type}/${ + data.userId + }/${DateTime.now().toFormat('yyyy-LL-dd')}/${uuidv4()}.json` - console.log('generate upload url') - - const uploadUrl = await generateUploadSignedUrl( - `${req.params.folder}/${data.type}/${ - data.userId - }/${DateTime.now().toFormat('yyyy-LL-dd')}/${uuidv4()}.json`, - contentType, - bucketName - ) - - console.log('start uploading', uploadUrl) - - await uploadToSignedUrl( - uploadUrl, + logger.info('uploading data to', filePath) + await uploadToBucket( + filePath, Buffer.from(msgStr, 'utf8'), - contentType + { contentType: 'application/json' }, + env.fileUpload.gcsUploadPrivateBucket ) + res.status(200).send('OK') } catch (err) { - console.log('upload page data failed', err) + logger.error('upload page data failed', err) res.status(500).send(err) } }) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 619a4fda5..7c9273f92 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -45,6 +45,7 @@ import { uploadServiceRouter } from './routers/svc/upload' import rateLimit from 'express-rate-limit' import { webhooksServiceRouter } from './routers/svc/webhooks' import { integrationsServiceRouter } from './routers/svc/integrations' +import { speechServiceRouter } from './routers/svc/speech' const PORT = process.env.PORT || 4000 @@ -119,6 +120,7 @@ export const createApp = (): { app.use('/svc/pubsub/integrations', integrationsServiceRouter()) app.use('/svc/reminders', remindersServiceRouter()) app.use('/svc/pdf-attachments', pdfAttachmentsRouter()) + app.use('/svc/text-to-speech', speechServiceRouter()) if (env.dev.isLocal) { app.use('/local/debug', localDebugRouter()) diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index f43a9a8b2..1ca344d02 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -63,6 +63,7 @@ interface BackendEnv { contentFetchGCFUrl: string reminderTaskHanderUrl: string integrationTaskHandlerUrl: string + textToSpeechTaskHandlerUrl: string } fileUpload: { gcsUploadBucket: string @@ -88,6 +89,10 @@ interface BackendEnv { readwise: { apiUrl: string } + azure: { + speechKey: string + speechRegion: string + } } /*** @@ -138,6 +143,9 @@ const nullableEnvVars = [ 'SENDGRID_INSTALLATION_TEMPLATE_ID', 'READWISE_API_URL', 'INTEGRATION_TASK_HANDLER_URL', + 'TEXT_TO_SPEECH_TASK_HANDLER_URL', + 'AZURE_SPEECH_KEY', + 'AZURE_SPEECH_REGION', ] // Allow some vars to be null/empty /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */ @@ -221,6 +229,7 @@ export function getEnv(): BackendEnv { contentFetchGCFUrl: parse('CONTENT_FETCH_GCF_URL'), reminderTaskHanderUrl: parse('REMINDER_TASK_HANDLER_URL'), integrationTaskHandlerUrl: parse('INTEGRATION_TASK_HANDLER_URL'), + textToSpeechTaskHandlerUrl: parse('TEXT_TO_SPEECH_TASK_HANDLER_URL'), } const imageProxy = { url: parse('IMAGE_PROXY_URL'), @@ -256,6 +265,11 @@ export function getEnv(): BackendEnv { apiUrl: parse('READWISE_API_URL'), } + const azure = { + speechKey: parse('AZURE_SPEECH_KEY'), + speechRegion: parse('AZURE_SPEECH_REGION'), + } + return { pg, client, @@ -274,6 +288,7 @@ export function getEnv(): BackendEnv { sender, sendgrid, readwise, + azure, } } diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 3dec00231..3f6ecfbce 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -9,9 +9,12 @@ import { buildLogger } from './logger' import { nanoid } from 'nanoid' import { google } from '@google-cloud/tasks/build/protos/protos' import { IntegrationType } from '../entity/integration' +import { promisify } from 'util' +import * as jwt from 'jsonwebtoken' import View = google.cloud.tasks.v2.Task.View const logger = buildLogger('app.dispatch') +const signToken = promisify(jwt.sign) // Instantiates a client. const client = new CloudTasksClient() @@ -325,4 +328,45 @@ export const enqueueSyncWithIntegration = async ( return createdTasks[0].name } +export const enqueueTextToSpeech = async ( + userId: string, + pageId: string +): Promise => { + const { GOOGLE_CLOUD_PROJECT } = process.env + const payload = { + userId, + pageId, + } + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const token = await signToken({ uid: userId }, env.server.jwtSecret, { + expiresIn: '1h', + }) + const taskHandlerUrl = `${env.queue.textToSpeechTaskHandlerUrl}?token=${token}` + // If there is no Google Cloud Project Id exposed, it means that we are in local environment + if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { + // Calling the handler function directly. + setTimeout(() => { + axios.post(taskHandlerUrl, payload).catch((error) => { + logger.error(error) + }) + }, 0) + return '' + } + const createdTasks = await createHttpTaskWithToken({ + project: GOOGLE_CLOUD_PROJECT, + payload, + taskHandlerUrl, + }) + + if (!createdTasks || !createdTasks[0].name) { + logger.error(`Unable to get the name of the task`, { + payload, + createdTasks, + }) + throw new CreateTaskError(`Unable to get the name of the task`) + } + return createdTasks[0].name +} + export default createHttpTaskWithToken diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 8900fc5d3..a46aed2bc 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -276,6 +276,45 @@ export const parsePreparedContent = async ( }) article.content = article.dom.outerHTML } + + const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ + 'omnivore-highlight-id', + 'data-twitter-tweet-id', + 'data-instagram-id', + ] + + // Get the top level element? + const pageNode = article.dom.firstElementChild as HTMLElement + const nodesToVisitStack: [HTMLElement] = [pageNode] + const visitedNodeList = [] + + while (nodesToVisitStack.length > 0) { + const currentNode = nodesToVisitStack.pop() + if ( + currentNode?.nodeType !== 1 || + // Avoiding dynamic elements from being counted as anchor-allowed elements + ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES.some((attrib) => + currentNode.hasAttribute(attrib) + ) + ) { + continue + } + visitedNodeList.push(currentNode) + ;[].slice + .call(currentNode.childNodes) + .reverse() + .forEach(function (node) { + nodesToVisitStack.push(node) + }) + } + + visitedNodeList.shift() + visitedNodeList.forEach((node, index) => { + // start from index 1, index 0 reserved for anchor unknown. + node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString()) + }) + + article.content = article.dom.outerHTML } const newWindow = parseHTML('') diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts new file mode 100644 index 000000000..42a057b3b --- /dev/null +++ b/packages/api/src/utils/textToSpeech.ts @@ -0,0 +1,296 @@ +import { buildLogger } from './logger' +import { createGCSFile, uploadToBucket } from './uploads' +import { + CancellationDetails, + CancellationReason, + ResultReason, + SpeechConfig, + SpeechSynthesisOutputFormat, + SpeechSynthesisResult, + SpeechSynthesizer, +} from 'microsoft-cognitiveservices-speech-sdk' +import { env } from '../env' +import { parseHTML } from 'linkedom' + +export interface TextToSpeechInput { + id: string + text: string + voice?: string + languageCode?: string + textType?: 'text' | 'ssml' + rate?: number + volume?: number +} + +export interface TextToSpeechOutput { + audioFileName: string + speechMarksFileName: string +} + +export interface SpeechMark { + time: number + start?: number + length?: number + word: string + type: 'word' | 'bookmark' +} + +const logger = buildLogger('app.dispatch') + +export const synthesizeTextToSpeech = async ( + input: TextToSpeechInput +): Promise => { + const audioFileName = `speech/${input.id}.mp3` + const audioFile = createGCSFile(audioFileName) + const writeStream = audioFile.createWriteStream({ + resumable: true, + }) + const speechConfig = SpeechConfig.fromSubscription( + env.azure.speechKey, + env.azure.speechRegion + ) + const textType = input.textType || 'text' + if (textType === 'text') { + speechConfig.speechSynthesisLanguage = input.languageCode || 'en-US' + speechConfig.speechSynthesisVoiceName = input.voice || 'en-US-JennyNeural' + } + speechConfig.speechSynthesisOutputFormat = + SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 + + // Create the speech synthesizer. + const synthesizer = new SpeechSynthesizer(speechConfig) + const speechMarks: SpeechMark[] = [] + let timeOffset = 0 + let characterOffset = 0 + + synthesizer.synthesizing = function (s, e) { + // convert arrayBuffer to stream and write to gcs file + writeStream.write(Buffer.from(e.result.audioData)) + } + + // The event synthesis completed signals that the synthesis is completed. + synthesizer.synthesisCompleted = (s, e) => { + logger.info( + `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${ + e.result.audioData.byteLength + }` + ) + } + + // The synthesis started event signals that the synthesis is started. + synthesizer.synthesisStarted = (s, e) => { + logger.info('(synthesis started)') + } + + // The event signals that the service has stopped processing speech. + // This can happen when an error is encountered. + synthesizer.SynthesisCanceled = (s, e) => { + const cancellationDetails = CancellationDetails.fromResult(e.result) + let str = + '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason] + if (cancellationDetails.reason === CancellationReason.Error) { + str += ': ' + e.result.errorDetails + } + logger.info(str) + } + + // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. + synthesizer.wordBoundary = (s, e) => { + speechMarks.push({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + start: characterOffset + e.textOffset, + length: e.wordLength, + type: 'word', + }) + } + + synthesizer.bookmarkReached = (s, e) => { + logger.debug( + `(Bookmark reached), Audio offset: ${ + e.audioOffset / 10000 + }ms, bookmark text: ${e.text}` + ) + speechMarks.push({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + type: 'bookmark', + }) + } + + const speakTextAsyncPromise = ( + text: string + ): Promise => { + return new Promise((resolve, reject) => { + synthesizer.speakTextAsync( + text, + (result) => { + resolve(result) + }, + (error) => { + synthesizer.close() + reject(error) + } + ) + }) + } + + const speakSsmlAsyncPromise = ( + text: string + ): Promise => { + return new Promise((resolve, reject) => { + synthesizer.speakSsmlAsync( + text, + (result) => { + resolve(result) + }, + (error) => { + synthesizer.close() + reject(error) + } + ) + }) + } + + if (textType === 'text') { + // slice the text into chunks of 5,000 characters + let currentTextChunk = '' + const textChunks = input.text.split('\n') + for (let i = 0; i < textChunks.length; i++) { + currentTextChunk += textChunks[i] + '\n' + if (currentTextChunk.length < 5000 && i < textChunks.length - 1) { + continue + } + logger.debug(`synthesizing ${currentTextChunk}`) + const result = await speakTextAsyncPromise(currentTextChunk) + timeOffset = timeOffset + result.audioDuration + characterOffset = characterOffset + currentTextChunk.length + currentTextChunk = '' + } + } else { + const document = parseHTML(input.text).document + const elements = document.querySelectorAll('h1, h2, h3, p, ul, ol') + // convert html elements to the ssml document + for (const e of Array.from(elements)) { + const htmlElement = e as HTMLElement + if (htmlElement.innerText) { + const ssml = htmlElementToSsml( + e, + input.languageCode || 'en-US', + input.voice || 'en-US-JennyNeural', + input.rate || 1, + input.volume || 100 + ) + logger.debug(`synthesizing ${ssml}`) + const result = await speakSsmlAsyncPromise(ssml) + timeOffset = timeOffset + result.audioDuration + // characterOffset = characterOffset + htmlElement.innerText.length + } + } + } + writeStream.end() + synthesizer.close() + + logger.debug(`audio file: ${audioFileName}`) + + // upload Speech Marks file to GCS + const speechMarksFileName = `speech/${input.id}.json` + await uploadToBucket( + speechMarksFileName, + Buffer.from(JSON.stringify(speechMarks)) + ) + + return { + audioFileName, + speechMarksFileName, + } +} + +export const htmlElementToSsml = ( + htmlElement: Element, + language = 'en-US', + voice = 'en-US-JennyNeural', + rate = 1, + volume = 100 +): string => { + const appendBookmarkElement = (parent: Element, element: Element) => { + const id = element.getAttribute('data-omnivore-anchor-idx') + if (id) { + const bookMark = ssml.createElement('bookmark') + bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`) + parent.appendChild(bookMark) + } + } + + const replaceEmphasisElement = (element: Element, level: string) => { + const parent = ssml.createDocumentFragment() as unknown as Element + appendBookmarkElement(parent, element) + const emphasisElement = ssml.createElement('emphasis') + emphasisElement.setAttribute('level', level) + emphasisElement.innerHTML = element.innerHTML.trim() + parent.appendChild(emphasisElement) + element?.parentNode?.replaceChild(parent, element) + } + + // create new ssml document + const ssml = parseHTML('').document + const speakElement = ssml.createElement('speak') + speakElement.setAttribute('version', '1.0') + speakElement.setAttribute('xmlns', 'http://www.w3.org/2001/10/synthesis') + speakElement.setAttribute('xml:lang', language) + const voiceElement = ssml.createElement('voice') + voiceElement.setAttribute('name', voice) + speakElement.appendChild(voiceElement) + const prosodyElement = ssml.createElement('prosody') + prosodyElement.setAttribute('rate', `${rate}`) + prosodyElement.setAttribute('volume', volume.toString()) + voiceElement.appendChild(prosodyElement) + // add each paragraph to the ssml document + appendBookmarkElement(prosodyElement, htmlElement) + // replace emphasis elements with ssml + htmlElement.querySelectorAll('*').forEach((e) => { + switch (e.tagName.toLowerCase()) { + case 's': + replaceEmphasisElement(e, 'reduced') + break + case 'sub': + if (e.getAttribute('alias') === null) { + replaceEmphasisElement(e, 'reduced') + } + break + case 'i': + case 'em': + case 'q': + case 'blockquote': + case 'cite': + case 'del': + case 'strike': + case 'sup': + case 'summary': + case 'caption': + case 'figcaption': + replaceEmphasisElement(e, 'reduced') + break + case 'b': + case 'strong': + case 'dt': + case 'dfn': + case 'u': + case 'li': + case 'mark': + case 'th': + case 'title': + case 'var': + replaceEmphasisElement(e, 'moderate') + break + default: { + const text = (e as HTMLElement).innerText.trim() + const textElement = ssml.createTextNode(text) + e.parentNode?.replaceChild(textElement, e) + } + } + }) + prosodyElement.appendChild(htmlElement) + + return speakElement.outerHTML.replace(/ /g, '') +} diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 6c4492479..8e631f3e1 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -1,8 +1,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { env } from '../env' -import { GetSignedUrlConfig, Storage } from '@google-cloud/storage' -import axios from 'axios' +import { File, GetSignedUrlConfig, Storage } from '@google-cloud/storage' /* On GAE/Prod, we shall rely on default app engine service account credentials. * Two changes needed: 1) add default service account to our uploads GCS Bucket @@ -102,21 +101,18 @@ export const generateUploadFilePathName = ( return `u/${id}/${fileName}` } -export const uploadToSignedUrl = async ( - uploadUrl: string, +export const uploadToBucket = async ( + filePath: string, data: Buffer, - contentType: string + options?: { contentType?: string; public?: boolean }, + selectedBucket?: string ): Promise => { - // if (env.dev.isLocal) { - // return - // } - - await axios.put(uploadUrl, data, { - headers: { - 'Content-Type': contentType, - }, - maxBodyLength: 1000000000, - maxContentLength: 100000000, - timeout: 30000, - }) + await storage + .bucket(selectedBucket || bucketName) + .file(filePath) + .save(data, options) +} + +export const createGCSFile = (filename: string): File => { + return storage.bucket(bucketName).file(filename) } diff --git a/packages/api/test/utils/data/text-to-speech.html b/packages/api/test/utils/data/text-to-speech.html new file mode 100644 index 000000000..65245fe69 --- /dev/null +++ b/packages/api/test/utils/data/text-to-speech.html @@ -0,0 +1 @@ +

An Instinct for Dragons is a book by University of Central Florida anthropologist, David E. Jones, in which he seeks to explain the universality of dragon images in the folklore of human societies. In the introduction, Jones conducts a survey of dragon myths from cultures around the world and argues that certain aspects of dragons or dragon-like mythical creatures are found very widely. He claims that even the Inuit have a reptilian dragon-like monster, even though (living in a frigid environment unsuited for cold-blooded animals) they had never seen an actual reptile.

Jones then argues against the common hypothesis that dragon myths might be motivated by primitive discoveries of dinosaur fossils (he argues that there are widespread traits of dragons in folklore which are not observable from fossils), and claims that the common traits of dragons seem to be an amalgam of the principal predators of our ancestral hominids, which he names as the raptors, great cats (especially leopards) and pythons.

The hypothesis to which Jones conforms is that over millions of years of evolution, members of a species will evolve an instinctive fear of their predators, and he proposes ways in which these fearful images may be merged in artistic or cultural expression to create the dragon image and, perhaps, other kinds of hybrid monster.

Finally he suggests sociological reasons for why such images may be perceived differently at different stages of a culture to try to explain why Chinese dragons are considered basically good and representative of government, but the great majority (although not all) European dragons are evil and often represent chaos.

Reception

Jones' theory was opposed in an article by Paul Jordan-Smith in the Spring 2002 issue of Western Folklore and by other authors. Jordan-Smith criticized the lack of evidence given to prove why dragon myths could not have been passed from culture to culture. He also notes that it cannot be demonstrated that the fears of ancestral hominids are coded into the human brain. He concludes his review by writing "One is tempted to say, as Dorothy Parker once did, that this is a book not to be tossed aside lightly but thrown violently. But no, it is not worth spending even that much energy on."[1]

D. Ogden writes that Jones' ideas "might offer pause for thought given the universality of dragon-slaying narratives". He adds, though, that the compound cat, snake, raptor creature imagined by Jones is mostly the Western stereotype based on mediaeval imagery, and that Jones has sought out similar images in a way that lacks rigor. In particular, Ogden notes that the dragons of Graeco-Roman myth do not fit with Jones's prototype, typically lacking one or more of the hybrid components (with the exception of Typhon, who, however, combines many more animals than Jones's three).[2]

References

  1. ^ Jordan-Smith, Paul (2002). "Review: An Instinct for Dragons". Western Folklore. JSTOR .
  2. ^ Ogden, Daniel (2013). Drakon: Dragon Myth and Serpent Cult in the Greek and Roman Worlds. Oxford University Press. pp. 24–25.
diff --git a/packages/api/test/utils/textToSpeech.test.ts b/packages/api/test/utils/textToSpeech.test.ts new file mode 100644 index 000000000..2276b2ac4 --- /dev/null +++ b/packages/api/test/utils/textToSpeech.test.ts @@ -0,0 +1,44 @@ +import 'mocha' +import { + htmlElementToSsml, + synthesizeTextToSpeech, + TextToSpeechInput, +} from '../../src/utils/textToSpeech' +import { expect } from 'chai' +import { generateFakeUuid } from '../util' +import { parseHTML } from 'linkedom' +import fs from 'fs' + +describe('textToSpeech', () => { + const load = (path: string): string => { + return fs.readFileSync(path, 'utf8') + } + + describe('synthesizeTextToSpeech', () => { + xit('should create an audio file with speech marks', async () => { + const html = load('./test/utils/data/text-to-speech.html') + const input: TextToSpeechInput = { + id: generateFakeUuid(), + text: html, + languageCode: 'en-US', + voice: 'en-US-JennyNeural', + textType: 'ssml', + } + const output = await synthesizeTextToSpeech(input) + expect(output.audioFileName).to.be.a('string') + expect(output.speechMarksFileName).to.be.a('string') + }) + }) + + describe('htmlElementToSsml', () => { + it('should convert Html Element to SSML', async () => { + const htmlElement = parseHTML( + `

Marry had a little lamb

` + ).document.documentElement + const ssml = htmlElementToSsml(htmlElement) + expect(ssml).to.equal( + `

Marry had a little lamb

` + ) + }) + }) +}) diff --git a/packages/db/migrations/0093.do.speech.sql b/packages/db/migrations/0093.do.speech.sql new file mode 100755 index 000000000..fcda74145 --- /dev/null +++ b/packages/db/migrations/0093.do.speech.sql @@ -0,0 +1,28 @@ +-- Type: DO +-- Name: speech +-- Description: Add speech table containing text to speech audio_url and speech_marks + +BEGIN; + +CREATE TABLE omnivore.speech ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + user_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE, + elastic_page_id TEXT NOT NULL, + voice text, + audio_url text NOT NULL, + speech_marks_url text NOT NULL, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp +); + +CREATE TRIGGER speech_modtime BEFORE UPDATE ON omnivore.speech FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +-- No permission to delete on the speech table, only superuser can delete. +GRANT SELECT, INSERT, UPDATE ON omnivore.speech TO omnivore_user; + +ALTER TABLE omnivore.user_personalization + ADD COLUMN speech_voice TEXT, + ADD COLUMN speech_rate INTEGER, + ADD COLUMN speech_volume INTEGER; + +COMMIT; diff --git a/packages/db/migrations/0093.undo.speech.sql b/packages/db/migrations/0093.undo.speech.sql new file mode 100755 index 000000000..9457f5bd8 --- /dev/null +++ b/packages/db/migrations/0093.undo.speech.sql @@ -0,0 +1,14 @@ +-- Type: UNDO +-- Name: speech +-- Description: Add speech table containing text to speech audio_url and speech_marks + +BEGIN; + +DROP TABLE IF EXISTS omnivore.speech; + +ALTER TABLE omnivore.user_personalization + DROP COLUMN IF EXISTS speech_voice, + DROP COLUMN IF EXISTS speech_rate, + DROP COLUMN IF EXISTS speech_volume; + +COMMIT; diff --git a/packages/db/migrations/0094.do.add_state_to_speech.sql b/packages/db/migrations/0094.do.add_state_to_speech.sql new file mode 100755 index 000000000..edd6d2eea --- /dev/null +++ b/packages/db/migrations/0094.do.add_state_to_speech.sql @@ -0,0 +1,16 @@ +-- Type: DO +-- Name: add_state_to_speech +-- Description: Add state field to speech table + +BEGIN; + +CREATE TYPE speech_state_type AS ENUM ('INITIALIZED', 'COMPLETED', 'FAILED', 'CANCELLED'); + +ALTER TABLE omnivore.speech + DROP COLUMN audio_url, + DROP COLUMN speech_marks_url, + ADD COLUMN audio_file_name VARCHAR(255) NOT NULL DEFAULT '', + ADD COLUMN speech_marks_file_name VARCHAR(255) NOT NULL DEFAULT '', + ADD COLUMN state speech_state_type NOT NULL DEFAULT 'INITIALIZED'; + +COMMIT; diff --git a/packages/db/migrations/0094.undo.add_state_to_speech.sql b/packages/db/migrations/0094.undo.add_state_to_speech.sql new file mode 100755 index 000000000..ff865b40a --- /dev/null +++ b/packages/db/migrations/0094.undo.add_state_to_speech.sql @@ -0,0 +1,17 @@ +-- Type: UNDO +-- Name: add_state_to_speech +-- Description: Add state field to speech table + +BEGIN; + +ALTER TABLE omnivore.speech + DROP COLUMN bucket, + DROP COLUMN audio_file_name, + DROP COLUMN speech_marks_file_name, + DROP COLUMN state, + ADD COLUMN audio_url text NOT NULL, + ADD COLUMN speech_marks_url text NOT NULL; + +DROP TYPE IF EXISTS speech_state_type CASCADE; + +COMMIT; diff --git a/yarn.lock b/yarn.lock index f5c4a4d79..075a3d86b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8739,6 +8739,11 @@ addressparser@^1.0.1: resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746" integrity sha512-aQX7AISOMM7HFE0iZ3+YnD07oIeJqWGVnJ+ZIKaBZAk03ftmVYVqsGas/rbXKR21n4D/hKCSHypvcyOkds/xzg== +agent-base@5: + version "5.1.1" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-5.1.1.tgz#e8fb3f242959db44d63be665db7a8e739537a32c" + integrity sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g== + agent-base@6: version "6.0.1" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.1.tgz#808007e4e5867decb0ab6ab2f928fbdb5a596db4" @@ -8746,7 +8751,7 @@ agent-base@6: dependencies: debug "4" -agent-base@^6.0.2: +agent-base@^6.0.1, agent-base@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== @@ -9360,7 +9365,21 @@ asap@^2.0.0, asap@~2.0.3: resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= -asn1.js@^5.2.0: +asn1.js-rfc2560@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/asn1.js-rfc2560/-/asn1.js-rfc2560-5.0.1.tgz#cff99b903e714756b29503ad49de01c72f131e60" + integrity sha512-1PrVg6kuBziDN3PGFmRk3QrjpKvP9h/Hv5yMrFZvC1kpzP6dQRzf5BpKstANqHBkaOUmTpakJWhicTATOA/SbA== + dependencies: + asn1.js-rfc5280 "^3.0.0" + +asn1.js-rfc5280@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/asn1.js-rfc5280/-/asn1.js-rfc5280-3.0.0.tgz#94e60498d5d4984b842d1a825485837574ccc902" + integrity sha512-Y2LZPOWeZ6qehv698ZgOGGCZXBQShObWnGthTrIFlIQjuV1gg2B8QOhWFRExq/MR1VnPpIIe7P9vX2vElxv+Pg== + dependencies: + asn1.js "^5.0.0" + +asn1.js@^5.0.0, asn1.js@^5.2.0: version "5.4.1" resolved "https://registry.yarnpkg.com/asn1.js/-/asn1.js-5.4.1.tgz#11a980b84ebb91781ce35b0fdc2ee294e3783f07" integrity sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA== @@ -9422,6 +9441,19 @@ astral-regex@^2.0.0: resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== +async-disk-cache@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/async-disk-cache/-/async-disk-cache-2.1.0.tgz#e0f37b187ed8c41a5991518a9556d206ae2843a2" + integrity sha512-iH+boep2xivfD9wMaZWkywYIURSmsL96d6MoqrC94BnGSvXE4Quf8hnJiHGFYhw/nLeIa1XyRaf4vvcvkwAefg== + dependencies: + debug "^4.1.1" + heimdalljs "^0.2.3" + istextorbinary "^2.5.1" + mkdirp "^0.5.0" + rimraf "^3.0.0" + rsvp "^4.8.5" + username-sync "^1.0.2" + async-each@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.3.tgz#b727dbf87d7651602f06f4d4ac387f47d91b0cbf" @@ -9829,6 +9861,15 @@ benchmark@^2.1.4: lodash "^4.17.4" platform "^1.3.3" +bent@^7.3.12: + version "7.3.12" + resolved "https://registry.yarnpkg.com/bent/-/bent-7.3.12.tgz#e0a2775d4425e7674c64b78b242af4f49da6b035" + integrity sha512-T3yrKnVGB63zRuoco/7Ybl7BwwGZR0lceoVG5XmQyMIH9s19SV5m+a8qam4if0zQuAmOQTyPTPmsQBdAorGK3w== + dependencies: + bytesish "^0.4.1" + caseless "~0.12.0" + is-stream "^2.0.0" + better-opn@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/better-opn/-/better-opn-2.1.1.tgz#94a55b4695dc79288f31d7d0e5f658320759f7c6" @@ -9856,6 +9897,11 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== +binaryextensions@^2.1.2: + version "2.3.0" + resolved "https://registry.yarnpkg.com/binaryextensions/-/binaryextensions-2.3.0.tgz#1d269cbf7e6243ea886aa41453c3651ccbe13c22" + integrity sha512-nAihlQsYGyc5Bwq6+EsubvANYGExeJKHDO3RjnvwU042fawQTQfM3Kxn7IHUXQOz4bzfwsGYYHGSvXyW4zOGLg== + bindings@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" @@ -10202,6 +10248,11 @@ bytes@3.1.1: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.1.tgz#3f018291cb4cbad9accb6e6970bca9c8889e879a" integrity sha512-dWe4nWO/ruEOY7HkUJ5gFt1DCFV9zPRoJr8pV0/ASQermOZjtq8jMjOprC0Kd10GLN+l7xaUPvxzJFWtxGu8Fg== +bytesish@^0.4.1: + version "0.4.4" + resolved "https://registry.yarnpkg.com/bytesish/-/bytesish-0.4.4.tgz#f3b535a0f1153747427aee27256748cff92347e6" + integrity sha512-i4uu6M4zuMUiyfZN4RU2+i9+peJh//pXhd9x1oSe1LBkZ3LEbCoygu8W0bXTukU1Jme2txKuotpCZRaC3FLxcQ== + c8@^7.6.0: version "7.11.0" resolved "https://registry.yarnpkg.com/c8/-/c8-7.11.0.tgz#b3ab4e9e03295a102c47ce11d4ef6d735d9a9ac9" @@ -12472,6 +12523,14 @@ ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: dependencies: safe-buffer "^5.0.1" +editions@^2.2.0: + version "2.3.1" + resolved "https://registry.yarnpkg.com/editions/-/editions-2.3.1.tgz#3bc9962f1978e801312fbd0aebfed63b49bfe698" + integrity sha512-ptGvkwTvGdGfC0hfhKg0MT+TRLRKGtUiWGBInxOm5pz7ssADezahjCUaYuZ8Dr+C05FW0AECIIPt4WBxVINEhA== + dependencies: + errlop "^2.0.0" + semver "^6.3.0" + editorconfig@^0.15.3: version "0.15.3" resolved "https://registry.yarnpkg.com/editorconfig/-/editorconfig-0.15.3.tgz#bef84c4e75fb8dcb0ce5cee8efd51c15999befc5" @@ -12648,6 +12707,11 @@ err-code@^2.0.2: resolved "https://registry.yarnpkg.com/err-code/-/err-code-2.0.3.tgz#23c2f3b756ffdfc608d30e27c9a941024807e7f9" integrity sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA== +errlop@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/errlop/-/errlop-2.2.0.tgz#1ff383f8f917ae328bebb802d6ca69666a42d21b" + integrity sha512-e64Qj9+4aZzjzzFpZC7p5kmm/ccCrbLhAJplhsDXQFs87XTsXwOpH4s1Io2s90Tau/8r2j9f4l/thhDevRjzxw== + errno@^0.1.3, errno@~0.1.7: version "0.1.8" resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.8.tgz#8bb3e9c7d463be4976ff888f76b4809ebc2e811f" @@ -14940,6 +15004,13 @@ header-case@^2.0.4: capital-case "^1.0.4" tslib "^2.0.3" +heimdalljs@^0.2.3: + version "0.2.6" + resolved "https://registry.yarnpkg.com/heimdalljs/-/heimdalljs-0.2.6.tgz#b0eebabc412813aeb9542f9cc622cb58dbdcd9fe" + integrity sha512-o9bd30+5vLBvBtzCPwwGqpry2+n0Hi6H1+qwt6y+0kwRHGGF8TFIhJPmnuM0xO97zaKrDZMwO/V56fAnn8m/tA== + dependencies: + rsvp "~3.2.1" + hexer@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/hexer/-/hexer-1.5.0.tgz#b86ce808598e8a9d1892c571f3cedd86fc9f0653" @@ -15288,6 +15359,14 @@ https-proxy-agent@5.0.1, https-proxy-agent@^5.0.0: agent-base "6" debug "4" +https-proxy-agent@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz#702b71fb5520a132a66de1f67541d9e62154d82b" + integrity sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg== + dependencies: + agent-base "5" + debug "4" + human-signals@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" @@ -16309,6 +16388,15 @@ istanbul-reports@^3.0.2, istanbul-reports@^3.1.3: html-escaper "^2.0.0" istanbul-lib-report "^3.0.0" +istextorbinary@^2.5.1: + version "2.6.0" + resolved "https://registry.yarnpkg.com/istextorbinary/-/istextorbinary-2.6.0.tgz#60776315fb0fa3999add276c02c69557b9ca28ab" + integrity sha512-+XRlFseT8B3L9KyjxxLjfXSLMuErKDsd8DBNrsaxoViABMEZlOSCstwmw0qpoFX3+U6yWU1yhLudAe6/lETGGA== + dependencies: + binaryextensions "^2.1.2" + editions "^2.2.0" + textextensions "^2.5.0" + iterall@^1.2.1: version "1.3.0" resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.3.0.tgz#afcb08492e2915cbd8a0884eb93a8c94d0d72fea" @@ -18224,6 +18312,21 @@ micromatch@^4.0.0, micromatch@^4.0.2, micromatch@^4.0.4: braces "^3.0.1" picomatch "^2.2.3" +microsoft-cognitiveservices-speech-sdk@^1.22.0: + version "1.22.0" + resolved "https://registry.yarnpkg.com/microsoft-cognitiveservices-speech-sdk/-/microsoft-cognitiveservices-speech-sdk-1.22.0.tgz#4c6f82147cbb364c5fa7478c7de691af781d6594" + integrity sha512-C1YV5jui3SD02DlmAlN+i7BKdBevETIbGxmkpFy/19yefja14Y7zOR/Hh0qb+ixuU49tPXWmTf2cL+FZE4YD6Q== + dependencies: + agent-base "^6.0.1" + asn1.js-rfc2560 "^5.0.1" + asn1.js-rfc5280 "^3.0.0" + async-disk-cache "^2.1.0" + bent "^7.3.12" + https-proxy-agent "^4.0.0" + simple-lru-cache "0.0.2" + uuid "^8.3.0" + ws "^7.5.6" + microtime@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/microtime/-/microtime-3.0.0.tgz#d140914bde88aa89b4f9fd2a18620b435af0f39b" @@ -18495,7 +18598,7 @@ mkdirp-infer-owner@^2.0.0: infer-owner "^1.0.4" mkdirp "^1.0.3" -mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.5: +mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.5: version "0.5.6" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== @@ -21963,11 +22066,16 @@ ripemd160@^2.0.0, ripemd160@^2.0.1: hash-base "^3.0.0" inherits "^2.0.1" -rsvp@^4.8.4: +rsvp@^4.8.4, rsvp@^4.8.5: version "4.8.5" resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-4.8.5.tgz#c8f155311d167f68f21e168df71ec5b083113734" integrity sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA== +rsvp@~3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-3.2.1.tgz#07cb4a5df25add9e826ebc67dcc9fd89db27d84a" + integrity sha512-Rf4YVNYpKjZ6ASAmibcwTNciQ5Co5Ztq6iZPEykHpkoflnD/K5ryE/rHehFsTm4NJj8nKDhbi3eKBWGogmNnkg== + run-async@^2.4.0: version "2.4.1" resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455" @@ -22387,6 +22495,11 @@ signedsource@^1.0.0: resolved "https://registry.yarnpkg.com/signedsource/-/signedsource-1.0.0.tgz#1ddace4981798f93bd833973803d80d52e93ad6a" integrity sha1-HdrOSYF5j5O9gzlzgD2A1S6TrWo= +simple-lru-cache@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz#d59cc3a193c1a5d0320f84ee732f6e4713e511dd" + integrity sha512-uEv/AFO0ADI7d99OHDmh1QfYzQk/izT1vCmu/riQfh7qjBVUUgRT87E5s5h7CxWCA/+YoZerykpEthzVrW3LIw== + simple-swizzle@^0.2.2: version "0.2.2" resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a" @@ -23525,6 +23638,11 @@ text-table@^0.2.0: resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= +textextensions@^2.5.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/textextensions/-/textextensions-2.6.0.tgz#d7e4ab13fe54e32e08873be40d51b74229b00fc4" + integrity sha512-49WtAWS+tcsy93dRt6P0P3AMD2m5PvXRhuEA0kaXos5ZLlujtYmpmFsB+QvWUSxE1ZsstmYXfQ7L40+EcQgpAQ== + thenify-all@^1.0.0: version "1.6.0" resolved "https://registry.yarnpkg.com/thenify-all/-/thenify-all-1.6.0.tgz#1a1918d402d8fc3f98fbf234db0bcc8cc10e9726" @@ -24469,6 +24587,11 @@ user-home@^1.1.1: resolved "https://registry.yarnpkg.com/user-home/-/user-home-1.1.1.tgz#2b5be23a32b63a7c9deb8d0f28d485724a3df190" integrity sha1-K1viOjK2Onyd640PKNSFcko98ZA= +username-sync@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/username-sync/-/username-sync-1.0.3.tgz#ae41c5c8a4c8c2ecc1443a7d0742742bd7e36732" + integrity sha512-m/7/FSqjJNAzF2La448c/aEom0gJy7HY7Y509h6l0ePvEkFictAGptwWaj1msWJ38JbfEDOUoE8kqFee9EHKdA== + util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" @@ -24535,7 +24658,7 @@ uuid@^3.2.1, uuid@^3.3.2, uuid@^3.3.3: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== -uuid@^8.0.0, uuid@^8.3.1, uuid@^8.3.2: +uuid@^8.0.0, uuid@^8.3.0, uuid@^8.3.1, uuid@^8.3.2: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== @@ -25290,6 +25413,11 @@ ws@8.8.1, ws@^8.2.3, ws@^8.3.0, ws@^8.4.2: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.7.tgz#9e0ac77ee50af70d58326ecff7e85eb3fa375e67" integrity sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A== +ws@^7.5.6: + version "7.5.9" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" + integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== + xdg-basedir@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13"