diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 3302cdf11..6d7b34e33 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -2,6 +2,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ import express from 'express' import { CreateArticleErrorCode } from '../generated/graphql' import { isSiteBlockedForParse } from '../utils/blocked' @@ -21,9 +22,28 @@ import { getPageById, updatePage } from '../elastic/pages' import { generateDownloadSignedUrl } from '../utils/uploads' import { enqueueTextToSpeech } from '../utils/createTask' import { createPubSubClient } from '../datalayer/pubsub' -import { htmlToSsml } from '@omnivore/text-to-speech-handler' +import { htmlToSsmlItems, SSMLItem } from '@omnivore/text-to-speech-handler' +import { WordPunctTokenizer } from 'natural' +import { htmlToText } from 'html-to-text' + +interface Utterance { + wordOffset: number + wordsCount: number + voice?: string + text: string + idx: number +} + +interface SSMLOutput { + wordsCount: number + averageWPM: number + language: string + defaultVoice: string + utterances: Utterance[] +} const logger = buildLogger('app.dispatch') +const WORDS_PER_MINUTE = 200 export function articleRouter() { const router = express.Router() @@ -105,14 +125,21 @@ export function articleRouter() { if (!page) { return res.status(404).send('Page not found') } - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const ssmlItems = htmlToSsml(page.content, { + const ssmlItems = htmlToSsmlItems(page.content, { primaryVoice: voice, secondaryVoice: 'en-US-GuyNeural', rate: '1', language: page.language || 'en-US', }) - return res.send({ ssmlItems }) + const [utterances, wordsCount] = ssmlItemsToUtterances(ssmlItems) + const ssmlOutput: SSMLOutput = { + wordsCount, + averageWPM: WORDS_PER_MINUTE, + language: page.language || 'en-US', + defaultVoice: voice, + utterances, + } + return res.send(ssmlOutput) } const existingSpeech = await getRepository(Speech).findOne({ @@ -192,3 +219,24 @@ const redirectUrl = async (speech: Speech, outputFormat: string) => { return generateDownloadSignedUrl(speech.audioFileName) } } + +const ssmlItemsToUtterances = (items: SSMLItem[]): [Utterance[], number] => { + const tokenizer = new WordPunctTokenizer() + let wordOffset = 0 + return [ + items.map((item) => { + const text = htmlToText(item.textItems.join(''), { wordwrap: false }) + const wordsCount = tokenizer.tokenize(text).length + const utterance: Utterance = { + wordOffset, + wordsCount, + text, + voice: item.voice, + idx: item.idx, + } + wordOffset += wordsCount + return utterance + }), + wordOffset, + ] +} diff --git a/packages/api/src/textToSpeech.d.ts b/packages/api/src/textToSpeech.d.ts index 5e438e05e..ab955ae1b 100644 --- a/packages/api/src/textToSpeech.d.ts +++ b/packages/api/src/textToSpeech.d.ts @@ -1,12 +1,21 @@ declare module '@omnivore/text-to-speech-handler' { - function htmlToSsml(html: string, options: SSMLOptions): string[] + export function htmlToSsmlItems( + html: string, + options: SSMLOptions + ): SSMLItem[] - interface SSMLOptions { + export interface SSMLOptions { primaryVoice: string secondaryVoice: string rate: string language: string } - export { htmlToSsml } + export interface SSMLItem { + open: string + close: string + textItems: string[] + idx: number + voice?: string + } } diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 9a9c93c31..a9b42a308 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -9,7 +9,7 @@ import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' import { File, Storage } from '@google-cloud/storage' -import { htmlToSsml } from './htmlToSsml' +import { htmlToSsmlItems } from './htmlToSsml' interface SSMLInput { text: string @@ -20,6 +20,7 @@ interface UtteranceInput { rate?: number language?: string text: string + idx: string } interface HTMLInput { @@ -177,6 +178,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( res.send({ audioData: audioData.toString('hex'), speechMarks, + idx: utteranceInput.idx, }) } catch (e) { console.error('Text to speech streaming error', e) @@ -186,7 +188,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( ) module.exports = { - htmlToSsml, + htmlToSsmlItems, textToSpeechStreamingHandler, textToSpeechHandler, }