From 62619115c8ade5a44e99cf3575dc98cf3c337cf3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 7 Sep 2022 17:59:41 +0800 Subject: [PATCH 1/6] Convert html to speech-file --- packages/text-to-speech/src/htmlToSsml.ts | 132 ++++++++++++++++++---- 1 file changed, 111 insertions(+), 21 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index e74f00ea5..ecc1cb87b 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -1,9 +1,47 @@ import { parseHTML } from 'linkedom' import * as _ from 'underscore' +import { WordPunctTokenizer } from 'natural' +import { htmlToText } from 'html-to-text' // this code needs to be kept in sync with the // frontend code in: useReadingProgressAnchor +export interface Utterance { + idx: number + text: string + wordOffset: number + wordCount: number + voice?: string +} + +export interface SpeechFile { + wordCount: number + averageWPM: number + language: string + defaultVoice: string + utterances: Utterance[] +} + +export type SSMLItem = { + open: string + close: string + textItems: string[] + idx: number + voice?: string +} + +export type SSMLOptions = { + primaryVoice?: string + secondaryVoice?: string + rate?: number + language?: string +} + +const WORDS_PER_MINUTE = 200 +const DEFAULT_LANGUAGE = 'en-US' +const DEFAULT_VOICE = 'en-US-JennyNeural' +const DEFAULT_RATE = 1.25 + const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ 'omnivore-highlight-id', 'data-twitter-tweet-id', @@ -140,30 +178,16 @@ function emitElement( return Number(maxVisitedIdx) } -export type SSMLItem = { - open: string - close: string - textItems: string[] - idx: number - voice?: string -} - -export type SSMLOptions = { - primaryVoice: string - secondaryVoice: string - rate: number - language: string -} - -export const startSsml = ( - element: Element | null, - options: SSMLOptions -): string => { +export const startSsml = (options: SSMLOptions, element?: Element): string => { const voice = element?.nodeName === 'BLOCKQUOTE' ? options.secondaryVoice : options.primaryVoice - return `` + return `` } export const endSsml = (): string => { @@ -210,7 +234,7 @@ export const htmlToSsmlItems = ( const idx = i i = emitElement(textItems, node, true) items.push({ - open: startSsml(node, options), + open: startSsml(options, node), close: endSsml(), textItems: textItems, idx, @@ -222,3 +246,69 @@ export const htmlToSsmlItems = ( return items } + +const htmlToUtterance = ( + tokenizer: WordPunctTokenizer, + idx: number, + htmlItems: string[], + wordOffset: number, + voice?: string +): Utterance => { + const text = htmlToText(htmlItems.join(''), { wordwrap: false }) + const wordCount = tokenizer.tokenize(text).length + return { + idx, + text, + wordOffset, + wordCount, + voice, + } +} + +export const htmlToSpeechFile = ( + html: string, + options: SSMLOptions +): SpeechFile => { + console.debug('creating speech file with options', options) + + const dom = parseHTML(html) + const body = dom.document.querySelector('#readability-page-1') + if (!body) { + throw new Error('Unable to parse HTML document') + } + + const parsedNodes = parseDomTree(body) + if (parsedNodes.length < 1) { + throw new Error('No HTML nodes found') + } + + const tokenizer = new WordPunctTokenizer() + const utterances: Utterance[] = [] + let wordOffset = 0 + for (let i = 2; i < parsedNodes.length + 2; i++) { + const textItems: string[] = [] + const node = parsedNodes[i - 2] + + if (TOP_LEVEL_TAGS.includes(node.nodeName) || hasSignificantText(node)) { + const idx = i + i = emitElement(textItems, node, true) + const utterance = htmlToUtterance( + tokenizer, + idx, + textItems, + wordOffset, + node.nodeName === 'BLOCKQUOTE' ? options.secondaryVoice : undefined + ) + utterances.push(utterance) + wordOffset += utterance.wordCount + } + } + + return { + wordCount: wordOffset, + averageWPM: WORDS_PER_MINUTE, + language: options.language || DEFAULT_LANGUAGE, + defaultVoice: options.primaryVoice || DEFAULT_VOICE, + utterances, + } +} From f1b379b96ef60a7414b86ccc64c01dc2d6a4f17f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 7 Sep 2022 18:00:32 +0800 Subject: [PATCH 2/6] Convert html to speech file and return --- packages/api/src/routers/article_router.ts | 69 ++++------------------ packages/api/src/textToSpeech.d.ts | 28 +++++---- packages/text-to-speech/src/index.ts | 10 +--- 3 files changed, 31 insertions(+), 76 deletions(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 027cf91cd..59fa773da 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -22,28 +22,9 @@ import { getPageById, updatePage } from '../elastic/pages' import { generateDownloadSignedUrl } from '../utils/uploads' import { enqueueTextToSpeech } from '../utils/createTask' import { createPubSubClient } from '../datalayer/pubsub' -import { htmlToSsmlItems, SSMLItem } from '@omnivore/text-to-speech-handler' -import { WordPunctTokenizer } from 'natural' -import { htmlToText } from 'html-to-text' - -interface Utterance { - wordOffset: number - wordCount: number - voice?: string - text: string - idx: number -} - -interface SSMLOutput { - wordCount: number - averageWPM: number - language: string - defaultVoice: string - utterances: Utterance[] -} +import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler' const logger = buildLogger('app.dispatch') -const WORDS_PER_MINUTE = 200 export function articleRouter() { const router = express.Router() @@ -93,16 +74,16 @@ export function articleRouter() { }) router.get( - '/:id/:outputFormat/:priority/:voice?', + '/:id/:outputFormat/:priority?/:voice?/:secondaryVoice?', cors(corsConfig), async (req, res) => { const articleId = req.params.id const outputFormat = req.params.outputFormat - const voice = req.params.voice || 'en-US-JennyNeural' - const priority = req.params.priority + const voice = req.params.voice + const priority = req.params.priority || 'high' if ( !articleId || - !['mp3', 'speech-marks', 'ssml'].includes(outputFormat) || + !['mp3', 'speech-marks', 'speech-file'].includes(outputFormat) || !['low', 'high'].includes(priority) ) { return res.status(400).send('Invalid data') @@ -120,26 +101,17 @@ export function articleRouter() { }, }) - if (outputFormat === 'ssml') { + if (outputFormat === 'speech-file') { const page = await getPageById(articleId) if (!page) { return res.status(404).send('Page not found') } - const ssmlItems = htmlToSsmlItems(page.content, { + const speechFile = htmlToSpeechFile(page.content, { primaryVoice: voice, - secondaryVoice: 'en-US-GuyNeural', - rate: '1', - language: page.language || 'en-US', + secondaryVoice: req.params.secondaryVoice, + language: page.language, }) - const [utterances, wordCount] = ssmlItemsToUtterances(ssmlItems) - const ssmlOutput: SSMLOutput = { - wordCount, - averageWPM: WORDS_PER_MINUTE, - language: page.language || 'en-US', - defaultVoice: voice, - utterances, - } - return res.send(ssmlOutput) + return res.send(speechFile) } const existingSpeech = await getRepository(Speech).findOne({ @@ -219,24 +191,3 @@ const redirectUrl = async (speech: Speech, outputFormat: string) => { return generateDownloadSignedUrl(speech.audioFileName) } } - -const ssmlItemsToUtterances = (items: SSMLItem[]): [Utterance[], number] => { - const tokenizer = new WordPunctTokenizer() - let wordOffset = 0 - return [ - items.map((item) => { - const text = htmlToText(item.textItems.join(''), { wordwrap: false }) - const wordCount = tokenizer.tokenize(text).length - const utterance: Utterance = { - wordOffset, - wordCount, - text, - voice: item.voice, - idx: item.idx, - } - wordOffset += wordCount - return utterance - }), - wordOffset, - ] -} diff --git a/packages/api/src/textToSpeech.d.ts b/packages/api/src/textToSpeech.d.ts index ab955ae1b..051b9be8f 100644 --- a/packages/api/src/textToSpeech.d.ts +++ b/packages/api/src/textToSpeech.d.ts @@ -1,21 +1,29 @@ declare module '@omnivore/text-to-speech-handler' { - export function htmlToSsmlItems( + export function htmlToSpeechFile( html: string, options: SSMLOptions - ): SSMLItem[] + ): SpeechFile export interface SSMLOptions { - primaryVoice: string - secondaryVoice: string - rate: string - language: string + primaryVoice?: string + secondaryVoice?: string + rate?: number + language?: string } - export interface SSMLItem { - open: string - close: string - textItems: string[] + interface Utterance { idx: number + wordOffset: number + wordCount: number voice?: string + text: string + } + + export interface SpeechFile { + wordCount: number + averageWPM: number + language: string + defaultVoice: string + utterances: Utterance[] } } diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index a9b42a308..5ca4d54f7 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -9,11 +9,7 @@ import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech' import { File, Storage } from '@google-cloud/storage' -import { htmlToSsmlItems } from './htmlToSsml' - -interface SSMLInput { - text: string -} +import { htmlToSpeechFile } from './htmlToSsml' interface UtteranceInput { voice?: string @@ -176,9 +172,9 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( return res.status(500).send({ errorCode: 'SYNTHESIZER_ERROR' }) } res.send({ + idx: utteranceInput.idx, audioData: audioData.toString('hex'), speechMarks, - idx: utteranceInput.idx, }) } catch (e) { console.error('Text to speech streaming error', e) @@ -188,7 +184,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( ) module.exports = { - htmlToSsmlItems, + htmlToSpeechFile, textToSpeechStreamingHandler, textToSpeechHandler, } From d9dc701de80f6c4a75d829dd1375a7fd56b3e67d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 7 Sep 2022 18:03:13 +0800 Subject: [PATCH 3/6] Stop counting the words in SSML tags --- packages/text-to-speech/src/textToSpeech.ts | 23 +++++++++++---------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 3bfe1be86..b7fef7368 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -13,7 +13,7 @@ export interface TextToSpeechInput { text: string voice?: string language?: string - textType?: 'html' | 'ssml' | 'utterance' + textType?: 'html' | 'utterance' rate?: number complimentaryVoice?: string audioStream?: NodeJS.ReadWriteStream @@ -51,6 +51,7 @@ export const synthesizeTextToSpeech = async ( const synthesizer = new SpeechSynthesizer(speechConfig) const speechMarks: SpeechMark[] = [] let timeOffset = 0 + let wordOffset = 0 synthesizer.synthesizing = function (s, e) { // convert arrayBuffer to stream and write to stream @@ -93,7 +94,7 @@ export const synthesizeTextToSpeech = async ( speechMarks.push({ word: e.text, time: (timeOffset + e.audioOffset) / 10000, - start: e.textOffset, + start: wordOffset + e.textOffset, length: e.wordLength, type: 'word', }) @@ -142,15 +143,19 @@ export const synthesizeTextToSpeech = async ( const result = await speakSsmlAsyncPromise(ssml) timeOffset = timeOffset + result.audioDuration } - } else { - // assemble ssml - const ssml = `${startSsml(null, ssmlOptions)}${input.text}${endSsml()}` - const result = await speakSsmlAsyncPromise(ssml) return { - audioData: Buffer.from(result.audioData), speechMarks, } } + // for utterance + const start = startSsml(ssmlOptions) + wordOffset = -start.length + const ssml = `${start}${input.text}${endSsml()}` + const result = await speakSsmlAsyncPromise(ssml) + return { + audioData: Buffer.from(result.audioData), + speechMarks, + } } catch (error) { console.error('synthesis error', error) throw error @@ -160,8 +165,4 @@ export const synthesizeTextToSpeech = async ( synthesizer.close() console.debug('synthesizer closed') } - - return { - speechMarks, - } } From bbbd3d10afca1796e61c458ade34d9716a9b3a06 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 7 Sep 2022 18:11:59 +0800 Subject: [PATCH 4/6] Update API endpoint --- packages/text-to-speech/src/textToSpeech.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index b7fef7368..f9679c6c4 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -15,7 +15,7 @@ export interface TextToSpeechInput { language?: string textType?: 'html' | 'utterance' rate?: number - complimentaryVoice?: string + secondaryVoice?: string audioStream?: NodeJS.ReadWriteStream } @@ -131,10 +131,10 @@ export const synthesizeTextToSpeech = async ( try { const ssmlOptions = { - primaryVoice: input.voice || 'en-US-JennyNeural', - secondaryVoice: input.complimentaryVoice || 'en-US-GuyNeural', - language: input.language || 'en-US', - rate: input.rate || 1.25, + primaryVoice: input.voice, + secondaryVoice: input.secondaryVoice, + language: input.language, + rate: input.rate, } if (textType === 'html') { const ssmlItems = htmlToSsmlItems(input.text, ssmlOptions) From 9117bd668d2f1d58cebce114b801faabb2e1058b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 7 Sep 2022 18:20:10 +0800 Subject: [PATCH 5/6] Update dependencies --- packages/api/package.json | 4 ---- packages/text-to-speech/package.json | 8 ++++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 0757848b2..5297027a2 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -61,7 +61,6 @@ "graphql-shield": "^7.5.0", "highlightjs": "^9.16.2", "html-entities": "^2.3.2", - "html-to-text": "^8.2.1", "intercom-client": "^3.1.4", "jsonwebtoken": "^8.5.1", "jwks-rsa": "^2.0.3", @@ -71,7 +70,6 @@ "luxon": "^2.3.1", "microsoft-cognitiveservices-speech-sdk": "^1.22.0", "nanoid": "^3.1.25", - "natural": "^5.2.3", "nodemailer": "^6.7.3", "normalize-url": "^6.1.0", "oauth": "^0.9.15", @@ -107,13 +105,11 @@ "@types/express": "^4.17.7", "@types/graphql-fields": "^1.3.4", "@types/highlightjs": "^9.12.2", - "@types/html-to-text": "^8.1.1", "@types/intercom-client": "^2.11.8", "@types/jsonwebtoken": "^8.5.0", "@types/luxon": "^1.25.0", "@types/mocha": "^8.2.2", "@types/nanoid": "^3.0.0", - "@types/natural": "^5.1.1", "@types/nodemailer": "^6.4.4", "@types/oauth": "^0.9.1", "@types/private-ip": "^1.0.0", diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json index 5416fd83e..279fca847 100644 --- a/packages/text-to-speech/package.json +++ b/packages/text-to-speech/package.json @@ -23,7 +23,9 @@ "devDependencies": { "@types/node": "^14.11.2", "@types/underscore": "^1.11.4", - "eslint-plugin-prettier": "^4.0.0" + "eslint-plugin-prettier": "^4.0.0", + "@types/html-to-text": "^8.1.1", + "@types/natural": "^5.1.1" }, "dependencies": { "@google-cloud/functions-framework": "3.1.2", @@ -34,6 +36,8 @@ "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.12", "microsoft-cognitiveservices-speech-sdk": "^1.22.0", - "underscore": "^1.13.4" + "underscore": "^1.13.4", + "natural": "^5.2.3", + "html-to-text": "^8.2.1" } } From aeb7e47d685b3e47cf1c85773f048361d1189f3e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 7 Sep 2022 18:42:46 +0800 Subject: [PATCH 6/6] Make voice, secondaryVoice and priority query params --- packages/api/src/routers/article_router.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 59fa773da..89f164838 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -24,6 +24,12 @@ import { enqueueTextToSpeech } from '../utils/createTask' import { createPubSubClient } from '../datalayer/pubsub' import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler' +interface SpeechInput { + voice?: string + secondaryVoice?: string + priority?: 'low' | 'high' +} +const outputFormats = ['mp3', 'speech-marks', 'speech-file'] const logger = buildLogger('app.dispatch') export function articleRouter() { @@ -74,18 +80,13 @@ export function articleRouter() { }) router.get( - '/:id/:outputFormat/:priority?/:voice?/:secondaryVoice?', + '/:id/:outputFormat', cors(corsConfig), async (req, res) => { const articleId = req.params.id const outputFormat = req.params.outputFormat - const voice = req.params.voice - const priority = req.params.priority || 'high' - if ( - !articleId || - !['mp3', 'speech-marks', 'speech-file'].includes(outputFormat) || - !['low', 'high'].includes(priority) - ) { + const { voice, priority, secondaryVoice } = req.query as SpeechInput + if (!articleId || outputFormats.indexOf(outputFormat) === -1) { return res.status(400).send('Invalid data') } const token = req.cookies?.auth || req.headers?.authorization @@ -108,7 +109,7 @@ export function articleRouter() { } const speechFile = htmlToSpeechFile(page.content, { primaryVoice: voice, - secondaryVoice: req.params.secondaryVoice, + secondaryVoice: secondaryVoice, language: page.language, }) return res.send(speechFile) @@ -171,7 +172,7 @@ export function articleRouter() { speechId: speech.id, text: page.content, voice: speech.voice, - priority: priority as 'low' | 'high', + priority: priority || 'high', }) logger.info('Start Text to speech task', { taskName }) res.status(202).send('Text to speech task started')