From e6706a6efb65be8dc9cd1775d0416004a45fe09d Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 29 Aug 2022 16:37:30 +0800 Subject: [PATCH 1/4] Add a new HTML to SSML function --- packages/text-to-speech/src/htmlToSsml.ts | 178 ++++++++++++++++++ .../text-to-speech/test/htmlToSsml.test.ts | 75 ++++++++ 2 files changed, 253 insertions(+) create mode 100644 packages/text-to-speech/src/htmlToSsml.ts create mode 100644 packages/text-to-speech/test/htmlToSsml.test.ts diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts new file mode 100644 index 000000000..581959543 --- /dev/null +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -0,0 +1,178 @@ + + + +import { parseHTML } from 'linkedom' + +// this code needs to be kept in sync with the +// frontend code in: useReadingProgressAnchor + +const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ + 'omnivore-highlight-id', + 'data-twitter-tweet-id', + 'data-instagram-id', +] + +function ssmlTagsForTopLevelElement(element: Element) { + // if (element.nodeName == 'BLOCKQUOTE') { + // return { + // opening: `

`, + // closing: `

` + // } + // } + return { + opening: `

`, + closing: `

` + } +} + +function parseDomTree(pageNode: Element) { + if (!pageNode || pageNode.childNodes.length == 0) { + console.log(' no child nodes found') + return [] + } + + const nodesToVisitStack = [pageNode] + const visitedNodeList = [] + + while (nodesToVisitStack.length > 0) { + const currentNode = nodesToVisitStack.pop() + if ( + currentNode?.nodeType !== 1 /* Node.ELEMENT_NODE */ || + // Avoiding dynamic elements from being counted as anchor-allowed elements + ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES.some((attrib) => + currentNode.hasAttribute(attrib) + ) + ) { + continue + } + + visitedNodeList.push(currentNode) + ;[].slice + .call(currentNode.childNodes) + .reverse() + .forEach(function (node) { + nodesToVisitStack.push(node) + }) + } + + visitedNodeList.shift() + visitedNodeList.forEach((node, index) => { + // start from index 1, index 0 reserved for anchor unknown. + node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString()) + }) + return visitedNodeList +} + +function emit(textItems: string[], text: string) { + textItems.push(text) +} + +function cleanTextNode(textNode: ChildNode): String { + return (textNode.textContent ?? '').replace(/\s+/g, ' ') +} + +function emitTextNode(textItems: string[], cleanedText: String, textNode: ChildNode) { + const ssmlElement = textNode.parentNode?.nodeName === 'B' ? "emphasis" : undefined + if (!cleanedText) { return } + + if (ssmlElement) { + emit(textItems, `<${ssmlElement}>`) + } + emit(textItems, `${cleanedText}`) + if (ssmlElement) { + emit(textItems, ``) + } +} + +function emitElement(textItems: string[], element: Element, isTopLevel: Boolean) { + const SKIP_TAGS = ['SCRIPT', 'STYLE', 'IMG', 'FIGURE', 'FIGCAPTION', 'IFRAME'] + + const topLevelTags = ssmlTagsForTopLevelElement(element) + const idx = element.getAttribute('data-omnivore-anchor-idx') + var maxVisitedIdx = Number(idx) + + if (isTopLevel) { + emit(textItems, topLevelTags.opening) + } + + for (const child of Array.from(element.childNodes)) { + if (SKIP_TAGS.indexOf(child.nodeName) >= 0) { + continue + } + + if (child.nodeType == 3 /* Node.TEXT_NODE */ && (child.textContent?.length ?? 0) > 0 ) { + const cleanedText = cleanTextNode(child) + if (cleanedText.length > 1) { // Make sure its more than just a space + emit(textItems, ``) + } + emitTextNode(textItems, cleanedText, child) + } + if (child.nodeType == 1 /* Node.ELEMENT_NODE */) { + maxVisitedIdx = emitElement(textItems, child as HTMLElement, false) + } + } + + if (isTopLevel) { + emit(textItems, topLevelTags.closing) + } + + return Number(maxVisitedIdx) +} + +export type SSMLItem = { + open: string + close: string + textItems: string[] +} + +export type VoiceOptions = { + primary: string + secondary: string +} + +const startSsml = (element: Element, voices: VoiceOptions): string => { + const voice = element.nodeName === 'BLOCKQUOTE' ? voices.secondary : voices.primary + return ` + + ` +} + +const endSsml = (): string => { + return `` +} + +export const ssmlItemText = (item: SSMLItem): string => { + return [ + item.open, + ...item.textItems, + item.close + ].join('') +} + +export const htmlToSsml = (html: string, voices: { primary: string, secondary: string}): SSMLItem[] => { + const dom = parseHTML(html) + var body = dom.document.querySelector('#readability-page-1') + if (!body) { + throw new Error('Unable to parse HTML document') + } + + var parsedNodes = parseDomTree(body) + if (parsedNodes.length < 1) { + throw new Error('No HTML nodes found') + } + + const items: SSMLItem[] = [] + for (var i = 1; i < parsedNodes.length + 1; i++) { + var textItems: string[] = [] + const node = parsedNodes[i - 1] + + i = emitElement(textItems, node, true) + items.push({ + open: startSsml(node, voices), + close: endSsml(), + textItems: textItems, + }) + } + + return items +} diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts new file mode 100644 index 000000000..424de5d9d --- /dev/null +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -0,0 +1,75 @@ +import 'mocha' +import { expect } from 'chai' +import { htmlToSsml } from '../src/htmlToSsml' + +describe('htmlToSsml', () => { + const TEST_VOCIES = { primary: 'test-primary', secondary: 'test-secondary' } + + describe('a simple html file', () => { + it('should convert Html to SSML', async () => { + const ssml = htmlToSsml(` +
+

this is some text

+
+ `, TEST_VOCIES + ) + const text = ssml[0].textItems.join('').trim() + expect(text).to.equal( + `

this is some text

` + ) + }) + }) + describe('a file with nested elements', () => { + it('should convert Html to SSML', async () => { + const ssml = htmlToSsml(` +
+

+this is in the first paragraph +this is in the second span +this is also in the first paragraph +

+
+ `, TEST_VOCIES + ) + const text = ssml[0].textItems.join('').trim() + expect(text).to.equal( + `

this is in the first paragraph this is in the second span this is also in the first paragraph

`.trim() + ) + }) + }) + describe('a file with blockquotes', () => { + it('should convert Html to SSML with complimentary voices', async () => { + const ssml = htmlToSsml(` +
+

first

+
second
+

third

+
+ `, TEST_VOCIES + ) + const first = ssml[0].textItems.join('').trim() + const second = ssml[1].textItems.join('').trim() + const third = ssml[2].textItems.join('').trim() + + expect(first).to.equal( + `

first

` + ) + expect(second).to.equal( + `

second

` + ) + expect(third).to.equal( + `

third

` + ) + + expect(ssml[0].open.trim()).to.equal( + `` + ) + expect(ssml[1].open.trim()).to.equal( + `` + ) + expect(ssml[2].open.trim()).to.equal( + `` + ) + }) + }) +}) From 8c3f7c57113071b208c59b482e11422afa77d320 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 29 Aug 2022 16:40:43 +0800 Subject: [PATCH 2/4] Use the new HTML -> SSML function --- packages/text-to-speech/src/index.ts | 45 ++++++++++------------------ 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 28833d879..aef261a3b 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -18,6 +18,7 @@ import { import axios from 'axios' import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import +import { htmlToSsml, ssmlItemText } from './htmlToSsml' dotenv.config() interface TextToSpeechInput { @@ -215,35 +216,21 @@ const synthesizeTextToSpeech = async ( currentTextChunk = '' } } else { - const document = parseHTML(input.text).document - const elements = document.querySelectorAll( - 'h1, h2, h3, p, ul, ol, blockquote' - ) - // convert html elements to the ssml document - for (const e of Array.from(elements)) { - const htmlElement = e as HTMLElement - if (htmlElement.innerText) { - // use complimentary voice for blockquote, hardcoded for now - const voice = - htmlElement.tagName.toLowerCase() === 'blockquote' - ? input.complimentaryVoice || 'en-US-AriaNeural' - : input.voice - const ssml = htmlElementToSsml({ - htmlElement: e, - language: input.languageCode, - rate: input.rate, - volume: input.volume, - voice, - }) - console.debug(`synthesizing ${ssml}`) - const result = await speakSsmlAsyncPromise(ssml) - // if (result.reason === ResultReason.Canceled) { - // synthesizer.close() - // throw new Error(result.errorDetails) - // } - timeOffset = timeOffset + result.audioDuration - // characterOffset = characterOffset + htmlElement.innerText.length - } + const ssmlItems = htmlToSsml(input.text, { + primary: speechConfig.speechSynthesisVoiceName, + secondary: 'en-US-GuyNeural' + }) + + for (const ssmlItem of Array.from(ssmlItems)) { + const ssml = ssmlItemText(ssmlItem) + console.debug(`synthesizing ${ssml}`) + const result = await speakSsmlAsyncPromise(ssml) + // if (result.reason === ResultReason.Canceled) { + // synthesizer.close() + // throw new Error(result.errorDetails) + // } + timeOffset = timeOffset + result.audioDuration + // characterOffset = characterOffset + htmlElement.innerText.length } } writeStream.end() From e298a49169b78455b5672792d2f9daf82363e672 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 29 Aug 2022 16:52:58 +0800 Subject: [PATCH 3/4] Linting fixes --- packages/text-to-speech/src/htmlToSsml.ts | 72 ++++++++++++----------- packages/text-to-speech/src/index.ts | 2 +- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index 581959543..19db626f1 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -1,9 +1,6 @@ - - - import { parseHTML } from 'linkedom' -// this code needs to be kept in sync with the +// this code needs to be kept in sync with the // frontend code in: useReadingProgressAnchor const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ @@ -12,16 +9,10 @@ const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ 'data-instagram-id', ] -function ssmlTagsForTopLevelElement(element: Element) { - // if (element.nodeName == 'BLOCKQUOTE') { - // return { - // opening: `

`, - // closing: `

` - // } - // } +function ssmlTagsForTopLevelElement() { return { opening: `

`, - closing: `

` + closing: `

`, } } @@ -67,13 +58,20 @@ function emit(textItems: string[], text: string) { textItems.push(text) } -function cleanTextNode(textNode: ChildNode): String { +function cleanTextNode(textNode: ChildNode): string { return (textNode.textContent ?? '').replace(/\s+/g, ' ') } -function emitTextNode(textItems: string[], cleanedText: String, textNode: ChildNode) { - const ssmlElement = textNode.parentNode?.nodeName === 'B' ? "emphasis" : undefined - if (!cleanedText) { return } +function emitTextNode( + textItems: string[], + cleanedText: string, + textNode: ChildNode +) { + const ssmlElement = + textNode.parentNode?.nodeName === 'B' ? 'emphasis' : undefined + if (!cleanedText) { + return + } if (ssmlElement) { emit(textItems, `<${ssmlElement}>`) @@ -84,12 +82,16 @@ function emitTextNode(textItems: string[], cleanedText: String, textNode: ChildN } } -function emitElement(textItems: string[], element: Element, isTopLevel: Boolean) { +function emitElement( + textItems: string[], + element: Element, + isTopLevel: boolean +) { const SKIP_TAGS = ['SCRIPT', 'STYLE', 'IMG', 'FIGURE', 'FIGCAPTION', 'IFRAME'] - - const topLevelTags = ssmlTagsForTopLevelElement(element) + + const topLevelTags = ssmlTagsForTopLevelElement() const idx = element.getAttribute('data-omnivore-anchor-idx') - var maxVisitedIdx = Number(idx) + let maxVisitedIdx = Number(idx) if (isTopLevel) { emit(textItems, topLevelTags.opening) @@ -100,9 +102,13 @@ function emitElement(textItems: string[], element: Element, isTopLevel: Boolean) continue } - if (child.nodeType == 3 /* Node.TEXT_NODE */ && (child.textContent?.length ?? 0) > 0 ) { + if ( + child.nodeType == 3 /* Node.TEXT_NODE */ && + (child.textContent?.length ?? 0) > 0 + ) { const cleanedText = cleanTextNode(child) - if (cleanedText.length > 1) { // Make sure its more than just a space + if (idx && cleanedText.length > 1) { + // Make sure its more than just a space emit(textItems, ``) } emitTextNode(textItems, cleanedText, child) @@ -131,7 +137,8 @@ export type VoiceOptions = { } const startSsml = (element: Element, voices: VoiceOptions): string => { - const voice = element.nodeName === 'BLOCKQUOTE' ? voices.secondary : voices.primary + const voice = + element.nodeName === 'BLOCKQUOTE' ? voices.secondary : voices.primary return ` ` @@ -142,28 +149,27 @@ const endSsml = (): string => { } export const ssmlItemText = (item: SSMLItem): string => { - return [ - item.open, - ...item.textItems, - item.close - ].join('') + return [item.open, ...item.textItems, item.close].join('') } -export const htmlToSsml = (html: string, voices: { primary: string, secondary: string}): SSMLItem[] => { +export const htmlToSsml = ( + html: string, + voices: { primary: string; secondary: string } +): SSMLItem[] => { const dom = parseHTML(html) - var body = dom.document.querySelector('#readability-page-1') + const body = dom.document.querySelector('#readability-page-1') if (!body) { throw new Error('Unable to parse HTML document') } - var parsedNodes = parseDomTree(body) + const parsedNodes = parseDomTree(body) if (parsedNodes.length < 1) { throw new Error('No HTML nodes found') } const items: SSMLItem[] = [] - for (var i = 1; i < parsedNodes.length + 1; i++) { - var textItems: string[] = [] + for (let i = 1; i < parsedNodes.length + 1; i++) { + const textItems: string[] = [] const node = parsedNodes[i - 1] i = emitElement(textItems, node, true) diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index aef261a3b..6d2e34736 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -218,7 +218,7 @@ const synthesizeTextToSpeech = async ( } else { const ssmlItems = htmlToSsml(input.text, { primary: speechConfig.speechSynthesisVoiceName, - secondary: 'en-US-GuyNeural' + secondary: 'en-US-GuyNeural', }) for (const ssmlItem of Array.from(ssmlItems)) { From 4f6d8cdfce5e787c1b5dbd0a795f8d70a922f408 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 29 Aug 2022 17:20:43 +0800 Subject: [PATCH 4/4] Set language/rate in prosody XML --- packages/text-to-speech/src/htmlToSsml.ts | 23 +++++------ packages/text-to-speech/src/index.ts | 6 ++- .../text-to-speech/test/htmlToSsml.test.ts | 38 +++++++++++++++---- 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index 19db626f1..6d4fb8040 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -131,16 +131,20 @@ export type SSMLItem = { textItems: string[] } -export type VoiceOptions = { - primary: string - secondary: string +export type SSMLOptions = { + primaryVoice: string + secondaryVoice: string + rate: string + language: string } -const startSsml = (element: Element, voices: VoiceOptions): string => { +const startSsml = (element: Element, options: SSMLOptions): string => { const voice = - element.nodeName === 'BLOCKQUOTE' ? voices.secondary : voices.primary + element.nodeName === 'BLOCKQUOTE' + ? options.secondaryVoice + : options.primaryVoice return ` - + ` } @@ -152,10 +156,7 @@ export const ssmlItemText = (item: SSMLItem): string => { return [item.open, ...item.textItems, item.close].join('') } -export const htmlToSsml = ( - html: string, - voices: { primary: string; secondary: string } -): SSMLItem[] => { +export const htmlToSsml = (html: string, options: SSMLOptions): SSMLItem[] => { const dom = parseHTML(html) const body = dom.document.querySelector('#readability-page-1') if (!body) { @@ -174,7 +175,7 @@ export const htmlToSsml = ( i = emitElement(textItems, node, true) items.push({ - open: startSsml(node, voices), + open: startSsml(node, options), close: endSsml(), textItems: textItems, }) diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 6d2e34736..2ff849856 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -217,8 +217,10 @@ const synthesizeTextToSpeech = async ( } } else { const ssmlItems = htmlToSsml(input.text, { - primary: speechConfig.speechSynthesisVoiceName, - secondary: 'en-US-GuyNeural', + primaryVoice: speechConfig.speechSynthesisVoiceName, + secondaryVoice: 'en-US-GuyNeural', + language: speechConfig.speechSynthesisLanguage, + rate: '1', }) for (const ssmlItem of Array.from(ssmlItems)) { diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index 424de5d9d..c751058de 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -1,9 +1,17 @@ import 'mocha' import { expect } from 'chai' + +import fs from 'fs' +import { glob } from 'glob' import { htmlToSsml } from '../src/htmlToSsml' describe('htmlToSsml', () => { - const TEST_VOCIES = { primary: 'test-primary', secondary: 'test-secondary' } + const TEST_OPTIONS = { + primaryVoice: 'test-primary', + secondaryVoice: 'test-secondary', + language: 'en-US', + rate: '1' + } describe('a simple html file', () => { it('should convert Html to SSML', async () => { @@ -11,7 +19,7 @@ describe('htmlToSsml', () => {

this is some text

- `, TEST_VOCIES + `, TEST_OPTIONS ) const text = ssml[0].textItems.join('').trim() expect(text).to.equal( @@ -29,7 +37,7 @@ this is in the first paragraph this is also in the first paragraph

- `, TEST_VOCIES + `, TEST_OPTIONS ) const text = ssml[0].textItems.join('').trim() expect(text).to.equal( @@ -45,7 +53,7 @@ this is also in the first paragraph
second

third

- `, TEST_VOCIES + `, TEST_OPTIONS ) const first = ssml[0].textItems.join('').trim() const second = ssml[1].textItems.join('').trim() @@ -62,14 +70,30 @@ this is also in the first paragraph ) expect(ssml[0].open.trim()).to.equal( - `` + `` ) expect(ssml[1].open.trim()).to.equal( - `` + `` ) expect(ssml[2].open.trim()).to.equal( - `` + `` ) }) }) + // For local testing: + // describe('readability test files', () => { + // it('should convert Html to SSML without throwing', async () => { + // const g = new glob.GlobSync('../readabilityjs/test/test-pages/*') + // console.log('glob: ', glob) + // for (const f of g.found) { + // const readablePath = `${f}/expected.html` + // if (!fs.existsSync(readablePath)) { + // continue + // } + // const html = fs.readFileSync(readablePath, { encoding: 'utf-8' }) + // const ssmlItems = htmlToSsml(html, TEST_OPTIONS) + // console.log('SSML ITEMS', ssmlItems) + // } + // }) + // }) })