diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts new file mode 100644 index 000000000..6d4fb8040 --- /dev/null +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -0,0 +1,185 @@ +import { parseHTML } from 'linkedom' + +// this code needs to be kept in sync with the +// frontend code in: useReadingProgressAnchor + +const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ + 'omnivore-highlight-id', + 'data-twitter-tweet-id', + 'data-instagram-id', +] + +function ssmlTagsForTopLevelElement() { + return { + opening: `

`, + closing: `

`, + } +} + +function parseDomTree(pageNode: Element) { + if (!pageNode || pageNode.childNodes.length == 0) { + console.log(' no child nodes found') + return [] + } + + const nodesToVisitStack = [pageNode] + const visitedNodeList = [] + + while (nodesToVisitStack.length > 0) { + const currentNode = nodesToVisitStack.pop() + if ( + currentNode?.nodeType !== 1 /* Node.ELEMENT_NODE */ || + // Avoiding dynamic elements from being counted as anchor-allowed elements + ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES.some((attrib) => + currentNode.hasAttribute(attrib) + ) + ) { + continue + } + + visitedNodeList.push(currentNode) + ;[].slice + .call(currentNode.childNodes) + .reverse() + .forEach(function (node) { + nodesToVisitStack.push(node) + }) + } + + visitedNodeList.shift() + visitedNodeList.forEach((node, index) => { + // start from index 1, index 0 reserved for anchor unknown. + node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString()) + }) + return visitedNodeList +} + +function emit(textItems: string[], text: string) { + textItems.push(text) +} + +function cleanTextNode(textNode: ChildNode): string { + return (textNode.textContent ?? '').replace(/\s+/g, ' ') +} + +function emitTextNode( + textItems: string[], + cleanedText: string, + textNode: ChildNode +) { + const ssmlElement = + textNode.parentNode?.nodeName === 'B' ? 'emphasis' : undefined + if (!cleanedText) { + return + } + + if (ssmlElement) { + emit(textItems, `<${ssmlElement}>`) + } + emit(textItems, `${cleanedText}`) + if (ssmlElement) { + emit(textItems, ``) + } +} + +function emitElement( + textItems: string[], + element: Element, + isTopLevel: boolean +) { + const SKIP_TAGS = ['SCRIPT', 'STYLE', 'IMG', 'FIGURE', 'FIGCAPTION', 'IFRAME'] + + const topLevelTags = ssmlTagsForTopLevelElement() + const idx = element.getAttribute('data-omnivore-anchor-idx') + let maxVisitedIdx = Number(idx) + + if (isTopLevel) { + emit(textItems, topLevelTags.opening) + } + + for (const child of Array.from(element.childNodes)) { + if (SKIP_TAGS.indexOf(child.nodeName) >= 0) { + continue + } + + if ( + child.nodeType == 3 /* Node.TEXT_NODE */ && + (child.textContent?.length ?? 0) > 0 + ) { + const cleanedText = cleanTextNode(child) + if (idx && cleanedText.length > 1) { + // Make sure its more than just a space + emit(textItems, ``) + } + emitTextNode(textItems, cleanedText, child) + } + if (child.nodeType == 1 /* Node.ELEMENT_NODE */) { + maxVisitedIdx = emitElement(textItems, child as HTMLElement, false) + } + } + + if (isTopLevel) { + emit(textItems, topLevelTags.closing) + } + + return Number(maxVisitedIdx) +} + +export type SSMLItem = { + open: string + close: string + textItems: string[] +} + +export type SSMLOptions = { + primaryVoice: string + secondaryVoice: string + rate: string + language: string +} + +const startSsml = (element: Element, options: SSMLOptions): string => { + const voice = + element.nodeName === 'BLOCKQUOTE' + ? options.secondaryVoice + : options.primaryVoice + return ` + + ` +} + +const endSsml = (): string => { + return `` +} + +export const ssmlItemText = (item: SSMLItem): string => { + return [item.open, ...item.textItems, item.close].join('') +} + +export const htmlToSsml = (html: string, options: SSMLOptions): SSMLItem[] => { + const dom = parseHTML(html) + const body = dom.document.querySelector('#readability-page-1') + if (!body) { + throw new Error('Unable to parse HTML document') + } + + const parsedNodes = parseDomTree(body) + if (parsedNodes.length < 1) { + throw new Error('No HTML nodes found') + } + + const items: SSMLItem[] = [] + for (let i = 1; i < parsedNodes.length + 1; i++) { + const textItems: string[] = [] + const node = parsedNodes[i - 1] + + i = emitElement(textItems, node, true) + items.push({ + open: startSsml(node, options), + close: endSsml(), + textItems: textItems, + }) + } + + return items +} diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 28833d879..2ff849856 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -18,6 +18,7 @@ import { import axios from 'axios' import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import +import { htmlToSsml, ssmlItemText } from './htmlToSsml' dotenv.config() interface TextToSpeechInput { @@ -215,35 +216,23 @@ const synthesizeTextToSpeech = async ( currentTextChunk = '' } } else { - const document = parseHTML(input.text).document - const elements = document.querySelectorAll( - 'h1, h2, h3, p, ul, ol, blockquote' - ) - // convert html elements to the ssml document - for (const e of Array.from(elements)) { - const htmlElement = e as HTMLElement - if (htmlElement.innerText) { - // use complimentary voice for blockquote, hardcoded for now - const voice = - htmlElement.tagName.toLowerCase() === 'blockquote' - ? input.complimentaryVoice || 'en-US-AriaNeural' - : input.voice - const ssml = htmlElementToSsml({ - htmlElement: e, - language: input.languageCode, - rate: input.rate, - volume: input.volume, - voice, - }) - console.debug(`synthesizing ${ssml}`) - const result = await speakSsmlAsyncPromise(ssml) - // if (result.reason === ResultReason.Canceled) { - // synthesizer.close() - // throw new Error(result.errorDetails) - // } - timeOffset = timeOffset + result.audioDuration - // characterOffset = characterOffset + htmlElement.innerText.length - } + const ssmlItems = htmlToSsml(input.text, { + primaryVoice: speechConfig.speechSynthesisVoiceName, + secondaryVoice: 'en-US-GuyNeural', + language: speechConfig.speechSynthesisLanguage, + rate: '1', + }) + + for (const ssmlItem of Array.from(ssmlItems)) { + const ssml = ssmlItemText(ssmlItem) + console.debug(`synthesizing ${ssml}`) + const result = await speakSsmlAsyncPromise(ssml) + // if (result.reason === ResultReason.Canceled) { + // synthesizer.close() + // throw new Error(result.errorDetails) + // } + timeOffset = timeOffset + result.audioDuration + // characterOffset = characterOffset + htmlElement.innerText.length } } writeStream.end() diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts new file mode 100644 index 000000000..c751058de --- /dev/null +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -0,0 +1,99 @@ +import 'mocha' +import { expect } from 'chai' + +import fs from 'fs' +import { glob } from 'glob' +import { htmlToSsml } from '../src/htmlToSsml' + +describe('htmlToSsml', () => { + const TEST_OPTIONS = { + primaryVoice: 'test-primary', + secondaryVoice: 'test-secondary', + language: 'en-US', + rate: '1' + } + + describe('a simple html file', () => { + it('should convert Html to SSML', async () => { + const ssml = htmlToSsml(` +
+

this is some text

+
+ `, TEST_OPTIONS + ) + const text = ssml[0].textItems.join('').trim() + expect(text).to.equal( + `

this is some text

` + ) + }) + }) + describe('a file with nested elements', () => { + it('should convert Html to SSML', async () => { + const ssml = htmlToSsml(` +
+

+this is in the first paragraph +this is in the second span +this is also in the first paragraph +

+
+ `, TEST_OPTIONS + ) + const text = ssml[0].textItems.join('').trim() + expect(text).to.equal( + `

this is in the first paragraph this is in the second span this is also in the first paragraph

`.trim() + ) + }) + }) + describe('a file with blockquotes', () => { + it('should convert Html to SSML with complimentary voices', async () => { + const ssml = htmlToSsml(` +
+

first

+
second
+

third

+
+ `, TEST_OPTIONS + ) + const first = ssml[0].textItems.join('').trim() + const second = ssml[1].textItems.join('').trim() + const third = ssml[2].textItems.join('').trim() + + expect(first).to.equal( + `

first

` + ) + expect(second).to.equal( + `

second

` + ) + expect(third).to.equal( + `

third

` + ) + + expect(ssml[0].open.trim()).to.equal( + `` + ) + expect(ssml[1].open.trim()).to.equal( + `` + ) + expect(ssml[2].open.trim()).to.equal( + `` + ) + }) + }) + // For local testing: + // describe('readability test files', () => { + // it('should convert Html to SSML without throwing', async () => { + // const g = new glob.GlobSync('../readabilityjs/test/test-pages/*') + // console.log('glob: ', glob) + // for (const f of g.found) { + // const readablePath = `${f}/expected.html` + // if (!fs.existsSync(readablePath)) { + // continue + // } + // const html = fs.readFileSync(readablePath, { encoding: 'utf-8' }) + // const ssmlItems = htmlToSsml(html, TEST_OPTIONS) + // console.log('SSML ITEMS', ssmlItems) + // } + // }) + // }) +})