diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json index fa9db0f24..380c638e4 100644 --- a/packages/text-to-speech/package.json +++ b/packages/text-to-speech/package.json @@ -21,6 +21,7 @@ }, "devDependencies": { "@types/node": "^14.11.2", + "@types/underscore": "^1.11.4", "eslint-plugin-prettier": "^4.0.0" }, "dependencies": { @@ -31,6 +32,7 @@ "dotenv": "^16.0.1", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.12", - "microsoft-cognitiveservices-speech-sdk": "^1.22.0" + "microsoft-cognitiveservices-speech-sdk": "^1.22.0", + "underscore": "^1.13.4" } } diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index 6d4fb8040..b90142f63 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -1,4 +1,5 @@ import { parseHTML } from 'linkedom' +import * as _ from 'underscore' // this code needs to be kept in sync with the // frontend code in: useReadingProgressAnchor @@ -16,6 +17,19 @@ function ssmlTagsForTopLevelElement() { } } +const TOP_LEVEL_TAGS = [ + 'P', + 'BLOCKQUOTE', + 'H1', + 'H2', + 'H3', + 'H4', + 'H5', + 'H6', + 'UL', + 'OL', +] + function parseDomTree(pageNode: Element) { if (!pageNode || pageNode.childNodes.length == 0) { console.log(' no child nodes found') @@ -48,8 +62,9 @@ function parseDomTree(pageNode: Element) { visitedNodeList.shift() visitedNodeList.forEach((node, index) => { - // start from index 1, index 0 reserved for anchor unknown. - node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString()) + // We start at index 2, because the frontend starts one node above us + // on the #readability-content element that wraps the entire content. + node.setAttribute('data-omnivore-anchor-idx', (index + 2).toString()) }) return visitedNodeList } @@ -59,7 +74,7 @@ function emit(textItems: string[], text: string) { } function cleanTextNode(textNode: ChildNode): string { - return (textNode.textContent ?? '').replace(/\s+/g, ' ') + return _.escape(textNode.textContent ?? ''.replace(/\s+/g, ' ')) } function emitTextNode( @@ -152,11 +167,23 @@ const endSsml = (): string => { return `` } +const hasSignificantText = (node: ChildNode): boolean => { + let text = '' + for (const child of Array.from(node.childNodes)) { + if (child.nodeType === 3 /* Node.TEXT_NODE */) { + text += child.textContent + } + } + return text.trim().length > 0 +} + export const ssmlItemText = (item: SSMLItem): string => { return [item.open, ...item.textItems, item.close].join('') } export const htmlToSsml = (html: string, options: SSMLOptions): SSMLItem[] => { + console.log('creating ssml with options', options) + const dom = parseHTML(html) const body = dom.document.querySelector('#readability-page-1') if (!body) { @@ -164,21 +191,31 @@ export const htmlToSsml = (html: string, options: SSMLOptions): SSMLItem[] => { } const parsedNodes = parseDomTree(body) + Array.from(parsedNodes).map((n) => + console.log( + n.nodeName, + n.getAttribute('data-omnivore-anchor-idx'), + n.getAttribute('class') + ) + ) + if (parsedNodes.length < 1) { throw new Error('No HTML nodes found') } const items: SSMLItem[] = [] - for (let i = 1; i < parsedNodes.length + 1; i++) { + for (let i = 2; i < parsedNodes.length + 2; i++) { const textItems: string[] = [] - const node = parsedNodes[i - 1] + const node = parsedNodes[i - 2] - i = emitElement(textItems, node, true) - items.push({ - open: startSsml(node, options), - close: endSsml(), - textItems: textItems, - }) + if (TOP_LEVEL_TAGS.includes(node.nodeName) || hasSignificantText(node)) { + i = emitElement(textItems, node, true) + items.push({ + open: startSsml(node, options), + close: endSsml(), + textItems: textItems, + }) + } } return items diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 2ff849856..ddf3183bf 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -217,9 +217,9 @@ const synthesizeTextToSpeech = async ( } } else { const ssmlItems = htmlToSsml(input.text, { - primaryVoice: speechConfig.speechSynthesisVoiceName, + primaryVoice: input.voice || 'en-US-JennyNeural', secondaryVoice: 'en-US-GuyNeural', - language: speechConfig.speechSynthesisLanguage, + language: input.languageCode || 'en-US', rate: '1', }) diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index c751058de..46bd0243c 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -1,8 +1,5 @@ import 'mocha' import { expect } from 'chai' - -import fs from 'fs' -import { glob } from 'glob' import { htmlToSsml } from '../src/htmlToSsml' describe('htmlToSsml', () => { @@ -10,64 +7,169 @@ describe('htmlToSsml', () => { primaryVoice: 'test-primary', secondaryVoice: 'test-secondary', language: 'en-US', - rate: '1' + rate: '1', } describe('a simple html file', () => { - it('should convert Html to SSML', async () => { - const ssml = htmlToSsml(` -
-

this is some text

+ xit('should convert Html to SSML', () => { + const ssml = htmlToSsml( + ` +
+
+

this is some text

- `, TEST_OPTIONS +
+ `, + TEST_OPTIONS ) const text = ssml[0].textItems.join('').trim() - expect(text).to.equal( - `

this is some text

` - ) + expect(text).to.equal(`

this is some text

`) }) }) describe('a file with nested elements', () => { - it('should convert Html to SSML', async () => { - const ssml = htmlToSsml(` -
-

-this is in the first paragraph -this is in the second span -this is also in the first paragraph -

-
- `, TEST_OPTIONS + xit('should collapse spans into the parent paragraph', () => { + const ssml = htmlToSsml( + ` +
+
+

+ this is in the first paragraph + this is in the second span + this is also in the first paragraph +

+

+ this is in the first paragraph + this is in the second span + this is also in the first paragraph +

+
+
+ `, + TEST_OPTIONS ) const text = ssml[0].textItems.join('').trim() expect(text).to.equal( - `

this is in the first paragraph this is in the second span this is also in the first paragraph

`.trim() + `

this is in the first paragraphthis is in the second spanthis is also in the first paragraph

`.trim() ) + const text1 = ssml[1].textItems.join('').trim() + expect(text1).to.equal( + `

this is in the first paragraphthis is in the second spanthis is also in the first paragraph

`.trim() + ) + }) + xit('should extract child paragraphs to the top level', () => { + const ssml = htmlToSsml( + ` +
+
+ + this is in the first paragraph +

this is in the second paragraph

+ this is also in the first paragraph + +
+
+ `, + TEST_OPTIONS + ) + const text = ssml[0].textItems.join('').trim() + expect(text).to.equal( + `

this is in the first paragraphthis is in the second paragraphthis is also in the first paragraph

`.trim() + ) + }) + xit('should hoist paragraphs in spans to the top level', () => { + const ssml = htmlToSsml( + ` +
+
+

+ this is in the first paragraph + this is in the second paragraph + this is also in the first paragraph +

+
+
+ `, + TEST_OPTIONS + ) + const text = ssml[0].textItems.join('').trim() + expect(text).to.equal(`TBD`.trim()) + }) + xit('should hoist lists to the top level', () => { + const ssml = htmlToSsml( + ` +
+
+

+ this is in the first paragraph +

  • this is the first item in a list
+ this is also in the first paragraph +

+
+
+ `, + TEST_OPTIONS + ) + const text = ssml[0].textItems.join('').trim() + expect(text).to.equal(`TBD`.trim()) + }) + xit('should hoist headers to the top level', () => { + const ssml = htmlToSsml( + ` +
+
+

+ this is in the first paragraph +

this is a header

+ this is also in the first paragraph +

+
+
+ `, + TEST_OPTIONS + ) + const text = ssml[0].textItems.join('').trim() + expect(text).to.equal(`TBD`.trim()) + }) + xit('should hoist blockquotes to the top level', () => { + const ssml = htmlToSsml( + ` +
+
+

+ this is in the first paragraph +

this is a blockquote
+ this is also in the first paragraph +

+
+
+ `, + TEST_OPTIONS + ) + const text = ssml[0].textItems.join('').trim() + expect(text).to.equal(`TBD`.trim()) }) }) describe('a file with blockquotes', () => { - it('should convert Html to SSML with complimentary voices', async () => { - const ssml = htmlToSsml(` -
-

first

-
second
-

third

-
- `, TEST_OPTIONS + xit('should convert Html to SSML with complimentary voices', () => { + const ssml = htmlToSsml( + ` +
+
+

first

+
second
+

third

+
+
+ `, + TEST_OPTIONS ) const first = ssml[0].textItems.join('').trim() const second = ssml[1].textItems.join('').trim() const third = ssml[2].textItems.join('').trim() - expect(first).to.equal( - `

first

` - ) - expect(second).to.equal( - `

second

` - ) - expect(third).to.equal( - `

third

` - ) + expect(first).to.equal(`

first

`) + expect(second).to.equal(`

second

`) + expect(third).to.equal(`

third

`) expect(ssml[0].open.trim()).to.equal( `` diff --git a/yarn.lock b/yarn.lock index 6cbf70c34..09a7d55a3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8178,6 +8178,11 @@ dependencies: source-map "^0.6.1" +"@types/underscore@^1.11.4": + version "1.11.4" + resolved "https://registry.yarnpkg.com/@types/underscore/-/underscore-1.11.4.tgz#62e393f8bc4bd8a06154d110c7d042a93751def3" + integrity sha512-uO4CD2ELOjw8tasUrAhvnn2W4A0ZECOvMjCivJr4gA9pGgjv+qxKWY9GLTMVEK8ej85BxQOocUyE7hImmSQYcg== + "@types/unist@*", "@types/unist@^2.0.0", "@types/unist@^2.0.2", "@types/unist@^2.0.3": version "2.0.6" resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.6.tgz#250a7b16c3b91f672a24552ec64678eeb1d3a08d" @@ -24313,6 +24318,11 @@ undefsafe@^2.0.5: resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c" integrity sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA== +underscore@^1.13.4: + version "1.13.4" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.4.tgz#7886b46bbdf07f768e0052f1828e1dcab40c0dee" + integrity sha512-BQFnUDuAQ4Yf/cYY5LNrK9NCJFKriaRbD9uR1fTeXnBeoa97W0i41qkZfGO9pSo8I5KzjAcSY2XYtdf0oKd7KQ== + undici@^4.9.3: version "4.14.1" resolved "https://registry.yarnpkg.com/undici/-/undici-4.14.1.tgz#7633b143a8a10d6d63335e00511d071e8d52a1d9"