From 8a86f19e61847c20cef9568a8f621c4fb844c35d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 17 Apr 2023 18:56:17 +0800 Subject: [PATCH 1/5] Upgrade natural nodejs module and use a new sentence tokenizer based on parsing --- packages/text-to-speech/package.json | 2 +- packages/text-to-speech/src/htmlToSsml.ts | 7 +++---- yarn.lock | 20 ++++++++++++++++---- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json index 2c4e8591b..6c40d7faf 100644 --- a/packages/text-to-speech/package.json +++ b/packages/text-to-speech/package.json @@ -42,7 +42,7 @@ "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.12", "microsoft-cognitiveservices-speech-sdk": "^1.22.0", - "natural": "^5.2.3", + "natural": "^6.2.0", "redis": "^4.3.1", "underscore": "^1.13.4" } diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index e6c1207d6..d3bfa63c2 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -1,6 +1,6 @@ -import { parseHTML } from 'linkedom' -import { SentenceTokenizer, WordPunctTokenizer } from 'natural' import { htmlToText } from 'html-to-text' +import { parseHTML } from 'linkedom' +import { SentenceTokenizerNew, WordPunctTokenizer } from 'natural' // this code needs to be kept in sync with the // frontend code in: useReadingProgressAnchor @@ -303,9 +303,8 @@ const textToUtterances = ({ text = parseHTML(text).document.documentElement.textContent ?? text console.info('Converted HTML to text:', text) } - const MAX_CHARS = 256 - const sentenceTokenizer = new SentenceTokenizer() + const sentenceTokenizer = new SentenceTokenizerNew() const sentences = sentenceTokenizer.tokenize(text) let currentText = '' // split text to max 256 chars per utterance and diff --git a/yarn.lock b/yarn.lock index 2154a36d6..d0d6c0934 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9470,6 +9470,11 @@ addressparser@^1.0.1: resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746" integrity sha512-aQX7AISOMM7HFE0iZ3+YnD07oIeJqWGVnJ+ZIKaBZAk03ftmVYVqsGas/rbXKR21n4D/hKCSHypvcyOkds/xzg== +afinn-165-financialmarketnews@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/afinn-165-financialmarketnews/-/afinn-165-financialmarketnews-3.0.0.tgz#cf422577775bf94f9bc156f3f001a1f29338c3d8" + integrity sha512-0g9A1S3ZomFIGDTzZ0t6xmv4AuokBvBmpes8htiyHpH7N4xDmvSQL6UxL/Zcs2ypRb3VwgCscaD8Q3zEawKYhw== + afinn-165@^1.0.2: version "1.0.4" resolved "https://registry.yarnpkg.com/afinn-165/-/afinn-165-1.0.4.tgz#3abf6b8922dd5db84d84e0abd155924381dd73a4" @@ -20594,14 +20599,16 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= -natural@^5.2.3: - version "5.2.3" - resolved "https://registry.yarnpkg.com/natural/-/natural-5.2.3.tgz#bfd9b9710139313edf6ec3172435e48998884535" - integrity sha512-fsGGpbU15YBc2oQCEsi0t7ZeF3VmKyxDhgWucQTPk4zaDFzeZtquRbZt4xlznN2ZUlH88215HcThMYaDHFM48Q== +natural@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/natural/-/natural-6.2.0.tgz#74a45c66336e5c35057aba0fafb0f5351c81a201" + integrity sha512-/+ceiLjldGcMgCtryGJV6jS2IslPeLE+bwjXry9jOFNl586J1rV5egDa4X8nnPCZmzCeXrE4uHajpkuB/ILH2Q== dependencies: afinn-165 "^1.0.2" + afinn-165-financialmarketnews "^3.0.0" apparatus "^0.0.10" safe-stable-stringify "^2.2.0" + stopwords-iso "^1.1.0" sylvester "^0.0.12" underscore "^1.9.1" wordnet-db "^3.1.11" @@ -25446,6 +25453,11 @@ statuses@2.0.1: resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= +stopwords-iso@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/stopwords-iso/-/stopwords-iso-1.1.0.tgz#dc303db6b0842d4290bc1339b4eaf37b94219395" + integrity sha512-I6GPS/E0zyieHehMRPQcqkiBMJKGgLta+1hREixhoLPqEA0AlVFiC43dl8uPpmkkeRdDMzYRWFWk5/l9x7nmNg== + store2@^2.12.0: version "2.13.2" resolved "https://registry.yarnpkg.com/store2/-/store2-2.13.2.tgz#01ad8802ca5b445b9c316b55e72645c13a3cd7e3" From c697b0cb4db61cd7b5c5bec9bacfde76cd038047 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 17 Apr 2023 19:13:09 +0800 Subject: [PATCH 2/5] Add test case --- .../text-to-speech/test/htmlToSsml.test.ts | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index a2673f465..af0aa46cc 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -1,12 +1,12 @@ -import 'mocha' import { expect } from 'chai' +import * as fs from 'fs' +import 'mocha' +import path from 'path' import { htmlToSpeechFile, htmlToSsmlItems, stripEmojis, } from '../src/htmlToSsml' -import * as fs from 'fs' -import path from 'path' const TEST_OPTIONS = { primaryVoice: 'test-primary', @@ -309,4 +309,18 @@ describe('convert HTML to Speech file', () => { 'I feel like I’m working on reading, listening, and speaking all at once, sometimes I feel like I’m just getting surface understanding. ' ) }) + + it('splits sentences in German correctly', () => { + const html = `
+

Q: „Die kürzliche Razzia in den BBC-Büros in Delhi sind ein weiterer Versuch der Regierung, kritische Medien-Kommentare zu unterdrücken. Man hat des Gefühl, Herr Modi hat Angst, in den Spiegel zu schauen!?“

+
` + const speechFile = htmlToSpeechFile({ + content: html, + options: TEST_OPTIONS, + }) + expect(speechFile.utterances).to.have.lengthOf(1) + expect(speechFile.utterances[0].text).to.eql( + 'Q: „Die kürzliche Razzia in den BBC-Büros in Delhi sind ein weiterer Versuch der Regierung, kritische Medien-Kommentare zu unterdrücken. Man hat des Gefühl, Herr Modi hat Angst, in den Spiegel zu schauen!? “' + ) + }) }) From c63ecc94fbce88596ac0f3fe04df869bccd7e99e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 18 Apr 2023 09:33:48 +0800 Subject: [PATCH 3/5] Replace SentenceTokenizerNew with SentenceTokenizer --- packages/text-to-speech/src/htmlToSsml.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index d3bfa63c2..8e03d6ff2 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -262,14 +262,14 @@ export const stripEmojis = (text: string): string => { } const textToUtterances = ({ - tokenizer, + wordTokenizer, idx, textItems, wordOffset, voice, isHtml = true, }: { - tokenizer: WordPunctTokenizer + wordTokenizer: WordPunctTokenizer idx: string textItems: string[] wordOffset: number @@ -284,7 +284,7 @@ const textToUtterances = ({ idx, text, wordOffset, - wordCount: tokenizer.tokenize(text).length, + wordCount: wordTokenizer.tokenize(text).length, voice, }, ] @@ -318,7 +318,7 @@ const textToUtterances = ({ const nextText = currentText + sentence if (nextText.length > MAX_CHARS) { if (currentText.length > 0) { - const wordCount = tokenizer.tokenize(currentText).length + const wordCount = wordTokenizer.tokenize(currentText).length utterances.push({ idx, text: currentText, @@ -329,7 +329,7 @@ const textToUtterances = ({ wordOffset += wordCount currentText = sentence } else { - const wordCount = tokenizer.tokenize(sentence).length + const wordCount = wordTokenizer.tokenize(sentence).length utterances.push({ idx, text: sentence, @@ -347,7 +347,7 @@ const textToUtterances = ({ idx, text: currentText, wordOffset, - wordCount: tokenizer.tokenize(currentText).length, + wordCount: wordTokenizer.tokenize(currentText).length, voice, }) } @@ -385,13 +385,13 @@ export const htmlToSpeechFile = (htmlInput: HtmlInput): SpeechFile => { } } - const tokenizer = new WordPunctTokenizer() + const wordTokenizer = new WordPunctTokenizer() const utterances: Utterance[] = [] let wordOffset = 0 if (title) { // first utterances is the title const titleUtterance = textToUtterances({ - tokenizer, + wordTokenizer, idx: '', textItems: [stripEmojis(title)], // title could have emoji wordOffset, @@ -412,7 +412,7 @@ export const htmlToSpeechFile = (htmlInput: HtmlInput): SpeechFile => { const idx = i.toString() i = emitElement(textItems, node, true) const newUtterances = textToUtterances({ - tokenizer, + wordTokenizer, idx, textItems, wordOffset, From 6337d60146d87047cd922fa21c48d300dfa970e5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 20 Apr 2023 17:10:32 +0800 Subject: [PATCH 4/5] Add test for Chinese --- packages/text-to-speech/test/htmlToSsml.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index af0aa46cc..61f1da66d 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -323,4 +323,18 @@ describe('convert HTML to Speech file', () => { 'Q: „Die kürzliche Razzia in den BBC-Büros in Delhi sind ein weiterer Versuch der Regierung, kritische Medien-Kommentare zu unterdrücken. Man hat des Gefühl, Herr Modi hat Angst, in den Spiegel zu schauen!? “' ) }) + + it('splits sentences in Chinese correctly', () => { + const html = `
+

这是一段中文，我想看看它是怎么分句的。如果买二手房有中介参与，要找相对大的、知名的中介。中介的收费、服务情况要先问清。还要和中介谈好，中介费的付款时间，一般来说是签完合同付一部分，过户后付一部分，省的太早付完钱，中介就不管事了。付完记得要发票。中介如果提供贷款服务，让他玩去。贷款之类的问题，别怕麻烦，自己去找银行。

+
` + const speechFile = htmlToSpeechFile({ + content: html, + options: TEST_OPTIONS, + }) + expect(speechFile.utterances).to.have.lengthOf(1) + expect(speechFile.utterances[0].text).to.eql( + '这是一段中文，我想看看它是怎么分句的。如果买二手房有中介参与，要找相对大的、知名的中介。中介的收费、服务情况要先问清。还要和中介谈好，中介费的付款时间，一般来说是签完合同付一部分，过户后付一部分，省的太早付完钱，中介就不管事了。付完记得要发票。中介如果提供贷款服务，让他玩去。贷款之类的问题，别怕麻烦，自己去找银行。' + ) + }) }) From 14b7ed5252705220302b0114d5cdab208f936878 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 20 Apr 2023 19:44:08 +0800 Subject: [PATCH 5/5] Fallback to old sentence tokenizer if new one is failed --- packages/text-to-speech/src/htmlToSsml.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index 8e03d6ff2..41e780011 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -1,6 +1,10 @@ import { htmlToText } from 'html-to-text' import { parseHTML } from 'linkedom' -import { SentenceTokenizerNew, WordPunctTokenizer } from 'natural' +import { + SentenceTokenizer, + SentenceTokenizerNew, + WordPunctTokenizer, +} from 'natural' // this code needs to be kept in sync with the // frontend code in: useReadingProgressAnchor @@ -304,8 +308,17 @@ const textToUtterances = ({ console.info('Converted HTML to text:', text) } const MAX_CHARS = 256 - const sentenceTokenizer = new SentenceTokenizerNew() - const sentences = sentenceTokenizer.tokenize(text) + let sentences: string[] = [] + try { + // use new sentence tokenizer + const sentenceTokenizer = new SentenceTokenizerNew() + sentences = sentenceTokenizer.tokenize(text) + } catch (err) { + console.debug('Unable to tokenize sentences, text:', text, ', error:', err) + // fallback to old sentence tokenizer + const sentenceTokenizer = new SentenceTokenizer() + sentences = sentenceTokenizer.tokenize(text) + } let currentText = '' // split text to max 256 chars per utterance and // use nlp lib to detect sentences and