prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #2063 from omnivore-app/fix/tts

Browse source 
Upgrade natural nodejs module and use a new sentence tokenizer based on parsing
This commit is contained in:
Hongbo Wu 2023-04-20 21:22:48 +08:00 committed by GitHub
commit 0125e4e340
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 74 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/text-to-speech/package.json
View file

@ -42,7 +42,7 @@
"jsonwebtoken": "^8.5.1",
"linkedom": "^0.14.12",
"microsoft-cognitiveservices-speech-sdk": "^1.22.0",
"natural": "^5.2.3",
"natural": "^6.2.0",
"redis": "^4.3.1",
"underscore": "^1.13.4"
}

40
packages/text-to-speech/src/htmlToSsml.ts
View file

@ -1,6 +1,10 @@
import { parseHTML } from 'linkedom'
import { SentenceTokenizer, WordPunctTokenizer } from 'natural'
import { htmlToText } from 'html-to-text'
import { parseHTML } from 'linkedom'
import {
SentenceTokenizer,
SentenceTokenizerNew,
WordPunctTokenizer,
} from 'natural'
// this code needs to be kept in sync with the
// frontend code in: useReadingProgressAnchor
@ -262,14 +266,14 @@ export const stripEmojis = (text: string): string => {
}
const textToUtterances = ({
tokenizer,
wordTokenizer,
idx,
textItems,
wordOffset,
voice,
isHtml = true,
}: {
tokenizer: WordPunctTokenizer
wordTokenizer: WordPunctTokenizer
idx: string
textItems: string[]
wordOffset: number
@ -284,7 +288,7 @@ const textToUtterances = ({
idx,
text,
wordOffset,
wordCount: tokenizer.tokenize(text).length,
wordCount: wordTokenizer.tokenize(text).length,
voice,
},
]
@ -303,10 +307,18 @@ const textToUtterances = ({
text = parseHTML(text).document.documentElement.textContent ?? text
console.info('Converted HTML to text:', text)
}
const MAX_CHARS = 256
const sentenceTokenizer = new SentenceTokenizer()
const sentences = sentenceTokenizer.tokenize(text)
let sentences: string[] = []
try {
// use new sentence tokenizer
const sentenceTokenizer = new SentenceTokenizerNew()
sentences = sentenceTokenizer.tokenize(text)
} catch (err) {
console.debug('Unable to tokenize sentences, text:', text, ', error:', err)
// fallback to old sentence tokenizer
const sentenceTokenizer = new SentenceTokenizer()
sentences = sentenceTokenizer.tokenize(text)
}
let currentText = ''
// split text to max 256 chars per utterance and
// use nlp lib to detect sentences and
@ -319,7 +331,7 @@ const textToUtterances = ({
const nextText = currentText + sentence
if (nextText.length > MAX_CHARS) {
if (currentText.length > 0) {
const wordCount = tokenizer.tokenize(currentText).length
const wordCount = wordTokenizer.tokenize(currentText).length
utterances.push({
idx,
text: currentText,
@ -330,7 +342,7 @@ const textToUtterances = ({
wordOffset += wordCount
currentText = sentence
} else {
const wordCount = tokenizer.tokenize(sentence).length
const wordCount = wordTokenizer.tokenize(sentence).length
utterances.push({
idx,
text: sentence,
@ -348,7 +360,7 @@ const textToUtterances = ({
idx,
text: currentText,
wordOffset,
wordCount: tokenizer.tokenize(currentText).length,
wordCount: wordTokenizer.tokenize(currentText).length,
voice,
})
}
@ -386,13 +398,13 @@ export const htmlToSpeechFile = (htmlInput: HtmlInput): SpeechFile => {
}
}
const tokenizer = new WordPunctTokenizer()
const wordTokenizer = new WordPunctTokenizer()
const utterances: Utterance[] = []
let wordOffset = 0
if (title) {
// first utterances is the title
const titleUtterance = textToUtterances({
tokenizer,
wordTokenizer,
idx: '',
textItems: [stripEmojis(title)], // title could have emoji
wordOffset,
@ -413,7 +425,7 @@ export const htmlToSpeechFile = (htmlInput: HtmlInput): SpeechFile => {
const idx = i.toString()
i = emitElement(textItems, node, true)
const newUtterances = textToUtterances({
tokenizer,
wordTokenizer,
idx,
textItems,
wordOffset,

34
packages/text-to-speech/test/htmlToSsml.test.ts
View file

@ -1,12 +1,12 @@
import 'mocha'
import { expect } from 'chai'
import * as fs from 'fs'
import 'mocha'
import path from 'path'
import {
htmlToSpeechFile,
htmlToSsmlItems,
stripEmojis,
} from '../src/htmlToSsml'
import * as fs from 'fs'
import path from 'path'
const TEST_OPTIONS = {
primaryVoice: 'test-primary',
@ -309,4 +309,32 @@ describe('convert HTML to Speech file', () => {
'I feel like I’m working on reading, listening, and speaking all at once, sometimes I feel like I’m just getting surface understanding. '
)
})
it('splits sentences in German correctly', () => {
const html = `<div class="page" id="readability-page-1" data-omnivore-anchor-idx="1">
<p data-omnivore-anchor-idx="2"><span data-omnivore-anchor-idx="3"><span data-omnivore-anchor-idx="4"><strong data-omnivore-anchor-idx="5"><em data-omnivore-anchor-idx="6"><span data-omnivore-anchor-idx="7" lang="DE" xml:lang="DE"><span data-omnivore-anchor-idx="8">Q</span></span></em></strong><em data-omnivore-anchor-idx="9"><span data-omnivore-anchor-idx="10" lang="DE" xml:lang="DE"><span data-omnivore-anchor-idx="11"><strong data-omnivore-anchor-idx="12">:</strong> „Die kürzliche Razzia in den BBC-Büros in Delhi sind ein weiterer Versuch der Regierung, kritische Medien-Kommentare zu unterdrücken. Man hat des Gefühl, Herr Modi hat Angst, in den Spiegel zu schauen!?“</span></span></em></span></span></p>
</div>`
const speechFile = htmlToSpeechFile({
content: html,
options: TEST_OPTIONS,
})
expect(speechFile.utterances).to.have.lengthOf(1)
expect(speechFile.utterances[0].text).to.eql(
'Q: „Die kürzliche Razzia in den BBC-Büros in Delhi sind ein weiterer Versuch der Regierung, kritische Medien-Kommentare zu unterdrücken. Man hat des Gefühl, Herr Modi hat Angst, in den Spiegel zu schauen!? “'
)
})
it('splits sentences in Chinese correctly', () => {
const html = `<div class="page" id="readability-page-1" data-omnivore-anchor-idx="1">
<p data-omnivore-anchor-idx="2">这是一段中文，我想看看它是怎么分句的。如果买二手房有中介参与，要找相对大的、知名的中介。中介的收费、服务情况要先问清。还要和中介谈好，中介费的付款时间，一般来说是签完合同付一部分，过户后付一部分，省的太早付完钱，中介就不管事了。付完记得要发票。中介如果提供贷款服务，让他玩去。贷款之类的问题，别怕麻烦，自己去找银行。</p>
</div>`
const speechFile = htmlToSpeechFile({
content: html,
options: TEST_OPTIONS,
})
expect(speechFile.utterances).to.have.lengthOf(1)
expect(speechFile.utterances[0].text).to.eql(
'这是一段中文，我想看看它是怎么分句的。如果买二手房有中介参与，要找相对大的、知名的中介。中介的收费、服务情况要先问清。还要和中介谈好，中介费的付款时间，一般来说是签完合同付一部分，过户后付一部分，省的太早付完钱，中介就不管事了。付完记得要发票。中介如果提供贷款服务，让他玩去。贷款之类的问题，别怕麻烦，自己去找银行。'
)
})
})

20
yarn.lock
View file

@ -9473,6 +9473,11 @@ addressparser@^1.0.1:
resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746"
integrity sha512-aQX7AISOMM7HFE0iZ3+YnD07oIeJqWGVnJ+ZIKaBZAk03ftmVYVqsGas/rbXKR21n4D/hKCSHypvcyOkds/xzg==
afinn-165-financialmarketnews@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/afinn-165-financialmarketnews/-/afinn-165-financialmarketnews-3.0.0.tgz#cf422577775bf94f9bc156f3f001a1f29338c3d8"
integrity sha512-0g9A1S3ZomFIGDTzZ0t6xmv4AuokBvBmpes8htiyHpH7N4xDmvSQL6UxL/Zcs2ypRb3VwgCscaD8Q3zEawKYhw==
afinn-165@^1.0.2:
version "1.0.4"
resolved "https://registry.yarnpkg.com/afinn-165/-/afinn-165-1.0.4.tgz#3abf6b8922dd5db84d84e0abd155924381dd73a4"
@ -20688,14 +20693,16 @@ natural-compare@^1.4.0:
resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
natural@^5.2.3:
version "5.2.3"
resolved "https://registry.yarnpkg.com/natural/-/natural-5.2.3.tgz#bfd9b9710139313edf6ec3172435e48998884535"
integrity sha512-fsGGpbU15YBc2oQCEsi0t7ZeF3VmKyxDhgWucQTPk4zaDFzeZtquRbZt4xlznN2ZUlH88215HcThMYaDHFM48Q==
natural@^6.2.0:
version "6.2.0"
resolved "https://registry.yarnpkg.com/natural/-/natural-6.2.0.tgz#74a45c66336e5c35057aba0fafb0f5351c81a201"
integrity sha512-/+ceiLjldGcMgCtryGJV6jS2IslPeLE+bwjXry9jOFNl586J1rV5egDa4X8nnPCZmzCeXrE4uHajpkuB/ILH2Q==
dependencies:
afinn-165 "^1.0.2"
afinn-165-financialmarketnews "^3.0.0"
apparatus "^0.0.10"
safe-stable-stringify "^2.2.0"
stopwords-iso "^1.1.0"
sylvester "^0.0.12"
underscore "^1.9.1"
wordnet-db "^3.1.11"
@ -25550,6 +25557,11 @@ statuses@2.0.1:
resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c"
integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=
stopwords-iso@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/stopwords-iso/-/stopwords-iso-1.1.0.tgz#dc303db6b0842d4290bc1339b4eaf37b94219395"
integrity sha512-I6GPS/E0zyieHehMRPQcqkiBMJKGgLta+1hREixhoLPqEA0AlVFiC43dl8uPpmkkeRdDMzYRWFWk5/l9x7nmNg==
store2@^2.12.0:
version "2.13.2"
resolved "https://registry.yarnpkg.com/store2/-/store2-2.13.2.tgz#01ad8802ca5b445b9c316b55e72645c13a3cd7e3"