diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index 2cfad13c1..ef9d6af6e 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -1,6 +1,6 @@ import { parseHTML } from 'linkedom' import * as _ from 'underscore' -import { WordPunctTokenizer } from 'natural' +import { SentenceTokenizer, WordPunctTokenizer } from 'natural' import { htmlToText } from 'html-to-text' // this code needs to be kept in sync with the @@ -69,7 +69,6 @@ const TOP_LEVEL_TAGS = [ 'H5', 'H6', 'LI', - 'CODE', ] function parseDomTree(pageNode: Element) { @@ -148,7 +147,15 @@ function emitElement( element: Element, isTopLevel: boolean ) { - const SKIP_TAGS = ['SCRIPT', 'STYLE', 'IMG', 'FIGURE', 'FIGCAPTION', 'IFRAME'] + const SKIP_TAGS = [ + 'SCRIPT', + 'STYLE', + 'IMG', + 'FIGURE', + 'FIGCAPTION', + 'IFRAME', + 'CODE', + ] const topLevelTags = ssmlTagsForTopLevelElement() const idx = element.getAttribute('data-omnivore-anchor-idx') @@ -297,38 +304,56 @@ const textToUtterances = ({ text = parseHTML(text).document.documentElement.textContent ?? text console.info('Converted HTML to text:', text) } - // if we hit 256, look back for first ending sentence within 80 chars + const MAX_CHARS = 256 - const MAX_LOOKBACK = 80 - while (text.length > MAX_CHARS) { - let end = MAX_CHARS - MAX_LOOKBACK - 1 - while (end < text.length && !text[end].match(/[.!?]/)) { - end++ + const sentenceTokenizer = new SentenceTokenizer() + const sentences = sentenceTokenizer.tokenize(text) + let currentText = '' + // split text to max 256 chars per utterance and + // use nlp lib to detect sentences and + // avoid splitting words and sentences + sentences.forEach((sentence, i) => { + if (i < sentences.length - 1) { + // add space to the end of sentence + sentence += ' ' } - - const utterance = text.substring(0, end + 1) - const wordCount = tokenizer.tokenize(utterance).length - utterances.push({ - idx, - text: utterance, - wordOffset, - wordCount, - voice, - }) - text = text.substring(end + 1) - wordOffset += wordCount - } - - if (text.length > 0) { - const wordCount = tokenizer.tokenize(text).length - utterances.push({ - idx, - text, - wordOffset, - wordCount, - voice, - }) - } + const nextText = currentText + sentence + if (nextText.length > MAX_CHARS) { + if (currentText.length > 0) { + const wordCount = tokenizer.tokenize(currentText).length + utterances.push({ + idx, + text: currentText, + wordOffset, + wordCount, + voice, + }) + wordOffset += wordCount + currentText = sentence + } else { + const wordCount = tokenizer.tokenize(sentence).length + utterances.push({ + idx, + text: sentence, + wordOffset, + wordCount, + voice, + }) + wordOffset += wordCount + } + } else { + currentText = nextText + } + if (i === sentences.length - 1 && currentText.length > 0) { + utterances.push({ + idx, + text: currentText, + wordOffset, + wordCount: tokenizer.tokenize(currentText).length, + voice, + }) + } + }) return utterances } diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 00cc06c58..da6c2e4c2 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -1,10 +1,8 @@ import { CancellationDetails, CancellationReason, - PropertyId, ResultReason, SpeechConfig, - SpeechSynthesisBoundaryType, SpeechSynthesisOutputFormat, SpeechSynthesisResult, SpeechSynthesizer, @@ -32,7 +30,7 @@ export interface SpeechMark { start?: number length?: number word: string - type: 'word' | 'bookmark' | 'punctuation' | 'sentence' + type: 'word' | 'bookmark' } export const synthesizeTextToSpeech = async ( @@ -49,17 +47,12 @@ export const synthesizeTextToSpeech = async ( ) speechConfig.speechSynthesisOutputFormat = SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 - // Required for sentence-level WordBoundary events - speechConfig.setProperty( - PropertyId.SpeechServiceResponse_RequestSentenceBoundary, - 'true' - ) // Create the speech synthesizer. const synthesizer = new SpeechSynthesizer(speechConfig) const speechMarks: SpeechMark[] = [] let timeOffset = 0 - // let wordOffset = 0 + let wordOffset = 0 synthesizer.synthesizing = function (s, e) { // convert arrayBuffer to stream and write to stream @@ -94,14 +87,13 @@ export const synthesizeTextToSpeech = async ( // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. synthesizer.wordBoundary = (s, e) => { - e.boundaryType === SpeechSynthesisBoundaryType.Sentence && - speechMarks.push({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - start: e.textOffset, - length: e.text.length, - type: 'sentence', - }) + speechMarks.push({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + start: wordOffset + e.textOffset, + length: e.wordLength, + type: 'word', + }) } synthesizer.bookmarkReached = (s, e) => { @@ -148,10 +140,9 @@ export const synthesizeTextToSpeech = async ( } // for ssml const startSsmlTag = startSsml(ssmlOptions) + wordOffset -= startSsmlTag.length const text = _.escape(input.text) const ssml = `${startSsmlTag}${text}${endSsml()}` - // set the text offset to be the end of SSML start tag - // wordOffset -= startSsmlTag.length const result = await speakSsmlAsyncPromise(ssml) if (result.reason === ResultReason.Canceled) { throw new Error(result.errorDetails) diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index 3d2350378..4f53bdaf4 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -235,7 +235,7 @@ describe('convert HTML to Speech file', () => { title: 'Wang Yi at the UN; Fu Zhenghua sentenced; Nvidia China sales', options: TEST_OPTIONS, }) - expect(speechFile.utterances).to.have.lengthOf(19) + expect(speechFile.utterances).to.have.lengthOf(20) }) it('converts long utterances to multiple utterances', () => { @@ -271,4 +271,23 @@ describe('convert HTML to Speech file', () => { }) expect(speechFile.utterances).to.have.lengthOf(2) }) + + it('does not break on not decimal point in sentences', () => { + const html = `
+
+
+ If terms of the original $12.5 billion financing package remain the same, bankers may struggle to sell the risky Twitter buyout debt just as credit markets begin to crack, with yields at multiyear highs, they’re potentially on the hook for hundreds of millions of dollars of losses on the unsecured portion alone should they try to unload it to investors. +
+
+
+` + const speechFile = htmlToSpeechFile({ + content: html, + title: 'Test long sentence with decimal point', + options: TEST_OPTIONS, + }) + expect(speechFile.utterances[1].text).to.eql( + 'If terms of the original $12.5 billion financing package remain the same, bankers may struggle to sell the risky Twitter buyout debt just as credit markets begin to crack, with yields at multiyear highs, they’re potentially on the hook for hundreds of millions of dollars of losses on the unsecured portion alone should they try to unload it to investors.' + ) + }) })