From 39dcab5076bb1a80ebd23253894a5f3d7ccb17a5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 4 Oct 2022 16:28:04 +0800 Subject: [PATCH 1/7] Fix tests --- packages/text-to-speech/test/htmlToSsml.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index 5e3270ced..f49810d9e 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -235,6 +235,6 @@ describe('convert HTML to Speech file', () => { title: 'Wang Yi at the UN; Fu Zhenghua sentenced; Nvidia China sales', options: TEST_OPTIONS, }) - expect(speechFile.utterances).to.have.lengthOf(12) + expect(speechFile.utterances).to.have.lengthOf(21) }) }) From 690ce05b0ec8adf71b3bd03b34797e4730cd0da8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 4 Oct 2022 17:16:26 +0800 Subject: [PATCH 2/7] if we hit 256, look back for first ending sentence within 80 chars --- packages/text-to-speech/src/htmlToSsml.ts | 45 ++++++++++++++++------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index a64a370bf..6935de2ef 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -298,21 +298,40 @@ const textToUtterances = ({ text = parseHTML(text).document.documentElement.textContent ?? text console.info('Converted HTML to text:', text) } - // split text into chunks of 256 characters to stream faster without breaking on words - const textChunks = text.match(/.{1,256}(?= |$)/g) - if (textChunks) { - for (const chunk of textChunks) { - const wordCount = tokenizer.tokenize(chunk).length - utterances.push({ - idx, - text: chunk, - wordOffset, - wordCount, - voice, - }) - wordOffset += wordCount + // if we hit 256, look back for first ending sentence within 80 chars + const MAX_CHARS = 256 + const MAX_LOOKBACK = 80 + while (text.length > MAX_CHARS) { + let lookback = MAX_LOOKBACK + let end = MAX_CHARS - lookback + while (lookback > 0) { + if (text[end] === '.' || text[end] === '!' || text[end] === '?') { + break + } + end++ + lookback-- } + const utterance = text.substring(0, end + 1) + const wordCount = tokenizer.tokenize(utterance).length + utterances.push({ + idx, + text: utterance, + wordOffset, + wordCount, + voice, + }) + text = text.substring(end + 1) + wordOffset += wordCount } + + const wordCount = tokenizer.tokenize(text).length + utterances.push({ + idx, + text, + wordOffset, + wordCount, + voice, + }) return utterances } From 851e2643008a7fcb04181a331ebe0a653798961a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 4 Oct 2022 19:35:14 +0800 Subject: [PATCH 3/7] Add test case --- .../test/fixtures/{large.html => li.html} | 0 .../text-to-speech/test/htmlToSsml.test.ts | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) rename packages/text-to-speech/test/fixtures/{large.html => li.html} (100%) diff --git a/packages/text-to-speech/test/fixtures/large.html b/packages/text-to-speech/test/fixtures/li.html similarity index 100% rename from packages/text-to-speech/test/fixtures/large.html rename to packages/text-to-speech/test/fixtures/li.html diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index f49810d9e..20f9a1d79 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -227,7 +227,7 @@ describe('htmlToSpeechFile', () => { describe('convert HTML to Speech file', () => { it('converts each
  • to an utterance', () => { const html = fs.readFileSync( - path.resolve(__dirname, './fixtures/large.html'), + path.resolve(__dirname, './fixtures/li.html'), { encoding: 'utf-8' } ) const speechFile = htmlToSpeechFile({ @@ -237,4 +237,21 @@ describe('convert HTML to Speech file', () => { }) expect(speechFile.utterances).to.have.lengthOf(21) }) + + it('converts long utterances to multiple utterances', () => { + const html = `
    +
    +
    + All neural voices are multilingual and fluent in their own language and English. For example, if the input text in English is "I'm excited to try text to speech" and you set es-ES-ElviraNeural, the text is spoken in English with a Spanish accent. If the voice doesn't speak the language of the input text, the Speech service won't output synthesized audio. See the full list of supported neural voices. +
    +
    +
    +` + const speechFile = htmlToSpeechFile({ + content: html, + title: 'How to synthesize speech from text', + options: TEST_OPTIONS, + }) + expect(speechFile.utterances).to.have.lengthOf(3) + }) }) From 3bd6b3d13729fd7f4f41af1459446f2ec509acbb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 5 Oct 2022 10:25:24 +0800 Subject: [PATCH 4/7] Do not break on words and long sentence if exceeds 256 chars --- packages/text-to-speech/src/htmlToSsml.ts | 32 +++++++++++------------ 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index 6935de2ef..2cfad13c1 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -273,13 +273,12 @@ const textToUtterances = ({ let text = textItems.join('') if (!isHtml) { // for title - const wordCount = tokenizer.tokenize(text).length return [ { idx, text, wordOffset, - wordCount, + wordCount: tokenizer.tokenize(text).length, voice, }, ] @@ -302,15 +301,11 @@ const textToUtterances = ({ const MAX_CHARS = 256 const MAX_LOOKBACK = 80 while (text.length > MAX_CHARS) { - let lookback = MAX_LOOKBACK - let end = MAX_CHARS - lookback - while (lookback > 0) { - if (text[end] === '.' || text[end] === '!' || text[end] === '?') { - break - } + let end = MAX_CHARS - MAX_LOOKBACK - 1 + while (end < text.length && !text[end].match(/[.!?]/)) { end++ - lookback-- } + const utterance = text.substring(0, end + 1) const wordCount = tokenizer.tokenize(utterance).length utterances.push({ @@ -324,14 +319,17 @@ const textToUtterances = ({ wordOffset += wordCount } - const wordCount = tokenizer.tokenize(text).length - utterances.push({ - idx, - text, - wordOffset, - wordCount, - voice, - }) + if (text.length > 0) { + const wordCount = tokenizer.tokenize(text).length + utterances.push({ + idx, + text, + wordOffset, + wordCount, + voice, + }) + } + return utterances } From e3959c4ab88ce9a23bd383e6a4d2658e0317956d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 5 Oct 2022 10:26:12 +0800 Subject: [PATCH 5/7] Fix tests --- packages/text-to-speech/test/htmlToSsml.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index 20f9a1d79..6f291ca86 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -235,7 +235,7 @@ describe('convert HTML to Speech file', () => { title: 'Wang Yi at the UN; Fu Zhenghua sentenced; Nvidia China sales', options: TEST_OPTIONS, }) - expect(speechFile.utterances).to.have.lengthOf(21) + expect(speechFile.utterances).to.have.lengthOf(19) }) it('converts long utterances to multiple utterances', () => { From e5c215fb9d41bf27f0852914d22408c7747f605d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 5 Oct 2022 10:34:51 +0800 Subject: [PATCH 6/7] Add test for long sentence --- packages/text-to-speech/test/htmlToSsml.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index 6f291ca86..3d2350378 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -254,4 +254,21 @@ describe('convert HTML to Speech file', () => { }) expect(speechFile.utterances).to.have.lengthOf(3) }) + + it('does not break long sentences', () => { + const html = `
    +
    +
    + This meeting did not offer any significant economic boosts, among other things it reviewed reports of the inspection teams sent to several provinces to check on implementation of economic stabilization measures, promised more administrative reforms, and cut toll fees for freight trucks by 10% and government-designated cargo port charges by 20% in Q4. +
    +
    +
    +` + const speechFile = htmlToSpeechFile({ + content: html, + title: 'Test long sentence', + options: TEST_OPTIONS, + }) + expect(speechFile.utterances).to.have.lengthOf(2) + }) }) From fd3047a8abd167a528a3243094151e5105d5e4e9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 5 Oct 2022 11:27:28 +0800 Subject: [PATCH 7/7] Escape HTML entities when synthesizing because we are sending raw text now --- packages/text-to-speech/src/textToSpeech.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index c7c35fed3..2c72a4852 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -8,6 +8,7 @@ import { SpeechSynthesizer, } from 'microsoft-cognitiveservices-speech-sdk' import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml' +import * as _ from 'underscore' export interface TextToSpeechInput { text: string @@ -139,7 +140,8 @@ export const synthesizeTextToSpeech = async ( } // for ssml const startSsmlTag = startSsml(ssmlOptions) - const ssml = `${startSsmlTag}${input.text}${endSsml()}` + const text = _.escape(input.text) + const ssml = `${startSsmlTag}${text}${endSsml()}` // set the text offset to be the end of SSML start tag wordOffset -= startSsmlTag.length const result = await speakSsmlAsyncPromise(ssml)