Merge pull request #1276 from omnivore-app/improve-utterance-break

improve utterance break

packages/text-to-speech/src/htmlToSsml.ts

@@ -273,13 +273,12 @@ const textToUtterances = ({
   let text = textItems.join('')
   if (!isHtml) {
     // for title
-    const wordCount = tokenizer.tokenize(text).length
     return [
       {
         idx,
         text,
         wordOffset,
-        wordCount,
+        wordCount: tokenizer.tokenize(text).length,
         voice,
       },
     ]
@@ -298,21 +297,39 @@ const textToUtterances = ({
     text = parseHTML(text).document.documentElement.textContent ?? text
     console.info('Converted HTML to text:', text)
   }
-  // split text into chunks of 256 characters to stream faster without breaking on words
-  const textChunks = text.match(/.{1,256}(?= |$)/g)
-  if (textChunks) {
-    for (const chunk of textChunks) {
-      const wordCount = tokenizer.tokenize(chunk).length
-      utterances.push({
-        idx,
-        text: chunk,
-        wordOffset,
-        wordCount,
-        voice,
-      })
-      wordOffset += wordCount
+  // if we hit 256, look back for first ending sentence within 80 chars
+  const MAX_CHARS = 256
+  const MAX_LOOKBACK = 80
+  while (text.length > MAX_CHARS) {
+    let end = MAX_CHARS - MAX_LOOKBACK - 1
+    while (end < text.length && !text[end].match(/[.!?]/)) {
+      end++
     }
+
+    const utterance = text.substring(0, end + 1)
+    const wordCount = tokenizer.tokenize(utterance).length
+    utterances.push({
+      idx,
+      text: utterance,
+      wordOffset,
+      wordCount,
+      voice,
+    })
+    text = text.substring(end + 1)
+    wordOffset += wordCount
   }
+
+  if (text.length > 0) {
+    const wordCount = tokenizer.tokenize(text).length
+    utterances.push({
+      idx,
+      text,
+      wordOffset,
+      wordCount,
+      voice,
+    })
+  }
+
   return utterances
 }
 

packages/text-to-speech/src/textToSpeech.ts

@@ -8,6 +8,7 @@ import {
   SpeechSynthesizer,
 } from 'microsoft-cognitiveservices-speech-sdk'
 import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml'
+import * as _ from 'underscore'
 
 export interface TextToSpeechInput {
   text: string
@@ -139,7 +140,8 @@ export const synthesizeTextToSpeech = async (
     }
     // for ssml
     const startSsmlTag = startSsml(ssmlOptions)
-    const ssml = `${startSsmlTag}${input.text}${endSsml()}`
+    const text = _.escape(input.text)
+    const ssml = `${startSsmlTag}${text}${endSsml()}`
     // set the text offset to be the end of SSML start tag
     wordOffset -= startSsmlTag.length
     const result = await speakSsmlAsyncPromise(ssml)

packages/text-to-speech/test/htmlToSsml.test.ts

@@ -227,7 +227,7 @@ describe('htmlToSpeechFile', () => {
 describe('convert HTML to Speech file', () => {
   it('converts each <li> to an utterance', () => {
     const html = fs.readFileSync(
-      path.resolve(__dirname, './fixtures/large.html'),
+      path.resolve(__dirname, './fixtures/li.html'),
       { encoding: 'utf-8' }
     )
     const speechFile = htmlToSpeechFile({
@@ -235,6 +235,40 @@ describe('convert HTML to Speech file', () => {
       title: 'Wang Yi at the UN; Fu Zhenghua sentenced; Nvidia China sales',
       options: TEST_OPTIONS,
     })
-    expect(speechFile.utterances).to.have.lengthOf(12)
+    expect(speechFile.utterances).to.have.lengthOf(19)
+  })
+
+  it('converts long utterances to multiple utterances', () => {
+    const html = `<div id="readability-content">
+  <div class="page" id="readability-page-1">
+    <div data-omnivore-anchor-idx="1">
+      All neural voices are multilingual and fluent in their own language and English. For example, if the input text in English is "I'm excited to try text to speech" and you set es-ES-ElviraNeural, the text is spoken in English with a Spanish accent. If the voice doesn't speak the language of the input text, the Speech service won't output synthesized audio. See the full list of supported neural voices.
+    </div>
+  </div>
+</div>
+`
+    const speechFile = htmlToSpeechFile({
+      content: html,
+      title: 'How to synthesize speech from text',
+      options: TEST_OPTIONS,
+    })
+    expect(speechFile.utterances).to.have.lengthOf(3)
+  })
+
+  it('does not break long sentences', () => {
+    const html = `<div id="readability-content">
+  <div class="page" id="readability-page-1">
+    <div data-omnivore-anchor-idx="1">
+      This meeting did not offer any significant economic boosts, among other things it reviewed reports of the inspection teams sent to several provinces to check on implementation of economic stabilization measures, promised more administrative reforms, and cut toll fees for freight trucks by 10% and government-designated cargo port charges by 20% in Q4.
+    </div>
+  </div>
+</div>
+`
+    const speechFile = htmlToSpeechFile({
+      content: html,
+      title: 'Test long sentence',
+      options: TEST_OPTIONS,
+    })
+    expect(speechFile.utterances).to.have.lengthOf(2)
   })
 })