Merge pull request #1279 from omnivore-app/use-nlp-to-detect-sentence-boundary

Use NLP lib to better detect sentence boundary and split utterances

packages/text-to-speech/src/htmlToSsml.ts

@@ -1,6 +1,6 @@
 import { parseHTML } from 'linkedom'
 import * as _ from 'underscore'
-import { WordPunctTokenizer } from 'natural'
+import { SentenceTokenizer, WordPunctTokenizer } from 'natural'
 import { htmlToText } from 'html-to-text'
 
 // this code needs to be kept in sync with the
@@ -69,7 +69,6 @@ const TOP_LEVEL_TAGS = [
   'H5',
   'H6',
   'LI',
-  'CODE',
 ]
 
 function parseDomTree(pageNode: Element) {
@@ -148,7 +147,15 @@ function emitElement(
   element: Element,
   isTopLevel: boolean
 ) {
-  const SKIP_TAGS = ['SCRIPT', 'STYLE', 'IMG', 'FIGURE', 'FIGCAPTION', 'IFRAME']
+  const SKIP_TAGS = [
+    'SCRIPT',
+    'STYLE',
+    'IMG',
+    'FIGURE',
+    'FIGCAPTION',
+    'IFRAME',
+    'CODE',
+  ]
 
   const topLevelTags = ssmlTagsForTopLevelElement()
   const idx = element.getAttribute('data-omnivore-anchor-idx')
@@ -297,38 +304,56 @@ const textToUtterances = ({
     text = parseHTML(text).document.documentElement.textContent ?? text
     console.info('Converted HTML to text:', text)
   }
-  // if we hit 256, look back for first ending sentence within 80 chars
+
   const MAX_CHARS = 256
-  const MAX_LOOKBACK = 80
-  while (text.length > MAX_CHARS) {
-    let end = MAX_CHARS - MAX_LOOKBACK - 1
-    while (end < text.length && !text[end].match(/[.!?]/)) {
-      end++
+  const sentenceTokenizer = new SentenceTokenizer()
+  const sentences = sentenceTokenizer.tokenize(text)
+  let currentText = ''
+  // split text to max 256 chars per utterance and
+  // use nlp lib to detect sentences and
+  // avoid splitting words and sentences
+  sentences.forEach((sentence, i) => {
+    if (i < sentences.length - 1) {
+      // add space to the end of sentence
+      sentence += ' '
     }
-
-    const utterance = text.substring(0, end + 1)
-    const wordCount = tokenizer.tokenize(utterance).length
-    utterances.push({
-      idx,
-      text: utterance,
-      wordOffset,
-      wordCount,
-      voice,
-    })
-    text = text.substring(end + 1)
-    wordOffset += wordCount
-  }
-
-  if (text.length > 0) {
-    const wordCount = tokenizer.tokenize(text).length
-    utterances.push({
-      idx,
-      text,
-      wordOffset,
-      wordCount,
-      voice,
-    })
-  }
+    const nextText = currentText + sentence
+    if (nextText.length > MAX_CHARS) {
+      if (currentText.length > 0) {
+        const wordCount = tokenizer.tokenize(currentText).length
+        utterances.push({
+          idx,
+          text: currentText,
+          wordOffset,
+          wordCount,
+          voice,
+        })
+        wordOffset += wordCount
+        currentText = sentence
+      } else {
+        const wordCount = tokenizer.tokenize(sentence).length
+        utterances.push({
+          idx,
+          text: sentence,
+          wordOffset,
+          wordCount,
+          voice,
+        })
+        wordOffset += wordCount
+      }
+    } else {
+      currentText = nextText
+    }
+    if (i === sentences.length - 1 && currentText.length > 0) {
+      utterances.push({
+        idx,
+        text: currentText,
+        wordOffset,
+        wordCount: tokenizer.tokenize(currentText).length,
+        voice,
+      })
+    }
+  })
 
   return utterances
 }

packages/text-to-speech/src/textToSpeech.ts

@@ -1,10 +1,8 @@
 import {
   CancellationDetails,
   CancellationReason,
-  PropertyId,
   ResultReason,
   SpeechConfig,
-  SpeechSynthesisBoundaryType,
   SpeechSynthesisOutputFormat,
   SpeechSynthesisResult,
   SpeechSynthesizer,
@@ -32,7 +30,7 @@ export interface SpeechMark {
   start?: number
   length?: number
   word: string
-  type: 'word' | 'bookmark' | 'punctuation' | 'sentence'
+  type: 'word' | 'bookmark'
 }
 
 export const synthesizeTextToSpeech = async (
@@ -49,17 +47,12 @@ export const synthesizeTextToSpeech = async (
   )
   speechConfig.speechSynthesisOutputFormat =
     SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3
-  // Required for sentence-level WordBoundary events
-  speechConfig.setProperty(
-    PropertyId.SpeechServiceResponse_RequestSentenceBoundary,
-    'true'
-  )
 
   // Create the speech synthesizer.
   const synthesizer = new SpeechSynthesizer(speechConfig)
   const speechMarks: SpeechMark[] = []
   let timeOffset = 0
-  // let wordOffset = 0
+  let wordOffset = 0
 
   synthesizer.synthesizing = function (s, e) {
     // convert arrayBuffer to stream and write to stream
@@ -94,14 +87,13 @@ export const synthesizeTextToSpeech = async (
 
   // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds.
   synthesizer.wordBoundary = (s, e) => {
-    e.boundaryType === SpeechSynthesisBoundaryType.Sentence &&
-      speechMarks.push({
-        word: e.text,
-        time: (timeOffset + e.audioOffset) / 10000,
-        start: e.textOffset,
-        length: e.text.length,
-        type: 'sentence',
-      })
+    speechMarks.push({
+      word: e.text,
+      time: (timeOffset + e.audioOffset) / 10000,
+      start: wordOffset + e.textOffset,
+      length: e.wordLength,
+      type: 'word',
+    })
   }
 
   synthesizer.bookmarkReached = (s, e) => {
@@ -148,10 +140,9 @@ export const synthesizeTextToSpeech = async (
     }
     // for ssml
     const startSsmlTag = startSsml(ssmlOptions)
+    wordOffset -= startSsmlTag.length
     const text = _.escape(input.text)
     const ssml = `${startSsmlTag}${text}${endSsml()}`
-    // set the text offset to be the end of SSML start tag
-    // wordOffset -= startSsmlTag.length
     const result = await speakSsmlAsyncPromise(ssml)
     if (result.reason === ResultReason.Canceled) {
       throw new Error(result.errorDetails)

packages/text-to-speech/test/htmlToSsml.test.ts

@@ -235,7 +235,7 @@ describe('convert HTML to Speech file', () => {
       title: 'Wang Yi at the UN; Fu Zhenghua sentenced; Nvidia China sales',
       options: TEST_OPTIONS,
     })
-    expect(speechFile.utterances).to.have.lengthOf(19)
+    expect(speechFile.utterances).to.have.lengthOf(20)
   })
 
   it('converts long utterances to multiple utterances', () => {
@@ -271,4 +271,23 @@ describe('convert HTML to Speech file', () => {
     })
     expect(speechFile.utterances).to.have.lengthOf(2)
   })
+
+  it('does not break on not decimal point in sentences', () => {
+    const html = `<div id="readability-content">
+  <div class="page" id="readability-page-1">
+    <div data-omnivore-anchor-idx="1">
+      If terms of the original $12.5 billion financing package remain the same, bankers may struggle to sell the risky Twitter buyout debt just as credit markets begin to crack, with yields at multiyear highs, they’re potentially on the hook for hundreds of millions of dollars of losses on the unsecured portion alone should they try to unload it to investors.
+    </div>
+  </div>
+</div>
+`
+    const speechFile = htmlToSpeechFile({
+      content: html,
+      title: 'Test long sentence with decimal point',
+      options: TEST_OPTIONS,
+    })
+    expect(speechFile.utterances[1].text).to.eql(
+      'If terms of the original $12.5 billion financing package remain the same, bankers may struggle to sell the risky Twitter buyout debt just as credit markets begin to crack, with yields at multiyear highs, they’re potentially on the hook for hundreds of millions of dollars of losses on the unsecured portion alone should they try to unload it to investors.'
+    )
+  })
 })