From 5e79529e11788d6b34bd8dd53dae3a03a9ae4fe6 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 4 Oct 2022 17:40:21 +0800
Subject: [PATCH] Add sentence-level speech marks

---
 packages/text-to-speech/src/textToSpeech.ts | 24 ++++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts
index 2c72a4852..4426c6f19 100644
--- a/packages/text-to-speech/src/textToSpeech.ts
+++ b/packages/text-to-speech/src/textToSpeech.ts
@@ -1,8 +1,10 @@
 import {
   CancellationDetails,
   CancellationReason,
+  PropertyId,
   ResultReason,
   SpeechConfig,
+  SpeechSynthesisBoundaryType,
   SpeechSynthesisOutputFormat,
   SpeechSynthesisResult,
   SpeechSynthesizer,
@@ -30,7 +32,7 @@ export interface SpeechMark {
   start?: number
   length?: number
   word: string
-  type: 'word' | 'bookmark'
+  type: 'word' | 'bookmark' | 'punctuation' | 'sentence'
 }
 
 export const synthesizeTextToSpeech = async (
@@ -47,6 +49,11 @@ export const synthesizeTextToSpeech = async (
   )
   speechConfig.speechSynthesisOutputFormat =
     SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3
+  // Required for sentence-level WordBoundary events
+  speechConfig.setProperty(
+    PropertyId.SpeechServiceResponse_RequestSentenceBoundary,
+    'true'
+  )
 
   // Create the speech synthesizer.
   const synthesizer = new SpeechSynthesizer(speechConfig)
@@ -87,13 +94,14 @@ export const synthesizeTextToSpeech = async (
 
   // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds.
   synthesizer.wordBoundary = (s, e) => {
-    speechMarks.push({
-      word: e.text,
-      time: (timeOffset + e.audioOffset) / 10000,
-      start: wordOffset + e.textOffset,
-      length: e.wordLength,
-      type: 'word',
-    })
+    e.boundaryType === SpeechSynthesisBoundaryType.Sentence &&
+      speechMarks.push({
+        word: e.text,
+        time: (timeOffset + e.audioOffset) / 10000,
+        start: wordOffset + e.textOffset,
+        length: e.wordLength,
+        type: 'sentence',
+      })
   }
 
   synthesizer.bookmarkReached = (s, e) => {