From 744c01df490162e8a6aa9932cee2f48eb8f52d2a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 26 Sep 2022 13:06:09 +0800 Subject: [PATCH 1/8] Make each
  • an utterance to shorter the utterance length and add pause after the utterance ends --- packages/text-to-speech/src/htmlToSsml.ts | 7 +- .../text-to-speech/test/fixtures/large.html | 181 ++++++++++++++++++ .../text-to-speech/test/htmlToSsml.test.ts | 39 +++- 3 files changed, 214 insertions(+), 13 deletions(-) create mode 100644 packages/text-to-speech/test/fixtures/large.html diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index 944a5f1e5..cc0506c76 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -69,8 +69,7 @@ const TOP_LEVEL_TAGS = [ 'H4', 'H5', 'H6', - 'UL', - 'OL', + 'LI', 'CODE', ] @@ -312,7 +311,7 @@ export const htmlToSpeechFile = (htmlInput: HtmlInput): SpeechFile => { const dom = parseHTML(content) const body = dom.document.querySelector('#readability-page-1') if (!body) { - console.log('No HTML body found:', content) + console.log('No HTML body found') return { wordCount: 0, language, @@ -323,7 +322,7 @@ export const htmlToSpeechFile = (htmlInput: HtmlInput): SpeechFile => { const parsedNodes = parseDomTree(body) if (parsedNodes.length < 1) { - console.log('No HTML nodes found:', body) + console.log('No HTML nodes found') return { wordCount: 0, language, diff --git a/packages/text-to-speech/test/fixtures/large.html b/packages/text-to-speech/test/fixtures/large.html new file mode 100644 index 000000000..868f1fa49 --- /dev/null +++ b/packages/text-to-speech/test/fixtures/large.html @@ -0,0 +1,181 @@ +
    +
    +
    +
    +

    Summary of today’s Essential Eight:

    +
      +
    1. +

      + Wang Yi at the UN + - Among Wang YI’s meetings was one with Russian Foreign Minister + Lavrov. There was nothing in the readout from the Lavrov meeting + that would indicate a shift in the PRC position in the Russian + invasion of Ukraine. Wang will meet US Secretary of State + Blinken Friday. + +

      +
      2. +
    2. +

      + Two more sentences in “Sun Lijun clique” case + - Authorities are wrapping up the Sun Lijun "clique" case before + the 20th. Today both Fu Zhenghua and Wang Like were sentenced + death with a two year reprieve, and both releases said they had + no possibility of parole or reduction in sentence. Sun has yet + to be sentenced but it feels like it will happen imminently. + Given his leadership role he should at least get life in jail, + if not the actual death penalty, though he was promoted for + years by people above him in the system so perhaps he performed + “meritorious service” and ratted out other senior officials. + +

      +
      3. +
    3. +

      + Weekly State Council Executive Meeting + - This meeting did not offer any significant economic boosts, + among other things it reviewed reports of the inspection teams + sent to several provinces to check on implementation of economic + stabilization measures, promised more administrative reforms, + and cut toll fees for freight trucks by 10% and + government-designated cargo port charges by 20% in Q4. + +

      +
      4. +
    4. +

      + Why this economic downturn may be different + - Two good pieces, one from Logan Wright and another from “China Charts”. The real estate boom is over and it is not coming back any + time soon, if ever. That is the outcome the policymakers have + been targeting for years, though they may have been + overconfident in their ability to rein in real estate without + creating dangerous domino effects throughout the economy. We are + all waiting for the 20th Party Congress outcomes, but I see no + reason to think there will be outcomes from that meeting that + reverse the trajectory of the real estate sector. + +

      +
      5. +
    5. +

      + PCAOB Audit inspections in Hong Kong + - The trial audits of PRC firms are underway, so far the signs + are positive that the PRC side understands the concessions + needed to keep the PRC firms listed in the US, but as the PCAOB + chair said today “The Holding Foreign Companies Accountable Act + demands complete access. The agreement we signed with our + Chinese counterparts guarantees complete access. And the PCAOB + will accept nothing less than complete access when we make our + determinations by the end of this year. When I say no loopholes + and no exceptions, I mean none.” Having a law that allows little + room for concessions has been very helpful to US negotiators. + +

      +
      6. +
    6. +

      + Nvidia CEO does not sound too worried about China sales + - The CEO told Caixin that ““There will be versions that are + going to be not restricted and serve the needs of the vast + majority of our market very comfortably” and he told Stratechery + that “The limitations and the restrictions are very specific to + a combination of computation level and multi-chip + interconnection level. That restriction gives us plenty of + envelope to go and run our business and for the vast majority of + our customers in China”. +

      +
      7. +
    7. +

      + US-PRC scientific relations + - There are two new reports of note, one on scientists who + worked at Los Alamos labs and then returned to the PRC and + contributed to PRC weapons development, and another on the + outflow of Chinese scientists from the US. + +

      +
      8. +
    8. +

      + Another scandal in the film and TV sector +

      +
      9. +
    +

    Thanks for reading.

    +
    +
    +

    This post is for paid subscribers

    +
    +
    +
    +
    diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index bfc86c55c..6159bb047 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -1,6 +1,19 @@ import 'mocha' import { expect } from 'chai' -import { htmlToSsmlItems, stripEmojis } from '../src/htmlToSsml' +import { + htmlToSpeechFile, + htmlToSsmlItems, + stripEmojis, +} from '../src/htmlToSsml' +import * as fs from 'fs' +import path from 'path' + +const TEST_OPTIONS = { + primaryVoice: 'test-primary', + secondaryVoice: 'test-secondary', + language: 'en-US', + rate: '1.0', +} describe('stripEmojis', () => { it('strips emojis from text and removes the extra space', () => { @@ -20,14 +33,7 @@ describe('stripEmojis', () => { }) }) -describe('htmlToSsmlItems', () => { - const TEST_OPTIONS = { - primaryVoice: 'test-primary', - secondaryVoice: 'test-secondary', - language: 'en-US', - rate: '1.0', - } - +describe('htmlToSpeechFile', () => { describe('a simple html file', () => { xit('should convert Html to SSML', () => { const ssml = htmlToSsmlItems( @@ -217,3 +223,18 @@ describe('htmlToSsmlItems', () => { // }) // }) }) + +describe('convert HTML to Speech file', () => { + it('should convert HTML to many utterances', () => { + const html = fs.readFileSync( + path.resolve(__dirname, './fixtures/large.html'), + { encoding: 'utf-8' } + ) + const speechFile = htmlToSpeechFile({ + content: html, + title: 'test', + options: TEST_OPTIONS, + }) + expect(speechFile.utterances).to.have.lengthOf(12) + }) +}) From c42996721ae75fa386966b934c12bd467bf1e8e6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 26 Sep 2022 15:46:03 +0800 Subject: [PATCH 2/8] Start at index 3, because the frontend starts two nodes above the backend --- packages/text-to-speech/src/htmlToSsml.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index cc0506c76..f7ec2cef0 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -105,9 +105,9 @@ function parseDomTree(pageNode: Element) { visitedNodeList.shift() visitedNodeList.forEach((node, index) => { - // We start at index 2, because the frontend starts one node above us - // on the #readability-content element that wraps the entire content. - node.setAttribute('data-omnivore-anchor-idx', (index + 2).toString()) + // We start at index 3, because the frontend starts two nodes above us + // on the #readability-page-1 element that wraps the entire content. + node.setAttribute('data-omnivore-anchor-idx', (index + 3).toString()) }) return visitedNodeList } From fbaa56c96f4302139d500891ba05435d497ede05 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 26 Sep 2022 16:33:29 +0800 Subject: [PATCH 3/8] Split big ssml into small chunks less than 200 chars --- packages/text-to-speech/src/htmlToSsml.ts | 1 + packages/text-to-speech/src/index.ts | 1 - packages/text-to-speech/src/textToSpeech.ts | 25 ++++++++++++++----- .../text-to-speech/test/htmlToSsml.test.ts | 2 +- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index f7ec2cef0..2a1417d34 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -347,6 +347,7 @@ export const htmlToSpeechFile = (htmlInput: HtmlInput): SpeechFile => { wordOffset += titleUtterance.wordCount } + // start at 2 to skip the #readability-content and #readability-page-1 elements for (let i = 2; i < parsedNodes.length + 2; i++) { const textItems: string[] = [] const node = parsedNodes[i - 2] diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 1bb8ff43e..d38adbf8b 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -205,7 +205,6 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( const input: TextToSpeechInput = { ...utteranceInput, textType: 'ssml', - text: ssml, } const { audioData, speechMarks } = await synthesizeTextToSpeech(input) if (!audioData) { diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 042e53454..541202871 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -7,7 +7,7 @@ import { SpeechSynthesisResult, SpeechSynthesizer, } from 'microsoft-cognitiveservices-speech-sdk' -import { htmlToSsmlItems, ssmlItemText } from './htmlToSsml' +import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml' export interface TextToSpeechInput { text: string @@ -51,7 +51,7 @@ export const synthesizeTextToSpeech = async ( const synthesizer = new SpeechSynthesizer(speechConfig) const speechMarks: SpeechMark[] = [] let timeOffset = 0 - const wordOffset = 0 + let wordOffset = 0 synthesizer.synthesizing = function (s, e) { // convert arrayBuffer to stream and write to stream @@ -137,12 +137,25 @@ export const synthesizeTextToSpeech = async ( speechMarks, } } - const result = await speakSsmlAsyncPromise(input.text) - if (result.reason === ResultReason.Canceled) { - throw new Error(result.errorDetails) + // for ssml + const audioData: Buffer = Buffer.from([]) + // split ssml into chunks of 200 characters to stream faster + const ssmlChunks = input.text.match(/.{1,200}/g) + if (ssmlChunks) { + for (const ssmlChunk of ssmlChunks) { + const ssml = `${startSsml(ssmlOptions)}${ssmlChunk}${endSsml()}` + const result = await speakSsmlAsyncPromise(ssml) + if (result.reason === ResultReason.Canceled) { + throw new Error(result.errorDetails) + } + timeOffset = timeOffset + result.audioDuration + wordOffset = wordOffset + ssmlChunk.length + Buffer.concat([audioData, Buffer.from(result.audioData)]) + } } + return { - audioData: Buffer.from(result.audioData), + audioData, speechMarks, } } catch (error) { diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index 6159bb047..cb06c2eb3 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -225,7 +225,7 @@ describe('htmlToSpeechFile', () => { }) describe('convert HTML to Speech file', () => { - it('should convert HTML to many utterances', () => { + it('converts each
  • to an utterance', () => { const html = fs.readFileSync( path.resolve(__dirname, './fixtures/large.html'), { encoding: 'utf-8' } From 8c6a18aed10e13fb237f396e7d5441c8b445c9b1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 26 Sep 2022 16:39:43 +0800 Subject: [PATCH 4/8] Add todo --- packages/text-to-speech/src/textToSpeech.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 541202871..b60fdfeda 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -141,6 +141,7 @@ export const synthesizeTextToSpeech = async ( const audioData: Buffer = Buffer.from([]) // split ssml into chunks of 200 characters to stream faster const ssmlChunks = input.text.match(/.{1,200}/g) + // TODO: not split on words and bookmarks if (ssmlChunks) { for (const ssmlChunk of ssmlChunks) { const ssml = `${startSsml(ssmlOptions)}${ssmlChunk}${endSsml()}` From e1123cc422bfd1785f72ec7dedbb8149d392b192 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 26 Sep 2022 16:56:12 +0800 Subject: [PATCH 5/8] Fix test --- packages/text-to-speech/test/htmlToSsml.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/text-to-speech/test/htmlToSsml.test.ts b/packages/text-to-speech/test/htmlToSsml.test.ts index cb06c2eb3..7adcc0fbe 100644 --- a/packages/text-to-speech/test/htmlToSsml.test.ts +++ b/packages/text-to-speech/test/htmlToSsml.test.ts @@ -232,9 +232,9 @@ describe('convert HTML to Speech file', () => { ) const speechFile = htmlToSpeechFile({ content: html, - title: 'test', + title: 'Wang Yi at the UN; Fu Zhenghua sentenced; Nvidia China sales', options: TEST_OPTIONS, }) - expect(speechFile.utterances).to.have.lengthOf(12) + expect(speechFile.utterances).to.have.lengthOf(11) }) }) From 9545d1d532255efce35b173778871ac6c6f02ad5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 26 Sep 2022 17:18:46 +0800 Subject: [PATCH 6/8] Replace

    with in ssml --- packages/text-to-speech/src/htmlToSsml.ts | 7 +------ packages/text-to-speech/src/textToSpeech.ts | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index 2a1417d34..ef55b09b1 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -55,8 +55,7 @@ const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ function ssmlTagsForTopLevelElement() { return { - opening: `

    `, - closing: `

    `, + opening: ``, } } @@ -180,10 +179,6 @@ function emitElement( } } - if (isTopLevel) { - emit(textItems, topLevelTags.closing) - } - return Number(maxVisitedIdx) } diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index b60fdfeda..5b35aae48 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -138,7 +138,7 @@ export const synthesizeTextToSpeech = async ( } } // for ssml - const audioData: Buffer = Buffer.from([]) + let audioData: Buffer = Buffer.from([]) // split ssml into chunks of 200 characters to stream faster const ssmlChunks = input.text.match(/.{1,200}/g) // TODO: not split on words and bookmarks @@ -151,7 +151,7 @@ export const synthesizeTextToSpeech = async ( } timeOffset = timeOffset + result.audioDuration wordOffset = wordOffset + ssmlChunk.length - Buffer.concat([audioData, Buffer.from(result.audioData)]) + audioData = Buffer.concat([audioData, Buffer.from(result.audioData)]) } } From 35f7b0b90b8f304d7ae550ec16bdb063303c7e5b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 26 Sep 2022 17:47:40 +0800 Subject: [PATCH 7/8] Do not breaking on words and bookmarks --- packages/text-to-speech/src/textToSpeech.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 5b35aae48..1da109085 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -139,11 +139,12 @@ export const synthesizeTextToSpeech = async ( } // for ssml let audioData: Buffer = Buffer.from([]) - // split ssml into chunks of 200 characters to stream faster - const ssmlChunks = input.text.match(/.{1,200}/g) - // TODO: not split on words and bookmarks + // split ssml into chunks of 2000 characters to stream faster + // both within limit & without breaking on words and bookmarks + const ssmlChunks = input.text.match(/.{1,2000}(?= |$)(?! mark=)/g) if (ssmlChunks) { for (const ssmlChunk of ssmlChunks) { + console.log('ssmlChunk', ssmlChunk) const ssml = `${startSsml(ssmlOptions)}${ssmlChunk}${endSsml()}` const result = await speakSsmlAsyncPromise(ssml) if (result.reason === ResultReason.Canceled) { From 629f898a36d8961a5872903fa85dc300a6d8f7fa Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 26 Sep 2022 18:01:03 +0800 Subject: [PATCH 8/8] Remove debugging logs --- packages/text-to-speech/src/textToSpeech.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index 1da109085..aa5f39713 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -140,12 +140,14 @@ export const synthesizeTextToSpeech = async ( // for ssml let audioData: Buffer = Buffer.from([]) // split ssml into chunks of 2000 characters to stream faster - // both within limit & without breaking on words and bookmarks + // both within limit & without breaking on words and bookmarks const ssmlChunks = input.text.match(/.{1,2000}(?= |$)(?! mark=)/g) if (ssmlChunks) { for (const ssmlChunk of ssmlChunks) { - console.log('ssmlChunk', ssmlChunk) - const ssml = `${startSsml(ssmlOptions)}${ssmlChunk}${endSsml()}` + const startSsmlChunk = startSsml(ssmlOptions) + const ssml = `${startSsmlChunk}${ssmlChunk}${endSsml()}` + // set the text offset to be the end of SSML start tag + wordOffset -= startSsmlChunk.length const result = await speakSsmlAsyncPromise(ssml) if (result.reason === ResultReason.Canceled) { throw new Error(result.errorDetails)