prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Create a cloud function to steam speech mp3

Browse source
This commit is contained in:
Hongbo Wu 2022-08-31 12:01:52 +08:00
parent 7cba83d13e
commit 7baafc96bb
3 changed files with 225 additions and 326 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
packages/text-to-speech/data/test.ssml Normal file
View file

388
packages/text-to-speech/src/index.ts
View file

@ -4,49 +4,15 @@
/* eslint-disable @typescript-eslint/no-unused-vars */
import * as Sentry from '@sentry/serverless'
import { parseHTML } from 'linkedom'
import { File, Storage } from '@google-cloud/storage'
import {
CancellationDetails,
CancellationReason,
ResultReason,
SpeechConfig,
SpeechSynthesisOutputFormat,
SpeechSynthesisResult,
SpeechSynthesizer,
} from 'microsoft-cognitiveservices-speech-sdk'
import axios from 'axios'
import * as jwt from 'jsonwebtoken'
import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
import { htmlToSsml, ssmlItemText } from './htmlToSsml'
import { synthesizeTextToSpeech, TextToSpeechInput } from './textToSpeech'
import { File, Storage } from '@google-cloud/storage'
import { createWriteStream } from 'fs'
dotenv.config()
interface TextToSpeechInput {
id: string
text: string
voice?: string
languageCode?: string
textType?: 'text' | 'ssml'
rate?: number
volume?: number
complimentaryVoice?: string
bucket: string
}
interface TextToSpeechOutput {
audioFileName: string
speechMarksFileName: string
}
interface SpeechMark {
time: number
start?: number
length?: number
word: string
type: 'word' | 'bookmark'
}
const storage = new Storage()
const uploadToBucket = async (
@ -85,293 +51,10 @@ const updateSpeech = async (
return response.status === 200
}
const synthesizeTextToSpeech = async (
input: TextToSpeechInput
): Promise<TextToSpeechOutput> => {
if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) {
throw new Error('Azure Speech Key or Region not set')
}
const audioFileName = `speech/${input.id}.mp3`
const audioFile = createGCSFile(input.bucket, audioFileName)
const writeStream = audioFile.createWriteStream({
resumable: true,
})
const speechConfig = SpeechConfig.fromSubscription(
process.env.AZURE_SPEECH_KEY,
process.env.AZURE_SPEECH_REGION
)
const textType = input.textType || 'text'
if (textType === 'text') {
speechConfig.speechSynthesisLanguage = input.languageCode || 'en-US'
speechConfig.speechSynthesisVoiceName = input.voice || 'en-US-JennyNeural'
}
speechConfig.speechSynthesisOutputFormat =
SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3
// Create the speech synthesizer.
const synthesizer = new SpeechSynthesizer(speechConfig)
const speechMarks: SpeechMark[] = []
let timeOffset = 0
let characterOffset = 0
synthesizer.synthesizing = function (s, e) {
// convert arrayBuffer to stream and write to gcs file
writeStream.write(Buffer.from(e.result.audioData))
}
// The event synthesis completed signals that the synthesis is completed.
synthesizer.synthesisCompleted = (s, e) => {
console.info(
`(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${
e.result.audioData.byteLength
}`
)
}
// The synthesis started event signals that the synthesis is started.
synthesizer.synthesisStarted = (s, e) => {
console.info('(synthesis started)')
}
// The event signals that the service has stopped processing speech.
// This can happen when an error is encountered.
synthesizer.SynthesisCanceled = (s, e) => {
const cancellationDetails = CancellationDetails.fromResult(e.result)
let str =
'(cancel) Reason: ' + CancellationReason[cancellationDetails.reason]
if (cancellationDetails.reason === CancellationReason.Error) {
str += ': ' + e.result.errorDetails
}
console.info(str)
}
// The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds.
synthesizer.wordBoundary = (s, e) => {
speechMarks.push({
word: e.text,
time: (timeOffset + e.audioOffset) / 10000,
start: characterOffset + e.textOffset,
length: e.wordLength,
type: 'word',
})
}
synthesizer.bookmarkReached = (s, e) => {
console.debug(
`(Bookmark reached), Audio offset: ${
e.audioOffset / 10000
}ms, bookmark text: ${e.text}`
)
speechMarks.push({
word: e.text,
time: (timeOffset + e.audioOffset) / 10000,
type: 'bookmark',
})
}
const speakTextAsyncPromise = (
text: string
): Promise<SpeechSynthesisResult> => {
return new Promise((resolve, reject) => {
synthesizer.speakTextAsync(
text,
(result) => {
resolve(result)
},
(error) => {
reject(error)
}
)
})
}
const speakSsmlAsyncPromise = (
text: string
): Promise<SpeechSynthesisResult> => {
return new Promise((resolve, reject) => {
synthesizer.speakSsmlAsync(
text,
(result) => {
resolve(result)
},
(error) => {
reject(error)
}
)
})
}
if (textType === 'text') {
// slice the text into chunks of 5,000 characters
let currentTextChunk = ''
const textChunks = input.text.split('\n')
for (let i = 0; i < textChunks.length; i++) {
currentTextChunk += textChunks[i] + '\n'
if (currentTextChunk.length < 5000 && i < textChunks.length - 1) {
continue
}
console.debug(`synthesizing ${currentTextChunk}`)
const result = await speakTextAsyncPromise(currentTextChunk)
timeOffset = timeOffset + result.audioDuration
characterOffset = characterOffset + currentTextChunk.length
currentTextChunk = ''
}
} else {
const ssmlItems = htmlToSsml(input.text, {
primaryVoice: input.voice || 'en-US-JennyNeural',
secondaryVoice: 'en-US-GuyNeural',
language: input.languageCode || 'en-US',
rate: '1',
})
for (const ssmlItem of Array.from(ssmlItems)) {
const ssml = ssmlItemText(ssmlItem)
console.debug(`synthesizing ${ssml}`)
const result = await speakSsmlAsyncPromise(ssml)
if (result.reason === ResultReason.Canceled) {
writeStream.end()
synthesizer.close()
throw new Error(result.errorDetails)
}
timeOffset = timeOffset + result.audioDuration
// characterOffset = characterOffset + htmlElement.innerText.length
}
}
writeStream.end()
synthesizer.close()
console.debug(`audio file: ${audioFileName}`)
// upload Speech Marks file to GCS
const speechMarksFileName = `speech/${input.id}.json`
await uploadToBucket(
speechMarksFileName,
Buffer.from(JSON.stringify(speechMarks)),
input.bucket
)
return {
audioFileName,
speechMarksFileName,
}
}
const htmlElementToSsml = ({
htmlElement,
language = 'en-US',
voice = 'en-US-JennyNeural',
rate = 1,
volume = 100,
}: {
htmlElement: Element
language?: string
voice?: string
rate?: number
volume?: number
}): string => {
const replaceElement = (newElement: Element, oldElement: Element) => {
const id = oldElement.getAttribute('data-omnivore-anchor-idx')
if (id) {
const e = htmlElement.querySelector(`[data-omnivore-anchor-idx="${id}"]`)
e?.parentNode?.replaceChild(newElement, e)
}
}
const appendBookmarkElement = (parent: Element, element: Element) => {
const id = element.getAttribute('data-omnivore-anchor-idx')
if (id) {
const bookMark = ssml.createElement('bookmark')
bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`)
parent.appendChild(bookMark)
}
}
const replaceWithEmphasis = (element: Element, level: string) => {
const parent = ssml.createDocumentFragment() as unknown as Element
appendBookmarkElement(parent, element)
const emphasisElement = ssml.createElement('emphasis')
emphasisElement.setAttribute('level', level)
emphasisElement.innerHTML = element.innerHTML.trim()
parent.appendChild(emphasisElement)
replaceElement(parent, element)
}
const replaceWithSentence = (element: Element) => {
const parent = ssml.createDocumentFragment() as unknown as Element
appendBookmarkElement(parent, element)
const sentenceElement = ssml.createElement('s')
sentenceElement.innerHTML = element.innerHTML.trim()
parent.appendChild(sentenceElement)
replaceElement(parent, element)
}
// create new ssml document
const ssml = parseHTML('').document
const speakElement = ssml.createElement('speak')
speakElement.setAttribute('version', '1.0')
speakElement.setAttribute('xmlns', 'http://www.w3.org/2001/10/synthesis')
speakElement.setAttribute('xml:lang', language)
const voiceElement = ssml.createElement('voice')
voiceElement.setAttribute('name', voice)
speakElement.appendChild(voiceElement)
const prosodyElement = ssml.createElement('prosody')
prosodyElement.setAttribute('rate', `${rate}`)
prosodyElement.setAttribute('volume', volume.toString())
voiceElement.appendChild(prosodyElement)
// add each paragraph to the ssml document
appendBookmarkElement(prosodyElement, htmlElement)
// replace emphasis elements with ssml
htmlElement.querySelectorAll('*').forEach((e) => {
switch (e.tagName.toLowerCase()) {
case 's':
replaceWithEmphasis(e, 'moderate')
break
case 'sub':
if (e.getAttribute('alias') === null) {
replaceWithEmphasis(e, 'moderate')
}
break
case 'i':
case 'em':
case 'q':
case 'blockquote':
case 'cite':
case 'del':
case 'strike':
case 'sup':
case 'summary':
case 'caption':
case 'figcaption':
replaceWithEmphasis(e, 'moderate')
break
case 'b':
case 'strong':
case 'dt':
case 'dfn':
case 'u':
case 'mark':
case 'th':
case 'title':
case 'var':
replaceWithEmphasis(e, 'moderate')
break
case 'li':
replaceWithSentence(e)
break
default: {
const parent = ssml.createDocumentFragment() as unknown as Element
appendBookmarkElement(parent, e)
const text = (e as HTMLElement).innerText.trim()
const textElement = ssml.createTextNode(text)
parent.appendChild(textElement)
replaceElement(parent, e)
}
}
})
prosodyElement.appendChild(htmlElement)
return speakElement.outerHTML.replace(/&nbsp;|\n/g, '')
}
Sentry.GCPFunction.init({
dsn: process.env.SENTRY_DSN,
tracesSampleRate: 0,
})
export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction(
async (req, res) => {
@ -389,8 +72,22 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction(
}
const input = req.body as TextToSpeechInput
try {
const { audioFileName, speechMarksFileName } =
await synthesizeTextToSpeech(input)
const audioFileName = `speech/${input.id}.mp3`
const audioFile = createGCSFile(input.bucket, audioFileName)
const writeStream = audioFile.createWriteStream({
resumable: true,
})
const { speechMarks } = await synthesizeTextToSpeech({
...input,
writeStream,
})
// upload Speech Marks file to GCS
const speechMarksFileName = `speech/${input.id}.json`
await uploadToBucket(
speechMarksFileName,
Buffer.from(JSON.stringify(speechMarks)),
input.bucket
)
const updated = await updateSpeech(
input.id,
token,
@ -411,3 +108,42 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction(
res.send('OK')
}
)
export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
async (req, res) => {
console.debug('Text to speech steaming request', req)
const token = req.query.token as string
if (!process.env.JWT_SECRET) {
console.error('JWT_SECRET not exists')
return res.status(500).send('JWT_SECRET not exists')
}
try {
jwt.verify(token, process.env.JWT_SECRET)
} catch (e) {
console.error(e)
return res.status(200).send('UNAUTHENTICATED')
}
try {
const audioFileName = `./tmp/speech-${Date.now()}.mp3`
const writeStream = createWriteStream(audioFileName)
const input: TextToSpeechInput = {
id: req.query.id as string,
text: 'text',
bucket: req.query.bucket as string,
textType: 'ssml',
writeStream,
}
await synthesizeTextToSpeech(input)
res.set({
'Content-Type': 'audio/mpeg',
'Transfer-Encoding': 'chunked',
})
writeStream.pipe(res)
} catch (e) {
console.error(e)
return res.status(500).send('Failed to synthesize')
}
}
)

163
packages/text-to-speech/src/textToSpeech.ts Normal file
View file

@ -0,0 +1,163 @@
import {
CancellationDetails,
CancellationReason,
ResultReason,
SpeechConfig,
SpeechSynthesisOutputFormat,
SpeechSynthesisResult,
SpeechSynthesizer,
} from 'microsoft-cognitiveservices-speech-sdk'
import { htmlToSsml, ssmlItemText } from './htmlToSsml'
export interface TextToSpeechInput {
id: string
text: string
voice?: string
languageCode?: string
textType?: 'text' | 'ssml'
rate?: number
volume?: number
complimentaryVoice?: string
bucket: string
writeStream: NodeJS.WritableStream
}
export interface TextToSpeechOutput {
speechMarks: SpeechMark[]
}
export interface SpeechMark {
time: number
start?: number
length?: number
word: string
type: 'word' | 'bookmark'
}
export const synthesizeTextToSpeech = async (
input: TextToSpeechInput
): Promise<TextToSpeechOutput> => {
if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) {
throw new Error('Azure Speech Key or Region not set')
}
const writeStream = input.writeStream
const speechConfig = SpeechConfig.fromSubscription(
process.env.AZURE_SPEECH_KEY,
process.env.AZURE_SPEECH_REGION
)
const textType = input.textType || 'text'
speechConfig.speechSynthesisOutputFormat =
SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3
// Create the speech synthesizer.
const synthesizer = new SpeechSynthesizer(speechConfig)
const speechMarks: SpeechMark[] = []
let timeOffset = 0
const characterOffset = 0
synthesizer.synthesizing = function (s, e) {
// convert arrayBuffer to stream and write to gcs file
writeStream.write(Buffer.from(e.result.audioData))
}
// The event synthesis completed signals that the synthesis is completed.
synthesizer.synthesisCompleted = (s, e) => {
console.info(
`(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${
e.result.audioData.byteLength
}`
)
}
// The synthesis started event signals that the synthesis is started.
synthesizer.synthesisStarted = (s, e) => {
console.info('(synthesis started)')
}
// The event signals that the service has stopped processing speech.
// This can happen when an error is encountered.
synthesizer.SynthesisCanceled = (s, e) => {
const cancellationDetails = CancellationDetails.fromResult(e.result)
let str =
'(cancel) Reason: ' + CancellationReason[cancellationDetails.reason]
if (cancellationDetails.reason === CancellationReason.Error) {
str += ': ' + e.result.errorDetails
}
console.info(str)
}
// The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds.
synthesizer.wordBoundary = (s, e) => {
speechMarks.push({
word: e.text,
time: (timeOffset + e.audioOffset) / 10000,
start: characterOffset + e.textOffset,
length: e.wordLength,
type: 'word',
})
}
synthesizer.bookmarkReached = (s, e) => {
console.debug(
`(Bookmark reached), Audio offset: ${
e.audioOffset / 10000
}ms, bookmark text: ${e.text}`
)
speechMarks.push({
word: e.text,
time: (timeOffset + e.audioOffset) / 10000,
type: 'bookmark',
})
}
const speakSsmlAsyncPromise = (
text: string
): Promise<SpeechSynthesisResult> => {
return new Promise((resolve, reject) => {
synthesizer.speakSsmlAsync(
text,
(result) => {
resolve(result)
},
(error) => {
reject(error)
}
)
})
}
if (textType === 'text') {
const ssmlItems = htmlToSsml(input.text, {
primaryVoice: input.voice || 'en-US-JennyNeural',
secondaryVoice: 'en-US-GuyNeural',
language: input.languageCode || 'en-US',
rate: '1',
})
for (const ssmlItem of Array.from(ssmlItems)) {
const ssml = ssmlItemText(ssmlItem)
console.debug(`synthesizing ${ssml}`)
const result = await speakSsmlAsyncPromise(ssml)
if (result.reason === ResultReason.Canceled) {
writeStream.end()
synthesizer.close()
throw new Error(result.errorDetails)
}
timeOffset = timeOffset + result.audioDuration
// characterOffset = characterOffset + htmlElement.innerText.length
}
} else {
const result = await speakSsmlAsyncPromise(input.text)
if (result.reason === ResultReason.Canceled) {
writeStream.end()
synthesizer.close()
throw new Error(result.errorDetails)
}
}
writeStream.end()
synthesizer.close()
return {
speechMarks,
}
}