prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Write speech marks to file stream

Browse source
This commit is contained in:
Hongbo Wu 2022-09-05 17:16:27 +08:00
parent b9eee9e4c5
commit 7fa092d95e
2 changed files with 37 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
packages/text-to-speech/src/index.ts
View file

@ -79,25 +79,24 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction(
try {
const audioFileName = `speech/${id}.mp3`
const audioFile = createGCSFile(bucket, audioFileName)
const writeStream = audioFile.createWriteStream({
const audioStream = audioFile.createWriteStream({
resumable: true,
}) as NodeJS.WriteStream
const speechMarksFileName = `speech/${id}.json`
const speechMarksFile = createGCSFile(bucket, speechMarksFileName)
const speechMarksStream = speechMarksFile.createWriteStream({
resumable: true,
}) as NodeJS.WriteStream
const startTime = Date.now()
const { speechMarks } = await synthesizeTextToSpeech({
await synthesizeTextToSpeech({
...input,
textType: 'html',
audioStream: writeStream,
audioStream,
speechMarksStream,
})
console.info(
`Synthesize text to speech completed in ${Date.now() - startTime} ms`
)
// upload Speech Marks file to GCS
const speechMarksFileName = `speech/${id}.json`
await uploadToBucket(
speechMarksFileName,
Buffer.from(JSON.stringify(speechMarks)),
bucket
)
const updated = await updateSpeech(
id,
token,
@ -146,11 +145,13 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
return res.status(200).send({ errorCode: 'INVALID_DATA' })
}
const audioStream = new PassThrough()
const speechMarksStream = new PassThrough()
const input: TextToSpeechInput = {
text: '',
textType: 'ssml',
audioStream,
ssmlItems,
speechMarksStream,
}
res.set({
'Content-Type': 'audio/mpeg',

41
packages/text-to-speech/src/textToSpeech.ts
View file

@ -21,6 +21,7 @@ export interface TextToSpeechInput {
bucket?: string
audioStream: NodeJS.ReadWriteStream
ssmlItems?: string[]
speechMarksStream: NodeJS.ReadWriteStream
}
export interface TextToSpeechOutput {
@ -42,7 +43,8 @@ export const synthesizeTextToSpeech = async (
throw new Error('Azure Speech Key or Region not set')
}
const textType = input.textType || 'html'
const writeStream = input.audioStream
const audioStream = input.audioStream
const speechMarksStream = input.speechMarksStream
const speechConfig = SpeechConfig.fromSubscription(
process.env.AZURE_SPEECH_KEY,
process.env.AZURE_SPEECH_REGION
@ -57,7 +59,7 @@ export const synthesizeTextToSpeech = async (
synthesizer.synthesizing = function (s, e) {
// convert arrayBuffer to stream and write to stream
writeStream.write(Buffer.from(e.result.audioData))
audioStream.write(Buffer.from(e.result.audioData))
}
// The event synthesis completed signals that the synthesis is completed.
@ -93,13 +95,17 @@ export const synthesizeTextToSpeech = async (
e.text
}`
)
speechMarks.push({
word: e.text,
time: (timeOffset + e.audioOffset) / 10000,
start: e.textOffset,
length: e.wordLength,
type: 'word',
})
speechMarksStream.write(
Buffer.from(
JSON.stringify({
word: e.text,
time: (timeOffset + e.audioOffset) / 10000,
start: e.textOffset,
length: e.wordLength,
type: 'word',
})
)
)
}
synthesizer.bookmarkReached = (s, e) => {
@ -108,11 +114,15 @@ export const synthesizeTextToSpeech = async (
e.audioOffset / 10000
}ms, bookmark text: ${e.text}`
)
speechMarks.push({
word: e.text,
time: (timeOffset + e.audioOffset) / 10000,
type: 'bookmark',
})
speechMarksStream.write(
Buffer.from(
JSON.stringify({
word: e.text,
time: (timeOffset + e.audioOffset) / 10000,
type: 'bookmark',
})
)
)
}
const speakSsmlAsyncPromise = (
@ -154,7 +164,8 @@ export const synthesizeTextToSpeech = async (
throw error
} finally {
console.debug('closing synthesizer')
writeStream.end()
audioStream.end()
speechMarksStream.end()
synthesizer.close()
console.debug('synthesizer closed')
}