diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index ab40d3365..cbc740894 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -18,7 +18,6 @@ import { Claims } from '../resolvers/types' import { getRepository } from '../entity/utils' import { Speech } from '../entity/speech' import { getPageById } from '../elastic/pages' -import { parseHTML } from 'linkedom' import { synthesizeTextToSpeech } from '../utils/textToSpeech' import { UserPersonalization } from '../entity/user_personalization' @@ -117,18 +116,19 @@ export function articleRouter() { return res.status(404).send('Page not found') } - const text = parseHTML(page.content).document.documentElement.innerText - if (!text) { - return res.status(404).send('Page has no text') - } + // const text = parseHTML(page.content).document.documentElement.innerText + // if (!text) { + // return res.status(404).send('Page has no text') + // } try { const startTime = Date.now() const speechOutput = await synthesizeTextToSpeech({ id, - text, + text: page.content, languageCode: page.language, voice: userPersonalization.speechVoice, + textType: 'ssml', }) logger.info('Created speech', { audioUrl: speechOutput.audioUrl, diff --git a/packages/api/src/routers/svc/speech.ts b/packages/api/src/routers/svc/speech.ts index 4899cecda..4aea9b2dc 100644 --- a/packages/api/src/routers/svc/speech.ts +++ b/packages/api/src/routers/svc/speech.ts @@ -5,7 +5,6 @@ import { getRepository } from '../../entity/utils' import { getPageById } from '../../elastic/pages' import { synthesizeTextToSpeech } from '../../utils/textToSpeech' import { Speech } from '../../entity/speech' -import { parseHTML } from 'linkedom' import { UserPersonalization } from '../../entity/user_personalization' import { buildLogger } from '../../utils/logger' @@ -39,12 +38,10 @@ export function speechServiceRouter() { if (!page) { return res.status(200).send('Page not found') } - - const text = parseHTML(page.content).document.documentElement.innerText - if (!text) { - return res.status(200).send('Page has no text') - } - + // const text = parseHTML(page.content).document.documentElement.innerText + // if (!text) { + // return res.status(200).send('Page has no text') + // } logger.info(`Create article speech`, { body: { userId, @@ -59,9 +56,10 @@ export function speechServiceRouter() { const startTime = Date.now() const speechOutput = await synthesizeTextToSpeech({ id: pageId, - text, + text: page.content, languageCode: page.language, voice: userPersonalization.speechVoice, + textType: 'ssml', }) logger.info('Created speech', { audioUrl: speechOutput.audioUrl, diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index 5d7eeb795..30352b133 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -107,7 +107,7 @@ export const synthesizeTextToSpeech = async ( } synthesizer.bookmarkReached = (s, e) => { - logger.info( + logger.debug( `(Bookmark reached), Audio offset: ${ e.audioOffset / 10000 }ms, bookmark text: ${e.text}` @@ -170,20 +170,20 @@ export const synthesizeTextToSpeech = async ( } } else { const document = parseHTML(input.text).document - const elements = document.querySelectorAll('h1, h2, h3, p, li') + const elements = document.querySelectorAll('h1, h2, h3, p, ul, ol') // convert html elements to the ssml document for (const e of Array.from(elements)) { const htmlElement = e as HTMLElement if (htmlElement.innerText) { - const result = await speakSsmlAsyncPromise( - htmlElementToSsml( - htmlElement, - input.languageCode, - input.voice, - input.rate, - input.volume - ) + const ssml = htmlElementToSsml( + e, + input.languageCode, + input.voice, + input.rate, + input.volume ) + logger.debug(`synthesizing ${ssml}`) + const result = await speakSsmlAsyncPromise(ssml) timeOffset = timeOffset + result.audioDuration characterOffset = characterOffset + htmlElement.innerText.length } @@ -211,12 +211,32 @@ export const synthesizeTextToSpeech = async ( } export const htmlElementToSsml = ( - htmlElement: HTMLElement, + htmlElement: Element, language = 'en-US', voice = 'en-US-JennyNeural', rate = 1, volume = 100 ): string => { + const appendBookmarkElement = (parent: Element, element: Element) => { + const id = element.getAttribute('data-omnivore-anchor-idx') + if (id) { + const bookMark = ssml.createElement('bookmark') + bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`) + parent.appendChild(bookMark) + } + } + + const replaceEmphasisElement = (element: Element, level: string) => { + logger.debug(`replaceEmphasisElement: ${element.innerHTML}`) + const parent = ssml.createDocumentFragment() as unknown as Element + appendBookmarkElement(parent, element) + const emphasisElement = ssml.createElement('emphasis') + emphasisElement.setAttribute('level', level) + emphasisElement.innerHTML = element.innerHTML.trim() + parent.appendChild(emphasisElement) + element?.parentNode?.replaceChild(parent, element) + } + // create new ssml document const ssml = parseHTML('').document const speakElement = ssml.createElement('speak') @@ -231,14 +251,53 @@ export const htmlElementToSsml = ( prosodyElement.setAttribute('volume', volume.toString()) voiceElement.appendChild(prosodyElement) // add each paragraph to the ssml document - const id = htmlElement.getAttribute('data-omnivore-anchor-idx') - if (id) { - const text = htmlElement.innerText - const bookMark = ssml.createElement('bookmark') - bookMark.setAttribute('mark', `data-omnivore-anchor-idx-${id}`) - prosodyElement.appendChild(bookMark) - prosodyElement.appendChild(ssml.createTextNode(text)) - } + appendBookmarkElement(prosodyElement, htmlElement) + // add text to the ssml document + htmlElement.querySelectorAll('*').forEach((e) => { + switch (e.tagName.toLowerCase()) { + case 's': + replaceEmphasisElement(e, 'reduced') + break + case 'sub': + if (e.getAttribute('alias') === null) { + replaceEmphasisElement(e, 'reduced') + } + break + case 'i': + case 'em': + case 'q': + case 'blockquote': + case 'cite': + case 'del': + case 'strike': + case 'sup': + case 'summary': + case 'caption': + case 'figcaption': + replaceEmphasisElement(e, 'reduced') + break + case 'b': + case 'strong': + case 'dt': + case 'dfn': + case 'u': + case 'li': + case 'mark': + case 'th': + case 'title': + case 'var': + replaceEmphasisElement(e, 'moderate') + break + default: { + const text = (e as HTMLElement).innerText.trim() + if (text) { + const textElement = ssml.createTextNode(text) + e.parentNode?.replaceChild(textElement, e) + } + } + } + }) + prosodyElement.appendChild(htmlElement) return speakElement.outerHTML } diff --git a/packages/api/test/utils/data/text-to-speech.html b/packages/api/test/utils/data/text-to-speech.html new file mode 100644 index 000000000..65245fe69 --- /dev/null +++ b/packages/api/test/utils/data/text-to-speech.html @@ -0,0 +1 @@ +
An Instinct for Dragons is a book by University of Central Florida anthropologist, David E. Jones, in which he seeks to explain the universality of dragon images in the folklore of human societies. In the introduction, Jones conducts a survey of dragon myths from cultures around the world and argues that certain aspects of dragons or dragon-like mythical creatures are found very widely. He claims that even the Inuit have a reptilian dragon-like monster, even though (living in a frigid environment unsuited for cold-blooded animals) they had never seen an actual reptile.
Jones then argues against the common hypothesis that dragon myths might be motivated by primitive discoveries of dinosaur fossils (he argues that there are widespread traits of dragons in folklore which are not observable from fossils), and claims that the common traits of dragons seem to be an amalgam of the principal predators of our ancestral hominids, which he names as the raptors, great cats (especially leopards) and pythons.
The hypothesis to which Jones conforms is that over millions of years of evolution, members of a species will evolve an instinctive fear of their predators, and he proposes ways in which these fearful images may be merged in artistic or cultural expression to create the dragon image and, perhaps, other kinds of hybrid monster.
Finally he suggests sociological reasons for why such images may be perceived differently at different stages of a culture to try to explain why Chinese dragons are considered basically good and representative of government, but the great majority (although not all) European dragons are evil and often represent chaos.
Jones' theory was opposed in an article by Paul Jordan-Smith in the Spring 2002 issue of Western Folklore and by other authors. Jordan-Smith criticized the lack of evidence given to prove why dragon myths could not have been passed from culture to culture. He also notes that it cannot be demonstrated that the fears of ancestral hominids are coded into the human brain. He concludes his review by writing "One is tempted to say, as Dorothy Parker once did, that this is a book not to be tossed aside lightly but thrown violently. But no, it is not worth spending even that much energy on."[1]
D. Ogden writes that Jones' ideas "might offer pause for thought given the universality of dragon-slaying narratives". He adds, though, that the compound cat, snake, raptor creature imagined by Jones is mostly the Western stereotype based on mediaeval imagery, and that Jones has sought out similar images in a way that lacks rigor. In particular, Ogden notes that the dragons of Graeco-Roman myth do not fit with Jones's prototype, typically lacking one or more of the hybrid components (with the exception of Typhon, who, however, combines many more animals than Jones's three).[2]
Marry had a little lamb