prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Strip Emojis from html text content

Browse source
This commit is contained in:
Hongbo Wu 2022-09-09 18:33:52 +08:00
parent f83435eb07
commit d04ad9d272
2 changed files with 25 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
packages/text-to-speech/src/htmlToSsml.ts
View file

@ -117,7 +117,7 @@ function emit(textItems: string[], text: string) {
}
function cleanTextNode(textNode: ChildNode): string {
return _.escape(textNode.textContent ?? ''.replace(/\s+/g, ' '))
return stripEmojis(_.escape(textNode.textContent ?? ''.replace(/\s+/g, ' ')))
}
function emitTextNode(
@ -252,9 +252,10 @@ export const htmlToSsmlItems = (
return items
}
const stripEmojis = (text: string): string => {
const emojiRegex = /[\u{1F600}-\u{1F64F}]/gu
return text.replace(emojiRegex, '').replace(/\s+/g, ' ').trim()
export const stripEmojis = (text: string): string => {
const emojiRegex =
/(?![*#0-9]+)[\p{Emoji}\p{Emoji_Modifier}\p{Emoji_Component}\p{Emoji_Modifier_Base}\p{Emoji_Presentation}]/gu
return text.replace(emojiRegex, '').replace(/\s+/g, ' ')
}
const textToUtterance = ({
@ -272,7 +273,7 @@ const textToUtterance = ({
voice?: string
isHtml?: boolean
}): Utterance => {
const text = stripEmojis(textItems.join(''))
const text = textItems.join('')
const plainText = isHtml ? htmlToText(text, { wordwrap: false }) : text
const wordCount = tokenizer.tokenize(plainText).length
return {

20
packages/text-to-speech/test/htmlToSsml.test.ts
View file

@ -1,6 +1,24 @@
import 'mocha'
import { expect } from 'chai'
import { htmlToSsmlItems } from '../src/htmlToSsml'
import { htmlToSsmlItems, stripEmojis } from '../src/htmlToSsml'
describe('stripEmojis', () => {
it('strips emojis from text and removes the extra space', () => {
const text = '🥛The Big Short guy is back with a new prediction'
expect(stripEmojis(text)).to.equal(
'The Big Short guy is back with a new prediction'
)
})
it('strips emojis from html and removes the extra space', () => {
const text = `<h2 data-omnivore-anchor-idx="37">🧠Brain food</h2>`
expect(stripEmojis(text)).to.equal(
`<h2 data-omnivore-anchor-idx="37">Brain food</h2>`
)
})
})
describe('htmlToSsmlItems', () => {
const TEST_OPTIONS = {