prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Parse online URLs for beehiiv newsletters

Browse source
This commit is contained in:
Jackson Harper 2022-03-07 15:49:44 -08:00
parent 2cb5cc065a
commit 2184c2a8d3
3 changed files with 34 additions and 869 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
packages/api/src/utils/parser.ts
View file

@ -432,9 +432,12 @@ export const isProbablyNewsletter = (html: string): boolean => {
return true
}
const beehiivUrl = beehiivNewsletterHref(dom.window)
if (beehiivUrl) {
return true
// Check if this is a beehiiv.net newsletter
if (dom.document.querySelectorAll('img[src*="beehiiv.net"]').length > 0) {
const beehiivUrl = beehiivNewsletterHref(dom.window)
if (beehiivUrl) {
return true
}
}
return false
@ -446,7 +449,6 @@ const beehiivNewsletterHref = (dom: DOMWindow): string | undefined => {
)
let res: string | undefined = undefined
readOnline.forEach((e) => {
console.log('text content', e.textContent)
if (e.textContent === 'Read Online') {
res = e.getAttribute('href') || undefined
}
@ -455,10 +457,18 @@ const beehiivNewsletterHref = (dom: DOMWindow): string | undefined => {
}
const findNewsletterHeaderHref = (dom: DOMWindow): string | undefined => {
// Substack header links
const postLink = dom.document.querySelector('h1 a ')
if (postLink) {
return postLink.getAttribute('href') || undefined
}
// Check if this is a beehiiv.net newsletter
const beehiiv = beehiivNewsletterHref(dom.window)
if (beehiiv) {
return beehiiv
}
return undefined
}
@ -468,6 +478,8 @@ export const findNewsletterUrl = async (
html: string
): Promise<string | undefined> => {
const dom = new JSDOM(html).window
// Check if this is a substack newsletter
const href = findNewsletterHeaderHref(dom.window)
if (href) {
// Try to make a HEAD request so we get the redirected URL, since these

878
packages/api/test/utils/data/beehiiv-newsletter.html
View file

File diff suppressed because one or more lines are too long

5
packages/api/test/utils/parser.test.ts
View file

@ -35,9 +35,10 @@ describe('findNewsletterUrl', async () => {
// Not sure if the redirects from substack expire, this test could eventually fail
expect(url).to.startWith('https://newsletter.slowchinese.net/p/companies-that-eat-people-217')
})
it('gets the URL from the header if it is a beehiiv newsletter', () => {
it('gets the URL from the header if it is a beehiiv newsletter', async () => {
const html = load('./test/utils/data/beehiiv-newsletter.html')
isProbablyNewsletter(html).should.be.true
const url = await findNewsletterUrl(html)
expect(url).to.startWith('https://www.milkroad.com/p/talked-guy-spent-30m-beeple')
})
it('returns undefined if it is not a newsletter', async () => {
const html = load('./test/utils/data/substack-forwarded-welcome-email.html')