prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add more matches on substack icons

Browse source
This commit is contained in:
Jackson Harper 2022-03-03 13:42:30 -08:00
parent 484cd78ac5
commit b326a5f8e7
3 changed files with 37 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
packages/api/src/utils/parser.ts
View file

@ -388,7 +388,6 @@ export const isProbablyNewsletter = (html: string): boolean => {
}).parse()
if (!article || !article.content) {
console.log('no article content')
return false
}
@ -397,30 +396,45 @@ export const isProbablyNewsletter = (html: string): boolean => {
return true
}
// If the article has a header link, and substack icons its probably a newsletter
const href = findNewsletterHeaderHref(dom.window)
const heartIcon = dom.document.querySelector(
'table tbody td span a img[src*="HeartIcon"]'
)
const recommendIcon = dom.document.querySelector(
'table tbody td span a img[src*="RecommendIconRounded"]'
)
if (href && (heartIcon || recommendIcon)) {
return true
}
return false
}
const findNewsletterHeaderHref = (dom: DOMWindow): string | undefined => {
const postLink = dom.document.querySelector('h1 a ')
if (postLink) {
return postLink.getAttribute('href') || undefined
}
return undefined
}
// Given an HTML blob tries to find a URL to use for
// a canonical URL.
export const findNewsletterUrl = async (
html: string
): Promise<string | undefined> => {
const dom = new JSDOM(html).window
// If there is an <h1 element with a URL, use that
const postLink = dom.document.querySelector('h1 a ')
if (postLink) {
const href = postLink.getAttribute('href')
if (href) {
// Try to make a HEAD request so we get the redirected URL, since these
// will usually be behind tracking url redirects
return axios({
method: 'HEAD',
url: href,
})
.then((res) => res.request.res.responseUrl as string | undefined)
.catch((e) => href)
}
const href = findNewsletterHeaderHref(dom.window)
if (href) {
// Try to make a HEAD request so we get the redirected URL, since these
// will usually be behind tracking url redirects
return axios({
method: 'HEAD',
url: href,
})
.then((res) => res.request.res.responseUrl as string | undefined)
.catch((e) => href)
}
return undefined

2
packages/api/test/utils/data/substack-private-forwarded-newsletter.html Normal file
View file

File diff suppressed because one or more lines are too long

5
packages/api/test/utils/parser.test.ts
View file

@ -14,6 +14,10 @@ describe('isProbablyNewsletter', () => {
const html = load('./test/utils/data/substack-forwarded-newsletter.html')
isProbablyNewsletter(html).should.be.true
})
it('returns true for private forwarded substack newsletter', () => {
const html = load('./test/utils/data/substack-private-forwarded-newsletter.html')
isProbablyNewsletter(html).should.be.true
})
it('returns false for substack welcome email', () => {
const html = load('./test/utils/data/substack-forwarded-welcome-email.html')
isProbablyNewsletter(html).should.be.false
@ -33,3 +37,4 @@ describe('findNewsletterUrl', async () => {
expect(url).to.be.undefined
})
})