From d8c5c97960ef9fcf1d369aa238951ec6d45d3315 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 17 Oct 2022 19:24:11 +0800 Subject: [PATCH] Save newsletters hosted by ghost.org correctly --- .../content-handler/src/content-handler.ts | 3 +- packages/content-handler/src/index.ts | 6 +- .../src/newsletters/beehiiv-handler.ts | 5 +- .../src/newsletters/convertkit-handler.ts | 5 +- .../src/newsletters/ghost-handler.ts | 32 ++ .../src/newsletters/revue-handler.ts | 5 +- .../src/newsletters/substack-handler.ts | 6 +- .../test/data/ghost-newsletter.html | 361 ++++++++++++++++++ .../content-handler/test/newsletter.test.ts | 56 ++- 9 files changed, 458 insertions(+), 21 deletions(-) create mode 100644 packages/content-handler/src/newsletters/ghost-handler.ts create mode 100644 packages/content-handler/test/data/ghost-newsletter.html diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts index 317f3d2d9..fd1fe5291 100644 --- a/packages/content-handler/src/content-handler.ts +++ b/packages/content-handler/src/content-handler.ts @@ -78,7 +78,8 @@ export abstract class ContentHandler { postHeader: string from: string unSubHeader: string - html?: string + html: string + dom: Document }): Promise { const re = new RegExp(this.senderRegex) return Promise.resolve( diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index 5a144b2ee..8e32f1cb3 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -24,6 +24,8 @@ import { BloombergNewsletterHandler } from './newsletters/bloomberg-newsletter-h import { BeehiivHandler } from './newsletters/beehiiv-handler' import { ConvertkitHandler } from './newsletters/convertkit-handler' import { RevueHandler } from './newsletters/revue-handler' +import { GhostHandler } from './newsletters/ghost-handler' +import { parseHTML } from 'linkedom' const validateUrlString = (url: string) => { const u = new URL(url) @@ -70,6 +72,7 @@ const newsletterHandlers: ContentHandler[] = [ new BeehiivHandler(), new ConvertkitHandler(), new RevueHandler(), + new GhostHandler(), ] export const preHandleContent = async ( @@ -122,8 +125,9 @@ export const preParseContent = async ( export const handleNewsletter = async ( input: NewsletterInput ): Promise => { + const dom = parseHTML(input.html).document for (const handler of newsletterHandlers) { - if (await handler.isNewsletter(input)) { + if (await handler.isNewsletter({ ...input, dom })) { return handler.handleNewsletter(input) } } diff --git a/packages/content-handler/src/newsletters/beehiiv-handler.ts b/packages/content-handler/src/newsletters/beehiiv-handler.ts index e0cf5c687..332e7a050 100644 --- a/packages/content-handler/src/newsletters/beehiiv-handler.ts +++ b/packages/content-handler/src/newsletters/beehiiv-handler.ts @@ -1,5 +1,4 @@ import { ContentHandler } from '../content-handler' -import { parseHTML } from 'linkedom' export class BeehiivHandler extends ContentHandler { constructor() { @@ -22,9 +21,9 @@ export class BeehiivHandler extends ContentHandler { postHeader: string from: string unSubHeader: string - html: string + dom: Document }): Promise { - const dom = parseHTML(input.html).document + const dom = input.dom if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) { const beehiivUrl = this.findNewsletterHeaderHref(dom) if (beehiivUrl) { diff --git a/packages/content-handler/src/newsletters/convertkit-handler.ts b/packages/content-handler/src/newsletters/convertkit-handler.ts index a77951178..15aebae26 100644 --- a/packages/content-handler/src/newsletters/convertkit-handler.ts +++ b/packages/content-handler/src/newsletters/convertkit-handler.ts @@ -1,5 +1,4 @@ import { ContentHandler } from '../content-handler' -import { parseHTML } from 'linkedom' export class ConvertkitHandler extends ContentHandler { constructor() { @@ -25,9 +24,9 @@ export class ConvertkitHandler extends ContentHandler { postHeader: string from: string unSubHeader: string - html: string + dom: Document }): Promise { - const dom = parseHTML(input.html).document + const dom = input.dom return Promise.resolve( dom.querySelectorAll( 'img[src*="convertkit.com"], img[src*="convertkit-mail"]' diff --git a/packages/content-handler/src/newsletters/ghost-handler.ts b/packages/content-handler/src/newsletters/ghost-handler.ts new file mode 100644 index 000000000..ca3e35a85 --- /dev/null +++ b/packages/content-handler/src/newsletters/ghost-handler.ts @@ -0,0 +1,32 @@ +import { ContentHandler } from '../content-handler' + +export class GhostHandler extends ContentHandler { + constructor() { + super() + this.name = 'ghost' + } + + findNewsletterHeaderHref(dom: Document): string | undefined { + const readOnline = dom.querySelector('.view-online-link') + return readOnline?.getAttribute('href') || undefined + } + + async isNewsletter(input: { + postHeader: string + from: string + unSubHeader: string + dom: Document + }): Promise { + const dom = input.dom + return Promise.resolve( + dom.querySelectorAll('img[src*="ghost.org"]').length > 0 + ) + } + + async parseNewsletterUrl( + postHeader: string, + html: string + ): Promise { + return this.findNewsletterUrl(html) + } +} diff --git a/packages/content-handler/src/newsletters/revue-handler.ts b/packages/content-handler/src/newsletters/revue-handler.ts index d8c8f911c..cd22d314c 100644 --- a/packages/content-handler/src/newsletters/revue-handler.ts +++ b/packages/content-handler/src/newsletters/revue-handler.ts @@ -1,5 +1,4 @@ import { ContentHandler } from '../content-handler' -import { parseHTML } from 'linkedom' export class RevueHandler extends ContentHandler { constructor() { @@ -22,9 +21,9 @@ export class RevueHandler extends ContentHandler { postHeader: string from: string unSubHeader: string - html: string + dom: Document }): Promise { - const dom = parseHTML(input.html).document + const dom = input.dom if ( dom.querySelectorAll('img[src*="getrevue.co"], img[src*="revue.email"]') .length > 0 diff --git a/packages/content-handler/src/newsletters/substack-handler.ts b/packages/content-handler/src/newsletters/substack-handler.ts index e90168c8e..e82d6d1cf 100644 --- a/packages/content-handler/src/newsletters/substack-handler.ts +++ b/packages/content-handler/src/newsletters/substack-handler.ts @@ -1,6 +1,5 @@ import addressparser from 'addressparser' import { ContentHandler } from '../content-handler' -import { parseHTML } from 'linkedom' export class SubstackHandler extends ContentHandler { constructor() { @@ -50,17 +49,16 @@ export class SubstackHandler extends ContentHandler { async isNewsletter({ postHeader, - html, + dom, }: { postHeader: string from: string unSubHeader: string - html: string + dom: Document }): Promise { if (postHeader) { return Promise.resolve(true) } - const dom = parseHTML(html).document // substack newsletter emails have tables with a *post-meta class if (dom.querySelector('table[class$="post-meta"]')) { return true diff --git a/packages/content-handler/test/data/ghost-newsletter.html b/packages/content-handler/test/data/ghost-newsletter.html new file mode 100644 index 000000000..0caf6932a --- /dev/null +++ b/packages/content-handler/test/data/ghost-newsletter.html @@ -0,0 +1,361 @@ + + + + + + + why ish / 2022-10-14 + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts index 4a9524498..f34760971 100644 --- a/packages/content-handler/test/newsletter.test.ts +++ b/packages/content-handler/test/newsletter.test.ts @@ -13,6 +13,8 @@ import { generateUniqueUrl } from '../src/content-handler' import fs from 'fs' import { BeehiivHandler } from '../src/newsletters/beehiiv-handler' import { ConvertkitHandler } from '../src/newsletters/convertkit-handler' +import { parseHTML } from 'linkedom' +import { GhostHandler } from '../src/newsletters/ghost-handler' chai.use(chaiAsPromised) chai.use(chaiString) @@ -93,9 +95,10 @@ describe('Newsletter email test', () => { describe('isProbablyNewsletter', () => { it('returns true for substack newsletter', async () => { const html = load('./test/data/substack-forwarded-newsletter.html') + const dom = parseHTML(html).document await expect( new SubstackHandler().isNewsletter({ - html, + dom, postHeader: '', from: '', unSubHeader: '', @@ -106,9 +109,10 @@ describe('Newsletter email test', () => { const html = load( './test/data/substack-private-forwarded-newsletter.html' ) + const dom = parseHTML(html).document await expect( new SubstackHandler().isNewsletter({ - html, + dom, postHeader: '', from: '', unSubHeader: '', @@ -117,9 +121,10 @@ describe('Newsletter email test', () => { }) it('returns false for substack welcome email', async () => { const html = load('./test/data/substack-forwarded-welcome-email.html') + const dom = parseHTML(html).document await expect( new SubstackHandler().isNewsletter({ - html, + dom, postHeader: '', from: '', unSubHeader: '', @@ -128,9 +133,10 @@ describe('Newsletter email test', () => { }) it('returns true for beehiiv.com newsletter', async () => { const html = load('./test/data/beehiiv-newsletter.html') + const dom = parseHTML(html).document await expect( new BeehiivHandler().isNewsletter({ - html, + dom, postHeader: '', from: '', unSubHeader: '', @@ -139,9 +145,22 @@ describe('Newsletter email test', () => { }) it('returns true for milkroad newsletter', async () => { const html = load('./test/data/milkroad-newsletter.html') + const dom = parseHTML(html).document await expect( new BeehiivHandler().isNewsletter({ - html, + dom, + postHeader: '', + from: '', + unSubHeader: '', + }) + ).to.eventually.be.true + }) + it('returns true for ghost newsletter', async () => { + const html = load('./test/data/ghost-newsletter.html') + const dom = parseHTML(html).document + await expect( + new GhostHandler().isNewsletter({ + dom, postHeader: '', from: '', unSubHeader: '', @@ -150,9 +169,10 @@ describe('Newsletter email test', () => { }) it('returns true for convertkit newsletter', async () => { const html = load('./test/data/convertkit-newsletter.html') + const dom = parseHTML(html).document await expect( new ConvertkitHandler().isNewsletter({ - html, + dom, postHeader: '', from: '', unSubHeader: '', @@ -244,6 +264,30 @@ describe('Newsletter email test', () => { const url = await new SubstackHandler().findNewsletterUrl(html) expect(url).to.be.undefined }) + + context('when email is from ghost', () => { + before(() => { + nock('https://u25184427.ct.sendgrid.net') + .head( + '/ls/click?upn=MnmHBiCwIPe9TmIJeskmA9nRLefEmmgrd5xWS-2Bc39wxPBpwDRny1FmWt1H0FpgKAz1dv_vVXscVLXlj5UtQe3aqo5RMTdTq2PepdZjP86UOmA8nzulL-2F3YyC-2FHgJV0JnOPtjNvgjHSaQVfisQ15hPQtnlo4t73zgTQL4QnDoer4qJ3-2F2Lf-2F2ElFMF3NyoUD4eqWCWwUM0w4P9Feaeo-2BolkySAB611BySXRt6V3Z-2F7mQcpcRX3D9zV-2B-2FdRY0Vn30aR-2BKY8qpTFuivxzF19UkQGjK5srg-3D-3D' + ) + .reply(302, undefined, { + Location: 'https://www.openml.fyi/2022-10-14/', + }) + .get('/2022-10-14/') + .reply(200, '') + }) + + after(() => { + nock.restore() + }) + + it('gets the URL from the header', async () => { + const html = load('./test/data/ghost-newsletter.html') + const url = await new GhostHandler().findNewsletterUrl(html) + expect(url).to.startWith('https://www.openml.fyi/2022-10-14/') + }).timeout(10000) + }) }) describe('generateUniqueUrl', () => {