Save newsletters hosted by ghost.org correctly

packages/content-handler/src/content-handler.ts

@@ -78,7 +78,8 @@ export abstract class ContentHandler {
     postHeader: string
     from: string
     unSubHeader: string
-    html?: string
+    html: string
+    dom: Document
   }): Promise<boolean> {
     const re = new RegExp(this.senderRegex)
     return Promise.resolve(

packages/content-handler/src/index.ts

@@ -24,6 +24,8 @@ import { BloombergNewsletterHandler } from './newsletters/bloomberg-newsletter-h
 import { BeehiivHandler } from './newsletters/beehiiv-handler'
 import { ConvertkitHandler } from './newsletters/convertkit-handler'
 import { RevueHandler } from './newsletters/revue-handler'
+import { GhostHandler } from './newsletters/ghost-handler'
+import { parseHTML } from 'linkedom'
 
 const validateUrlString = (url: string) => {
   const u = new URL(url)
@@ -70,6 +72,7 @@ const newsletterHandlers: ContentHandler[] = [
   new BeehiivHandler(),
   new ConvertkitHandler(),
   new RevueHandler(),
+  new GhostHandler(),
 ]
 
 export const preHandleContent = async (
@@ -122,8 +125,9 @@ export const preParseContent = async (
 export const handleNewsletter = async (
   input: NewsletterInput
 ): Promise<NewsletterResult | undefined> => {
+  const dom = parseHTML(input.html).document
   for (const handler of newsletterHandlers) {
-    if (await handler.isNewsletter(input)) {
+    if (await handler.isNewsletter({ ...input, dom })) {
       return handler.handleNewsletter(input)
     }
   }

packages/content-handler/src/newsletters/beehiiv-handler.ts

@@ -1,5 +1,4 @@
 import { ContentHandler } from '../content-handler'
-import { parseHTML } from 'linkedom'
 
 export class BeehiivHandler extends ContentHandler {
   constructor() {
@@ -22,9 +21,9 @@ export class BeehiivHandler extends ContentHandler {
     postHeader: string
     from: string
     unSubHeader: string
-    html: string
+    dom: Document
   }): Promise<boolean> {
-    const dom = parseHTML(input.html).document
+    const dom = input.dom
     if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) {
       const beehiivUrl = this.findNewsletterHeaderHref(dom)
       if (beehiivUrl) {

packages/content-handler/src/newsletters/convertkit-handler.ts

@@ -1,5 +1,4 @@
 import { ContentHandler } from '../content-handler'
-import { parseHTML } from 'linkedom'
 
 export class ConvertkitHandler extends ContentHandler {
   constructor() {
@@ -25,9 +24,9 @@ export class ConvertkitHandler extends ContentHandler {
     postHeader: string
     from: string
     unSubHeader: string
-    html: string
+    dom: Document
   }): Promise<boolean> {
-    const dom = parseHTML(input.html).document
+    const dom = input.dom
     return Promise.resolve(
       dom.querySelectorAll(
         'img[src*="convertkit.com"], img[src*="convertkit-mail"]'

packages/content-handler/src/newsletters/ghost-handler.ts

@@ -0,0 +1,32 @@
+import { ContentHandler } from '../content-handler'
+
+export class GhostHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'ghost'
+  }
+
+  findNewsletterHeaderHref(dom: Document): string | undefined {
+    const readOnline = dom.querySelector('.view-online-link')
+    return readOnline?.getAttribute('href') || undefined
+  }
+
+  async isNewsletter(input: {
+    postHeader: string
+    from: string
+    unSubHeader: string
+    dom: Document
+  }): Promise<boolean> {
+    const dom = input.dom
+    return Promise.resolve(
+      dom.querySelectorAll('img[src*="ghost.org"]').length > 0
+    )
+  }
+
+  async parseNewsletterUrl(
+    postHeader: string,
+    html: string
+  ): Promise<string | undefined> {
+    return this.findNewsletterUrl(html)
+  }
+}

packages/content-handler/src/newsletters/revue-handler.ts

@@ -1,5 +1,4 @@
 import { ContentHandler } from '../content-handler'
-import { parseHTML } from 'linkedom'
 
 export class RevueHandler extends ContentHandler {
   constructor() {
@@ -22,9 +21,9 @@ export class RevueHandler extends ContentHandler {
     postHeader: string
     from: string
     unSubHeader: string
-    html: string
+    dom: Document
   }): Promise<boolean> {
-    const dom = parseHTML(input.html).document
+    const dom = input.dom
     if (
       dom.querySelectorAll('img[src*="getrevue.co"], img[src*="revue.email"]')
         .length > 0

packages/content-handler/src/newsletters/substack-handler.ts

@@ -1,6 +1,5 @@
 import addressparser from 'addressparser'
 import { ContentHandler } from '../content-handler'
-import { parseHTML } from 'linkedom'
 
 export class SubstackHandler extends ContentHandler {
   constructor() {
@@ -50,17 +49,16 @@ export class SubstackHandler extends ContentHandler {
 
   async isNewsletter({
     postHeader,
-    html,
+    dom,
   }: {
     postHeader: string
     from: string
     unSubHeader: string
-    html: string
+    dom: Document
   }): Promise<boolean> {
     if (postHeader) {
       return Promise.resolve(true)
     }
-    const dom = parseHTML(html).document
     // substack newsletter emails have tables with a *post-meta class
     if (dom.querySelector('table[class$="post-meta"]')) {
       return true

packages/content-handler/test/newsletter.test.ts

@@ -13,6 +13,8 @@ import { generateUniqueUrl } from '../src/content-handler'
 import fs from 'fs'
 import { BeehiivHandler } from '../src/newsletters/beehiiv-handler'
 import { ConvertkitHandler } from '../src/newsletters/convertkit-handler'
+import { parseHTML } from 'linkedom'
+import { GhostHandler } from '../src/newsletters/ghost-handler'
 
 chai.use(chaiAsPromised)
 chai.use(chaiString)
@@ -93,9 +95,10 @@ describe('Newsletter email test', () => {
   describe('isProbablyNewsletter', () => {
     it('returns true for substack newsletter', async () => {
       const html = load('./test/data/substack-forwarded-newsletter.html')
+      const dom = parseHTML(html).document
       await expect(
         new SubstackHandler().isNewsletter({
-          html,
+          dom,
           postHeader: '',
           from: '',
           unSubHeader: '',
@@ -106,9 +109,10 @@ describe('Newsletter email test', () => {
       const html = load(
         './test/data/substack-private-forwarded-newsletter.html'
       )
+      const dom = parseHTML(html).document
       await expect(
         new SubstackHandler().isNewsletter({
-          html,
+          dom,
           postHeader: '',
           from: '',
           unSubHeader: '',
@@ -117,9 +121,10 @@ describe('Newsletter email test', () => {
     })
     it('returns false for substack welcome email', async () => {
       const html = load('./test/data/substack-forwarded-welcome-email.html')
+      const dom = parseHTML(html).document
       await expect(
         new SubstackHandler().isNewsletter({
-          html,
+          dom,
           postHeader: '',
           from: '',
           unSubHeader: '',
@@ -128,9 +133,10 @@ describe('Newsletter email test', () => {
     })
     it('returns true for beehiiv.com newsletter', async () => {
       const html = load('./test/data/beehiiv-newsletter.html')
+      const dom = parseHTML(html).document
       await expect(
         new BeehiivHandler().isNewsletter({
-          html,
+          dom,
           postHeader: '',
           from: '',
           unSubHeader: '',
@@ -139,9 +145,22 @@ describe('Newsletter email test', () => {
     })
     it('returns true for milkroad newsletter', async () => {
       const html = load('./test/data/milkroad-newsletter.html')
+      const dom = parseHTML(html).document
       await expect(
         new BeehiivHandler().isNewsletter({
-          html,
+          dom,
+          postHeader: '',
+          from: '',
+          unSubHeader: '',
+        })
+      ).to.eventually.be.true
+    })
+    it('returns true for ghost newsletter', async () => {
+      const html = load('./test/data/ghost-newsletter.html')
+      const dom = parseHTML(html).document
+      await expect(
+        new GhostHandler().isNewsletter({
+          dom,
           postHeader: '',
           from: '',
           unSubHeader: '',
@@ -150,9 +169,10 @@ describe('Newsletter email test', () => {
     })
     it('returns true for convertkit newsletter', async () => {
       const html = load('./test/data/convertkit-newsletter.html')
+      const dom = parseHTML(html).document
       await expect(
         new ConvertkitHandler().isNewsletter({
-          html,
+          dom,
           postHeader: '',
           from: '',
           unSubHeader: '',
@@ -244,6 +264,30 @@ describe('Newsletter email test', () => {
       const url = await new SubstackHandler().findNewsletterUrl(html)
       expect(url).to.be.undefined
     })
+
+    context('when email is from ghost', () => {
+      before(() => {
+        nock('https://u25184427.ct.sendgrid.net')
+          .head(
+            '/ls/click?upn=MnmHBiCwIPe9TmIJeskmA9nRLefEmmgrd5xWS-2Bc39wxPBpwDRny1FmWt1H0FpgKAz1dv_vVXscVLXlj5UtQe3aqo5RMTdTq2PepdZjP86UOmA8nzulL-2F3YyC-2FHgJV0JnOPtjNvgjHSaQVfisQ15hPQtnlo4t73zgTQL4QnDoer4qJ3-2F2Lf-2F2ElFMF3NyoUD4eqWCWwUM0w4P9Feaeo-2BolkySAB611BySXRt6V3Z-2F7mQcpcRX3D9zV-2B-2FdRY0Vn30aR-2BKY8qpTFuivxzF19UkQGjK5srg-3D-3D'
+          )
+          .reply(302, undefined, {
+            Location: 'https://www.openml.fyi/2022-10-14/',
+          })
+          .get('/2022-10-14/')
+          .reply(200, '')
+      })
+
+      after(() => {
+        nock.restore()
+      })
+
+      it('gets the URL from the header', async () => {
+        const html = load('./test/data/ghost-newsletter.html')
+        const url = await new GhostHandler().findNewsletterUrl(html)
+        expect(url).to.startWith('https://www.openml.fyi/2022-10-14/')
+      }).timeout(10000)
+    })
   })
 
   describe('generateUniqueUrl', () => {