Merge pull request #1748 from omnivore-app/fix/milkroad-newsletters-not-detected

fix/milkroad newsletters not detected

packages/content-handler/src/content-handler.ts

@@ -11,12 +11,11 @@ interface Unsubscribe {
 }
 
 export interface NewsletterInput {
-  postHeader: string
   from: string
-  unSubHeader: string
-  email: string
+  to: string
+  subject: string
   html: string
-  title: string
+  headers: Record<string, string | string[]>
 }
 
 export interface NewsletterResult {
@@ -76,15 +75,16 @@ export abstract class ContentHandler {
   }
 
   async isNewsletter(input: {
-    postHeader: string
     from: string
-    unSubHeader: string
     html: string
+    headers: Record<string, string | string[]>
     dom: Document
   }): Promise<boolean> {
     const re = new RegExp(this.senderRegex)
+    const postHeader = input.headers['list-post']
+    const unSubHeader = input.headers['list-unsubscribe']
     return Promise.resolve(
-      re.test(input.from) && (!!input.postHeader || !!input.unSubHeader)
+      re.test(input.from) && (!!postHeader || !!unSubHeader)
     )
   }
 
@@ -118,7 +118,7 @@ export abstract class ContentHandler {
   }
 
   async parseNewsletterUrl(
-    _postHeader: string,
+    headers: Record<string, string | string[]>,
     html: string
   ): Promise<string | undefined> {
     // get newsletter url from html
@@ -151,16 +151,15 @@ export abstract class ContentHandler {
   }
 
   async handleNewsletter({
-    email,
-    html,
-    postHeader,
-    title,
     from,
-    unSubHeader,
+    to,
+    subject,
+    html,
+    headers,
   }: NewsletterInput): Promise<NewsletterResult> {
-    console.log('handleNewsletter', email, postHeader, title, from)
+    console.log('handleNewsletter', from, to, subject, headers, from)
 
-    if (!email || !html || !title || !from) {
+    if (!from || !html || !subject || !to) {
       console.log('invalid newsletter email')
       throw new Error('invalid newsletter email')
     }
@@ -168,15 +167,17 @@ export abstract class ContentHandler {
     // fallback to default url if newsletter url does not exist
     // assign a random uuid to the default url to avoid duplicate url
     const url =
-      (await this.parseNewsletterUrl(postHeader, html)) || generateUniqueUrl()
+      (await this.parseNewsletterUrl(headers, html)) || generateUniqueUrl()
     const author = this.parseAuthor(from)
-    const unsubscribe = this.parseUnsubscribe(unSubHeader)
+    const unsubscribe = this.parseUnsubscribe(
+      headers['list-unsubscribe']?.toString()
+    )
 
     return {
-      email,
+      email: to,
       content: html,
       url,
-      title,
+      title: subject,
       author,
       unsubMailTo: unsubscribe.mailTo || '',
       unsubHttpUrl: unsubscribe.httpUrl || '',

packages/content-handler/src/index.ts

@@ -131,10 +131,9 @@ export const preParseContent = async (
 }
 
 export const getNewsletterHandler = async (input: {
-  postHeader: string
   from: string
-  unSubHeader: string
   html: string
+  headers: Record<string, string | string[]>
 }): Promise<ContentHandler | undefined> => {
   const dom = parseHTML(input.html).document
   for (const handler of newsletterHandlers) {
@@ -151,7 +150,7 @@ export const handleNewsletter = async (
 ): Promise<NewsletterResult | undefined> => {
   const handler = await getNewsletterHandler(input)
   if (handler) {
-    console.log('handleNewsletter', handler.name, input.title)
+    console.log('handleNewsletter', handler.name, input.subject)
     return handler.handleNewsletter(input)
   }
 

packages/content-handler/src/newsletters/beehiiv-handler.ts

@@ -6,37 +6,19 @@ export class BeehiivHandler extends ContentHandler {
     this.name = 'beehiiv'
   }
 
-  findNewsletterHeaderHref(dom: Document): string | undefined {
-    const readOnline = dom.querySelectorAll('table tr td a')
-    let res: string | undefined = undefined
-    readOnline.forEach((e) => {
-      if (e.textContent === 'Read Online') {
-        res = e.getAttribute('href') || undefined
-      }
-    })
-    return res
-  }
-
   async isNewsletter(input: {
-    postHeader: string
     from: string
-    unSubHeader: string
-    dom: Document
+    headers: Record<string, string | string[]>
   }): Promise<boolean> {
-    const dom = input.dom
-    if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) {
-      const beehiivUrl = this.findNewsletterHeaderHref(dom)
-      if (beehiivUrl) {
-        return Promise.resolve(true)
-      }
-    }
-    return false
+    return Promise.resolve(
+      input.headers['x-beehiiv-type']?.toString() === 'newsletter'
+    )
   }
 
   async parseNewsletterUrl(
-    postHeader: string,
+    headers: Record<string, string | string[]>,
     html: string
   ): Promise<string | undefined> {
-    return this.findNewsletterUrl(html)
+    return Promise.resolve(headers['x-newsletter']?.toString())
   }
 }

packages/content-handler/src/newsletters/convertkit-handler.ts

@@ -21,10 +21,9 @@ export class ConvertkitHandler extends ContentHandler {
   }
 
   async isNewsletter(input: {
-    postHeader: string
     from: string
-    unSubHeader: string
     dom: Document
+    headers: Record<string, string | string[]>
   }): Promise<boolean> {
     const dom = input.dom
     const icons = dom.querySelectorAll(
@@ -45,7 +44,7 @@ export class ConvertkitHandler extends ContentHandler {
   }
 
   async parseNewsletterUrl(
-    postHeader: string,
+    headers: Record<string, string | string[]>,
     html: string
   ): Promise<string | undefined> {
     return this.findNewsletterUrl(html)

packages/content-handler/src/newsletters/cooper-press-handler.ts

@@ -18,10 +18,9 @@ export class CooperPressHandler extends ContentHandler {
   }
 
   async isNewsletter(input: {
-    postHeader: string
     from: string
-    unSubHeader: string
     dom: Document
+    headers: Record<string, string | string[]>
   }): Promise<boolean> {
     const dom = input.dom
     return Promise.resolve(
@@ -30,7 +29,7 @@ export class CooperPressHandler extends ContentHandler {
   }
 
   async parseNewsletterUrl(
-    postHeader: string,
+    headers: Record<string, string | string[]>,
     html: string
   ): Promise<string | undefined> {
     return this.findNewsletterUrl(html)

packages/content-handler/src/newsletters/ghost-handler.ts

@@ -12,10 +12,9 @@ export class GhostHandler extends ContentHandler {
   }
 
   async isNewsletter(input: {
-    postHeader: string
     from: string
-    unSubHeader: string
     dom: Document
+    headers: Record<string, string | string[]>
   }): Promise<boolean> {
     const dom = input.dom
     return Promise.resolve(
@@ -24,7 +23,7 @@ export class GhostHandler extends ContentHandler {
   }
 
   async parseNewsletterUrl(
-    postHeader: string,
+    headers: Record<string, string | string[]>,
     html: string
   ): Promise<string | undefined> {
     return this.findNewsletterUrl(html)

packages/content-handler/src/newsletters/hey-world-handler.ts

@@ -19,7 +19,7 @@ export class HeyWorldHandler extends ContentHandler {
   }
 
   async parseNewsletterUrl(
-    postHeader: string,
+    headers: Record<string, string | string[]>,
     html: string
   ): Promise<string | undefined> {
     return this.findNewsletterUrl(html)

packages/content-handler/src/newsletters/revue-handler.ts

@@ -18,10 +18,9 @@ export class RevueHandler extends ContentHandler {
   }
 
   async isNewsletter(input: {
-    postHeader: string
     from: string
-    unSubHeader: string
     dom: Document
+    headers: Record<string, string | string[]>
   }): Promise<boolean> {
     const dom = input.dom
     if (
@@ -37,7 +36,7 @@ export class RevueHandler extends ContentHandler {
   }
 
   async parseNewsletterUrl(
-    postHeader: string,
+    headers: Record<string, string | string[]>,
     html: string
   ): Promise<string | undefined> {
     return this.findNewsletterUrl(html)

packages/content-handler/src/newsletters/substack-handler.ts

@@ -51,15 +51,14 @@ export class SubstackHandler extends ContentHandler {
   }
 
   async isNewsletter({
-    postHeader,
+    headers,
     dom,
   }: {
-    postHeader: string
     from: string
-    unSubHeader: string
+    headers: Record<string, string | string[]>
     dom: Document
   }): Promise<boolean> {
-    if (postHeader) {
+    if (headers['list-post']) {
       return Promise.resolve(true)
     }
     // substack newsletter emails have tables with a *post-meta class
@@ -85,11 +84,12 @@ export class SubstackHandler extends ContentHandler {
   }
 
   async parseNewsletterUrl(
-    postHeader: string,
+    headers: Record<string, string | string[]>,
     html: string
   ): Promise<string | undefined> {
     // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
     // we need to get the real url from the raw url
+    const postHeader = headers['list-post']?.toString()
     if (postHeader && addressparser(postHeader).length > 0) {
       return Promise.resolve(addressparser(postHeader)[0].name)
     }

packages/content-handler/test/newsletter.test.ts

@@ -1,23 +1,23 @@
-import 'mocha'
-import * as chai from 'chai'
-import { expect } from 'chai'
-import chaiAsPromised from 'chai-as-promised'
-import chaiString from 'chai-string'
-import { SubstackHandler } from '../src/newsletters/substack-handler'
-import { AxiosHandler } from '../src/newsletters/axios-handler'
-import { BloombergNewsletterHandler } from '../src/newsletters/bloomberg-newsletter-handler'
-import { GolangHandler } from '../src/newsletters/golang-handler'
-import { MorningBrewHandler } from '../src/newsletters/morning-brew-handler'
-import nock from 'nock'
-import { generateUniqueUrl } from '../src/content-handler'
-import fs from 'fs'
-import { BeehiivHandler } from '../src/newsletters/beehiiv-handler'
-import { ConvertkitHandler } from '../src/newsletters/convertkit-handler'
-import { GhostHandler } from '../src/newsletters/ghost-handler'
-import { CooperPressHandler } from '../src/newsletters/cooper-press-handler'
-import { getNewsletterHandler } from '../src'
-import { parseHTML } from 'linkedom'
-import { HeyWorldHandler } from '../src/newsletters/hey-world-handler'
+import "mocha";
+import * as chai from "chai";
+import { expect } from "chai";
+import chaiAsPromised from "chai-as-promised";
+import chaiString from "chai-string";
+import { SubstackHandler } from "../src/newsletters/substack-handler";
+import { AxiosHandler } from "../src/newsletters/axios-handler";
+import { BloombergNewsletterHandler } from "../src/newsletters/bloomberg-newsletter-handler";
+import { GolangHandler } from "../src/newsletters/golang-handler";
+import { MorningBrewHandler } from "../src/newsletters/morning-brew-handler";
+import nock from "nock";
+import { generateUniqueUrl } from "../src/content-handler";
+import fs from "fs";
+import { BeehiivHandler } from "../src/newsletters/beehiiv-handler";
+import { ConvertkitHandler } from "../src/newsletters/convertkit-handler";
+import { GhostHandler } from "../src/newsletters/ghost-handler";
+import { CooperPressHandler } from "../src/newsletters/cooper-press-handler";
+import { getNewsletterHandler } from "../src";
+import { parseHTML } from "linkedom";
+import { HeyWorldHandler } from "../src/newsletters/hey-world-handler";
 
 chai.use(chaiAsPromised)
 chai.use(chaiString)
@@ -29,10 +29,10 @@ const load = (path: string): string => {
 describe('Newsletter email test', () => {
   describe('#getNewsletterUrl()', () => {
     it('returns url when email is from SubStack', async () => {
-      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
+      const headers = { 'list-post': '<https://hongbo130.substack.com/p/tldr>' }
 
       await expect(
-        new SubstackHandler().parseNewsletterUrl(rawUrl, '')
+        new SubstackHandler().parseNewsletterUrl(headers, '')
       ).to.eventually.equal('https://hongbo130.substack.com/p/tldr')
     })
 
@@ -41,7 +41,7 @@ describe('Newsletter email test', () => {
       const html = `View in browser at <a>${url}</a>`
 
       await expect(
-        new AxiosHandler().parseNewsletterUrl('', html)
+        new AxiosHandler().parseNewsletterUrl({}, html)
       ).to.eventually.equal(url)
     })
 
@@ -54,7 +54,7 @@ describe('Newsletter email test', () => {
       `
 
       await expect(
-        new BloombergNewsletterHandler().parseNewsletterUrl('', html)
+        new BloombergNewsletterHandler().parseNewsletterUrl({}, html)
       ).to.eventually.equal(url)
     })
 
@@ -65,7 +65,7 @@ describe('Newsletter email test', () => {
       `
 
       await expect(
-        new GolangHandler().parseNewsletterUrl('', html)
+        new GolangHandler().parseNewsletterUrl({}, html)
       ).to.eventually.equal(url)
     })
 
@@ -76,7 +76,7 @@ describe('Newsletter email test', () => {
       `
 
       await expect(
-        new MorningBrewHandler().parseNewsletterUrl('', html)
+        new MorningBrewHandler().parseNewsletterUrl({}, html)
       ).to.eventually.equal(url)
     })
   })
@@ -105,9 +105,8 @@ describe('Newsletter email test', () => {
       const html = load('./test/data/substack-forwarded-newsletter.html')
       const handler = await getNewsletterHandler({
         html,
-        postHeader: '',
         from: '',
-        unSubHeader: '',
+        headers: {},
       })
       expect(handler).to.be.instanceOf(SubstackHandler)
     })
@@ -118,9 +117,8 @@ describe('Newsletter email test', () => {
       )
       const handler = await getNewsletterHandler({
         html,
-        postHeader: '',
         from: '',
-        unSubHeader: '',
+        headers: {},
       })
       expect(handler).to.be.instanceOf(SubstackHandler)
     })
@@ -129,9 +127,8 @@ describe('Newsletter email test', () => {
       const html = load('./test/data/substack-forwarded-welcome-email.html')
       const handler = await getNewsletterHandler({
         html,
-        postHeader: '',
         from: '',
-        unSubHeader: '',
+        headers: {},
       })
       expect(handler).to.be.undefined
     })
@@ -142,9 +139,8 @@ describe('Newsletter email test', () => {
       )
       const handler = await getNewsletterHandler({
         html,
-        postHeader: '',
         from: '',
-        unSubHeader: '',
+        headers: {},
       })
       expect(handler).to.be.instanceOf(SubstackHandler)
     })
@@ -157,9 +153,8 @@ describe('Newsletter email test', () => {
       )
       const handler = await getNewsletterHandler({
         html,
-        postHeader: '',
         from: '',
-        unSubHeader: '',
+        headers: {},
       })
       expect(handler).to.be.instanceOf(SubstackHandler)
 
@@ -175,23 +170,13 @@ describe('Newsletter email test', () => {
     })
 
     it('returns BeehiivHandler for beehiiv.com newsletter', async () => {
-      const html = load('./test/data/beehiiv-newsletter.html')
       const handler = await getNewsletterHandler({
-        html,
-        postHeader: '',
+        html: '',
         from: '',
-        unSubHeader: '',
-      })
-      expect(handler).to.be.instanceOf(BeehiivHandler)
-    })
-
-    it('returns BeehiivHandler for milkroad newsletter', async () => {
-      const html = load('./test/data/milkroad-newsletter.html')
-      const handler = await getNewsletterHandler({
-        html,
-        postHeader: '',
-        from: '',
-        unSubHeader: '',
+        headers: {
+          'x-newsletter': 'https://www.milkroad.com/p/bored-ape-amazon',
+          'x-beehiiv-type': 'newsletter'
+        },
       })
       expect(handler).to.be.instanceOf(BeehiivHandler)
     })
@@ -200,9 +185,8 @@ describe('Newsletter email test', () => {
       const html = load('./test/data/ghost-newsletter.html')
       const handler = await getNewsletterHandler({
         html,
-        postHeader: '',
         from: '',
-        unSubHeader: '',
+        headers: {},
       })
       expect(handler).to.be.instanceOf(GhostHandler)
     })
@@ -211,9 +195,8 @@ describe('Newsletter email test', () => {
       const html = load('./test/data/convertkit-newsletter.html')
       const handler = await getNewsletterHandler({
         html,
-        postHeader: '',
         from: '',
-        unSubHeader: '',
+        headers: {},
       })
       expect(handler).to.be.instanceOf(ConvertkitHandler)
     })
@@ -222,9 +205,8 @@ describe('Newsletter email test', () => {
       const html = load('./test/data/node-weekly-newsletter.html')
       const handler = await getNewsletterHandler({
         html,
-        postHeader: '',
         from: '',
-        unSubHeader: '',
+        headers: {},
       })
       expect(handler).to.be.instanceOf(CooperPressHandler)
     })
@@ -233,10 +215,8 @@ describe('Newsletter email test', () => {
       const html = load('./test/data/hey-world-newsletter.html')
       const handler = await getNewsletterHandler({
         html,
-        postHeader: '',
         from: 'Hongbo Wu <hw@world.hey.com>',
-        unSubHeader:
-          '<https://world.hey.com/dhh/subscribers/MtuoW9TvSJK9o5c7ohB72V2s/unsubscribe>',
+        headers: {'list-unsubscribe': '<https://world.hey.com/dhh/subscribers/MtuoW9TvSJK9o5c7ohB72V2s/unsubscribe>'},
       })
       expect(handler).to.be.instanceOf(HeyWorldHandler)
     })
@@ -245,9 +225,8 @@ describe('Newsletter email test', () => {
       const html = load('./test/data/tomasz-tunguz-newsletter.html')
       const handler = await getNewsletterHandler({
         html,
-        postHeader: '',
         from: '',
-        unSubHeader: '',
+        headers: {},
       })
       expect(handler).to.be.instanceOf(ConvertkitHandler)
     })
@@ -256,9 +235,8 @@ describe('Newsletter email test', () => {
       const html = load('./test/data/convertkit-confirmation.html')
       const handler = await getNewsletterHandler({
         html,
-        postHeader: '',
         from: '',
-        unSubHeader: '',
+        headers: {},
       })
       expect(handler).to.be.undefined
     })
@@ -291,24 +269,12 @@ describe('Newsletter email test', () => {
     })
 
     context('when email is from beehiiv', () => {
-      before(() => {
-        nock('https://u23463625.ct.sendgrid.net')
-          .head(
-            '/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno'
-          )
-          .reply(301, undefined, {
-            Location: 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple',
-          })
-        nock('https://www.milkroad.com')
-          .head('/p/talked-guy-spent-30m-beeple')
-          .reply(200, '')
-      })
-
       it('gets the URL from the header', async () => {
-        const html = load('./test/data/beehiiv-newsletter.html')
-        const url = await new BeehiivHandler().findNewsletterUrl(html)
+        const url = await new BeehiivHandler().parseNewsletterUrl({
+          'x-newsletter': 'https://www.milkroad.com/p/bored-ape-amazon',
+        }, '')
         expect(url).to.startWith(
-          'https://www.milkroad.com/p/talked-guy-spent-30m-beeple'
+          'https://www.milkroad.com/p/bored-ape-amazon'
         )
       })
     })

packages/inbound-email-handler/src/index.ts

@@ -109,7 +109,6 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       // if an email is forwarded to the inbox, the to is the forwarding email recipient
       const to = forwardedTo || parsed['to']
-      const postHeader = headers['list-post']?.toString()
       const unSubHeader = headers['list-unsubscribe']?.toString()
 
       const { id: receivedEmailId } = await saveReceivedEmail(to, {
@@ -124,11 +123,10 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
         // check if it is a confirmation email or forwarding newsletter
         const newsletterMessage = await handleNewsletter({
           from,
+          to,
+          subject,
           html,
-          postHeader,
-          unSubHeader,
-          email: to,
-          title: subject,
+          headers,
         })
         if (newsletterMessage) {
           await publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, {