Import content-handler in inbound-email-handler

packages/content-handler/src/index.ts

@@ -84,7 +84,7 @@ export const preHandleContent = async (
   return undefined
 }
 
-export const handlerNewsletter = (
+export const handleNewsletter = (
   input: NewsletterInput
 ): NewsletterResult | undefined => {
   for (const handler of contentHandlers) {
@@ -98,5 +98,5 @@ export const handlerNewsletter = (
 
 module.exports = {
   preHandleContent,
-  handlerNewsletter,
+  handleNewsletter,
 }

packages/inbound-email-handler/package.json

@@ -31,6 +31,7 @@
     "@google-cloud/pubsub": "^2.18.4",
     "@sendgrid/client": "^7.6.0",
     "@sentry/serverless": "^6.16.1",
+    "@omnivore/content-handler": "1.0.0",
     "addressparser": "^1.0.1",
     "axios": "^0.27.2",
     "jsonwebtoken": "^8.5.1",

packages/inbound-email-handler/src/axios-handler.ts

@@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class AxiosHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /<.+@axios.com>/
-    this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
-    this.defaultUrl = 'https://axios.com'
-  }
-}

packages/inbound-email-handler/src/bloomberg-handler.ts

@@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class BloombergHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /<.+@mail.bloomberg.*.com>/
-    this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
-    this.defaultUrl = 'https://www.bloomberg.com'
-  }
-}

packages/inbound-email-handler/src/golang-handler.ts

@@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class GolangHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /<.+@golangweekly.com>/
-    this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
-    this.defaultUrl = 'https://golangweekly.com'
-  }
-}

packages/inbound-email-handler/src/index.ts

@@ -9,35 +9,27 @@ import * as multipart from 'parse-multipart-data'
 import {
   handleConfirmation,
   isConfirmationEmail,
-  NewsletterHandler,
   parseUnsubscribe,
 } from './newsletter'
 import { PubSub } from '@google-cloud/pubsub'
 import { handlePdfAttachment } from './pdf'
-import { SubstackHandler } from './substack-handler'
-import { AxiosHandler } from './axios-handler'
-import { BloombergHandler } from './bloomberg-handler'
-import { GolangHandler } from './golang-handler'
-import { MorningBrewHandler } from './morning-brew-handler'
+import { handleNewsletter } from '@omnivore/content-handler'
 
+const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
 const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived'
 const pubsub = new PubSub()
-const NEWSLETTER_HANDLERS = [
-  new SubstackHandler(),
-  new AxiosHandler(),
-  new BloombergHandler(),
-  new GolangHandler(),
-  new MorningBrewHandler(),
-]
 
-export const getNewsletterHandler = (
-  postHeader: string,
-  from: string,
-  unSubHeader: string
-): NewsletterHandler | undefined => {
-  return NEWSLETTER_HANDLERS.find((h) => {
-    return h.isNewsletter(postHeader, from, unSubHeader)
-  })
+export const publishMessage = async (
+  topic: string,
+  message: any
+): Promise<string | undefined> => {
+  return pubsub
+    .topic(topic)
+    .publishMessage({ json: message })
+    .catch((err) => {
+      console.log('error publishing message:', err)
+      return undefined
+    })
 }
 
 export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
@@ -86,23 +78,20 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       try {
         // check if it is a confirmation email or forwarding newsletter
-        const newsletterHandler = getNewsletterHandler(
-          postHeader,
+        const newsletterMessage = handleNewsletter({
           from,
-          unSubHeader
-        )
-
-        if (newsletterHandler) {
-          console.log('handleNewsletter', from, to)
-          await newsletterHandler.handleNewsletter(
-            to,
-            html,
-            postHeader,
-            subject,
-            from,
-            unSubHeader
+          html,
+          postHeader,
+          unSubHeader,
+          email: to,
+          title: subject,
+        })
+        if (newsletterMessage) {
+          await publishMessage(
+            NEWSLETTER_EMAIL_RECEIVED_TOPIC,
+            newsletterMessage
           )
-          return res.send('ok')
+          return res.status(200).send('newsletter received')
         }
 
         console.log('non-newsletter email from', from, 'to', to)

packages/inbound-email-handler/src/morning-brew-handler.ts

@@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class MorningBrewHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /Morning Brew <crew@morningbrew.com>/
-    this.urlRegex = /<a.* href=["']([^"']*)["'].*>View Online<\/a>/
-    this.defaultUrl = 'https://www.morningbrew.com'
-  }
-}

packages/inbound-email-handler/src/newsletter.ts

@@ -1,15 +1,12 @@
-import { PubSub } from '@google-cloud/pubsub'
-import { v4 as uuidv4 } from 'uuid'
 import addressparser from 'addressparser'
 import rfc2047 from 'rfc2047'
+import { publishMessage } from './index'
 
 interface Unsubscribe {
   mailTo?: string
   httpUrl?: string
 }
 
-const pubsub = new PubSub()
-const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
 const EMAIL_CONFIRMATION_CODE_RECEIVED_TOPIC = 'emailConfirmationCodeReceived'
 const CONFIRMATION_EMAIL_SENDER_ADDRESS = 'forwarding-noreply@google.com'
 // check unicode parentheses too
@@ -35,73 +32,6 @@ const parseAddress = (address: string): string => {
   return ''
 }
 
-export class NewsletterHandler {
-  protected senderRegex = /NEWSLETTER_SENDER_REGEX/
-  protected urlRegex = /NEWSLETTER_URL_REGEX/
-  protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
-
-  parseNewsletterUrl(_postHeader: string, html: string): string | undefined {
-    // get newsletter url from html
-    const matches = html.match(this.urlRegex)
-    if (matches) {
-      return matches[1]
-    }
-    return undefined
-  }
-
-  parseAuthor(from: string): string {
-    // get author name from email
-    // e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
-    // or 'Mike Allen <mike@axios.com>'
-    const parsed = addressparser(from)
-    if (parsed.length > 0) {
-      return parsed[0].name
-    }
-    return from
-  }
-
-  async handleNewsletter(
-    email: string,
-    html: string,
-    postHeader: string,
-    title: string,
-    from: string,
-    unSubHeader: string
-  ): Promise<string | undefined> {
-    console.log('handleNewsletter', email, postHeader, title, from)
-
-    if (!email || !html || !title || !from) {
-      console.log('invalid newsletter email')
-      throw new Error('invalid newsletter email')
-    }
-
-    // fallback to default url if newsletter url does not exist
-    // assign a random uuid to the default url to avoid duplicate url
-    const url =
-      this.parseNewsletterUrl(postHeader, html) ||
-      `${this.defaultUrl}?source=newsletters&id=${uuidv4()}`
-    const author = this.parseAuthor(from)
-    const unsubscribe = parseUnsubscribe(unSubHeader)
-    const message = {
-      email,
-      content: html,
-      url,
-      title,
-      author,
-      unsubMailTo: unsubscribe.mailTo || '',
-      unsubHttpUrl: unsubscribe.httpUrl || '',
-    }
-
-    return publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, message)
-  }
-}
-
 export const handleConfirmation = async (email: string, subject: string) => {
   console.log('confirmation email', email, subject)
 
@@ -136,16 +66,3 @@ export const isConfirmationEmail = (from: string, subject: string): boolean => {
     CONFIRMATION_CODE_PATTERN.test(subject)
   )
 }
-
-const publishMessage = async (
-  topic: string,
-  message: Record<string, string>
-): Promise<string | undefined> => {
-  return pubsub
-    .topic(topic)
-    .publishMessage({ json: message })
-    .catch((err) => {
-      console.log('error publishing message:', err)
-      return undefined
-    })
-}

packages/inbound-email-handler/src/substack-handler.ts

@@ -1,25 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-import addressparser from 'addressparser'
-
-export class SubstackHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.defaultUrl = 'https://www.substack.com'
-  }
-
-  parseNewsletterUrl(postHeader: string, _html: string): string | undefined {
-    // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
-    // we need to get the real url from the raw url
-    return addressparser(postHeader).length > 0
-      ? addressparser(postHeader)[0].name
-      : undefined
-  }
-
-  isNewsletter(
-    postHeader: string,
-    _from: string,
-    _unSubHeader: string
-  ): boolean {
-    return !!postHeader
-  }
-}

packages/inbound-email-handler/test/newsletter.test.ts

@@ -2,15 +2,8 @@ import { expect } from 'chai'
 import {
   getConfirmationCode,
   isConfirmationEmail,
-  NewsletterHandler,
   parseUnsubscribe,
 } from '../src/newsletter'
-import { SubstackHandler } from '../src/substack-handler'
-import { AxiosHandler } from '../src/axios-handler'
-import { BloombergHandler } from '../src/bloomberg-handler'
-import { GolangHandler } from '../src/golang-handler'
-import { getNewsletterHandler } from '../src'
-import { MorningBrewHandler } from '../src/morning-brew-handler'
 
 describe('Confirmation email test', () => {
   describe('#isConfirmationEmail()', () => {
@@ -54,126 +47,6 @@ describe('Confirmation email test', () => {
 })
 
 describe('Newsletter email test', () => {
-  describe('#getNewsletterHandler()', () => {
-    it('returns SubstackHandler when email is from SubStack', () => {
-      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
-
-      expect(getNewsletterHandler(rawUrl, '', '')).to.be.instanceof(
-        SubstackHandler
-      )
-    })
-
-    it('returns AxiosHandler when email is from Axios', () => {
-      const from = 'Mike Allen <mike@axios.com>'
-      const unSubRawUrl =
-        '<https://axios.com/unsubscribe?email=mike%40axios.com&code=593781109>'
-
-      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-        AxiosHandler
-      )
-    })
-
-    context('when email is from Bloomberg', () => {
-      it('should return BloombergHandler when email is from Bloomberg Business', () => {
-        const from = 'From: Bloomberg <noreply@mail.bloombergbusiness.com>'
-        const unSubRawUrl = '<https://bloomberg.com/unsubscribe>'
-
-        expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-          BloombergHandler
-        )
-      })
-
-      it('should return BloombergHandler when email is from Bloomberg View', () => {
-        const from = 'From: Bloomberg <noreply@mail.bloombergview.com>'
-        const unSubRawUrl = '<https://bloomberg.com/unsubscribe>'
-
-        expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-          BloombergHandler
-        )
-      })
-    })
-
-    it('should return GolangHandler when email is from Golang Weekly', () => {
-      const from = 'Golang Weekly <peter@golangweekly.com>'
-      const unSubRawUrl = '<https://golangweekly.com/unsubscribe>'
-
-      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-        GolangHandler
-      )
-    })
-
-    it('should return MorningBrewHandler when email is from Morning Brew', () => {
-      const from = 'Morning Brew <crew@morningbrew.com>'
-      const unSubRawUrl = '<https://morningbrew.com/unsubscribe>'
-
-      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-        MorningBrewHandler
-      )
-    })
-  })
-
-  describe('#getNewsletterUrl()', () => {
-    it('returns url when email is from SubStack', () => {
-      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
-
-      expect(new SubstackHandler().parseNewsletterUrl(rawUrl, '')).to.equal(
-        'https://hongbo130.substack.com/p/tldr'
-      )
-    })
-
-    it('returns url when email is from Axios', () => {
-      const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app'
-      const html = `View in browser at <a>${url}</a>`
-
-      expect(new AxiosHandler().parseNewsletterUrl('', html)).to.equal(url)
-    })
-
-    it('returns url when email is from Bloomberg', () => {
-      const url = 'https://www.bloomberg.com/news/google-is-now-a-partner'
-      const html = `
-        <a class="view-in-browser__url" href="${url}">
-        View in browser
-        </a>
-      `
-
-      expect(new BloombergHandler().parseNewsletterUrl('', html)).to.equal(url)
-    })
-
-    it('returns url when email is from Golang Weekly', () => {
-      const url = 'https://www.golangweekly.com/first'
-      const html = `
-        <a href="${url}" style="text-decoration: none">Read on the Web</a>
-      `
-
-      expect(new GolangHandler().parseNewsletterUrl('', html)).to.equal(url)
-    })
-
-    it('returns url when email is from Morning Brew', () => {
-      const url = 'https://www.morningbrew.com/daily/issues/first'
-      const html = `
-        <a style="color: #000000; text-decoration: none;" target="_blank" rel="noopener" href="${url}">View Online</a>
-      `
-
-      expect(new MorningBrewHandler().parseNewsletterUrl('', html)).to.equal(
-        url
-      )
-    })
-  })
-
-  describe('get author from email address', () => {
-    it('returns author when email is from Substack', () => {
-      const from = 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
-      expect(new NewsletterHandler().parseAuthor(from)).to.equal(
-        'Jackson Harper from Omnivore App'
-      )
-    })
-
-    it('returns author when email is from Axios', () => {
-      const from = 'Mike Allen <mike@axios.com>'
-      expect(new NewsletterHandler().parseAuthor(from)).to.equal('Mike Allen')
-    })
-  })
-
   describe('get unsubscribe from header', () => {
     const mailTo = 'unsub@omnivore.com'
     const httpUrl = 'https://omnivore.com/unsubscribe'