Add support for Golang Weekly newsletters (#96)

* abstract different newsletter handler to a class

* create a generic newsletter handler class

* add support for golang-weekly newsletters

* add tests for golang weekly newsletters

* use addressparser lib to parse author from from address

packages/inbound-email-handler/package.json

@@ -20,6 +20,7 @@
     "deploy": "yarn build && yarn gcloud-deploy"
   },
   "devDependencies": {
+    "@types/addressparser": "^1.0.1",
     "@types/json-bigint": "^1.0.1",
     "@types/node": "^14.11.2",
     "eslint-plugin-prettier": "^4.0.0"
@@ -29,6 +30,7 @@
     "@google-cloud/pubsub": "^2.18.4",
     "@sendgrid/client": "^7.6.0",
     "@sentry/serverless": "^6.16.1",
+    "addressparser": "^1.0.1",
     "axios": "^0.26.0",
     "jsonwebtoken": "^8.5.1",
     "parse-headers": "^2.0.4",

packages/inbound-email-handler/src/axios-handler.ts

@@ -0,0 +1,9 @@
+import { NewsletterHandler } from './newsletter'
+
+export class AxiosHandler extends NewsletterHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@axios.com>/
+    this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
+  }
+}

packages/inbound-email-handler/src/bloomberg-handler.ts

@@ -0,0 +1,9 @@
+import { NewsletterHandler } from './newsletter'
+
+export class BloombergHandler extends NewsletterHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@mail.bloombergbusiness.com>/
+    this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
+  }
+}

packages/inbound-email-handler/src/golang-handler.ts

@@ -0,0 +1,9 @@
+import { NewsletterHandler } from './newsletter'
+
+export class GolangHandler extends NewsletterHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@golangweekly.com>/
+    this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
+  }
+}

packages/inbound-email-handler/src/index.ts

@@ -8,16 +8,35 @@ import parseHeaders from 'parse-headers'
 import * as multipart from 'parse-multipart-data'
 import {
   handleConfirmation,
-  handleNewsletter,
   isConfirmationEmail,
-  isNewsletter,
+  NewsletterHandler,
 } from './newsletter'
 import { PubSub } from '@google-cloud/pubsub'
 import { handlePdfAttachment } from './pdf'
+import { SubstackHandler } from './substack-handler'
+import { AxiosHandler } from './axios-handler'
+import { BloombergHandler } from './bloomberg-handler'
+import { GolangHandler } from './golang-handler'
 
 const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived'
 const pubsub = new PubSub()
 
+const NEWSLETTER_HANDLERS = [
+  new SubstackHandler(),
+  new AxiosHandler(),
+  new BloombergHandler(),
+  new GolangHandler(),
+]
+
+export const getNewsletterHandler = (
+  rawUrl: string,
+  from: string
+): NewsletterHandler | undefined => {
+  return NEWSLETTER_HANDLERS.find((h) => {
+    return h.isNewsletter(rawUrl, from)
+  })
+}
+
 export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
   async (req, res) => {
     const parts = multipart.parse(req.body, 'xYzZY')
@@ -55,10 +74,17 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
       const rawUrl = headers['list-post'] ? headers['list-post'].toString() : ''
 
       // check if it is a forwarding confirmation email or newsletter
-      if (isNewsletter(rawUrl, from)) {
+      const newsletterHandler = getNewsletterHandler(rawUrl, from)
+      if (newsletterHandler) {
         try {
           console.log('handleNewsletter', from, recipientAddress)
-          await handleNewsletter(recipientAddress, html, rawUrl, subject, from)
+          await newsletterHandler.handleNewsletter(
+            recipientAddress,
+            html,
+            rawUrl,
+            subject,
+            from
+          )
         } catch (error) {
           console.log(
             'error handling newsletter, will forward.',

packages/inbound-email-handler/src/newsletter.ts

@@ -1,4 +1,5 @@
 import { PubSub } from '@google-cloud/pubsub'
+import addressparser from 'addressparser'
 
 const pubsub = new PubSub()
 const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
@@ -6,12 +7,71 @@ const EMAIL_CONFIRMATION_CODE_RECEIVED_TOPIC = 'emailConfirmationCodeReceived'
 const EMAIL_FORWARDING_SENDER_ADDRESSES = [
   'Gmail Team <forwarding-noreply@google.com>',
 ]
-const NEWSLETTER_SENDER_REGEX =
-  /<.+@((axios.com)|(mail.bloombergbusiness.com))>/
 const CONFIRMATION_CODE_PATTERN = /^\\(#\\d+\\)/
-const AXIOS_URL_PATTERN = /View in browser at <a.*>(.*)<\/a>/
-const BLOOMBERG_URL_PATTERN =
-  /<a class="view-in-browser__url" href=["']([^"']*)["']/
+
+export class NewsletterHandler {
+  protected senderRegex = /NEWSLETTER_SENDER_REGEX/
+  protected urlRegex = /NEWSLETTER_URL_REGEX/
+
+  isNewsletter(_rawUrl: string, from: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from)
+  }
+
+  getNewsletterUrl(rawUrl: string, html: string): string | undefined {
+    // get newsletter url from html
+    const matches = html.match(this.urlRegex)
+    if (matches) {
+      return matches[1]
+    }
+    return undefined
+  }
+
+  getAuthor(from: string): string {
+    // get author name from email
+    // e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
+    // or 'Mike Allen <mike@axios.com>'
+    const parsed = addressparser(from)
+    if (parsed.length > 0) {
+      return parsed[0].name
+    }
+    return from
+  }
+
+  async handleNewsletter(
+    email: string,
+    html: string,
+    rawUrl: string,
+    title: string,
+    from: string
+  ): Promise<[string] | undefined> {
+    console.log('handleNewsletter', email, rawUrl, title, from)
+
+    if (!email || !html || !title || !from) {
+      console.log('invalid newsletter email')
+      throw new Error('invalid newsletter email')
+    }
+
+    const url = this.getNewsletterUrl(rawUrl, html)
+    console.log('url', url)
+    if (!url) {
+      console.log('invalid newsletter url', url)
+      throw new Error('invalid newsletter url')
+    }
+
+    const author = this.getAuthor(from)
+
+    const message = {
+      email: email,
+      content: html,
+      url: url,
+      title: title,
+      author: author,
+    }
+    return publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, message)
+  }
+}
 
 export const handleConfirmation = async (email: string, subject: string) => {
   console.log('confirmation email')
@@ -38,49 +98,6 @@ export const handleConfirmation = async (email: string, subject: string) => {
   return publishMessage(EMAIL_CONFIRMATION_CODE_RECEIVED_TOPIC, message)
 }
 
-export const handleNewsletter = async (
-  email: string,
-  html: string,
-  rawUrl: string,
-  title: string,
-  from: string
-) => {
-  console.log('handleNewsletter', email, rawUrl, title, from)
-
-  if (!email || !html || !title || !from) {
-    console.log('invalid newsletter email')
-    throw new Error('invalid newsletter email')
-  }
-
-  const url = getNewsletterUrl(rawUrl, html)
-  console.log('url', url)
-  if (!url) {
-    console.log('invalid newsletter url', url)
-    throw new Error('invalid newsletter url')
-  }
-
-  // get author name from email
-  // e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
-  // or 'Mike Allen <mike@axios.com>'
-  const authors = from.includes(' from ')
-    ? from.split(' from')
-    : from.split(' <')
-  if (!authors) {
-    console.log('invalid from', from)
-    throw new Error('invalid from')
-  }
-  const author = authors[0]
-
-  const message = {
-    email: email,
-    content: html,
-    url: url,
-    title: title,
-    author: author,
-  }
-  return publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, message)
-}
-
 const publishMessage = async (
   topic: string,
   message: Record<string, string>
@@ -94,38 +111,6 @@ const publishMessage = async (
     })
 }
 
-// SubStack newsletter has raw Url in the email
-// url is like <https://hongbo130.substack.com/p/tldr>
-// Axios newsletter is from <xx@axios.com>
-export const isNewsletter = (rawUrl: string, from: string): boolean => {
-  const re = new RegExp(NEWSLETTER_SENDER_REGEX)
-  return !!rawUrl || re.test(from)
-}
-
 export const isConfirmationEmail = (from: string): boolean => {
   return EMAIL_FORWARDING_SENDER_ADDRESSES.includes(from)
 }
-
-export const getNewsletterUrl = (
-  rawUrl: string,
-  html: string
-): string | undefined => {
-  // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
-  // we need to get the real url
-  if (rawUrl.startsWith('<')) {
-    return rawUrl.slice(1, -1)
-  }
-
-  // axios newsletter url from html
-  let matches = html.match(AXIOS_URL_PATTERN)
-  if (matches) {
-    return matches[1]
-  }
-
-  // bloomberg newsletter url from html
-  matches = html.match(BLOOMBERG_URL_PATTERN)
-  if (matches) {
-    return matches[1]
-  }
-  return undefined
-}

packages/inbound-email-handler/src/substack-handler.ts

@@ -0,0 +1,13 @@
+import { NewsletterHandler } from './newsletter'
+
+export class SubstackHandler extends NewsletterHandler {
+  getNewsletterUrl(rawUrl: string, _html: string): string | undefined {
+    // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
+    // we need to get the real url
+    return rawUrl.slice(1, -1)
+  }
+
+  isNewsletter(rawUrl: string, _from: string): boolean {
+    return !!rawUrl
+  }
+}

packages/inbound-email-handler/test/newsletter.test.ts

@@ -1,9 +1,10 @@
 import { expect } from 'chai'
-import {
-  getNewsletterUrl,
-  isConfirmationEmail,
-  isNewsletter,
-} from '../src/newsletter'
+import { isConfirmationEmail, NewsletterHandler } from '../src/newsletter'
+import { SubstackHandler } from '../src/substack-handler'
+import { AxiosHandler } from '../src/axios-handler'
+import { BloombergHandler } from '../src/bloomberg-handler'
+import { GolangHandler } from '../src/golang-handler'
+import { getNewsletterHandler } from '../src'
 
 describe('Confirmation email test', () => {
   describe('#isConfirmationEmail()', () => {
@@ -16,22 +17,27 @@ describe('Confirmation email test', () => {
 })
 
 describe('Newsletter email test', () => {
-  describe('#isNewsletter()', () => {
-    it('returns true when email is from SubStack', () => {
+  describe('#getNewsletterHandler()', () => {
+    it('returns SubstackHandler when email is from SubStack', () => {
       const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
 
-      expect(isNewsletter(rawUrl, '')).to.be.true
+      expect(getNewsletterHandler(rawUrl, '')).to.be.instanceof(SubstackHandler)
     })
 
-    it('returns true when email is from Axios', () => {
+    it('returns AxiosHandler when email is from Axios', () => {
       const from = 'Mike Allen <mike@axios.com>'
 
-      expect(isNewsletter('', from)).to.be.true
+      expect(getNewsletterHandler('', from)).to.be.instanceof(AxiosHandler)
     })
 
-    it('should return true when email is from bloomberg', () => {
+    it('should return BloombergHandler when email is from Bloomberg', () => {
       const from = 'From: Bloomberg <noreply@mail.bloombergbusiness.com>'
-      expect(isNewsletter('', from)).to.be.true
+      expect(getNewsletterHandler('', from)).to.be.instanceof(BloombergHandler)
+    })
+
+    it('should return GolangHandler when email is from Golang Weekly', () => {
+      const from = 'Golang Weekly <peter@golangweekly.com>'
+      expect(getNewsletterHandler('', from)).to.be.instanceof(GolangHandler)
     })
   })
 
@@ -39,31 +45,50 @@ describe('Newsletter email test', () => {
     it('returns url when email is from SubStack', () => {
       const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
 
-      expect(getNewsletterUrl(rawUrl, '')).to.equal(
+      expect(new SubstackHandler().getNewsletterUrl(rawUrl, '')).to.equal(
         'https://hongbo130.substack.com/p/tldr'
       )
     })
 
     it('returns url when email is from Axios', () => {
-      const rawUrl = ''
-      const html = `View in browser at <a>https://axios.com/blog/the-best-way-to-build-a-web-app</a>`
+      const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app'
+      const html = `View in browser at <a>${url}</a>`
 
-      expect(getNewsletterUrl(rawUrl, html)).to.equal(
-        'https://axios.com/blog/the-best-way-to-build-a-web-app'
-      )
+      expect(new AxiosHandler().getNewsletterUrl('', html)).to.equal(url)
     })
 
     it('returns url when email is from Bloomberg', () => {
-      const rawUrl = ''
+      const url = 'https://www.bloomberg.com/news/google-is-now-a-partner'
       const html = `
-        <a class="view-in-browser__url" href="https://www.bloomberg.com/news/google-is-now-a-partner">
+        <a class="view-in-browser__url" href="${url}">
         View in browser
         </a>
       `
 
-      expect(getNewsletterUrl(rawUrl, html)).to.equal(
-        'https://www.bloomberg.com/news/google-is-now-a-partner'
+      expect(new BloombergHandler().getNewsletterUrl('', html)).to.equal(url)
+    })
+
+    it('returns url when email is from Golang Weekly', () => {
+      const url = 'https://www.golangweekly.com/first'
+      const html = `
+        <a href="${url}" style="text-decoration: none">Read on the Web</a>
+      `
+
+      expect(new GolangHandler().getNewsletterUrl('', html)).to.equal(url)
+    })
+  })
+
+  describe('get author from email address', () => {
+    it('returns author when email is from Substack', () => {
+      const from = 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
+      expect(new NewsletterHandler().getAuthor(from)).to.equal(
+        'Jackson Harper from Omnivore App'
       )
     })
+
+    it('returns author when email is from Axios', () => {
+      const from = 'Mike Allen <mike@axios.com>'
+      expect(new NewsletterHandler().getAuthor(from)).to.equal('Mike Allen')
+    })
   })
 })

yarn.lock

@@ -4195,6 +4195,13 @@
   dependencies:
     "@types/node" "*"
 
+"@types/addressparser@^1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@types/addressparser/-/addressparser-1.0.1.tgz#e34e8ca186c96a5bc3d9bfbf93a7258d7a03b413"
+  integrity sha512-iw+cRQy5XcLGWhyyuYJ2Fnu8dyJ1y0QoaODfnbJCRzv/qYtAWH5yK6H688kGIyqm1VowwKUlA8mTH9qUkPOd7A==
+  dependencies:
+    "@types/node" "*"
+
 "@types/analytics-node@^3.1.7":
   version "3.1.7"
   resolved "https://registry.yarnpkg.com/@types/analytics-node/-/analytics-node-3.1.7.tgz#cb97c80ee505094e44a0188c3ad25f70c67e3c65"
@@ -5025,6 +5032,11 @@ add-stream@^1.0.0:
   resolved "https://registry.yarnpkg.com/add-stream/-/add-stream-1.0.0.tgz#6a7990437ca736d5e1288db92bd3266d5f5cb2aa"
   integrity sha1-anmQQ3ynNtXhKI25K9MmbV9csqo=
 
+addressparser@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746"
+  integrity sha1-R6++GiqSYhkdtoOOT9HTm0CCF0Y=
+
 agent-base@6:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.1.tgz#808007e4e5867decb0ab6ab2f928fbdb5a596db4"