add support for axios newsletters (#49)

* add support for axios newsletters

* fix raw url

* fix getting newsletter url from axios

* add test for url

* add axios parser

packages/api/src/routers/svc/newsletters.ts

@@ -116,7 +116,6 @@ export function newsletterServiceRouter() {
       if (
         !('email' in data) ||
         !('content' in data) ||
-        !('url' in data) ||
         !('title' in data) ||
         !('author' in data)
       ) {

packages/api/src/utils/axios-handler.ts

@@ -0,0 +1,36 @@
+import { DOMWindow } from 'jsdom'
+
+export class AxiosHandler {
+  name = 'axios'
+
+  shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => {
+    const host = this.name + '.com'
+    // check if url ends with axios.com
+    return url.hostname.endsWith(host)
+  }
+
+  prehandle = (url: URL, dom: DOMWindow): Promise<DOMWindow> => {
+    const body = dom.document.querySelector('table')
+
+    // this removes ads and replaces table with a div
+    body?.querySelectorAll('table').forEach((el, k) => {
+      if (k > 0) {
+        el.remove()
+      } else {
+        // remove the last two rows of the table (they are ads)
+        el.querySelectorAll('tr').forEach((tr, i) => {
+          if (i >= el.querySelectorAll('tr').length - 2) {
+            console.log('removing', tr)
+            tr.remove()
+          }
+        })
+        // replace the table with a div
+        const div = dom.document.createElement('div')
+        div.innerHTML = el.innerHTML
+        el.parentNode?.replaceChild(div, el)
+      }
+    })
+
+    return Promise.resolve(dom)
+  }
+}

packages/api/src/utils/parser.ts

@@ -10,6 +10,7 @@ import { createImageProxyUrl } from './imageproxy'
 import axios from 'axios'
 import { WikipediaHandler } from './wikipedia-handler'
 import { SubstackHandler } from './substack-handler'
+import { AxiosHandler } from './axios-handler'
 
 const logger = buildLogger('utils.parse')
 
@@ -40,7 +41,11 @@ interface ContentHandler {
   prehandle: (url: URL, document: DOMWindow) => Promise<DOMWindow>
 }
 
-const HANDLERS = [new WikipediaHandler(), new SubstackHandler()]
+const HANDLERS = [
+  new WikipediaHandler(),
+  new SubstackHandler(),
+  new AxiosHandler(),
+]
 
 /** Hook that prevents DOMPurify from removing youtube iframes */
 const domPurifySanitizeHook = (

packages/inbound-email-handler/src/index.ts

@@ -10,6 +10,7 @@ import {
   handleConfirmation,
   handleNewsletter,
   isConfirmationEmail,
+  isNewsletter,
 } from './newsletter'
 import { PubSub } from '@google-cloud/pubsub'
 
@@ -44,10 +45,10 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
       const recipientAddress = forwardedAddress
         ? forwardedAddress.toString()
         : parsed.to
-      const rawUrl = headers['list-post']?.toString()
+      const rawUrl = headers['list-post'] ? headers['list-post'].toString() : ''
 
       // check if it is a forwarding confirmation email or newsletter
-      if (rawUrl) {
+      if (isNewsletter(rawUrl, from)) {
         try {
           console.log('handleNewsletter', from, recipientAddress)
           await handleNewsletter(recipientAddress, html, rawUrl, subject, from)

packages/inbound-email-handler/src/newsletter.ts

@@ -6,8 +6,9 @@ const EMAIL_CONFIRMATION_CODE_RECEIVED_TOPIC = 'emailConfirmationCodeReceived'
 const EMAIL_FORWARDING_SENDER_ADDRESSES = [
   'Gmail Team <forwarding-noreply@google.com>',
 ]
-const NEWSLETTER_SENDER_REGEX = '<.+@substack.com>'
+const NEWSLETTER_SENDER_REGEX = '<.+@axios.com>'
 const CONFIRMATION_CODE_PATTERN = '^\\(#\\d+\\)'
+const AXIOS_URL_PATTERN = 'View in browser at <.+>'
 
 export const handleConfirmation = async (email: string, subject: string) => {
   console.log('confirmation email')
@@ -41,20 +42,26 @@ export const handleNewsletter = async (
   title: string,
   from: string
 ) => {
-  console.log('handleNewsletter')
+  console.log('handleNewsletter', email, rawUrl, title, from)
 
-  if (!email || !html || !rawUrl || !title || !from) {
+  if (!email || !html || !title || !from) {
     console.log('invalid newsletter email')
     throw new Error('invalid newsletter email')
   }
 
-  // raw newsletter url is like <https://hongbo130.substack.com/p/tldr>
-  // we need to get the real url
-  const url = rawUrl.slice(1, -1)
+  const url = getNewsletterUrl(rawUrl, html)
+  console.log('url', url)
+  if (!url) {
+    console.log('invalid newsletter url', url)
+    throw new Error('invalid newsletter url')
+  }
 
   // get author name from email
-  // e.g. 'Jackson Harper from Omnivore App'
-  const authors = from.split(' from ')
+  // e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
+  // or 'Mike Allen <mike@axios.com>'
+  const authors = from.includes(' from ')
+    ? from.split(' from')
+    : from.split(' <')
   if (!authors) {
     console.log('invalid from', from)
     throw new Error('invalid from')
@@ -84,11 +91,34 @@ const publishMessage = async (
     })
 }
 
-export const isNewsletter = (from: string, messageId: string): boolean => {
+// SubStack newsletter has raw Url in the email
+// url is like <https://hongbo130.substack.com/p/tldr>
+// Axios newsletter is from <xx@axios.com>
+export const isNewsletter = (rawUrl: string, from: string): boolean => {
   const re = new RegExp(NEWSLETTER_SENDER_REGEX)
-  return re.test(from) || messageId.includes('substack.com')
+  return !!rawUrl || re.test(from)
 }
 
 export const isConfirmationEmail = (from: string): boolean => {
   return EMAIL_FORWARDING_SENDER_ADDRESSES.includes(from)
 }
+
+export const getNewsletterUrl = (
+  rawUrl: string,
+  html: string
+): string | undefined => {
+  // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
+  // we need to get the real url
+  if (rawUrl.startsWith('<')) {
+    return rawUrl.slice(1, -1)
+  }
+
+  // axios newsletter url from html
+  const re = new RegExp(AXIOS_URL_PATTERN)
+  const matches = html.match(re)
+  if (matches) {
+    const match = matches[0]
+    return match.slice(match.indexOf('>') + 1, match.lastIndexOf('<'))
+  }
+  return undefined
+}

packages/inbound-email-handler/test/newsletter.test.ts

@@ -1,10 +1,14 @@
 import { expect } from 'chai'
-import { isConfirmationEmail, isNewsletter } from '../src/newsletter'
+import {
+  getNewsletterUrl,
+  isConfirmationEmail,
+  isNewsletter,
+} from '../src/newsletter'
 
 describe('Confirmation email test', () => {
   describe('#isConfirmationEmail()', () => {
     it('returns true when email is from Gmail Team', () => {
-      const from = `Gmail Team <forwarding-noreply@google.com>`
+      const from = 'Gmail Team <forwarding-noreply@google.com>'
 
       expect(isConfirmationEmail(from)).to.be.true
     })
@@ -13,10 +17,35 @@ describe('Confirmation email test', () => {
 
 describe('Newsletter email test', () => {
   describe('#isNewsletter()', () => {
-    it('returns true when email is from substack', () => {
-      const from = `Hongbo from Hongbo’s Newsletter <hongbo130@substack.com>`
+    it('returns true when email is from SubStack', () => {
+      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
 
-      expect(isNewsletter(from, '')).to.be.true
+      expect(isNewsletter(rawUrl, '')).to.be.true
+    })
+
+    it('returns true when email is from Axios', () => {
+      const from = 'Mike Allen <mike@axios.com>'
+
+      expect(isNewsletter('', from)).to.be.true
+    })
+  })
+
+  describe('#getNewsletterUrl()', () => {
+    it('returns url when email is from SubStack', () => {
+      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
+
+      expect(getNewsletterUrl(rawUrl, '')).to.equal(
+        'https://hongbo130.substack.com/p/tldr'
+      )
+    })
+
+    it('returns url when email is from Axios', () => {
+      const rawUrl = ''
+      const html = `View in browser at <a>https://axios.com/blog/2019/02/28/the-best-way-to-build-a-web-app</a>`
+
+      expect(getNewsletterUrl(rawUrl, html)).to.equal(
+        'https://axios.com/blog/2019/02/28/the-best-way-to-build-a-web-app'
+      )
     })
   })
 })