Merge pull request #388 from omnivore-app/fix/confirmation-email-not-forward

identify a newsletter by checking both list-url and unsubscribe-url i…

packages/api/src/services/newsletters.ts

@@ -3,9 +3,7 @@ import { nanoid } from 'nanoid'
 import { User } from '../entity/user'
 import { CreateNewsletterEmailErrorCode } from '../generated/graphql'
 import { env } from '../env'
-import { AppDataSource } from '../server'
 import { getRepository } from '../entity/utils'
-
 import addressparser = require('nodemailer/lib/addressparser')
 
 const parsedAddress = (emailAddress: string): string | undefined => {

packages/inbound-email-handler/src/bloomberg-handler.ts

@@ -5,6 +5,6 @@ export class BloombergHandler extends NewsletterHandler {
     super()
     this.senderRegex = /<.+@mail.bloomberg.*.com>/
     this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
-    this.defaultUrl = 'https://www.bloomberg.com/'
+    this.defaultUrl = 'https://www.bloomberg.com'
   }
 }

packages/inbound-email-handler/src/golang-handler.ts

@@ -5,6 +5,6 @@ export class GolangHandler extends NewsletterHandler {
     super()
     this.senderRegex = /<.+@golangweekly.com>/
     this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
-    this.defaultUrl = 'https://golangweekly.com/'
+    this.defaultUrl = 'https://golangweekly.com'
   }
 }

packages/inbound-email-handler/src/index.ts

@@ -30,10 +30,11 @@ const NEWSLETTER_HANDLERS = [
 
 export const getNewsletterHandler = (
   rawUrl: string,
-  from: string
+  from: string,
+  unSubRawUrl: string
 ): NewsletterHandler | undefined => {
   return NEWSLETTER_HANDLERS.find((h) => {
-    return h.isNewsletter(rawUrl, from)
+    return h.isNewsletter(rawUrl, from, unSubRawUrl)
   })
 }
 
@@ -72,9 +73,12 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
         ? forwardedAddress.toString()
         : parsed.to
       const rawUrl = headers['list-post'] ? headers['list-post'].toString() : ''
+      const unSubRawUrl = headers['list-unsubscribe']
+        ? headers['list-unsubscribe'].toString()
+        : ''
 
       // check if it is a forwarding confirmation email or newsletter
-      const newsletterHandler = getNewsletterHandler(rawUrl, from)
+      const newsletterHandler = getNewsletterHandler(rawUrl, from, unSubRawUrl)
       try {
         if (newsletterHandler) {
           console.log('handleNewsletter', from, recipientAddress)

packages/inbound-email-handler/src/newsletter.ts

@@ -1,4 +1,5 @@
 import { PubSub } from '@google-cloud/pubsub'
+import { v4 as uuidv4 } from 'uuid'
 import addressparser from 'addressparser'
 
 const pubsub = new PubSub()
@@ -14,10 +15,10 @@ export class NewsletterHandler {
   protected urlRegex = /NEWSLETTER_URL_REGEX/
   protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
 
-  isNewsletter(_rawUrl: string, from: string): boolean {
+  isNewsletter(rawUrl: string, from: string, unSubRawUrl: string): boolean {
     // Axios newsletter is from <xx@axios.com>
     const re = new RegExp(this.senderRegex)
-    return re.test(from)
+    return re.test(from) && (!!rawUrl || !!unSubRawUrl)
   }
 
   getNewsletterUrl(_rawUrl: string, html: string): string | undefined {
@@ -55,8 +56,11 @@ export class NewsletterHandler {
     }
 
     // fallback to default url if newsletter url does not exist
-    const url = this.getNewsletterUrl(rawUrl, html) || this.defaultUrl
-    const author = this.getAuthor(from)
+    // assign a random uuid to the default url to avoid duplicate url
+    const url =
+      this.getNewsletterUrl(rawUrl, html) ||
+      `${this.defaultUrl}?source=newsletters&id=${uuidv4()}`
+    const author = this.getAuthor(from) || 'Unknown'
 
     const message = {
       email: email,

packages/inbound-email-handler/src/substack-handler.ts

@@ -4,7 +4,7 @@ import addressparser from 'addressparser'
 export class SubstackHandler extends NewsletterHandler {
   constructor() {
     super()
-    this.defaultUrl = 'https://www.substack.com/'
+    this.defaultUrl = 'https://www.substack.com'
   }
 
   getNewsletterUrl(rawUrl: string, _html: string): string | undefined {
@@ -15,7 +15,7 @@ export class SubstackHandler extends NewsletterHandler {
       : undefined
   }
 
-  isNewsletter(rawUrl: string, _from: string): boolean {
+  isNewsletter(rawUrl: string, _from: string, _unSubRawUrl: string): boolean {
     return !!rawUrl
   }
 }

packages/inbound-email-handler/test/newsletter.test.ts

@@ -34,26 +34,36 @@ describe('Newsletter email test', () => {
     it('returns SubstackHandler when email is from SubStack', () => {
       const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
 
-      expect(getNewsletterHandler(rawUrl, '')).to.be.instanceof(SubstackHandler)
+      expect(getNewsletterHandler(rawUrl, '', '')).to.be.instanceof(
+        SubstackHandler
+      )
     })
 
     it('returns AxiosHandler when email is from Axios', () => {
       const from = 'Mike Allen <mike@axios.com>'
+      const unSubRawUrl =
+        '<https://axios.com/unsubscribe?email=mike%40axios.com&code=593781109>'
 
-      expect(getNewsletterHandler('', from)).to.be.instanceof(AxiosHandler)
+      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
+        AxiosHandler
+      )
     })
 
     context('when email is from Bloomberg', () => {
       it('should return BloombergHandler when email is from Bloomberg Business', () => {
         const from = 'From: Bloomberg <noreply@mail.bloombergbusiness.com>'
-        expect(getNewsletterHandler('', from)).to.be.instanceof(
+        const unSubRawUrl = '<https://bloomberg.com/unsubscribe>'
+
+        expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
           BloombergHandler
         )
       })
 
       it('should return BloombergHandler when email is from Bloomberg View', () => {
         const from = 'From: Bloomberg <noreply@mail.bloombergview.com>'
-        expect(getNewsletterHandler('', from)).to.be.instanceof(
+        const unSubRawUrl = '<https://bloomberg.com/unsubscribe>'
+
+        expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
           BloombergHandler
         )
       })
@@ -61,7 +71,11 @@ describe('Newsletter email test', () => {
 
     it('should return GolangHandler when email is from Golang Weekly', () => {
       const from = 'Golang Weekly <peter@golangweekly.com>'
-      expect(getNewsletterHandler('', from)).to.be.instanceof(GolangHandler)
+      const unSubRawUrl = '<https://golangweekly.com/unsubscribe>'
+
+      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
+        GolangHandler
+      )
     })
   })