identify a newsletter by checking both list-url and unsubscribe-url in headers and email sender address

packages/api/src/services/newsletters.ts

@@ -3,9 +3,7 @@ import { nanoid } from 'nanoid'
 import { User } from '../entity/user'
 import { CreateNewsletterEmailErrorCode } from '../generated/graphql'
 import { env } from '../env'
-import { AppDataSource } from '../server'
 import { getRepository } from '../entity/utils'
-
 import addressparser = require('nodemailer/lib/addressparser')
 
 const parsedAddress = (emailAddress: string): string | undefined => {

packages/inbound-email-handler/src/bloomberg-handler.ts

@@ -5,6 +5,6 @@ export class BloombergHandler extends NewsletterHandler {
     super()
     this.senderRegex = /<.+@mail.bloomberg.*.com>/
     this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
-    this.defaultUrl = 'https://www.bloomberg.com/'
+    this.defaultUrl = 'https://www.bloomberg.com'
   }
 }

packages/inbound-email-handler/src/golang-handler.ts

@@ -5,6 +5,6 @@ export class GolangHandler extends NewsletterHandler {
     super()
     this.senderRegex = /<.+@golangweekly.com>/
     this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
-    this.defaultUrl = 'https://golangweekly.com/'
+    this.defaultUrl = 'https://golangweekly.com'
   }
 }

packages/inbound-email-handler/src/index.ts

@@ -30,10 +30,11 @@ const NEWSLETTER_HANDLERS = [
 
 export const getNewsletterHandler = (
   rawUrl: string,
-  from: string
+  from: string,
+  rawUnSubUrl: string
 ): NewsletterHandler | undefined => {
   return NEWSLETTER_HANDLERS.find((h) => {
-    return h.isNewsletter(rawUrl, from)
+    return h.isNewsletter(rawUrl, from, rawUnSubUrl)
   })
 }
 
@@ -72,9 +73,12 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
         ? forwardedAddress.toString()
         : parsed.to
       const rawUrl = headers['list-post'] ? headers['list-post'].toString() : ''
+      const rawUnSubUrl = headers['list-unsubscribe']
+        ? headers['list-unsubscribe'].toString()
+        : ''
 
       // check if it is a forwarding confirmation email or newsletter
-      const newsletterHandler = getNewsletterHandler(rawUrl, from)
+      const newsletterHandler = getNewsletterHandler(rawUrl, from, rawUnSubUrl)
       try {
         if (newsletterHandler) {
           console.log('handleNewsletter', from, recipientAddress)

packages/inbound-email-handler/src/newsletter.ts

@@ -1,4 +1,5 @@
 import { PubSub } from '@google-cloud/pubsub'
+import { v4 as uuidv4 } from 'uuid'
 import addressparser from 'addressparser'
 
 const pubsub = new PubSub()
@@ -14,10 +15,10 @@ export class NewsletterHandler {
   protected urlRegex = /NEWSLETTER_URL_REGEX/
   protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
 
-  isNewsletter(_rawUrl: string, from: string): boolean {
+  isNewsletter(rawUrl: string, from: string, rawUnSubUrl: string): boolean {
     // Axios newsletter is from <xx@axios.com>
     const re = new RegExp(this.senderRegex)
-    return re.test(from)
+    return re.test(from) && (!!rawUrl || !!rawUnSubUrl)
   }
 
   getNewsletterUrl(_rawUrl: string, html: string): string | undefined {
@@ -55,8 +56,11 @@ export class NewsletterHandler {
     }
 
     // fallback to default url if newsletter url does not exist
-    const url = this.getNewsletterUrl(rawUrl, html) || this.defaultUrl
-    const author = this.getAuthor(from)
+    // assign a random uuid to the default url to avoid duplicate url
+    const url =
+      this.getNewsletterUrl(rawUrl, html) ||
+      `${this.defaultUrl}?source=newsletters&id=${uuidv4()}`
+    const author = this.getAuthor(from) || 'Unknown'
 
     const message = {
       email: email,

packages/inbound-email-handler/src/substack-handler.ts

@@ -4,7 +4,7 @@ import addressparser from 'addressparser'
 export class SubstackHandler extends NewsletterHandler {
   constructor() {
     super()
-    this.defaultUrl = 'https://www.substack.com/'
+    this.defaultUrl = 'https://www.substack.com'
   }
 
   getNewsletterUrl(rawUrl: string, _html: string): string | undefined {
@@ -15,7 +15,7 @@ export class SubstackHandler extends NewsletterHandler {
       : undefined
   }
 
-  isNewsletter(rawUrl: string, _from: string): boolean {
-    return !!rawUrl
+  isNewsletter(rawUrl: string, _from: string, rawUnSubUrl: string): boolean {
+    return !!rawUrl || !!rawUnSubUrl
   }
 }