Save subscriptions when saving newsletters (#447)

* parse unsub mailTo or httpUrl and send to pubsub

* save subscription from newsletter

* fix tests

packages/api/index_settings.json

@@ -97,6 +97,9 @@
       },
       "siteName": {
         "type": "text"
+      },
+      "subscription": {
+        "type": "keyword"
       }
     }
   }

packages/api/src/elastic/types.ts

@@ -191,6 +191,9 @@ export interface Page {
   _id?: string
   siteIcon?: string
   highlights?: Highlight[]
+  subscription?: string
+  unsubMailTo?: string
+  unsubHttpUrl?: string
 }
 
 export interface SearchItem {

packages/api/src/services/save_email.ts

@@ -20,6 +20,8 @@ export type SaveEmailInput = {
   url: string
   title: string
   author: string
+  unsubMailTo?: string
+  unsubHttpUrl?: string
 }
 
 export const saveEmail = async (
@@ -66,6 +68,9 @@ export const saveEmail = async (
     createdAt: new Date(),
     readingProgressAnchorIndex: 0,
     readingProgressPercent: 0,
+    subscription: input.author,
+    unsubHttpUrl: input.unsubHttpUrl,
+    unsubMailTo: input.unsubMailTo,
   }
 
   const page = await getPageByParam({ userId: ctx.uid, url: articleToSave.url })

packages/api/src/services/save_newsletter_email.ts

@@ -10,6 +10,7 @@ import { SaveContext, saveEmail, SaveEmailInput } from './save_email'
 import { getDeviceTokensByUserId } from './user_device_tokens'
 import { Page } from '../elastic/types'
 import { addLabelToPage } from './labels'
+import { saveSubscription } from './subscriptions'
 
 interface NewsletterMessage {
   email: string
@@ -17,6 +18,8 @@ interface NewsletterMessage {
   url: string
   title: string
   author: string
+  unsubMailTo?: string
+  unsubHttpUrl?: string
 }
 
 // Returns true if the link was created successfully. Can still fail to
@@ -54,6 +57,8 @@ export const saveNewsletterEmail = async (
     originalContent: data.content,
     title: data.title,
     author: data.author,
+    unsubMailTo: data.unsubMailTo,
+    unsubHttpUrl: data.unsubHttpUrl,
   }
 
   const page = await saveEmail(saveCtx, input)
@@ -62,14 +67,23 @@ export const saveNewsletterEmail = async (
     return false
   }
 
-  // add newsletters label to page
+  // creates or updates subscription
+  const subscription = await saveSubscription(
+    newsletterEmail.user.id,
+    data.author,
+    data.unsubMailTo,
+    data.unsubHttpUrl
+  )
+  console.log('subscription', subscription)
+
+  // adds newsletters label to page
   const result = await addLabelToPage(saveCtx, page.id, {
     name: 'Newsletter',
     color: '#07D2D1',
   })
   console.log('newsletter label added:', result)
 
-  // send push notification
+  // sends push notification
   const deviceTokens = await getDeviceTokensByUserId(newsletterEmail.user.id)
 
   if (!deviceTokens) {

packages/api/src/services/subscriptions.ts

@@ -0,0 +1,33 @@
+import { Subscription } from '../entity/subscription'
+import { getRepository } from '../entity/utils'
+import { SubscriptionStatus } from '../generated/graphql'
+
+export const saveSubscription = async (
+  userId: string,
+  name: string,
+  unsubscribeMailTo?: string,
+  unsubscribeHttpUrl?: string
+): Promise<Subscription> => {
+  const subscription = await getRepository(Subscription).findOneBy({
+    name,
+    user: { id: userId },
+  })
+
+  if (subscription) {
+    // if subscription already exists, updates updatedAt
+    subscription.updatedAt = new Date()
+    subscription.status = SubscriptionStatus.Active
+    unsubscribeMailTo && (subscription.unsubscribeMailTo = unsubscribeMailTo)
+    unsubscribeHttpUrl && (subscription.unsubscribeHttpUrl = unsubscribeHttpUrl)
+    return getRepository(Subscription).save(subscription)
+  }
+
+  // create new subscription
+  return getRepository(Subscription).save({
+    name,
+    user: { id: userId },
+    status: SubscriptionStatus.Active,
+    unsubscribeHttpUrl,
+    unsubscribeMailTo,
+  })
+}

packages/inbound-email-handler/src/index.ts

@@ -29,12 +29,12 @@ const NEWSLETTER_HANDLERS = [
 ]
 
 export const getNewsletterHandler = (
-  rawUrl: string,
+  postHeader: string,
   from: string,
-  unSubRawUrl: string
+  unSubHeader: string
 ): NewsletterHandler | undefined => {
   return NEWSLETTER_HANDLERS.find((h) => {
-    return h.isNewsletter(rawUrl, from, unSubRawUrl)
+    return h.isNewsletter(postHeader, from, unSubHeader)
   })
 }
 
@@ -72,22 +72,29 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
       const recipientAddress = forwardedAddress
         ? forwardedAddress.toString()
         : parsed.to
-      const rawUrl = headers['list-post'] ? headers['list-post'].toString() : ''
-      const unSubRawUrl = headers['list-unsubscribe']
+      const postHeader = headers['list-post']
+        ? headers['list-post'].toString()
+        : ''
+      const unSubHeader = headers['list-unsubscribe']
         ? headers['list-unsubscribe'].toString()
         : ''
 
       // check if it is a forwarding confirmation email or newsletter
-      const newsletterHandler = getNewsletterHandler(rawUrl, from, unSubRawUrl)
+      const newsletterHandler = getNewsletterHandler(
+        postHeader,
+        from,
+        unSubHeader
+      )
       try {
         if (newsletterHandler) {
           console.log('handleNewsletter', from, recipientAddress)
           await newsletterHandler.handleNewsletter(
             recipientAddress,
             html,
-            rawUrl,
+            postHeader,
             subject,
-            from
+            from,
+            unSubHeader
           )
         } else {
           console.log('non-newsletter email from:', from, recipientAddress)

packages/inbound-email-handler/src/newsletter.ts

@@ -9,19 +9,26 @@ const EMAIL_FORWARDING_SENDER_ADDRESSES = [
   'Gmail Team <forwarding-noreply@google.com>',
 ]
 const CONFIRMATION_CODE_PATTERN = /^\(#\d+\)/
+const UNSUBSCRIBE_HTTP_URL_PATTERN = /<(https?:\/\/[^>]*)>/
+const UNSUBSCRIBE_MAIL_TO_PATTERN = /<mailto:([^>]*)>/
+
+interface Unsubscribe {
+  mailTo?: string
+  httpUrl?: string
+}
 
 export class NewsletterHandler {
   protected senderRegex = /NEWSLETTER_SENDER_REGEX/
   protected urlRegex = /NEWSLETTER_URL_REGEX/
   protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
 
-  isNewsletter(rawUrl: string, from: string, unSubRawUrl: string): boolean {
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
     // Axios newsletter is from <xx@axios.com>
     const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!rawUrl || !!unSubRawUrl)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
   }
 
-  getNewsletterUrl(_rawUrl: string, html: string): string | undefined {
+  parseNewsletterUrl(_postHeader: string, html: string): string | undefined {
     // get newsletter url from html
     const matches = html.match(this.urlRegex)
     if (matches) {
@@ -30,7 +37,7 @@ export class NewsletterHandler {
     return undefined
   }
 
-  getAuthor(from: string): string {
+  parseAuthor(from: string): string {
     // get author name from email
     // e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
     // or 'Mike Allen <mike@axios.com>'
@@ -41,14 +48,24 @@ export class NewsletterHandler {
     return from
   }
 
+  parseUnsubscribe(unSubHeader: string): Unsubscribe {
+    // parse list-unsubscribe header
+    // e.g. List-Unsubscribe: <https://omnivore.com/unsub>, <mailto:unsub@omnivore.com>
+    return {
+      mailTo: unSubHeader.match(UNSUBSCRIBE_MAIL_TO_PATTERN)?.[1],
+      httpUrl: unSubHeader.match(UNSUBSCRIBE_HTTP_URL_PATTERN)?.[1],
+    }
+  }
+
   async handleNewsletter(
     email: string,
     html: string,
-    rawUrl: string,
+    postHeader: string,
     title: string,
-    from: string
+    from: string,
+    unSubHeader: string
   ): Promise<string | undefined> {
-    console.log('handleNewsletter', email, rawUrl, title, from)
+    console.log('handleNewsletter', email, postHeader, title, from)
 
     if (!email || !html || !title || !from) {
       console.log('invalid newsletter email')
@@ -58,17 +75,20 @@ export class NewsletterHandler {
     // fallback to default url if newsletter url does not exist
     // assign a random uuid to the default url to avoid duplicate url
     const url =
-      this.getNewsletterUrl(rawUrl, html) ||
+      this.parseNewsletterUrl(postHeader, html) ||
       `${this.defaultUrl}?source=newsletters&id=${uuidv4()}`
-    const author = this.getAuthor(from) || 'Unknown'
-
+    const author = this.parseAuthor(from) || 'Unknown'
+    const unsubscribe = this.parseUnsubscribe(unSubHeader)
     const message = {
-      email: email,
+      email,
       content: html,
-      url: url,
-      title: title,
-      author: author,
+      url,
+      title,
+      author,
+      unsubMailTo: unsubscribe.mailTo || '',
+      unsubHttpUrl: unsubscribe.httpUrl || '',
     }
+
     return publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, message)
   }
 }

packages/inbound-email-handler/src/substack-handler.ts

@@ -7,15 +7,19 @@ export class SubstackHandler extends NewsletterHandler {
     this.defaultUrl = 'https://www.substack.com'
   }
 
-  getNewsletterUrl(rawUrl: string, _html: string): string | undefined {
+  getNewsletterUrl(postHeader: string, _html: string): string | undefined {
     // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
     // we need to get the real url from the raw url
-    return addressparser(rawUrl).length > 0
-      ? addressparser(rawUrl)[0].name
+    return addressparser(postHeader).length > 0
+      ? addressparser(postHeader)[0].name
       : undefined
   }
 
-  isNewsletter(rawUrl: string, _from: string, _unSubRawUrl: string): boolean {
-    return !!rawUrl
+  isNewsletter(
+    postHeader: string,
+    _from: string,
+    _unSubHeader: string
+  ): boolean {
+    return !!postHeader
   }
 }

packages/inbound-email-handler/test/newsletter.test.ts

@@ -92,7 +92,7 @@ describe('Newsletter email test', () => {
       const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app'
       const html = `View in browser at <a>${url}</a>`
 
-      expect(new AxiosHandler().getNewsletterUrl('', html)).to.equal(url)
+      expect(new AxiosHandler().parseNewsletterUrl('', html)).to.equal(url)
     })
 
     it('returns url when email is from Bloomberg', () => {
@@ -103,7 +103,7 @@ describe('Newsletter email test', () => {
         </a>
       `
 
-      expect(new BloombergHandler().getNewsletterUrl('', html)).to.equal(url)
+      expect(new BloombergHandler().parseNewsletterUrl('', html)).to.equal(url)
     })
 
     it('returns url when email is from Golang Weekly', () => {
@@ -112,21 +112,42 @@ describe('Newsletter email test', () => {
         <a href="${url}" style="text-decoration: none">Read on the Web</a>
       `
 
-      expect(new GolangHandler().getNewsletterUrl('', html)).to.equal(url)
+      expect(new GolangHandler().parseNewsletterUrl('', html)).to.equal(url)
     })
   })
 
   describe('get author from email address', () => {
     it('returns author when email is from Substack', () => {
       const from = 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
-      expect(new NewsletterHandler().getAuthor(from)).to.equal(
+      expect(new NewsletterHandler().parseAuthor(from)).to.equal(
         'Jackson Harper from Omnivore App'
       )
     })
 
     it('returns author when email is from Axios', () => {
       const from = 'Mike Allen <mike@axios.com>'
-      expect(new NewsletterHandler().getAuthor(from)).to.equal('Mike Allen')
+      expect(new NewsletterHandler().parseAuthor(from)).to.equal('Mike Allen')
+    })
+  })
+
+  describe('get unsubscribe from header', () => {
+    const mailTo = 'unsub@omnivore.com'
+    const httpUrl = 'https://omnivore.com/unsubscribe'
+
+    it('returns mail to address if exists', () => {
+      const header = `<https://omnivore.com/unsub>, <mailto:${mailTo}>`
+
+      expect(new NewsletterHandler().parseUnsubscribe(header).mailTo).to.equal(
+        mailTo
+      )
+    })
+
+    it('returns http url if exists', () => {
+      const header = `<${httpUrl}>`
+
+      expect(new NewsletterHandler().parseUnsubscribe(header).httpUrl).to.equal(
+        httpUrl
+      )
     })
   })
 })