diff --git a/packages/api/src/services/newsletters.ts b/packages/api/src/services/newsletters.ts index d091a18c3..ac6391146 100644 --- a/packages/api/src/services/newsletters.ts +++ b/packages/api/src/services/newsletters.ts @@ -3,9 +3,7 @@ import { nanoid } from 'nanoid' import { User } from '../entity/user' import { CreateNewsletterEmailErrorCode } from '../generated/graphql' import { env } from '../env' -import { AppDataSource } from '../server' import { getRepository } from '../entity/utils' - import addressparser = require('nodemailer/lib/addressparser') const parsedAddress = (emailAddress: string): string | undefined => { diff --git a/packages/inbound-email-handler/src/bloomberg-handler.ts b/packages/inbound-email-handler/src/bloomberg-handler.ts index a60736461..3239ab176 100644 --- a/packages/inbound-email-handler/src/bloomberg-handler.ts +++ b/packages/inbound-email-handler/src/bloomberg-handler.ts @@ -5,6 +5,6 @@ export class BloombergHandler extends NewsletterHandler { super() this.senderRegex = /<.+@mail.bloomberg.*.com>/ this.urlRegex = // this.urlRegex = /Read on the Web<\/a>/ - this.defaultUrl = 'https://golangweekly.com/' + this.defaultUrl = 'https://golangweekly.com' } } diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index 6f4caa502..724e2c064 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -30,10 +30,11 @@ const NEWSLETTER_HANDLERS = [ export const getNewsletterHandler = ( rawUrl: string, - from: string + from: string, + rawUnSubUrl: string ): NewsletterHandler | undefined => { return NEWSLETTER_HANDLERS.find((h) => { - return h.isNewsletter(rawUrl, from) + return h.isNewsletter(rawUrl, from, rawUnSubUrl) }) } @@ -72,9 +73,12 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( ? forwardedAddress.toString() : parsed.to const rawUrl = headers['list-post'] ? headers['list-post'].toString() : '' + const rawUnSubUrl = headers['list-unsubscribe'] + ? headers['list-unsubscribe'].toString() + : '' // check if it is a forwarding confirmation email or newsletter - const newsletterHandler = getNewsletterHandler(rawUrl, from) + const newsletterHandler = getNewsletterHandler(rawUrl, from, rawUnSubUrl) try { if (newsletterHandler) { console.log('handleNewsletter', from, recipientAddress) diff --git a/packages/inbound-email-handler/src/newsletter.ts b/packages/inbound-email-handler/src/newsletter.ts index 858bdcf97..8c59f4fc4 100644 --- a/packages/inbound-email-handler/src/newsletter.ts +++ b/packages/inbound-email-handler/src/newsletter.ts @@ -1,4 +1,5 @@ import { PubSub } from '@google-cloud/pubsub' +import { v4 as uuidv4 } from 'uuid' import addressparser from 'addressparser' const pubsub = new PubSub() @@ -14,10 +15,10 @@ export class NewsletterHandler { protected urlRegex = /NEWSLETTER_URL_REGEX/ protected defaultUrl = 'NEWSLETTER_DEFAULT_URL' - isNewsletter(_rawUrl: string, from: string): boolean { + isNewsletter(rawUrl: string, from: string, rawUnSubUrl: string): boolean { // Axios newsletter is from const re = new RegExp(this.senderRegex) - return re.test(from) + return re.test(from) && (!!rawUrl || !!rawUnSubUrl) } getNewsletterUrl(_rawUrl: string, html: string): string | undefined { @@ -55,8 +56,11 @@ export class NewsletterHandler { } // fallback to default url if newsletter url does not exist - const url = this.getNewsletterUrl(rawUrl, html) || this.defaultUrl - const author = this.getAuthor(from) + // assign a random uuid to the default url to avoid duplicate url + const url = + this.getNewsletterUrl(rawUrl, html) || + `${this.defaultUrl}?source=newsletters&id=${uuidv4()}` + const author = this.getAuthor(from) || 'Unknown' const message = { email: email, diff --git a/packages/inbound-email-handler/src/substack-handler.ts b/packages/inbound-email-handler/src/substack-handler.ts index 02530c30b..1870a17e4 100644 --- a/packages/inbound-email-handler/src/substack-handler.ts +++ b/packages/inbound-email-handler/src/substack-handler.ts @@ -4,7 +4,7 @@ import addressparser from 'addressparser' export class SubstackHandler extends NewsletterHandler { constructor() { super() - this.defaultUrl = 'https://www.substack.com/' + this.defaultUrl = 'https://www.substack.com' } getNewsletterUrl(rawUrl: string, _html: string): string | undefined { @@ -15,7 +15,7 @@ export class SubstackHandler extends NewsletterHandler { : undefined } - isNewsletter(rawUrl: string, _from: string): boolean { - return !!rawUrl + isNewsletter(rawUrl: string, _from: string, rawUnSubUrl: string): boolean { + return !!rawUrl || !!rawUnSubUrl } }