diff --git a/packages/inbound-email-handler/package.json b/packages/inbound-email-handler/package.json index a3b9231db..b80f4e6d6 100644 --- a/packages/inbound-email-handler/package.json +++ b/packages/inbound-email-handler/package.json @@ -20,6 +20,7 @@ "deploy": "yarn build && yarn gcloud-deploy" }, "devDependencies": { + "@types/addressparser": "^1.0.1", "@types/json-bigint": "^1.0.1", "@types/node": "^14.11.2", "eslint-plugin-prettier": "^4.0.0" @@ -29,6 +30,7 @@ "@google-cloud/pubsub": "^2.18.4", "@sendgrid/client": "^7.6.0", "@sentry/serverless": "^6.16.1", + "addressparser": "^1.0.1", "axios": "^0.26.0", "jsonwebtoken": "^8.5.1", "parse-headers": "^2.0.4", diff --git a/packages/inbound-email-handler/src/axios-handler.ts b/packages/inbound-email-handler/src/axios-handler.ts new file mode 100644 index 000000000..7af57daa9 --- /dev/null +++ b/packages/inbound-email-handler/src/axios-handler.ts @@ -0,0 +1,9 @@ +import { NewsletterHandler } from './newsletter' + +export class AxiosHandler extends NewsletterHandler { + constructor() { + super() + this.senderRegex = /<.+@axios.com>/ + this.urlRegex = /View in browser at (.*)<\/a>/ + } +} diff --git a/packages/inbound-email-handler/src/bloomberg-handler.ts b/packages/inbound-email-handler/src/bloomberg-handler.ts new file mode 100644 index 000000000..d3dad1961 --- /dev/null +++ b/packages/inbound-email-handler/src/bloomberg-handler.ts @@ -0,0 +1,9 @@ +import { NewsletterHandler } from './newsletter' + +export class BloombergHandler extends NewsletterHandler { + constructor() { + super() + this.senderRegex = /<.+@mail.bloombergbusiness.com>/ + this.urlRegex = // + this.urlRegex = /Read on the Web<\/a>/ + } +} diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index e60170fbb..055a18778 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -8,16 +8,35 @@ import parseHeaders from 'parse-headers' import * as multipart from 'parse-multipart-data' import { handleConfirmation, - handleNewsletter, isConfirmationEmail, - isNewsletter, + NewsletterHandler, } from './newsletter' import { PubSub } from '@google-cloud/pubsub' import { handlePdfAttachment } from './pdf' +import { SubstackHandler } from './substack-handler' +import { AxiosHandler } from './axios-handler' +import { BloombergHandler } from './bloomberg-handler' +import { GolangHandler } from './golang-handler' const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived' const pubsub = new PubSub() +const NEWSLETTER_HANDLERS = [ + new SubstackHandler(), + new AxiosHandler(), + new BloombergHandler(), + new GolangHandler(), +] + +export const getNewsletterHandler = ( + rawUrl: string, + from: string +): NewsletterHandler | undefined => { + return NEWSLETTER_HANDLERS.find((h) => { + return h.isNewsletter(rawUrl, from) + }) +} + export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { const parts = multipart.parse(req.body, 'xYzZY') @@ -55,10 +74,17 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( const rawUrl = headers['list-post'] ? headers['list-post'].toString() : '' // check if it is a forwarding confirmation email or newsletter - if (isNewsletter(rawUrl, from)) { + const newsletterHandler = getNewsletterHandler(rawUrl, from) + if (newsletterHandler) { try { console.log('handleNewsletter', from, recipientAddress) - await handleNewsletter(recipientAddress, html, rawUrl, subject, from) + await newsletterHandler.handleNewsletter( + recipientAddress, + html, + rawUrl, + subject, + from + ) } catch (error) { console.log( 'error handling newsletter, will forward.', diff --git a/packages/inbound-email-handler/src/newsletter.ts b/packages/inbound-email-handler/src/newsletter.ts index 2c384c485..322f3bf57 100644 --- a/packages/inbound-email-handler/src/newsletter.ts +++ b/packages/inbound-email-handler/src/newsletter.ts @@ -1,4 +1,5 @@ import { PubSub } from '@google-cloud/pubsub' +import addressparser from 'addressparser' const pubsub = new PubSub() const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived' @@ -6,12 +7,71 @@ const EMAIL_CONFIRMATION_CODE_RECEIVED_TOPIC = 'emailConfirmationCodeReceived' const EMAIL_FORWARDING_SENDER_ADDRESSES = [ 'Gmail Team ', ] -const NEWSLETTER_SENDER_REGEX = - /<.+@((axios.com)|(mail.bloombergbusiness.com))>/ const CONFIRMATION_CODE_PATTERN = /^\\(#\\d+\\)/ -const AXIOS_URL_PATTERN = /View in browser at (.*)<\/a>/ -const BLOOMBERG_URL_PATTERN = - / + const re = new RegExp(this.senderRegex) + return re.test(from) + } + + getNewsletterUrl(rawUrl: string, html: string): string | undefined { + // get newsletter url from html + const matches = html.match(this.urlRegex) + if (matches) { + return matches[1] + } + return undefined + } + + getAuthor(from: string): string { + // get author name from email + // e.g. 'Jackson Harper from Omnivore App ' + // or 'Mike Allen ' + const parsed = addressparser(from) + if (parsed.length > 0) { + return parsed[0].name + } + return from + } + + async handleNewsletter( + email: string, + html: string, + rawUrl: string, + title: string, + from: string + ): Promise<[string] | undefined> { + console.log('handleNewsletter', email, rawUrl, title, from) + + if (!email || !html || !title || !from) { + console.log('invalid newsletter email') + throw new Error('invalid newsletter email') + } + + const url = this.getNewsletterUrl(rawUrl, html) + console.log('url', url) + if (!url) { + console.log('invalid newsletter url', url) + throw new Error('invalid newsletter url') + } + + const author = this.getAuthor(from) + + const message = { + email: email, + content: html, + url: url, + title: title, + author: author, + } + return publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, message) + } +} export const handleConfirmation = async (email: string, subject: string) => { console.log('confirmation email') @@ -38,49 +98,6 @@ export const handleConfirmation = async (email: string, subject: string) => { return publishMessage(EMAIL_CONFIRMATION_CODE_RECEIVED_TOPIC, message) } -export const handleNewsletter = async ( - email: string, - html: string, - rawUrl: string, - title: string, - from: string -) => { - console.log('handleNewsletter', email, rawUrl, title, from) - - if (!email || !html || !title || !from) { - console.log('invalid newsletter email') - throw new Error('invalid newsletter email') - } - - const url = getNewsletterUrl(rawUrl, html) - console.log('url', url) - if (!url) { - console.log('invalid newsletter url', url) - throw new Error('invalid newsletter url') - } - - // get author name from email - // e.g. 'Jackson Harper from Omnivore App ' - // or 'Mike Allen ' - const authors = from.includes(' from ') - ? from.split(' from') - : from.split(' <') - if (!authors) { - console.log('invalid from', from) - throw new Error('invalid from') - } - const author = authors[0] - - const message = { - email: email, - content: html, - url: url, - title: title, - author: author, - } - return publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, message) -} - const publishMessage = async ( topic: string, message: Record @@ -94,38 +111,6 @@ const publishMessage = async ( }) } -// SubStack newsletter has raw Url in the email -// url is like -// Axios newsletter is from -export const isNewsletter = (rawUrl: string, from: string): boolean => { - const re = new RegExp(NEWSLETTER_SENDER_REGEX) - return !!rawUrl || re.test(from) -} - export const isConfirmationEmail = (from: string): boolean => { return EMAIL_FORWARDING_SENDER_ADDRESSES.includes(from) } - -export const getNewsletterUrl = ( - rawUrl: string, - html: string -): string | undefined => { - // raw SubStack newsletter url is like - // we need to get the real url - if (rawUrl.startsWith('<')) { - return rawUrl.slice(1, -1) - } - - // axios newsletter url from html - let matches = html.match(AXIOS_URL_PATTERN) - if (matches) { - return matches[1] - } - - // bloomberg newsletter url from html - matches = html.match(BLOOMBERG_URL_PATTERN) - if (matches) { - return matches[1] - } - return undefined -} diff --git a/packages/inbound-email-handler/src/substack-handler.ts b/packages/inbound-email-handler/src/substack-handler.ts new file mode 100644 index 000000000..ac0e54d1b --- /dev/null +++ b/packages/inbound-email-handler/src/substack-handler.ts @@ -0,0 +1,13 @@ +import { NewsletterHandler } from './newsletter' + +export class SubstackHandler extends NewsletterHandler { + getNewsletterUrl(rawUrl: string, _html: string): string | undefined { + // raw SubStack newsletter url is like + // we need to get the real url + return rawUrl.slice(1, -1) + } + + isNewsletter(rawUrl: string, _from: string): boolean { + return !!rawUrl + } +} diff --git a/packages/inbound-email-handler/test/newsletter.test.ts b/packages/inbound-email-handler/test/newsletter.test.ts index 4fa65485b..3cd6d7824 100644 --- a/packages/inbound-email-handler/test/newsletter.test.ts +++ b/packages/inbound-email-handler/test/newsletter.test.ts @@ -1,9 +1,10 @@ import { expect } from 'chai' -import { - getNewsletterUrl, - isConfirmationEmail, - isNewsletter, -} from '../src/newsletter' +import { isConfirmationEmail, NewsletterHandler } from '../src/newsletter' +import { SubstackHandler } from '../src/substack-handler' +import { AxiosHandler } from '../src/axios-handler' +import { BloombergHandler } from '../src/bloomberg-handler' +import { GolangHandler } from '../src/golang-handler' +import { getNewsletterHandler } from '../src' describe('Confirmation email test', () => { describe('#isConfirmationEmail()', () => { @@ -16,22 +17,27 @@ describe('Confirmation email test', () => { }) describe('Newsletter email test', () => { - describe('#isNewsletter()', () => { - it('returns true when email is from SubStack', () => { + describe('#getNewsletterHandler()', () => { + it('returns SubstackHandler when email is from SubStack', () => { const rawUrl = '' - expect(isNewsletter(rawUrl, '')).to.be.true + expect(getNewsletterHandler(rawUrl, '')).to.be.instanceof(SubstackHandler) }) - it('returns true when email is from Axios', () => { + it('returns AxiosHandler when email is from Axios', () => { const from = 'Mike Allen ' - expect(isNewsletter('', from)).to.be.true + expect(getNewsletterHandler('', from)).to.be.instanceof(AxiosHandler) }) - it('should return true when email is from bloomberg', () => { + it('should return BloombergHandler when email is from Bloomberg', () => { const from = 'From: Bloomberg ' - expect(isNewsletter('', from)).to.be.true + expect(getNewsletterHandler('', from)).to.be.instanceof(BloombergHandler) + }) + + it('should return GolangHandler when email is from Golang Weekly', () => { + const from = 'Golang Weekly ' + expect(getNewsletterHandler('', from)).to.be.instanceof(GolangHandler) }) }) @@ -39,31 +45,50 @@ describe('Newsletter email test', () => { it('returns url when email is from SubStack', () => { const rawUrl = '' - expect(getNewsletterUrl(rawUrl, '')).to.equal( + expect(new SubstackHandler().getNewsletterUrl(rawUrl, '')).to.equal( 'https://hongbo130.substack.com/p/tldr' ) }) it('returns url when email is from Axios', () => { - const rawUrl = '' - const html = `View in browser at https://axios.com/blog/the-best-way-to-build-a-web-app` + const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app' + const html = `View in browser at ${url}` - expect(getNewsletterUrl(rawUrl, html)).to.equal( - 'https://axios.com/blog/the-best-way-to-build-a-web-app' - ) + expect(new AxiosHandler().getNewsletterUrl('', html)).to.equal(url) }) it('returns url when email is from Bloomberg', () => { - const rawUrl = '' + const url = 'https://www.bloomberg.com/news/google-is-now-a-partner' const html = ` - + View in browser ` - expect(getNewsletterUrl(rawUrl, html)).to.equal( - 'https://www.bloomberg.com/news/google-is-now-a-partner' + expect(new BloombergHandler().getNewsletterUrl('', html)).to.equal(url) + }) + + it('returns url when email is from Golang Weekly', () => { + const url = 'https://www.golangweekly.com/first' + const html = ` + Read on the Web + ` + + expect(new GolangHandler().getNewsletterUrl('', html)).to.equal(url) + }) + }) + + describe('get author from email address', () => { + it('returns author when email is from Substack', () => { + const from = 'Jackson Harper from Omnivore App ' + expect(new NewsletterHandler().getAuthor(from)).to.equal( + 'Jackson Harper from Omnivore App' ) }) + + it('returns author when email is from Axios', () => { + const from = 'Mike Allen ' + expect(new NewsletterHandler().getAuthor(from)).to.equal('Mike Allen') + }) }) }) diff --git a/yarn.lock b/yarn.lock index 18d387a35..3fe060a40 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4195,6 +4195,13 @@ dependencies: "@types/node" "*" +"@types/addressparser@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@types/addressparser/-/addressparser-1.0.1.tgz#e34e8ca186c96a5bc3d9bfbf93a7258d7a03b413" + integrity sha512-iw+cRQy5XcLGWhyyuYJ2Fnu8dyJ1y0QoaODfnbJCRzv/qYtAWH5yK6H688kGIyqm1VowwKUlA8mTH9qUkPOd7A== + dependencies: + "@types/node" "*" + "@types/analytics-node@^3.1.7": version "3.1.7" resolved "https://registry.yarnpkg.com/@types/analytics-node/-/analytics-node-3.1.7.tgz#cb97c80ee505094e44a0188c3ad25f70c67e3c65" @@ -5025,6 +5032,11 @@ add-stream@^1.0.0: resolved "https://registry.yarnpkg.com/add-stream/-/add-stream-1.0.0.tgz#6a7990437ca736d5e1288db92bd3266d5f5cb2aa" integrity sha1-anmQQ3ynNtXhKI25K9MmbV9csqo= +addressparser@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746" + integrity sha1-R6++GiqSYhkdtoOOT9HTm0CCF0Y= + agent-base@6: version "6.0.1" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.1.tgz#808007e4e5867decb0ab6ab2f928fbdb5a596db4"