diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index 88509c3e8..baa1a9ddd 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -84,7 +84,7 @@ export const preHandleContent = async ( return undefined } -export const handlerNewsletter = ( +export const handleNewsletter = ( input: NewsletterInput ): NewsletterResult | undefined => { for (const handler of contentHandlers) { @@ -98,5 +98,5 @@ export const handlerNewsletter = ( module.exports = { preHandleContent, - handlerNewsletter, + handleNewsletter, } diff --git a/packages/inbound-email-handler/package.json b/packages/inbound-email-handler/package.json index f3048a3b2..ed22a170c 100644 --- a/packages/inbound-email-handler/package.json +++ b/packages/inbound-email-handler/package.json @@ -31,6 +31,7 @@ "@google-cloud/pubsub": "^2.18.4", "@sendgrid/client": "^7.6.0", "@sentry/serverless": "^6.16.1", + "@omnivore/content-handler": "1.0.0", "addressparser": "^1.0.1", "axios": "^0.27.2", "jsonwebtoken": "^8.5.1", diff --git a/packages/inbound-email-handler/src/axios-handler.ts b/packages/inbound-email-handler/src/axios-handler.ts deleted file mode 100644 index 706d11047..000000000 --- a/packages/inbound-email-handler/src/axios-handler.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { NewsletterHandler } from './newsletter' - -export class AxiosHandler extends NewsletterHandler { - constructor() { - super() - this.senderRegex = /<.+@axios.com>/ - this.urlRegex = /View in browser at (.*)<\/a>/ - this.defaultUrl = 'https://axios.com' - } -} diff --git a/packages/inbound-email-handler/src/bloomberg-handler.ts b/packages/inbound-email-handler/src/bloomberg-handler.ts deleted file mode 100644 index 3239ab176..000000000 --- a/packages/inbound-email-handler/src/bloomberg-handler.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { NewsletterHandler } from './newsletter' - -export class BloombergHandler extends NewsletterHandler { - constructor() { - super() - this.senderRegex = /<.+@mail.bloomberg.*.com>/ - this.urlRegex = // - this.urlRegex = /Read on the Web<\/a>/ - this.defaultUrl = 'https://golangweekly.com' - } -} diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index ee11511dc..8e3a69874 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -9,35 +9,27 @@ import * as multipart from 'parse-multipart-data' import { handleConfirmation, isConfirmationEmail, - NewsletterHandler, parseUnsubscribe, } from './newsletter' import { PubSub } from '@google-cloud/pubsub' import { handlePdfAttachment } from './pdf' -import { SubstackHandler } from './substack-handler' -import { AxiosHandler } from './axios-handler' -import { BloombergHandler } from './bloomberg-handler' -import { GolangHandler } from './golang-handler' -import { MorningBrewHandler } from './morning-brew-handler' +import { handleNewsletter } from '@omnivore/content-handler' +const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived' const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived' const pubsub = new PubSub() -const NEWSLETTER_HANDLERS = [ - new SubstackHandler(), - new AxiosHandler(), - new BloombergHandler(), - new GolangHandler(), - new MorningBrewHandler(), -] -export const getNewsletterHandler = ( - postHeader: string, - from: string, - unSubHeader: string -): NewsletterHandler | undefined => { - return NEWSLETTER_HANDLERS.find((h) => { - return h.isNewsletter(postHeader, from, unSubHeader) - }) +export const publishMessage = async ( + topic: string, + message: any +): Promise => { + return pubsub + .topic(topic) + .publishMessage({ json: message }) + .catch((err) => { + console.log('error publishing message:', err) + return undefined + }) } export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( @@ -86,23 +78,20 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( try { // check if it is a confirmation email or forwarding newsletter - const newsletterHandler = getNewsletterHandler( - postHeader, + const newsletterMessage = handleNewsletter({ from, - unSubHeader - ) - - if (newsletterHandler) { - console.log('handleNewsletter', from, to) - await newsletterHandler.handleNewsletter( - to, - html, - postHeader, - subject, - from, - unSubHeader + html, + postHeader, + unSubHeader, + email: to, + title: subject, + }) + if (newsletterMessage) { + await publishMessage( + NEWSLETTER_EMAIL_RECEIVED_TOPIC, + newsletterMessage ) - return res.send('ok') + return res.status(200).send('newsletter received') } console.log('non-newsletter email from', from, 'to', to) diff --git a/packages/inbound-email-handler/src/morning-brew-handler.ts b/packages/inbound-email-handler/src/morning-brew-handler.ts deleted file mode 100644 index 6f5478c63..000000000 --- a/packages/inbound-email-handler/src/morning-brew-handler.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { NewsletterHandler } from './newsletter' - -export class MorningBrewHandler extends NewsletterHandler { - constructor() { - super() - this.senderRegex = /Morning Brew / - this.urlRegex = /View Online<\/a>/ - this.defaultUrl = 'https://www.morningbrew.com' - } -} diff --git a/packages/inbound-email-handler/src/newsletter.ts b/packages/inbound-email-handler/src/newsletter.ts index cf031e905..1f3294541 100644 --- a/packages/inbound-email-handler/src/newsletter.ts +++ b/packages/inbound-email-handler/src/newsletter.ts @@ -1,15 +1,12 @@ -import { PubSub } from '@google-cloud/pubsub' -import { v4 as uuidv4 } from 'uuid' import addressparser from 'addressparser' import rfc2047 from 'rfc2047' +import { publishMessage } from './index' interface Unsubscribe { mailTo?: string httpUrl?: string } -const pubsub = new PubSub() -const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived' const EMAIL_CONFIRMATION_CODE_RECEIVED_TOPIC = 'emailConfirmationCodeReceived' const CONFIRMATION_EMAIL_SENDER_ADDRESS = 'forwarding-noreply@google.com' // check unicode parentheses too @@ -35,73 +32,6 @@ const parseAddress = (address: string): string => { return '' } -export class NewsletterHandler { - protected senderRegex = /NEWSLETTER_SENDER_REGEX/ - protected urlRegex = /NEWSLETTER_URL_REGEX/ - protected defaultUrl = 'NEWSLETTER_DEFAULT_URL' - - isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean { - // Axios newsletter is from - const re = new RegExp(this.senderRegex) - return re.test(from) && (!!postHeader || !!unSubHeader) - } - - parseNewsletterUrl(_postHeader: string, html: string): string | undefined { - // get newsletter url from html - const matches = html.match(this.urlRegex) - if (matches) { - return matches[1] - } - return undefined - } - - parseAuthor(from: string): string { - // get author name from email - // e.g. 'Jackson Harper from Omnivore App ' - // or 'Mike Allen ' - const parsed = addressparser(from) - if (parsed.length > 0) { - return parsed[0].name - } - return from - } - - async handleNewsletter( - email: string, - html: string, - postHeader: string, - title: string, - from: string, - unSubHeader: string - ): Promise { - console.log('handleNewsletter', email, postHeader, title, from) - - if (!email || !html || !title || !from) { - console.log('invalid newsletter email') - throw new Error('invalid newsletter email') - } - - // fallback to default url if newsletter url does not exist - // assign a random uuid to the default url to avoid duplicate url - const url = - this.parseNewsletterUrl(postHeader, html) || - `${this.defaultUrl}?source=newsletters&id=${uuidv4()}` - const author = this.parseAuthor(from) - const unsubscribe = parseUnsubscribe(unSubHeader) - const message = { - email, - content: html, - url, - title, - author, - unsubMailTo: unsubscribe.mailTo || '', - unsubHttpUrl: unsubscribe.httpUrl || '', - } - - return publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, message) - } -} - export const handleConfirmation = async (email: string, subject: string) => { console.log('confirmation email', email, subject) @@ -136,16 +66,3 @@ export const isConfirmationEmail = (from: string, subject: string): boolean => { CONFIRMATION_CODE_PATTERN.test(subject) ) } - -const publishMessage = async ( - topic: string, - message: Record -): Promise => { - return pubsub - .topic(topic) - .publishMessage({ json: message }) - .catch((err) => { - console.log('error publishing message:', err) - return undefined - }) -} diff --git a/packages/inbound-email-handler/src/substack-handler.ts b/packages/inbound-email-handler/src/substack-handler.ts deleted file mode 100644 index 10160b76e..000000000 --- a/packages/inbound-email-handler/src/substack-handler.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { NewsletterHandler } from './newsletter' -import addressparser from 'addressparser' - -export class SubstackHandler extends NewsletterHandler { - constructor() { - super() - this.defaultUrl = 'https://www.substack.com' - } - - parseNewsletterUrl(postHeader: string, _html: string): string | undefined { - // raw SubStack newsletter url is like - // we need to get the real url from the raw url - return addressparser(postHeader).length > 0 - ? addressparser(postHeader)[0].name - : undefined - } - - isNewsletter( - postHeader: string, - _from: string, - _unSubHeader: string - ): boolean { - return !!postHeader - } -} diff --git a/packages/inbound-email-handler/test/newsletter.test.ts b/packages/inbound-email-handler/test/newsletter.test.ts index 8c20d9747..253c294ab 100644 --- a/packages/inbound-email-handler/test/newsletter.test.ts +++ b/packages/inbound-email-handler/test/newsletter.test.ts @@ -2,15 +2,8 @@ import { expect } from 'chai' import { getConfirmationCode, isConfirmationEmail, - NewsletterHandler, parseUnsubscribe, } from '../src/newsletter' -import { SubstackHandler } from '../src/substack-handler' -import { AxiosHandler } from '../src/axios-handler' -import { BloombergHandler } from '../src/bloomberg-handler' -import { GolangHandler } from '../src/golang-handler' -import { getNewsletterHandler } from '../src' -import { MorningBrewHandler } from '../src/morning-brew-handler' describe('Confirmation email test', () => { describe('#isConfirmationEmail()', () => { @@ -54,126 +47,6 @@ describe('Confirmation email test', () => { }) describe('Newsletter email test', () => { - describe('#getNewsletterHandler()', () => { - it('returns SubstackHandler when email is from SubStack', () => { - const rawUrl = '' - - expect(getNewsletterHandler(rawUrl, '', '')).to.be.instanceof( - SubstackHandler - ) - }) - - it('returns AxiosHandler when email is from Axios', () => { - const from = 'Mike Allen ' - const unSubRawUrl = - '' - - expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof( - AxiosHandler - ) - }) - - context('when email is from Bloomberg', () => { - it('should return BloombergHandler when email is from Bloomberg Business', () => { - const from = 'From: Bloomberg ' - const unSubRawUrl = '' - - expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof( - BloombergHandler - ) - }) - - it('should return BloombergHandler when email is from Bloomberg View', () => { - const from = 'From: Bloomberg ' - const unSubRawUrl = '' - - expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof( - BloombergHandler - ) - }) - }) - - it('should return GolangHandler when email is from Golang Weekly', () => { - const from = 'Golang Weekly ' - const unSubRawUrl = '' - - expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof( - GolangHandler - ) - }) - - it('should return MorningBrewHandler when email is from Morning Brew', () => { - const from = 'Morning Brew ' - const unSubRawUrl = '' - - expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof( - MorningBrewHandler - ) - }) - }) - - describe('#getNewsletterUrl()', () => { - it('returns url when email is from SubStack', () => { - const rawUrl = '' - - expect(new SubstackHandler().parseNewsletterUrl(rawUrl, '')).to.equal( - 'https://hongbo130.substack.com/p/tldr' - ) - }) - - it('returns url when email is from Axios', () => { - const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app' - const html = `View in browser at ${url}` - - expect(new AxiosHandler().parseNewsletterUrl('', html)).to.equal(url) - }) - - it('returns url when email is from Bloomberg', () => { - const url = 'https://www.bloomberg.com/news/google-is-now-a-partner' - const html = ` - - View in browser - - ` - - expect(new BloombergHandler().parseNewsletterUrl('', html)).to.equal(url) - }) - - it('returns url when email is from Golang Weekly', () => { - const url = 'https://www.golangweekly.com/first' - const html = ` - Read on the Web - ` - - expect(new GolangHandler().parseNewsletterUrl('', html)).to.equal(url) - }) - - it('returns url when email is from Morning Brew', () => { - const url = 'https://www.morningbrew.com/daily/issues/first' - const html = ` - View Online - ` - - expect(new MorningBrewHandler().parseNewsletterUrl('', html)).to.equal( - url - ) - }) - }) - - describe('get author from email address', () => { - it('returns author when email is from Substack', () => { - const from = 'Jackson Harper from Omnivore App ' - expect(new NewsletterHandler().parseAuthor(from)).to.equal( - 'Jackson Harper from Omnivore App' - ) - }) - - it('returns author when email is from Axios', () => { - const from = 'Mike Allen ' - expect(new NewsletterHandler().parseAuthor(from)).to.equal('Mike Allen') - }) - }) - describe('get unsubscribe from header', () => { const mailTo = 'unsub@omnivore.com' const httpUrl = 'https://omnivore.com/unsubscribe'