From e85273d87a82d1f8067714f7cc17aa557977fddf Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 15 Feb 2022 11:56:03 +0800 Subject: [PATCH] add support for axios newsletters (#49) * add support for axios newsletters * fix raw url * fix getting newsletter url from axios * add test for url * add axios parser --- packages/api/src/routers/svc/newsletters.ts | 1 - packages/api/src/utils/axios-handler.ts | 36 +++++++++++++ packages/api/src/utils/parser.ts | 7 ++- packages/inbound-email-handler/src/index.ts | 5 +- .../inbound-email-handler/src/newsletter.ts | 50 +++++++++++++++---- .../test/newsletter.test.ts | 39 +++++++++++++-- 6 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 packages/api/src/utils/axios-handler.ts diff --git a/packages/api/src/routers/svc/newsletters.ts b/packages/api/src/routers/svc/newsletters.ts index a8abb28e7..2a668115b 100644 --- a/packages/api/src/routers/svc/newsletters.ts +++ b/packages/api/src/routers/svc/newsletters.ts @@ -116,7 +116,6 @@ export function newsletterServiceRouter() { if ( !('email' in data) || !('content' in data) || - !('url' in data) || !('title' in data) || !('author' in data) ) { diff --git a/packages/api/src/utils/axios-handler.ts b/packages/api/src/utils/axios-handler.ts new file mode 100644 index 000000000..973379770 --- /dev/null +++ b/packages/api/src/utils/axios-handler.ts @@ -0,0 +1,36 @@ +import { DOMWindow } from 'jsdom' + +export class AxiosHandler { + name = 'axios' + + shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => { + const host = this.name + '.com' + // check if url ends with axios.com + return url.hostname.endsWith(host) + } + + prehandle = (url: URL, dom: DOMWindow): Promise => { + const body = dom.document.querySelector('table') + + // this removes ads and replaces table with a div + body?.querySelectorAll('table').forEach((el, k) => { + if (k > 0) { + el.remove() + } else { + // remove the last two rows of the table (they are ads) + el.querySelectorAll('tr').forEach((tr, i) => { + if (i >= el.querySelectorAll('tr').length - 2) { + console.log('removing', tr) + tr.remove() + } + }) + // replace the table with a div + const div = dom.document.createElement('div') + div.innerHTML = el.innerHTML + el.parentNode?.replaceChild(div, el) + } + }) + + return Promise.resolve(dom) + } +} diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index db2396967..88160a413 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -10,6 +10,7 @@ import { createImageProxyUrl } from './imageproxy' import axios from 'axios' import { WikipediaHandler } from './wikipedia-handler' import { SubstackHandler } from './substack-handler' +import { AxiosHandler } from './axios-handler' const logger = buildLogger('utils.parse') @@ -40,7 +41,11 @@ interface ContentHandler { prehandle: (url: URL, document: DOMWindow) => Promise } -const HANDLERS = [new WikipediaHandler(), new SubstackHandler()] +const HANDLERS = [ + new WikipediaHandler(), + new SubstackHandler(), + new AxiosHandler(), +] /** Hook that prevents DOMPurify from removing youtube iframes */ const domPurifySanitizeHook = ( diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index 3b37d45d7..121c1194a 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -10,6 +10,7 @@ import { handleConfirmation, handleNewsletter, isConfirmationEmail, + isNewsletter, } from './newsletter' import { PubSub } from '@google-cloud/pubsub' @@ -44,10 +45,10 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( const recipientAddress = forwardedAddress ? forwardedAddress.toString() : parsed.to - const rawUrl = headers['list-post']?.toString() + const rawUrl = headers['list-post'] ? headers['list-post'].toString() : '' // check if it is a forwarding confirmation email or newsletter - if (rawUrl) { + if (isNewsletter(rawUrl, from)) { try { console.log('handleNewsletter', from, recipientAddress) await handleNewsletter(recipientAddress, html, rawUrl, subject, from) diff --git a/packages/inbound-email-handler/src/newsletter.ts b/packages/inbound-email-handler/src/newsletter.ts index 8300f5270..b50354cf6 100644 --- a/packages/inbound-email-handler/src/newsletter.ts +++ b/packages/inbound-email-handler/src/newsletter.ts @@ -6,8 +6,9 @@ const EMAIL_CONFIRMATION_CODE_RECEIVED_TOPIC = 'emailConfirmationCodeReceived' const EMAIL_FORWARDING_SENDER_ADDRESSES = [ 'Gmail Team ', ] -const NEWSLETTER_SENDER_REGEX = '<.+@substack.com>' +const NEWSLETTER_SENDER_REGEX = '<.+@axios.com>' const CONFIRMATION_CODE_PATTERN = '^\\(#\\d+\\)' +const AXIOS_URL_PATTERN = 'View in browser at <.+>' export const handleConfirmation = async (email: string, subject: string) => { console.log('confirmation email') @@ -41,20 +42,26 @@ export const handleNewsletter = async ( title: string, from: string ) => { - console.log('handleNewsletter') + console.log('handleNewsletter', email, rawUrl, title, from) - if (!email || !html || !rawUrl || !title || !from) { + if (!email || !html || !title || !from) { console.log('invalid newsletter email') throw new Error('invalid newsletter email') } - // raw newsletter url is like - // we need to get the real url - const url = rawUrl.slice(1, -1) + const url = getNewsletterUrl(rawUrl, html) + console.log('url', url) + if (!url) { + console.log('invalid newsletter url', url) + throw new Error('invalid newsletter url') + } // get author name from email - // e.g. 'Jackson Harper from Omnivore App' - const authors = from.split(' from ') + // e.g. 'Jackson Harper from Omnivore App ' + // or 'Mike Allen ' + const authors = from.includes(' from ') + ? from.split(' from') + : from.split(' <') if (!authors) { console.log('invalid from', from) throw new Error('invalid from') @@ -84,11 +91,34 @@ const publishMessage = async ( }) } -export const isNewsletter = (from: string, messageId: string): boolean => { +// SubStack newsletter has raw Url in the email +// url is like +// Axios newsletter is from +export const isNewsletter = (rawUrl: string, from: string): boolean => { const re = new RegExp(NEWSLETTER_SENDER_REGEX) - return re.test(from) || messageId.includes('substack.com') + return !!rawUrl || re.test(from) } export const isConfirmationEmail = (from: string): boolean => { return EMAIL_FORWARDING_SENDER_ADDRESSES.includes(from) } + +export const getNewsletterUrl = ( + rawUrl: string, + html: string +): string | undefined => { + // raw SubStack newsletter url is like + // we need to get the real url + if (rawUrl.startsWith('<')) { + return rawUrl.slice(1, -1) + } + + // axios newsletter url from html + const re = new RegExp(AXIOS_URL_PATTERN) + const matches = html.match(re) + if (matches) { + const match = matches[0] + return match.slice(match.indexOf('>') + 1, match.lastIndexOf('<')) + } + return undefined +} diff --git a/packages/inbound-email-handler/test/newsletter.test.ts b/packages/inbound-email-handler/test/newsletter.test.ts index b95681d8e..920262792 100644 --- a/packages/inbound-email-handler/test/newsletter.test.ts +++ b/packages/inbound-email-handler/test/newsletter.test.ts @@ -1,10 +1,14 @@ import { expect } from 'chai' -import { isConfirmationEmail, isNewsletter } from '../src/newsletter' +import { + getNewsletterUrl, + isConfirmationEmail, + isNewsletter, +} from '../src/newsletter' describe('Confirmation email test', () => { describe('#isConfirmationEmail()', () => { it('returns true when email is from Gmail Team', () => { - const from = `Gmail Team ` + const from = 'Gmail Team ' expect(isConfirmationEmail(from)).to.be.true }) @@ -13,10 +17,35 @@ describe('Confirmation email test', () => { describe('Newsletter email test', () => { describe('#isNewsletter()', () => { - it('returns true when email is from substack', () => { - const from = `Hongbo from Hongbo’s Newsletter ` + it('returns true when email is from SubStack', () => { + const rawUrl = '' - expect(isNewsletter(from, '')).to.be.true + expect(isNewsletter(rawUrl, '')).to.be.true + }) + + it('returns true when email is from Axios', () => { + const from = 'Mike Allen ' + + expect(isNewsletter('', from)).to.be.true + }) + }) + + describe('#getNewsletterUrl()', () => { + it('returns url when email is from SubStack', () => { + const rawUrl = '' + + expect(getNewsletterUrl(rawUrl, '')).to.equal( + 'https://hongbo130.substack.com/p/tldr' + ) + }) + + it('returns url when email is from Axios', () => { + const rawUrl = '' + const html = `View in browser at https://axios.com/blog/2019/02/28/the-best-way-to-build-a-web-app` + + expect(getNewsletterUrl(rawUrl, html)).to.equal( + 'https://axios.com/blog/2019/02/28/the-best-way-to-build-a-web-app' + ) }) }) })