diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts index ebdf81d3e..1a27344f0 100644 --- a/packages/content-handler/src/content-handler.ts +++ b/packages/content-handler/src/content-handler.ts @@ -121,12 +121,17 @@ export abstract class ContentHandler { headers: Record, html: string ): Promise { + // get url from dom + const url = await this.findNewsletterUrl(html) + if (url) { + return url + } // get newsletter url from html const matches = html.match(this.urlRegex) if (matches) { - return Promise.resolve(matches[1]) + return matches[1] } - return Promise.resolve(undefined) + return undefined } parseAuthor(from: string): string { diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index 366fc48fb..2993fa1d9 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -31,6 +31,7 @@ import { HeyWorldHandler } from './newsletters/hey-world-handler' import { Browser } from 'puppeteer-core' import { StackOverflowHandler } from './websites/stack-overflow-handler' import { GenericHandler } from './newsletters/generic-handler' +import { EveryIoHandler } from './newsletters/every-io-handler' const validateUrlString = (url: string) => { const u = new URL(url) @@ -82,6 +83,7 @@ const newsletterHandlers: ContentHandler[] = [ new CooperPressHandler(), new HeyWorldHandler(), new GenericHandler(), + new EveryIoHandler(), ] export const preHandleContent = async ( diff --git a/packages/content-handler/src/newsletters/every-io-handler.ts b/packages/content-handler/src/newsletters/every-io-handler.ts new file mode 100644 index 000000000..3ac836f00 --- /dev/null +++ b/packages/content-handler/src/newsletters/every-io-handler.ts @@ -0,0 +1,22 @@ +import { ContentHandler } from '../content-handler' + +export class EveryIoHandler extends ContentHandler { + constructor() { + super() + this.name = 'Every.io' + } + + async isNewsletter(input: { + from: string + html: string + headers: Record + dom: Document + }): Promise { + return Promise.resolve(input.from === 'Every ') + } + + findNewsletterHeaderHref(dom: Document): string | undefined { + const readOnline = dom.querySelector('.newsletter-email .title a') + return readOnline?.getAttribute('href') || undefined + } +} diff --git a/packages/content-handler/test/data/every-io-newsletter.html b/packages/content-handler/test/data/every-io-newsletter.html new file mode 100644 index 000000000..5424cb5a1 --- /dev/null +++ b/packages/content-handler/test/data/every-io-newsletter.html @@ -0,0 +1,163 @@ + + + + + The Endgame for AI-generated Writing + + + + + + + + + + + +
+
+ + +
+ + + +
+ + +
+ + +

+ You received this email because you signed up for emails from Every. No longer interested in receiving emails from us? Click here to unsubscribe. +

+ + +
+ 221 Canal St 5th floor, New York, NY 10013 +
+
+ +
+
+ + diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts index a404495f8..0630783a2 100644 --- a/packages/content-handler/test/newsletter.test.ts +++ b/packages/content-handler/test/newsletter.test.ts @@ -19,6 +19,7 @@ import { getNewsletterHandler } from '../src' import { parseHTML } from 'linkedom' import { HeyWorldHandler } from '../src/newsletters/hey-world-handler' import { GenericHandler } from '../src/newsletters/generic-handler' +import { EveryIoHandler } from '../src/newsletters/every-io-handler' chai.use(chaiAsPromised) chai.use(chaiString) @@ -102,6 +103,32 @@ describe('Newsletter email test', () => { ).to.eventually.equal(url) }) }) + + context('when email is from Every.io', () => { + before(() => { + nock('https://u25184427.ct.sendgrid.net') + .head( + '/ls/click?upn=MnmHBiCwIPe9TmIJeskmAzMyFAOgfmWxTSMhUO2y3cX7SH8TU8fE7Bob959Q5SX8kPrOyAS87mVlVX7RClBX5O3kASzJW2uCz02hYzOF2YQ0jOrkqrBElBIkfeLl7ZERdDGVCD8VSijl4E0Tqgyrzl2vhAQyRsnJuYNo38k-2FROTfvWd85434dJ5Ajc284NO0JH4cM9g-2BdWlh0bID30oroznIHSP8Lg4qTxIVRNMq8zYyOtmR3fanZTexF5IrLPDqQMXmm0Wz0QK23ojkpC8yL3gexm2dDFZNOtKvgYlprJoR6QhevN9bulX1wI8ht5dtNccOAcexw3K4a3WMuoJp4thIKz0hIzOup38HTJPLtjRojXWKOyRBqE-2FdX17d4wY3lSqAN80IHqCic3WUBTenbU6Uo0Qi0FEVj0Iar7TuplOEENppNCZVJ5vpWMlMmQ8Wio7L_vVXscVLXlj5UtQe3aqo5RMTdTq2PepdZjP86UOmA8nyAZZ0kujTWt86aLf2utm1bD-2FSj2DcbTCh3O3HD32SBIzeJdlXzKUs-2FRqjzbupd0J1Y8z5xnvQV5k1D5zyWOAHSxLv0Ezts-2FF2V2Y974g-2FABq5B151S2LbhNv8zTY6syz-2B1z-2BrGm8iNYGAPN1C9KbPpr0x08ptebSaNu86Stmdovg-3D-3D' + ) + .reply(301, undefined, { + Location: + 'https://every.to/divinations/the-endgame-for-ai-generated-writing', + }) + nock('https://every.to/divinations') + .head('/the-endgame-for-ai-generated-writing') + .reply(200, '') + }) + + it('returns url when email is from Every.io', async () => { + const url = + 'https://every.to/divinations/the-endgame-for-ai-generated-writing' + const html = load('./test/data/every-io-newsletter.html') + + await expect( + new EveryIoHandler().parseNewsletterUrl({}, html) + ).to.eventually.equal(url) + }) + }) }) describe('get author from email address', () => { @@ -267,7 +294,7 @@ describe('Newsletter email test', () => { expect(handler).to.be.undefined }) - it('returns ContentHandler for TTSO newsletter', async () => { + it('returns GenericHandler for TTSO newsletter', async () => { const html = load('./test/data/ttso-newsletter.html') const handler = await getNewsletterHandler({ html, @@ -278,7 +305,17 @@ describe('Newsletter email test', () => { ', ', }, }) - expect(handler).to.be.instanceOf(ContentHandler) + expect(handler).to.be.instanceOf(GenericHandler) + }) + + it('returns EveryIoHandler for every.io newsletter', async () => { + const html = load('./test/data/every-io-newsletter.html') + const handler = await getNewsletterHandler({ + html, + from: 'Every ', + headers: {}, + }) + expect(handler).to.be.instanceOf(EveryIoHandler) }) })