From 363489f5be85dabf3901efd35f5cd78fceb0a462 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Feb 2023 14:46:24 +0800 Subject: [PATCH] Add generic newsletter handler --- .../content-handler/src/content-handler.ts | 37 +++----------- packages/content-handler/src/index.ts | 2 + .../src/newsletters/generic-handler.ts | 49 +++++++++++++++++++ .../content-handler/test/newsletter.test.ts | 5 +- 4 files changed, 62 insertions(+), 31 deletions(-) create mode 100644 packages/content-handler/src/newsletters/generic-handler.ts diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts index 78c9f5ffd..ebdf81d3e 100644 --- a/packages/content-handler/src/content-handler.ts +++ b/packages/content-handler/src/content-handler.ts @@ -39,16 +39,12 @@ export interface PreHandleResult { export const FAKE_URL_PREFIX = 'https://omnivore.app/no_url?q=' export const generateUniqueUrl = () => FAKE_URL_PREFIX + uuid() -export class ContentHandler { +export abstract class ContentHandler { protected senderRegex: RegExp protected urlRegex: RegExp name: string - // newsletter url text regex for newsletters that don't have a newsletter header - NEWSLETTER_URL_TEXT_REGEX = - /((View|Read)(.*)(email|post)?(.*)(in your browser|online|on (FS|the Web))|Lire en ligne)/i - - constructor() { + protected constructor() { this.senderRegex = new RegExp(/NEWSLETTER_SENDER_REGEX/) this.urlRegex = new RegExp(/NEWSLETTER_URL_REGEX/) this.name = 'Handler name' @@ -84,20 +80,16 @@ export class ContentHandler { headers: Record dom: Document }): Promise { - const postHeader = input.headers['list-post'] || input.headers['list-id'] + const re = new RegExp(this.senderRegex) + const postHeader = input.headers['list-post'] const unSubHeader = input.headers['list-unsubscribe'] - return Promise.resolve(!!postHeader || !!unSubHeader) + return Promise.resolve( + re.test(input.from) && (!!postHeader || !!unSubHeader) + ) } findNewsletterHeaderHref(dom: Document): string | undefined { - const readOnline = dom.querySelectorAll('a') - let res: string | undefined = undefined - readOnline.forEach((e) => { - if (e.textContent && this.NEWSLETTER_URL_TEXT_REGEX.test(e.textContent)) { - res = e.getAttribute('href') || undefined - } - }) - return res + return undefined } // Given an HTML blob tries to find a URL to use for @@ -129,24 +121,11 @@ export class ContentHandler { headers: Record, html: string ): Promise { - // raw SubStack newsletter url is like - // we need to get the real url from the raw url - const postHeader = headers['list-post']?.toString() - if (postHeader && addressparser(postHeader).length > 0) { - return addressparser(postHeader)[0].name - } - - const url = await this.findNewsletterUrl(html) - if (url) { - return url - } - // get newsletter url from html const matches = html.match(this.urlRegex) if (matches) { return Promise.resolve(matches[1]) } - return Promise.resolve(undefined) } diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index fb6491f59..366fc48fb 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -30,6 +30,7 @@ import { CooperPressHandler } from './newsletters/cooper-press-handler' import { HeyWorldHandler } from './newsletters/hey-world-handler' import { Browser } from 'puppeteer-core' import { StackOverflowHandler } from './websites/stack-overflow-handler' +import { GenericHandler } from './newsletters/generic-handler' const validateUrlString = (url: string) => { const u = new URL(url) @@ -80,6 +81,7 @@ const newsletterHandlers: ContentHandler[] = [ new GhostHandler(), new CooperPressHandler(), new HeyWorldHandler(), + new GenericHandler(), ] export const preHandleContent = async ( diff --git a/packages/content-handler/src/newsletters/generic-handler.ts b/packages/content-handler/src/newsletters/generic-handler.ts new file mode 100644 index 000000000..40f894705 --- /dev/null +++ b/packages/content-handler/src/newsletters/generic-handler.ts @@ -0,0 +1,49 @@ +import { ContentHandler } from '../content-handler' +import addressparser from 'addressparser' + +export class GenericHandler extends ContentHandler { + // newsletter url text regex for newsletters that don't have a newsletter header + NEWSLETTER_URL_TEXT_REGEX = + /((View|Read)(.*)(email|post)?(.*)(in your browser|online|on (FS|the Web))|Lire en ligne)/i + + constructor() { + super() + this.name = 'Generic Newsletter' + } + + async isNewsletter(input: { + from: string + html: string + headers: Record + dom: Document + }): Promise { + const postHeader = input.headers['list-post'] || input.headers['list-id'] + const unSubHeader = input.headers['list-unsubscribe'] + return Promise.resolve(!!postHeader || !!unSubHeader) + } + + findNewsletterHeaderHref(dom: Document): string | undefined { + const readOnline = dom.querySelectorAll('a') + let res: string | undefined = undefined + readOnline.forEach((e) => { + if (e.textContent && this.NEWSLETTER_URL_TEXT_REGEX.test(e.textContent)) { + res = e.getAttribute('href') || undefined + } + }) + return res + } + + async parseNewsletterUrl( + headers: Record, + html: string + ): Promise { + // raw SubStack newsletter url is like + // we need to get the real url from the raw url + const postHeader = headers['list-post']?.toString() + if (postHeader && addressparser(postHeader).length > 0) { + return addressparser(postHeader)[0].name + } + + return this.findNewsletterUrl(html) + } +} diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts index 74d78e0ff..a404495f8 100644 --- a/packages/content-handler/test/newsletter.test.ts +++ b/packages/content-handler/test/newsletter.test.ts @@ -18,6 +18,7 @@ import { CooperPressHandler } from '../src/newsletters/cooper-press-handler' import { getNewsletterHandler } from '../src' import { parseHTML } from 'linkedom' import { HeyWorldHandler } from '../src/newsletters/hey-world-handler' +import { GenericHandler } from '../src/newsletters/generic-handler' chai.use(chaiAsPromised) chai.use(chaiString) @@ -97,7 +98,7 @@ describe('Newsletter email test', () => { const html = load('./test/data/ttso-newsletter.html') await expect( - new ContentHandler().parseNewsletterUrl({}, html) + new GenericHandler().parseNewsletterUrl({}, html) ).to.eventually.equal(url) }) }) @@ -414,7 +415,7 @@ describe('Newsletter email test', () => { it('gets the URL from the header', async () => { const html = load('./test/data/ttso-newsletter.html') - const url = await new ContentHandler().findNewsletterUrl(html) + const url = await new GenericHandler().findNewsletterUrl(html) expect(url).to.startWith('https://ttso.paris/2023-01-31') }) })