diff --git a/packages/api/package.json b/packages/api/package.json index 5297027a2..0631570ed 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -18,6 +18,7 @@ "@google-cloud/pubsub": "^2.16.0", "@google-cloud/storage": "^5.18.1", "@google-cloud/tasks": "^2.3.0", + "@omnivore/content-handler": "1.0.0", "@omnivore/readability": "1.0.0", "@omnivore/text-to-speech-handler": "1.0.0", "@opentelemetry/api": "^1.0.1", diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 6eb4237e2..eee86b417 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -15,6 +15,7 @@ import { User } from '../entity/user' import { ILike } from 'typeorm' import { v4 as uuid } from 'uuid' import addressparser from 'addressparser' +import { preParseContent } from '@omnivore/content-handler' const logger = buildLogger('utils.parse') @@ -194,7 +195,11 @@ export const parsePreparedContent = async ( } } - const dom = parseHTML(document).document + let dom = parseHTML(document).document + + // preParse content + const preParsedDom = await preParseContent(url, dom) + preParsedDom && (dom = preParsedDom) try { article = await getReadabilityResult(url, document, dom, isNewsletter) diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts index 22216fabe..317f3d2d9 100644 --- a/packages/content-handler/src/content-handler.ts +++ b/packages/content-handler/src/content-handler.ts @@ -58,12 +58,20 @@ export abstract class ContentHandler { return Promise.resolve(url) } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreHandle(url: string): boolean { return false } - async preHandle(url: string, dom?: Document): Promise { - return Promise.resolve({ url, dom }) + async preHandle(url: string): Promise { + return Promise.resolve({ url }) + } + + shouldPreParse(url: string, dom: Document): boolean { + return false + } + + async preParse(url: string, dom: Document): Promise { + return Promise.resolve(dom) } async isNewsletter(input: { diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index e41c811c4..ba67d2d9d 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -52,6 +52,11 @@ const contentHandlers: ContentHandler[] = [ new TwitterHandler(), new YoutubeHandler(), new WikipediaHandler(), + new AxiosHandler(), + new GolangHandler(), + new MorningBrewHandler(), + new BloombergNewsletterHandler(), + new SubstackHandler(), ] const newsletterHandlers: ContentHandler[] = [ @@ -60,15 +65,13 @@ const newsletterHandlers: ContentHandler[] = [ new GolangHandler(), new SubstackHandler(), new MorningBrewHandler(), - new SubstackHandler(), new BeehiivHandler(), new ConvertkitHandler(), new RevueHandler(), ] export const preHandleContent = async ( - url: string, - dom?: Document + url: string ): Promise => { // Before we run the regular handlers we check to see if we need tp // pre-resolve the URL. TODO: This should probably happen recursively, @@ -90,9 +93,25 @@ export const preHandleContent = async ( // to perform a prefetch action that can modify our requests. // enumerate the handlers and see if any of them want to handle the request for (const handler of contentHandlers) { - if (handler.shouldPreHandle(url, dom)) { + if (handler.shouldPreHandle(url)) { console.log('preHandleContent', handler.name, url) - return handler.preHandle(url, dom) + return handler.preHandle(url) + } + } + return undefined +} + +export const preParseContent = async ( + url: string, + dom: Document +): Promise => { + // Before we parse the page we check the handlers, to see if they want + // to perform a preParse action that can modify our dom. + // enumerate the handlers and see if any of them want to handle the dom + for (const handler of contentHandlers) { + if (handler.shouldPreParse(url, dom)) { + console.log('preParseContent', handler.name, url) + return handler.preParse(url, dom) } } return undefined @@ -113,4 +132,5 @@ export const handleNewsletter = async ( module.exports = { preHandleContent, handleNewsletter, + preParseContent, } diff --git a/packages/content-handler/src/newsletters/axios-handler.ts b/packages/content-handler/src/newsletters/axios-handler.ts index cd783c30e..0fc6dc3b3 100644 --- a/packages/content-handler/src/newsletters/axios-handler.ts +++ b/packages/content-handler/src/newsletters/axios-handler.ts @@ -1,4 +1,4 @@ -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' export class AxiosHandler extends ContentHandler { constructor() { @@ -8,13 +8,13 @@ export class AxiosHandler extends ContentHandler { this.name = 'axios' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { const host = this.name + '.com' // check if url ends with axios.com return new URL(url).hostname.endsWith(host) } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { const body = dom.querySelector('table') let isFooter = false @@ -41,6 +41,6 @@ export class AxiosHandler extends ContentHandler { } }) - return Promise.resolve({ dom }) + return Promise.resolve(dom) } } diff --git a/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts b/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts index a5f84f076..5ccd31466 100644 --- a/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts +++ b/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts @@ -1,4 +1,4 @@ -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' export class BloombergNewsletterHandler extends ContentHandler { constructor() { @@ -8,7 +8,7 @@ export class BloombergNewsletterHandler extends ContentHandler { this.name = 'bloomberg' } - shouldPreHandle(url: string, dom: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { const host = this.name + '.com' // check if url ends with bloomberg.com return ( @@ -18,7 +18,7 @@ export class BloombergNewsletterHandler extends ContentHandler { ) } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { const body = dom.querySelector('.wrapper') // this removes header @@ -32,6 +32,6 @@ export class BloombergNewsletterHandler extends ContentHandler { body?.querySelector('.component-wrapper')?.remove() body?.querySelector('.footer')?.remove() - return Promise.resolve({ dom }) + return Promise.resolve(dom) } } diff --git a/packages/content-handler/src/newsletters/golang-handler.ts b/packages/content-handler/src/newsletters/golang-handler.ts index 7d4724004..aeafec852 100644 --- a/packages/content-handler/src/newsletters/golang-handler.ts +++ b/packages/content-handler/src/newsletters/golang-handler.ts @@ -1,4 +1,4 @@ -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' export class GolangHandler extends ContentHandler { constructor() { @@ -8,13 +8,13 @@ export class GolangHandler extends ContentHandler { this.name = 'golangweekly' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { const host = this.name + '.com' // check if url ends with golangweekly.com return new URL(url).hostname.endsWith(host) } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { const body = dom.querySelector('body') // this removes the "Subscribe" button @@ -22,6 +22,6 @@ export class GolangHandler extends ContentHandler { // this removes the title body?.querySelector('.el-masthead')?.remove() - return Promise.resolve({ dom }) + return Promise.resolve(dom) } } diff --git a/packages/content-handler/src/newsletters/morning-brew-handler.ts b/packages/content-handler/src/newsletters/morning-brew-handler.ts index f187ac0dc..0a69292fe 100644 --- a/packages/content-handler/src/newsletters/morning-brew-handler.ts +++ b/packages/content-handler/src/newsletters/morning-brew-handler.ts @@ -1,4 +1,4 @@ -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' export class MorningBrewHandler extends ContentHandler { constructor() { @@ -8,13 +8,13 @@ export class MorningBrewHandler extends ContentHandler { this.name = 'morningbrew' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { const host = this.name + '.com' // check if url ends with morningbrew.com return new URL(url).hostname.endsWith(host) } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { // retain the width of the cells in the table of market info dom.querySelectorAll('.markets-arrow-cell').forEach((td) => { const table = td.closest('table') @@ -30,6 +30,6 @@ export class MorningBrewHandler extends ContentHandler { } }) - return Promise.resolve({ dom }) + return Promise.resolve(dom) } } diff --git a/packages/content-handler/src/newsletters/substack-handler.ts b/packages/content-handler/src/newsletters/substack-handler.ts index f66a85a52..e90168c8e 100644 --- a/packages/content-handler/src/newsletters/substack-handler.ts +++ b/packages/content-handler/src/newsletters/substack-handler.ts @@ -1,5 +1,5 @@ import addressparser from 'addressparser' -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' import { parseHTML } from 'linkedom' export class SubstackHandler extends ContentHandler { @@ -8,7 +8,7 @@ export class SubstackHandler extends ContentHandler { this.name = 'substack' } - shouldPreHandle(url: string, dom: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { const host = this.name + '.com' // check if url ends with substack.com // or has a profile image hosted at substack.com @@ -21,7 +21,7 @@ export class SubstackHandler extends ContentHandler { ) } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { const body = dom.querySelector('.email-body-container') // this removes header and profile avatar diff --git a/packages/content-handler/src/websites/wikipedia-handler.ts b/packages/content-handler/src/websites/wikipedia-handler.ts index 8c3a176fd..37005a18a 100644 --- a/packages/content-handler/src/websites/wikipedia-handler.ts +++ b/packages/content-handler/src/websites/wikipedia-handler.ts @@ -1,4 +1,4 @@ -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' export class WikipediaHandler extends ContentHandler { constructor() { @@ -6,15 +6,15 @@ export class WikipediaHandler extends ContentHandler { this.name = 'wikipedia' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { return new URL(url).hostname.endsWith('wikipedia.org') } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { // This removes the [edit] anchors from wikipedia pages dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove()) // this removes the sidebar dom.querySelector('.infobox')?.remove() - return Promise.resolve({ dom }) + return Promise.resolve(dom) } }