diff --git a/packages/api/src/utils/axios-handler.ts b/packages/api/src/utils/axios-handler.ts index 6e44e1e90..868181c85 100644 --- a/packages/api/src/utils/axios-handler.ts +++ b/packages/api/src/utils/axios-handler.ts @@ -1,17 +1,15 @@ -import { DOMWindow } from 'jsdom' - export class AxiosHandler { name = 'axios' // eslint-disable-next-line @typescript-eslint/no-unused-vars - shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => { + shouldPrehandle = (url: URL, _dom: Document): boolean => { const host = this.name + '.com' // check if url ends with axios.com return url.hostname.endsWith(host) } - prehandle = (url: URL, dom: DOMWindow): Promise => { - const body = dom.document.querySelector('table') + prehandle = (url: URL, dom: Document): Promise => { + const body = dom.querySelector('table') // this removes ads and replaces table with a div body?.querySelectorAll('table').forEach((el, k) => { @@ -27,7 +25,7 @@ export class AxiosHandler { } }) // replace the table with a div - const div = dom.document.createElement('div') + const div = dom.createElement('div') div.innerHTML = el.innerHTML el.parentNode?.replaceChild(div, el) } diff --git a/packages/api/src/utils/bloomberg-handler.ts b/packages/api/src/utils/bloomberg-handler.ts index 2fbae0ef5..c03af3f4b 100644 --- a/packages/api/src/utils/bloomberg-handler.ts +++ b/packages/api/src/utils/bloomberg-handler.ts @@ -1,22 +1,18 @@ -import { DOMWindow } from 'jsdom' - export class BloombergHandler { name = 'bloomberg' - shouldPrehandle = (url: URL, dom: DOMWindow): boolean => { + shouldPrehandle = (url: URL, dom: Document): boolean => { const host = this.name + '.com' // check if url ends with bloomberg.com return ( url.hostname.endsWith(host) || - dom.document - .querySelector('.logo-image') - ?.getAttribute('alt') - ?.toLowerCase() === this.name + dom.querySelector('.logo-image')?.getAttribute('alt')?.toLowerCase() === + this.name ) } - prehandle = (_url: URL, dom: DOMWindow): Promise => { - const body = dom.document.querySelector('.wrapper') + prehandle = (_url: URL, dom: Document): Promise => { + const body = dom.querySelector('.wrapper') // this removes header body?.querySelector('.sailthru-variables')?.remove() diff --git a/packages/api/src/utils/golang-handler.ts b/packages/api/src/utils/golang-handler.ts index 3a3037a59..6e6e29d02 100644 --- a/packages/api/src/utils/golang-handler.ts +++ b/packages/api/src/utils/golang-handler.ts @@ -1,17 +1,15 @@ -import { DOMWindow } from 'jsdom' - export class GolangHandler { name = 'golangweekly' // eslint-disable-next-line @typescript-eslint/no-unused-vars - shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => { + shouldPrehandle = (url: URL, _dom: Document): boolean => { const host = this.name + '.com' // check if url ends with golangweekly.com return url.hostname.endsWith(host) } - prehandle = (url: URL, dom: DOMWindow): Promise => { - const body = dom.document.querySelector('body') + prehandle = (url: URL, dom: Document): Promise => { + const body = dom.querySelector('body') // this removes the "Subscribe" button body?.querySelector('.el-splitbar')?.remove() diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index d5dfc19d0..878f801b1 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -2,7 +2,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/no-unused-vars */ import { Readability } from '@omnivore/readability' -import { DOMWindow, JSDOM, VirtualConsole } from 'jsdom' import createDOMPurify, { SanitizeElementHookEvent } from 'dompurify' import { PageType, PreparedDocumentInput } from '../generated/graphql' import { buildLogger, LogRecord } from './logger' @@ -42,8 +41,8 @@ const DOM_PURIFY_CONFIG = { } interface ContentHandler { - shouldPrehandle: (url: URL, dom: DOMWindow) => boolean - prehandle: (url: URL, document: DOMWindow) => Promise + shouldPrehandle: (url: URL, dom: Document) => boolean + prehandle: (url: URL, document: Document) => Promise } const HANDLERS = [ @@ -178,12 +177,15 @@ const getReadabilityResult = ( return null } -const applyHandlers = async (url: string, window: DOMWindow): Promise => { +const applyHandlers = async ( + url: string, + document: Document +): Promise => { try { const u = new URL(url) const handler = HANDLERS.find((h) => { try { - return h.shouldPrehandle(u, window) + return h.shouldPrehandle(u, document) } catch (e) { console.log('error with handler: ', h.name, e) } @@ -192,7 +194,7 @@ const applyHandlers = async (url: string, window: DOMWindow): Promise => { if (handler) { try { console.log('pre-handling url or content with handler: ', handler.name) - await handler.prehandle(u, window) + await handler.prehandle(u, document) } catch (e) { console.log('error with handler: ', handler, e) } @@ -240,7 +242,7 @@ export const parsePreparedContent = async ( // }) const { document: doc } = parseHTML(document) - await applyHandlers(url, window) + await applyHandlers(url, doc) try { article = getReadabilityResult(url, document, doc, isNewsletter) diff --git a/packages/api/src/utils/substack-handler.ts b/packages/api/src/utils/substack-handler.ts index 0f8f2176b..ea6b69bc3 100644 --- a/packages/api/src/utils/substack-handler.ts +++ b/packages/api/src/utils/substack-handler.ts @@ -1,23 +1,21 @@ -import { DOMWindow } from 'jsdom' - export class SubstackHandler { name = 'substack' - shouldPrehandle = (url: URL, dom: DOMWindow): boolean => { + shouldPrehandle = (url: URL, dom: Document): boolean => { const host = this.name + '.com' // check if url ends with substack.com // or has a profile image hosted at substack.com return ( url.hostname.endsWith(host) || - !!dom.document + !!dom .querySelector('.email-body img') ?.getAttribute('src') ?.includes(host) ) } - prehandle = (url: URL, dom: DOMWindow): Promise => { - const body = dom.document.querySelector('.email-body-container') + prehandle = (url: URL, dom: Document): Promise => { + const body = dom.querySelector('.email-body-container') // this removes header and profile avatar body?.querySelector('.header')?.remove() diff --git a/packages/api/src/utils/wikipedia-handler.ts b/packages/api/src/utils/wikipedia-handler.ts index 05fc4b5d4..ce30517c9 100644 --- a/packages/api/src/utils/wikipedia-handler.ts +++ b/packages/api/src/utils/wikipedia-handler.ts @@ -1,18 +1,16 @@ -import { DOMWindow } from 'jsdom' - export class WikipediaHandler { name = 'wikipedia' // eslint-disable-next-line @typescript-eslint/no-unused-vars - shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => { + shouldPrehandle = (url: URL, _dom: Document): boolean => { return url.hostname.endsWith('wikipedia.org') } - prehandle = (url: URL, dom: DOMWindow): Promise => { + prehandle = (url: URL, dom: Document): Promise => { // This removes the [edit] anchors from wikipedia pages - dom.document.querySelectorAll('.mw-editsection').forEach((e) => e.remove()) + dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove()) // this removes the sidebar - dom.document.querySelector('.infobox')?.remove() + dom.querySelector('.infobox')?.remove() return Promise.resolve(dom) } }