Merge pull request #1285 from omnivore-app/use-content-handler-in-api

use content handler in api

.github/workflows/run-tests.yaml

@@ -88,3 +88,7 @@ jobs:
         run: 'docker build --file packages/api/Dockerfile .'
       - name: Build the content-fetch docker image
         run: 'docker build --file packages/content-fetch/Dockerfile .'
+      - name: Build the inbound-email-handler docker image
+        run: 'docker build --file packages/inbound-email-handler/Dockerfile .'
+      - name: Build the puppeteer-parse docker image
+        run: 'docker build --file packages/puppeteer-parse/Dockerfile .'

packages/api/Dockerfile

@@ -14,15 +14,17 @@ COPY .eslintrc .
 COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json
 COPY /packages/api/package.json ./packages/api/package.json
 COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json
+COPY /packages/content-handler/package.json ./packages/content-handler/package.json
 
 RUN yarn install --pure-lockfile
 
 ADD /packages/readabilityjs ./packages/readabilityjs
 ADD /packages/api ./packages/api
 ADD /packages/text-to-speech ./packages/text-to-speech
+ADD /packages/content-handler ./packages/content-handler
 
-RUN yarn
 RUN yarn workspace @omnivore/text-to-speech-handler build
+RUN yarn workspace @omnivore/content-handler build
 RUN yarn workspace @omnivore/api build
 
 # After building, fetch the production dependencies

packages/api/package.json

@@ -18,6 +18,7 @@
     "@google-cloud/pubsub": "^2.16.0",
     "@google-cloud/storage": "^5.18.1",
     "@google-cloud/tasks": "^2.3.0",
+    "@omnivore/content-handler": "1.0.0",
     "@omnivore/readability": "1.0.0",
     "@omnivore/text-to-speech-handler": "1.0.0",
     "@opentelemetry/api": "^1.0.1",

packages/api/src/utils/axios-handler.ts

@@ -1,40 +0,0 @@
-export class AxiosHandler {
-  name = 'axios'
-
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  shouldPrehandle = (url: URL, _dom: Document): boolean => {
-    const host = this.name + '.com'
-    // check if url ends with axios.com
-    return url.hostname.endsWith(host)
-  }
-
-  prehandle = (url: URL, dom: Document): Promise<Document> => {
-    const body = dom.querySelector('table')
-
-    let isFooter = false
-    // this removes ads and replaces table with a div
-    body?.querySelectorAll('table').forEach((el) => {
-      // remove the footer and the ads
-      if (!el.textContent || el.textContent.length < 20 || isFooter) {
-        el.remove()
-      } else {
-        // removes the first few rows of the table (the header)
-        // remove the last two rows of the table (they are ads)
-        el.querySelectorAll('tr').forEach((tr, i) => {
-          if (i <= 7 || i >= el.querySelectorAll('tr').length - 2) {
-            console.log('removing', tr)
-            tr.remove()
-          }
-        })
-        // replace the table with a div
-        const div = dom.createElement('div')
-        div.innerHTML = el.innerHTML
-        el.parentNode?.replaceChild(div, el)
-        // set the isFooter flag to true because the next table is the footer
-        isFooter = true
-      }
-    })
-
-    return Promise.resolve(dom)
-  }
-}

packages/api/src/utils/bloomberg-handler.ts

@@ -1,30 +0,0 @@
-export class BloombergHandler {
-  name = 'bloomberg'
-
-  shouldPrehandle = (url: URL, dom: Document): boolean => {
-    const host = this.name + '.com'
-    // check if url ends with bloomberg.com
-    return (
-      url.hostname.endsWith(host) ||
-      dom.querySelector('.logo-image')?.getAttribute('alt')?.toLowerCase() ===
-        this.name
-    )
-  }
-
-  prehandle = (_url: URL, dom: Document): Promise<Document> => {
-    const body = dom.querySelector('.wrapper')
-
-    // this removes header
-    body?.querySelector('.sailthru-variables')?.remove()
-    body?.querySelector('.preview-text')?.remove()
-    body?.querySelector('.logo-wrapper')?.remove()
-    body?.querySelector('.by-the-number-wrapper')?.remove()
-    // this removes footer
-    body?.querySelector('.quote-box-wrapper')?.remove()
-    body?.querySelector('.header-wrapper')?.remove()
-    body?.querySelector('.component-wrapper')?.remove()
-    body?.querySelector('.footer')?.remove()
-
-    return Promise.resolve(dom)
-  }
-}

packages/api/src/utils/morning-brew-handler.ts

@@ -1,29 +0,0 @@
-export class MorningBrewHandler {
-  name = 'morningbrew'
-
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  shouldPrehandle = (url: URL, _dom: Document): boolean => {
-    const host = this.name + '.com'
-    // check if url ends with morningbrew.com
-    return url.hostname.endsWith(host)
-  }
-
-  prehandle = (url: URL, dom: Document): Promise<Document> => {
-    // retain the width of the cells in the table of market info
-    dom.querySelectorAll('.markets-arrow-cell').forEach((td) => {
-      const table = td.closest('table')
-      if (table) {
-        const bubbleTable = table.querySelector('.markets-bubble')
-        if (bubbleTable) {
-          // replace the nested table with the text
-          const e = bubbleTable.querySelector('.markets-table-text')
-          e && bubbleTable.parentNode?.replaceChild(e, bubbleTable)
-        }
-        // set custom class for the table
-        table.className = 'morning-brew-markets'
-      }
-    })
-
-    return Promise.resolve(dom)
-  }
-}

packages/api/src/utils/parser.ts

@@ -7,11 +7,6 @@ import { PageType, PreparedDocumentInput } from '../generated/graphql'
 import { buildLogger, LogRecord } from './logger'
 import { createImageProxyUrl } from './imageproxy'
 import axios from 'axios'
-import { WikipediaHandler } from './wikipedia-handler'
-import { SubstackHandler } from './substack-handler'
-import { AxiosHandler } from './axios-handler'
-import { BloombergHandler } from './bloomberg-handler'
-import { GolangHandler } from './golang-handler'
 import * as hljs from 'highlightjs'
 import { decode } from 'html-entities'
 import { parseHTML } from 'linkedom'
@@ -20,7 +15,7 @@ import { User } from '../entity/user'
 import { ILike } from 'typeorm'
 import { v4 as uuid } from 'uuid'
 import addressparser from 'addressparser'
-import { MorningBrewHandler } from './morning-brew-handler'
+import { preParseContent } from '@omnivore/content-handler'
 
 const logger = buildLogger('utils.parse')
 
@@ -47,20 +42,6 @@ const ARTICLE_PREFIX = 'omnivore:'
 
 export const FAKE_URL_PREFIX = 'https://omnivore.app/no_url?q='
 
-interface ContentHandler {
-  shouldPrehandle: (url: URL, dom: Document) => boolean
-  prehandle: (url: URL, document: Document) => Promise<Document>
-}
-
-const HANDLERS = [
-  new WikipediaHandler(),
-  new SubstackHandler(),
-  new AxiosHandler(),
-  new BloombergHandler(),
-  new GolangHandler(),
-  new MorningBrewHandler(),
-]
-
 /** Hook that prevents DOMPurify from removing youtube iframes */
 const domPurifySanitizeHook = (
   node: Element,
@@ -185,33 +166,6 @@ const getReadabilityResult = async (
   return null
 }
 
-const applyHandlers = async (
-  url: string,
-  document: Document
-): Promise<void> => {
-  try {
-    const u = new URL(url)
-    const handler = HANDLERS.find((h) => {
-      try {
-        return h.shouldPrehandle(u, document)
-      } catch (e) {
-        console.log('error with handler: ', h.name, e)
-      }
-      return false
-    })
-    if (handler) {
-      try {
-        console.log('pre-handling url or content with handler: ', handler.name)
-        await handler.prehandle(u, document)
-      } catch (e) {
-        console.log('error with handler: ', handler, e)
-      }
-    }
-  } catch (error) {
-    logger.error('Error prehandling url', url, error)
-  }
-}
-
 export const parsePreparedContent = async (
   url: string,
   preparedDocument: PreparedDocumentInput,
@@ -241,9 +195,11 @@ export const parsePreparedContent = async (
     }
   }
 
-  const dom = parseHTML(document).document
+  let dom = parseHTML(document).document
 
-  await applyHandlers(url, dom)
+  // preParse content
+  const preParsedDom = await preParseContent(url, dom)
+  preParsedDom && (dom = preParsedDom)
 
   try {
     article = await getReadabilityResult(url, document, dom, isNewsletter)

packages/api/src/utils/substack-handler.ts

@@ -1,33 +0,0 @@
-export class SubstackHandler {
-  name = 'substack'
-
-  shouldPrehandle = (url: URL, dom: Document): boolean => {
-    const host = this.name + '.com'
-    // check if url ends with substack.com
-    // or has a profile image hosted at substack.com
-    return (
-      url.hostname.endsWith(host) ||
-      !!dom
-        .querySelector('.email-body img')
-        ?.getAttribute('src')
-        ?.includes(host)
-    )
-  }
-
-  prehandle = (url: URL, dom: Document): Promise<Document> => {
-    const body = dom.querySelector('.email-body-container')
-
-    // this removes header and profile avatar
-    body?.querySelector('.header')?.remove()
-    body?.querySelector('.preamble')?.remove()
-    body?.querySelector('.meta-author-wrap')?.remove()
-    // this removes meta button
-    body?.querySelector('.post-meta')?.remove()
-    // this removes footer
-    body?.querySelector('.post-cta')?.remove()
-    body?.querySelector('.container-border')?.remove()
-    body?.querySelector('.footer')?.remove()
-
-    return Promise.resolve(dom)
-  }
-}

packages/api/src/utils/wikipedia-handler.ts

@@ -1,16 +0,0 @@
-export class WikipediaHandler {
-  name = 'wikipedia'
-
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  shouldPrehandle = (url: URL, _dom: Document): boolean => {
-    return url.hostname.endsWith('wikipedia.org')
-  }
-
-  prehandle = (url: URL, dom: Document): Promise<Document> => {
-    // This removes the [edit] anchors from wikipedia pages
-    dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove())
-    // this removes the sidebar
-    dom.querySelector('.infobox')?.remove()
-    return Promise.resolve(dom)
-  }
-}

packages/content-handler/src/content-handler.ts

@@ -58,12 +58,20 @@ export abstract class ContentHandler {
     return Promise.resolve(url)
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreHandle(url: string): boolean {
     return false
   }
 
-  async preHandle(url: string, dom?: Document): Promise<PreHandleResult> {
-    return Promise.resolve({ url, dom })
+  async preHandle(url: string): Promise<PreHandleResult> {
+    return Promise.resolve({ url })
+  }
+
+  shouldPreParse(url: string, dom: Document): boolean {
+    return false
+  }
+
+  async preParse(url: string, dom: Document): Promise<Document> {
+    return Promise.resolve(dom)
   }
 
   async isNewsletter(input: {

packages/content-handler/src/index.ts

@@ -52,6 +52,11 @@ const contentHandlers: ContentHandler[] = [
   new TwitterHandler(),
   new YoutubeHandler(),
   new WikipediaHandler(),
+  new AxiosHandler(),
+  new GolangHandler(),
+  new MorningBrewHandler(),
+  new BloombergNewsletterHandler(),
+  new SubstackHandler(),
 ]
 
 const newsletterHandlers: ContentHandler[] = [
@@ -60,15 +65,13 @@ const newsletterHandlers: ContentHandler[] = [
   new GolangHandler(),
   new SubstackHandler(),
   new MorningBrewHandler(),
-  new SubstackHandler(),
   new BeehiivHandler(),
   new ConvertkitHandler(),
   new RevueHandler(),
 ]
 
 export const preHandleContent = async (
-  url: string,
-  dom?: Document
+  url: string
 ): Promise<PreHandleResult | undefined> => {
   // Before we run the regular handlers we check to see if we need tp
   // pre-resolve the URL. TODO: This should probably happen recursively,
@@ -90,9 +93,25 @@ export const preHandleContent = async (
   // to perform a prefetch action that can modify our requests.
   // enumerate the handlers and see if any of them want to handle the request
   for (const handler of contentHandlers) {
-    if (handler.shouldPreHandle(url, dom)) {
+    if (handler.shouldPreHandle(url)) {
       console.log('preHandleContent', handler.name, url)
-      return handler.preHandle(url, dom)
+      return handler.preHandle(url)
+    }
+  }
+  return undefined
+}
+
+export const preParseContent = async (
+  url: string,
+  dom: Document
+): Promise<Document | undefined> => {
+  // Before we parse the page we check the handlers, to see if they want
+  // to perform a preParse action that can modify our dom.
+  // enumerate the handlers and see if any of them want to handle the dom
+  for (const handler of contentHandlers) {
+    if (handler.shouldPreParse(url, dom)) {
+      console.log('preParseContent', handler.name, url)
+      return handler.preParse(url, dom)
     }
   }
   return undefined
@@ -113,4 +132,5 @@ export const handleNewsletter = async (
 module.exports = {
   preHandleContent,
   handleNewsletter,
+  preParseContent,
 }

packages/content-handler/src/newsletters/axios-handler.ts

@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 
 export class AxiosHandler extends ContentHandler {
   constructor() {
@@ -8,13 +8,13 @@ export class AxiosHandler extends ContentHandler {
     this.name = 'axios'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     const host = this.name + '.com'
     // check if url ends with axios.com
     return new URL(url).hostname.endsWith(host)
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     const body = dom.querySelector('table')
 
     let isFooter = false
@@ -41,6 +41,6 @@ export class AxiosHandler extends ContentHandler {
       }
     })
 
-    return Promise.resolve({ dom })
+    return Promise.resolve(dom)
   }
 }

packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts

@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 
 export class BloombergNewsletterHandler extends ContentHandler {
   constructor() {
@@ -8,7 +8,7 @@ export class BloombergNewsletterHandler extends ContentHandler {
     this.name = 'bloomberg'
   }
 
-  shouldPreHandle(url: string, dom: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     const host = this.name + '.com'
     // check if url ends with bloomberg.com
     return (
@@ -18,7 +18,7 @@ export class BloombergNewsletterHandler extends ContentHandler {
     )
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     const body = dom.querySelector('.wrapper')
 
     // this removes header
@@ -32,6 +32,6 @@ export class BloombergNewsletterHandler extends ContentHandler {
     body?.querySelector('.component-wrapper')?.remove()
     body?.querySelector('.footer')?.remove()
 
-    return Promise.resolve({ dom })
+    return Promise.resolve(dom)
   }
 }

packages/content-handler/src/newsletters/golang-handler.ts

@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 
 export class GolangHandler extends ContentHandler {
   constructor() {
@@ -8,13 +8,13 @@ export class GolangHandler extends ContentHandler {
     this.name = 'golangweekly'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     const host = this.name + '.com'
     // check if url ends with golangweekly.com
     return new URL(url).hostname.endsWith(host)
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     const body = dom.querySelector('body')
 
     // this removes the "Subscribe" button
@@ -22,6 +22,6 @@ export class GolangHandler extends ContentHandler {
     // this removes the title
     body?.querySelector('.el-masthead')?.remove()
 
-    return Promise.resolve({ dom })
+    return Promise.resolve(dom)
   }
 }

packages/content-handler/src/newsletters/morning-brew-handler.ts

@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 
 export class MorningBrewHandler extends ContentHandler {
   constructor() {
@@ -8,13 +8,13 @@ export class MorningBrewHandler extends ContentHandler {
     this.name = 'morningbrew'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     const host = this.name + '.com'
     // check if url ends with morningbrew.com
     return new URL(url).hostname.endsWith(host)
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     // retain the width of the cells in the table of market info
     dom.querySelectorAll('.markets-arrow-cell').forEach((td) => {
       const table = td.closest('table')
@@ -30,6 +30,6 @@ export class MorningBrewHandler extends ContentHandler {
       }
     })
 
-    return Promise.resolve({ dom })
+    return Promise.resolve(dom)
   }
 }

packages/content-handler/src/newsletters/substack-handler.ts

@@ -1,5 +1,5 @@
 import addressparser from 'addressparser'
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 import { parseHTML } from 'linkedom'
 
 export class SubstackHandler extends ContentHandler {
@@ -8,7 +8,7 @@ export class SubstackHandler extends ContentHandler {
     this.name = 'substack'
   }
 
-  shouldPreHandle(url: string, dom: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     const host = this.name + '.com'
     // check if url ends with substack.com
     // or has a profile image hosted at substack.com
@@ -21,7 +21,7 @@ export class SubstackHandler extends ContentHandler {
     )
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     const body = dom.querySelector('.email-body-container')
 
     // this removes header and profile avatar

packages/content-handler/src/websites/apple-news-handler.ts

@@ -8,12 +8,12 @@ export class AppleNewsHandler extends ContentHandler {
     this.name = 'Apple News'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreHandle(url: string): boolean {
     const u = new URL(url)
     return u.hostname === 'apple.news'
   }
 
-  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+  async preHandle(url: string): Promise<PreHandleResult> {
     const MOBILE_USER_AGENT =
       'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
     const response = await axios.get(url, {

packages/content-handler/src/websites/bloomberg-handler.ts

@@ -8,13 +8,13 @@ export class BloombergHandler extends ContentHandler {
     this.name = 'Bloomberg'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreHandle(url: string): boolean {
     const BLOOMBERG_URL_MATCH =
       /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)/
     return BLOOMBERG_URL_MATCH.test(url.toString())
   }
 
-  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+  async preHandle(url: string): Promise<PreHandleResult> {
     console.log('prehandling bloomberg url', url)
 
     try {

packages/content-handler/src/websites/derstandard-handler.ts

@@ -8,12 +8,12 @@ export class DerstandardHandler extends ContentHandler {
     this.name = 'Derstandard'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreHandle(url: string): boolean {
     const u = new URL(url)
     return u.hostname === 'www.derstandard.at'
   }
 
-  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+  async preHandle(url: string): Promise<PreHandleResult> {
     const response = await axios.get(url, {
       // set cookie to give consent to get the article
       headers: {

packages/content-handler/src/websites/image-handler.ts

@@ -6,12 +6,12 @@ export class ImageHandler extends ContentHandler {
     this.name = 'Image'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreHandle(url: string): boolean {
     const IMAGE_URL_PATTERN = /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
     return IMAGE_URL_PATTERN.test(url.toString())
   }
 
-  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+  async preHandle(url: string): Promise<PreHandleResult> {
     const title = url.toString().split('/').pop() || 'Image'
     const content = `
       <html>

packages/content-handler/src/websites/medium-handler.ts

@@ -6,12 +6,12 @@ export class MediumHandler extends ContentHandler {
     this.name = 'Medium'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreHandle(url: string): boolean {
     const u = new URL(url)
     return u.hostname.endsWith('medium.com')
   }
 
-  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+  async preHandle(url: string): Promise<PreHandleResult> {
     console.log('prehandling medium url', url)
 
     try {

packages/content-handler/src/websites/pdf-handler.ts

@@ -6,13 +6,13 @@ export class PdfHandler extends ContentHandler {
     this.name = 'PDF'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreHandle(url: string): boolean {
     const u = new URL(url)
     const path = u.pathname.replace(u.search, '')
     return path.endsWith('.pdf')
   }
 
-  async preHandle(_url: string, document?: Document): Promise<PreHandleResult> {
+  async preHandle(url: string): Promise<PreHandleResult> {
     return Promise.resolve({ contentType: 'application/pdf' })
   }
 }

packages/content-handler/src/websites/scrapingBee-handler.ts

@@ -8,14 +8,14 @@ export class ScrapingBeeHandler extends ContentHandler {
     this.name = 'ScrapingBee'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreHandle(url: string): boolean {
     const u = new URL(url)
     const hostnames = ['nytimes.com', 'news.google.com']
 
     return hostnames.some((h) => u.hostname.endsWith(h))
   }
 
-  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+  async preHandle(url: string): Promise<PreHandleResult> {
     console.log('prehandling url with scrapingbee', url)
 
     try {

packages/content-handler/src/websites/twitter-handler.ts

@@ -140,11 +140,11 @@ export class TwitterHandler extends ContentHandler {
     this.name = 'Twitter'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreHandle(url: string): boolean {
     return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
   }
 
-  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+  async preHandle(url: string): Promise<PreHandleResult> {
     const tweetId = tweetIdFromStatusUrl(url)
     if (!tweetId) {
       throw new Error('could not find tweet id in url')

packages/content-handler/src/websites/wikipedia-handler.ts

@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 
 export class WikipediaHandler extends ContentHandler {
   constructor() {
@@ -6,15 +6,15 @@ export class WikipediaHandler extends ContentHandler {
     this.name = 'wikipedia'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     return new URL(url).hostname.endsWith('wikipedia.org')
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     // This removes the [edit] anchors from wikipedia pages
     dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove())
     // this removes the sidebar
     dom.querySelector('.infobox')?.remove()
-    return Promise.resolve({ dom })
+    return Promise.resolve(dom)
   }
 }

packages/content-handler/src/websites/youtube-handler.ts

@@ -24,11 +24,11 @@ export class YoutubeHandler extends ContentHandler {
     this.name = 'Youtube'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreHandle(url: string): boolean {
     return YOUTUBE_URL_MATCH.test(url.toString())
   }
 
-  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+  async preHandle(url: string): Promise<PreHandleResult> {
     const videoId = getYoutubeVideoId(url)
     if (!videoId) {
       return {}