Calls preParseContent function in content-handler package before parsing content

packages/api/package.json

@@ -18,6 +18,7 @@
     "@google-cloud/pubsub": "^2.16.0",
     "@google-cloud/storage": "^5.18.1",
     "@google-cloud/tasks": "^2.3.0",
+    "@omnivore/content-handler": "1.0.0",
     "@omnivore/readability": "1.0.0",
     "@omnivore/text-to-speech-handler": "1.0.0",
     "@opentelemetry/api": "^1.0.1",

packages/api/src/utils/parser.ts

@@ -15,6 +15,7 @@ import { User } from '../entity/user'
 import { ILike } from 'typeorm'
 import { v4 as uuid } from 'uuid'
 import addressparser from 'addressparser'
+import { preParseContent } from '@omnivore/content-handler'
 
 const logger = buildLogger('utils.parse')
 
@@ -194,7 +195,11 @@ export const parsePreparedContent = async (
     }
   }
 
-  const dom = parseHTML(document).document
+  let dom = parseHTML(document).document
+
+  // preParse content
+  const preParsedDom = await preParseContent(url, dom)
+  preParsedDom && (dom = preParsedDom)
 
   try {
     article = await getReadabilityResult(url, document, dom, isNewsletter)

packages/content-handler/src/content-handler.ts

@@ -58,12 +58,20 @@ export abstract class ContentHandler {
     return Promise.resolve(url)
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreHandle(url: string): boolean {
     return false
   }
 
-  async preHandle(url: string, dom?: Document): Promise<PreHandleResult> {
-    return Promise.resolve({ url, dom })
+  async preHandle(url: string): Promise<PreHandleResult> {
+    return Promise.resolve({ url })
+  }
+
+  shouldPreParse(url: string, dom: Document): boolean {
+    return false
+  }
+
+  async preParse(url: string, dom: Document): Promise<Document> {
+    return Promise.resolve(dom)
   }
 
   async isNewsletter(input: {

packages/content-handler/src/index.ts

@@ -52,6 +52,11 @@ const contentHandlers: ContentHandler[] = [
   new TwitterHandler(),
   new YoutubeHandler(),
   new WikipediaHandler(),
+  new AxiosHandler(),
+  new GolangHandler(),
+  new MorningBrewHandler(),
+  new BloombergNewsletterHandler(),
+  new SubstackHandler(),
 ]
 
 const newsletterHandlers: ContentHandler[] = [
@@ -60,15 +65,13 @@ const newsletterHandlers: ContentHandler[] = [
   new GolangHandler(),
   new SubstackHandler(),
   new MorningBrewHandler(),
-  new SubstackHandler(),
   new BeehiivHandler(),
   new ConvertkitHandler(),
   new RevueHandler(),
 ]
 
 export const preHandleContent = async (
-  url: string,
-  dom?: Document
+  url: string
 ): Promise<PreHandleResult | undefined> => {
   // Before we run the regular handlers we check to see if we need tp
   // pre-resolve the URL. TODO: This should probably happen recursively,
@@ -90,9 +93,25 @@ export const preHandleContent = async (
   // to perform a prefetch action that can modify our requests.
   // enumerate the handlers and see if any of them want to handle the request
   for (const handler of contentHandlers) {
-    if (handler.shouldPreHandle(url, dom)) {
+    if (handler.shouldPreHandle(url)) {
       console.log('preHandleContent', handler.name, url)
-      return handler.preHandle(url, dom)
+      return handler.preHandle(url)
+    }
+  }
+  return undefined
+}
+
+export const preParseContent = async (
+  url: string,
+  dom: Document
+): Promise<Document | undefined> => {
+  // Before we parse the page we check the handlers, to see if they want
+  // to perform a preParse action that can modify our dom.
+  // enumerate the handlers and see if any of them want to handle the dom
+  for (const handler of contentHandlers) {
+    if (handler.shouldPreParse(url, dom)) {
+      console.log('preParseContent', handler.name, url)
+      return handler.preParse(url, dom)
     }
   }
   return undefined
@@ -113,4 +132,5 @@ export const handleNewsletter = async (
 module.exports = {
   preHandleContent,
   handleNewsletter,
+  preParseContent,
 }

packages/content-handler/src/newsletters/axios-handler.ts

@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 
 export class AxiosHandler extends ContentHandler {
   constructor() {
@@ -8,13 +8,13 @@ export class AxiosHandler extends ContentHandler {
     this.name = 'axios'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     const host = this.name + '.com'
     // check if url ends with axios.com
     return new URL(url).hostname.endsWith(host)
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     const body = dom.querySelector('table')
 
     let isFooter = false
@@ -41,6 +41,6 @@ export class AxiosHandler extends ContentHandler {
       }
     })
 
-    return Promise.resolve({ dom })
+    return Promise.resolve(dom)
   }
 }

packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts

@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 
 export class BloombergNewsletterHandler extends ContentHandler {
   constructor() {
@@ -8,7 +8,7 @@ export class BloombergNewsletterHandler extends ContentHandler {
     this.name = 'bloomberg'
   }
 
-  shouldPreHandle(url: string, dom: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     const host = this.name + '.com'
     // check if url ends with bloomberg.com
     return (
@@ -18,7 +18,7 @@ export class BloombergNewsletterHandler extends ContentHandler {
     )
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     const body = dom.querySelector('.wrapper')
 
     // this removes header
@@ -32,6 +32,6 @@ export class BloombergNewsletterHandler extends ContentHandler {
     body?.querySelector('.component-wrapper')?.remove()
     body?.querySelector('.footer')?.remove()
 
-    return Promise.resolve({ dom })
+    return Promise.resolve(dom)
   }
 }

packages/content-handler/src/newsletters/golang-handler.ts

@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 
 export class GolangHandler extends ContentHandler {
   constructor() {
@@ -8,13 +8,13 @@ export class GolangHandler extends ContentHandler {
     this.name = 'golangweekly'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     const host = this.name + '.com'
     // check if url ends with golangweekly.com
     return new URL(url).hostname.endsWith(host)
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     const body = dom.querySelector('body')
 
     // this removes the "Subscribe" button
@@ -22,6 +22,6 @@ export class GolangHandler extends ContentHandler {
     // this removes the title
     body?.querySelector('.el-masthead')?.remove()
 
-    return Promise.resolve({ dom })
+    return Promise.resolve(dom)
   }
 }

packages/content-handler/src/newsletters/morning-brew-handler.ts

@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 
 export class MorningBrewHandler extends ContentHandler {
   constructor() {
@@ -8,13 +8,13 @@ export class MorningBrewHandler extends ContentHandler {
     this.name = 'morningbrew'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     const host = this.name + '.com'
     // check if url ends with morningbrew.com
     return new URL(url).hostname.endsWith(host)
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     // retain the width of the cells in the table of market info
     dom.querySelectorAll('.markets-arrow-cell').forEach((td) => {
       const table = td.closest('table')
@@ -30,6 +30,6 @@ export class MorningBrewHandler extends ContentHandler {
       }
     })
 
-    return Promise.resolve({ dom })
+    return Promise.resolve(dom)
   }
 }

packages/content-handler/src/newsletters/substack-handler.ts

@@ -1,5 +1,5 @@
 import addressparser from 'addressparser'
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 import { parseHTML } from 'linkedom'
 
 export class SubstackHandler extends ContentHandler {
@@ -8,7 +8,7 @@ export class SubstackHandler extends ContentHandler {
     this.name = 'substack'
   }
 
-  shouldPreHandle(url: string, dom: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     const host = this.name + '.com'
     // check if url ends with substack.com
     // or has a profile image hosted at substack.com
@@ -21,7 +21,7 @@ export class SubstackHandler extends ContentHandler {
     )
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     const body = dom.querySelector('.email-body-container')
 
     // this removes header and profile avatar

packages/content-handler/src/websites/wikipedia-handler.ts

@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from '../content-handler'
+import { ContentHandler } from '../content-handler'
 
 export class WikipediaHandler extends ContentHandler {
   constructor() {
@@ -6,15 +6,15 @@ export class WikipediaHandler extends ContentHandler {
     this.name = 'wikipedia'
   }
 
-  shouldPreHandle(url: string, dom?: Document): boolean {
+  shouldPreParse(url: string, dom: Document): boolean {
     return new URL(url).hostname.endsWith('wikipedia.org')
   }
 
-  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+  async preParse(url: string, dom: Document): Promise<Document> {
     // This removes the [edit] anchors from wikipedia pages
     dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove())
     // this removes the sidebar
     dom.querySelector('.infobox')?.remove()
-    return Promise.resolve({ dom })
+    return Promise.resolve(dom)
   }
 }