From e4c847ac4cc8c5bd76c757015ae102bb39798312 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 28 Sep 2022 12:35:42 +0800
Subject: [PATCH 01/15] Init newsletter-handler

---
 packages/newsletter-handler/.eslintignore     |  1 +
 packages/newsletter-handler/.eslintrc         |  6 ++++++
 packages/newsletter-handler/.gitignore        |  2 ++
 packages/newsletter-handler/.npmignore        |  7 +++++++
 packages/newsletter-handler/index.js          |  0
 packages/newsletter-handler/mocha-config.json |  5 +++++
 packages/newsletter-handler/package.json      | 20 +++++++++++++++++++
 packages/newsletter-handler/src/index.ts      |  0
 .../newsletter-handler/test/babel-register.js |  3 +++
 packages/newsletter-handler/test/stub.test.ts | 13 ++++++++++++
 packages/newsletter-handler/tsconfig.json     | 10 ++++++++++
 11 files changed, 67 insertions(+)
 create mode 100644 packages/newsletter-handler/.eslintignore
 create mode 100644 packages/newsletter-handler/.eslintrc
 create mode 100644 packages/newsletter-handler/.gitignore
 create mode 100644 packages/newsletter-handler/.npmignore
 create mode 100644 packages/newsletter-handler/index.js
 create mode 100644 packages/newsletter-handler/mocha-config.json
 create mode 100644 packages/newsletter-handler/package.json
 create mode 100644 packages/newsletter-handler/src/index.ts
 create mode 100644 packages/newsletter-handler/test/babel-register.js
 create mode 100644 packages/newsletter-handler/test/stub.test.ts
 create mode 100644 packages/newsletter-handler/tsconfig.json

diff --git a/packages/newsletter-handler/.eslintignore b/packages/newsletter-handler/.eslintignore
new file mode 100644
index 000000000..c2658d7d1
--- /dev/null
+++ b/packages/newsletter-handler/.eslintignore
@@ -0,0 +1 @@
+node_modules/
diff --git a/packages/newsletter-handler/.eslintrc b/packages/newsletter-handler/.eslintrc
new file mode 100644
index 000000000..e006282a6
--- /dev/null
+++ b/packages/newsletter-handler/.eslintrc
@@ -0,0 +1,6 @@
+{
+  "extends": "../../.eslintrc",
+  "parserOptions": {
+    "project": "tsconfig.json"
+  }
+}
\ No newline at end of file
diff --git a/packages/newsletter-handler/.gitignore b/packages/newsletter-handler/.gitignore
new file mode 100644
index 000000000..0ae7e5c9e
--- /dev/null
+++ b/packages/newsletter-handler/.gitignore
@@ -0,0 +1,2 @@
+node_modules
+/lib
diff --git a/packages/newsletter-handler/.npmignore b/packages/newsletter-handler/.npmignore
new file mode 100644
index 000000000..b5e2b8569
--- /dev/null
+++ b/packages/newsletter-handler/.npmignore
@@ -0,0 +1,7 @@
+/test/
+src
+tsconfig.json
+.eslintrc
+.eslintignore
+.gitignore
+mocha-config.json
diff --git a/packages/newsletter-handler/index.js b/packages/newsletter-handler/index.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/packages/newsletter-handler/mocha-config.json b/packages/newsletter-handler/mocha-config.json
new file mode 100644
index 000000000..44d1d24c1
--- /dev/null
+++ b/packages/newsletter-handler/mocha-config.json
@@ -0,0 +1,5 @@
+{
+    "extension": ["ts"],
+    "spec": "test/**/*.test.ts",
+    "require": "test/babel-register.js"
+  }
\ No newline at end of file
diff --git a/packages/newsletter-handler/package.json b/packages/newsletter-handler/package.json
new file mode 100644
index 000000000..c998a2d58
--- /dev/null
+++ b/packages/newsletter-handler/package.json
@@ -0,0 +1,20 @@
+{
+  "name": "@omnivore/newsletter-handler",
+  "version": "1.0.0",
+  "description": "A standalone version of newsletter handler to parse and format each newsletter",
+  "main": "build/src/index.js",
+  "types": "build/src/index.d.ts",
+  "files": [
+    "build/src"
+  ],
+  "license": "Apache-2.0",
+  "scripts": {
+    "test": "yarn mocha -r ts-node/register --config mocha-config.json",
+    "lint": "eslint src --ext ts,js,tsx,jsx",
+    "compile": "tsc",
+    "build": "tsc"
+  },
+  "devDependencies": {
+    "eslint-plugin-prettier": "^4.0.0"
+  }
+}
diff --git a/packages/newsletter-handler/src/index.ts b/packages/newsletter-handler/src/index.ts
new file mode 100644
index 000000000..e69de29bb
diff --git a/packages/newsletter-handler/test/babel-register.js b/packages/newsletter-handler/test/babel-register.js
new file mode 100644
index 000000000..a6f65f60a
--- /dev/null
+++ b/packages/newsletter-handler/test/babel-register.js
@@ -0,0 +1,3 @@
+const register = require('@babel/register').default
+
+register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/newsletter-handler/test/stub.test.ts b/packages/newsletter-handler/test/stub.test.ts
new file mode 100644
index 000000000..173ca4917
--- /dev/null
+++ b/packages/newsletter-handler/test/stub.test.ts
@@ -0,0 +1,13 @@
+import 'mocha'
+import * as chai from 'chai'
+import { expect } from 'chai'
+import 'chai/register-should'
+import chaiString from 'chai-string'
+
+chai.use(chaiString)
+
+describe('Stub test', () => {
+  it('should pass', () => {
+    expect(true).to.be.true
+  })
+})
diff --git a/packages/newsletter-handler/tsconfig.json b/packages/newsletter-handler/tsconfig.json
new file mode 100644
index 000000000..a6b8377a0
--- /dev/null
+++ b/packages/newsletter-handler/tsconfig.json
@@ -0,0 +1,10 @@
+{
+  "extends": "@tsconfig/node14/tsconfig.json",
+  "compilerOptions": {
+    "rootDir": ".",
+    "declaration": true,
+    "outDir": "build",
+    "lib": ["dom"]
+  },
+  "include": ["src", "test"]
+}

From 91952e587b0f1d197407ca665d5d978b5d816738 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 28 Sep 2022 16:06:35 +0800
Subject: [PATCH 02/15] Replace newsletter-handler with content-handler

---
 .../.eslintignore                             |   0
 .../.eslintrc                                 |   0
 .../.gitignore                                |   0
 .../.npmignore                                |   0
 .../mocha-config.json                         |   0
 .../package.json                              |   4 +-
 packages/content-handler/src/index.ts         | 103 ++++++++++++++++++
 .../test/babel-register.js                    |   0
 .../test/stub.test.ts                         |   0
 .../tsconfig.json                             |   2 +-
 packages/newsletter-handler/index.js          |   0
 packages/newsletter-handler/src/index.ts      |   0
 12 files changed, 106 insertions(+), 3 deletions(-)
 rename packages/{newsletter-handler => content-handler}/.eslintignore (100%)
 rename packages/{newsletter-handler => content-handler}/.eslintrc (100%)
 rename packages/{newsletter-handler => content-handler}/.gitignore (100%)
 rename packages/{newsletter-handler => content-handler}/.npmignore (100%)
 rename packages/{newsletter-handler => content-handler}/mocha-config.json (100%)
 rename packages/{newsletter-handler => content-handler}/package.json (73%)
 create mode 100644 packages/content-handler/src/index.ts
 rename packages/{newsletter-handler => content-handler}/test/babel-register.js (100%)
 rename packages/{newsletter-handler => content-handler}/test/stub.test.ts (100%)
 rename packages/{newsletter-handler => content-handler}/tsconfig.json (85%)
 delete mode 100644 packages/newsletter-handler/index.js
 delete mode 100644 packages/newsletter-handler/src/index.ts

diff --git a/packages/newsletter-handler/.eslintignore b/packages/content-handler/.eslintignore
similarity index 100%
rename from packages/newsletter-handler/.eslintignore
rename to packages/content-handler/.eslintignore
diff --git a/packages/newsletter-handler/.eslintrc b/packages/content-handler/.eslintrc
similarity index 100%
rename from packages/newsletter-handler/.eslintrc
rename to packages/content-handler/.eslintrc
diff --git a/packages/newsletter-handler/.gitignore b/packages/content-handler/.gitignore
similarity index 100%
rename from packages/newsletter-handler/.gitignore
rename to packages/content-handler/.gitignore
diff --git a/packages/newsletter-handler/.npmignore b/packages/content-handler/.npmignore
similarity index 100%
rename from packages/newsletter-handler/.npmignore
rename to packages/content-handler/.npmignore
diff --git a/packages/newsletter-handler/mocha-config.json b/packages/content-handler/mocha-config.json
similarity index 100%
rename from packages/newsletter-handler/mocha-config.json
rename to packages/content-handler/mocha-config.json
diff --git a/packages/newsletter-handler/package.json b/packages/content-handler/package.json
similarity index 73%
rename from packages/newsletter-handler/package.json
rename to packages/content-handler/package.json
index c998a2d58..5cbda3c86 100644
--- a/packages/newsletter-handler/package.json
+++ b/packages/content-handler/package.json
@@ -1,7 +1,7 @@
 {
-  "name": "@omnivore/newsletter-handler",
+  "name": "@omnivore/content-handler",
   "version": "1.0.0",
-  "description": "A standalone version of newsletter handler to parse and format each newsletter",
+  "description": "A standalone version of content handler to parse and format each type of content",
   "main": "build/src/index.js",
   "types": "build/src/index.d.ts",
   "files": [
diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts
new file mode 100644
index 000000000..24f3e874e
--- /dev/null
+++ b/packages/content-handler/src/index.ts
@@ -0,0 +1,103 @@
+import addressparser from 'addressparser'
+import { v4 as uuidv4 } from 'uuid'
+import rfc2047 from 'rfc2047'
+
+interface Unsubscribe {
+  mailTo?: string
+  httpUrl?: string
+}
+
+interface NewsletterMessage {
+  email: string
+  content: string
+  url: string
+  title: string
+  author: string
+  unsubMailTo?: string
+  unsubHttpUrl?: string
+}
+
+export class ContentHandler {
+  protected senderRegex = /NEWSLETTER_SENDER_REGEX/
+  protected urlRegex = /NEWSLETTER_URL_REGEX/
+  protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
+  protected name = ''
+
+  shouldPrehandle(url: URL, dom: Document): boolean {
+    return false
+  }
+
+  prehandle(url: URL, document: Document): Promise<Document> {
+    return Promise.resolve(document)
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+
+  parseNewsletterUrl(_postHeader: string, html: string): string | undefined {
+    // get newsletter url from html
+    const matches = html.match(this.urlRegex)
+    if (matches) {
+      return matches[1]
+    }
+    return undefined
+  }
+
+  parseAuthor(from: string): string {
+    // get author name from email
+    // e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
+    // or 'Mike Allen <mike@axios.com>'
+    const parsed = addressparser(from)
+    if (parsed.length > 0) {
+      return parsed[0].name
+    }
+    return from
+  }
+
+  parseUnsubscribe(unSubHeader: string): Unsubscribe {
+    // parse list-unsubscribe header
+    // e.g. List-Unsubscribe: <https://omnivore.com/unsub>, <mailto:unsub@omnivore.com>
+    const decoded = rfc2047.decode(unSubHeader)
+    return {
+      mailTo: decoded.match(/<(https?:\/\/[^>]*)>/)?.[1],
+      httpUrl: decoded.match(/<mailto:([^>]*)>/)?.[1],
+    }
+  }
+
+  async handleNewsletter(
+    email: string,
+    html: string,
+    postHeader: string,
+    title: string,
+    from: string,
+    unSubHeader: string
+  ): Promise<NewsletterMessage> {
+    console.log('handleNewsletter', email, postHeader, title, from)
+
+    if (!email || !html || !title || !from) {
+      console.log('invalid newsletter email')
+      throw new Error('invalid newsletter email')
+    }
+
+    // fallback to default url if newsletter url does not exist
+    // assign a random uuid to the default url to avoid duplicate url
+    const url =
+      this.parseNewsletterUrl(postHeader, html) ||
+      `${this.defaultUrl}?source=newsletters&id=${uuidv4()}`
+    const author = this.parseAuthor(from)
+    const unsubscribe = this.parseUnsubscribe(unSubHeader)
+
+    return {
+      email,
+      content: html,
+      url,
+      title,
+      author,
+      unsubMailTo: unsubscribe.mailTo || '',
+      unsubHttpUrl: unsubscribe.httpUrl || '',
+    }
+  }
+}
diff --git a/packages/newsletter-handler/test/babel-register.js b/packages/content-handler/test/babel-register.js
similarity index 100%
rename from packages/newsletter-handler/test/babel-register.js
rename to packages/content-handler/test/babel-register.js
diff --git a/packages/newsletter-handler/test/stub.test.ts b/packages/content-handler/test/stub.test.ts
similarity index 100%
rename from packages/newsletter-handler/test/stub.test.ts
rename to packages/content-handler/test/stub.test.ts
diff --git a/packages/newsletter-handler/tsconfig.json b/packages/content-handler/tsconfig.json
similarity index 85%
rename from packages/newsletter-handler/tsconfig.json
rename to packages/content-handler/tsconfig.json
index a6b8377a0..aeb8d2c3a 100644
--- a/packages/newsletter-handler/tsconfig.json
+++ b/packages/content-handler/tsconfig.json
@@ -6,5 +6,5 @@
     "outDir": "build",
     "lib": ["dom"]
   },
-  "include": ["src", "test"]
+  "include": ["src"]
 }
diff --git a/packages/newsletter-handler/index.js b/packages/newsletter-handler/index.js
deleted file mode 100644
index e69de29bb..000000000
diff --git a/packages/newsletter-handler/src/index.ts b/packages/newsletter-handler/src/index.ts
deleted file mode 100644
index e69de29bb..000000000

From 6deb62d98322dc90985b0ab433229c6fff4f9659 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 29 Sep 2022 13:28:07 +0800
Subject: [PATCH 03/15] Add handlers for content-fetching

---
 packages/content-handler/package.json         |   6 +
 .../src/handlers/apple-news-handler.ts        |  26 ++++
 .../src/handlers/bloomberg-handler.ts         |  36 +++++
 .../src/handlers/derstandard-handler.ts       |  29 ++++
 .../src/handlers/image-handler.ts             |  27 ++++
 .../src/handlers/medium-handler.ts            |  21 +++
 .../src/handlers/pdf-handler.ts               |  13 ++
 .../src/handlers/scrapingBee-handler.ts       |  33 ++++
 .../src/handlers/t-dot-co-handler.ts          |  21 +++
 .../src/handlers/twitter-handler.ts           | 142 ++++++++++++++++++
 .../src/handlers/youtube-handler.ts           |  66 ++++++++
 packages/content-handler/src/index.ts         |  18 ++-
 yarn.lock                                     |  21 +++
 13 files changed, 454 insertions(+), 5 deletions(-)
 create mode 100644 packages/content-handler/src/handlers/apple-news-handler.ts
 create mode 100644 packages/content-handler/src/handlers/bloomberg-handler.ts
 create mode 100644 packages/content-handler/src/handlers/derstandard-handler.ts
 create mode 100644 packages/content-handler/src/handlers/image-handler.ts
 create mode 100644 packages/content-handler/src/handlers/medium-handler.ts
 create mode 100644 packages/content-handler/src/handlers/pdf-handler.ts
 create mode 100644 packages/content-handler/src/handlers/scrapingBee-handler.ts
 create mode 100644 packages/content-handler/src/handlers/t-dot-co-handler.ts
 create mode 100644 packages/content-handler/src/handlers/twitter-handler.ts
 create mode 100644 packages/content-handler/src/handlers/youtube-handler.ts

diff --git a/packages/content-handler/package.json b/packages/content-handler/package.json
index 5cbda3c86..bae0e5953 100644
--- a/packages/content-handler/package.json
+++ b/packages/content-handler/package.json
@@ -16,5 +16,11 @@
   },
   "devDependencies": {
     "eslint-plugin-prettier": "^4.0.0"
+  },
+  "dependencies": {
+    "axios": "^0.27.2",
+    "linkedom": "^0.14.16",
+    "luxon": "^3.0.4",
+    "underscore": "^1.13.6"
   }
 }
diff --git a/packages/content-handler/src/handlers/apple-news-handler.ts b/packages/content-handler/src/handlers/apple-news-handler.ts
new file mode 100644
index 000000000..a53c8ef02
--- /dev/null
+++ b/packages/content-handler/src/handlers/apple-news-handler.ts
@@ -0,0 +1,26 @@
+import { ContentHandler, PreHandleResult } from '../index'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+export class AppleNewsHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const u = new URL(url)
+    return u.hostname === 'apple.news'
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    const MOBILE_USER_AGENT =
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
+    const response = await axios.get(url, {
+      headers: { 'User-Agent': MOBILE_USER_AGENT },
+    })
+    const data = response.data
+    const dom = parseHTML(data).document
+    // make sure it's a valid URL by wrapping in new URL
+    const href = dom
+      .querySelector('span.click-here')
+      ?.parentElement?.getAttribute('href')
+    const u = href ? new URL(href) : undefined
+    return { url: u?.href }
+  }
+}
diff --git a/packages/content-handler/src/handlers/bloomberg-handler.ts b/packages/content-handler/src/handlers/bloomberg-handler.ts
new file mode 100644
index 000000000..05100429c
--- /dev/null
+++ b/packages/content-handler/src/handlers/bloomberg-handler.ts
@@ -0,0 +1,36 @@
+import { ContentHandler, PreHandleResult } from '../index'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+class BloombergHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const BLOOMBERG_URL_MATCH =
+      /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
+    return BLOOMBERG_URL_MATCH.test(url.toString())
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    console.log('prehandling bloomberg url', url)
+
+    try {
+      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+        params: {
+          api_key: process.env.SCRAPINGBEE_API_KEY,
+          url: url,
+          return_page_source: true,
+          block_ads: true,
+          block_resources: false,
+        },
+      })
+      const dom = parseHTML(response.data).document
+      return {
+        title: dom.title,
+        content: dom.querySelector('body')?.innerHTML,
+        url: url,
+      }
+    } catch (error) {
+      console.error('error prehandling bloomberg url', error)
+      throw error
+    }
+  }
+}
diff --git a/packages/content-handler/src/handlers/derstandard-handler.ts b/packages/content-handler/src/handlers/derstandard-handler.ts
new file mode 100644
index 000000000..fd77ce108
--- /dev/null
+++ b/packages/content-handler/src/handlers/derstandard-handler.ts
@@ -0,0 +1,29 @@
+import { ContentHandler, PreHandleResult } from '../index'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+class DerstandardHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const u = new URL(url)
+    return u.hostname === 'www.derstandard.at'
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    const response = await axios.get(url, {
+      // set cookie to give consent to get the article
+      headers: {
+        cookie: `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`,
+      },
+    })
+    const content = response.data
+
+    const dom = parseHTML(content).document
+    const titleElement = dom.querySelector('.article-title')
+    titleElement && titleElement.remove()
+
+    return {
+      content: dom.body.outerHTML,
+      title: titleElement?.textContent || undefined,
+    }
+  }
+}
diff --git a/packages/content-handler/src/handlers/image-handler.ts b/packages/content-handler/src/handlers/image-handler.ts
new file mode 100644
index 000000000..3f994332b
--- /dev/null
+++ b/packages/content-handler/src/handlers/image-handler.ts
@@ -0,0 +1,27 @@
+import { ContentHandler, PreHandleResult } from '../index'
+
+class ImageHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const IMAGE_URL_PATTERN = /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
+    return IMAGE_URL_PATTERN.test(url.toString())
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    const title = url.toString().split('/').pop()
+    const content = `
+      <html>
+        <head>
+          <title>${title}</title>
+          <meta property="og:image" content="${url}" />
+          <meta property="og:title" content="${title}" />
+        </head>
+        <body>
+          <div>
+            <img src="${url}" alt="${title}">
+          </div>
+        </body>
+      </html>`
+
+    return { title, content }
+  }
+}
diff --git a/packages/content-handler/src/handlers/medium-handler.ts b/packages/content-handler/src/handlers/medium-handler.ts
new file mode 100644
index 000000000..d87d4338f
--- /dev/null
+++ b/packages/content-handler/src/handlers/medium-handler.ts
@@ -0,0 +1,21 @@
+import { ContentHandler, PreHandleResult } from '../index'
+
+class MediumHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const u = new URL(url)
+    return u.hostname.endsWith('medium.com')
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    console.log('prehandling medium url', url)
+
+    try {
+      const res = new URL(url)
+      res.searchParams.delete('source')
+      return { url: res.toString() }
+    } catch (error) {
+      console.error('error prehandling medium url', error)
+      throw error
+    }
+  }
+}
diff --git a/packages/content-handler/src/handlers/pdf-handler.ts b/packages/content-handler/src/handlers/pdf-handler.ts
new file mode 100644
index 000000000..54c9e6bf4
--- /dev/null
+++ b/packages/content-handler/src/handlers/pdf-handler.ts
@@ -0,0 +1,13 @@
+import { ContentHandler, PreHandleResult } from '../index'
+
+class PdfHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const u = new URL(url)
+    const path = u.pathname.replace(u.search, '')
+    return path.endsWith('.pdf')
+  }
+
+  async preHandle(_url: string, _document: Document): Promise<PreHandleResult> {
+    return { contentType: 'application/pdf' }
+  }
+}
diff --git a/packages/content-handler/src/handlers/scrapingBee-handler.ts b/packages/content-handler/src/handlers/scrapingBee-handler.ts
new file mode 100644
index 000000000..1c84ba06b
--- /dev/null
+++ b/packages/content-handler/src/handlers/scrapingBee-handler.ts
@@ -0,0 +1,33 @@
+import { ContentHandler, PreHandleResult } from '../index'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+class ScrapingBeeHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const u = new URL(url)
+    const hostnames = ['nytimes.com', 'news.google.com']
+
+    return hostnames.some((h) => u.hostname.endsWith(h))
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    console.log('prehandling url with scrapingbee', url)
+
+    try {
+      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+        params: {
+          api_key: process.env.SCRAPINGBEE_API_KEY,
+          url: url,
+          return_page_source: true,
+          block_ads: true,
+          block_resources: false,
+        },
+      })
+      const dom = parseHTML(response.data).document
+      return { title: dom.title, content: response.data, url: url }
+    } catch (error) {
+      console.error('error prehandling url w/scrapingbee', error)
+      throw error
+    }
+  }
+}
diff --git a/packages/content-handler/src/handlers/t-dot-co-handler.ts b/packages/content-handler/src/handlers/t-dot-co-handler.ts
new file mode 100644
index 000000000..e83443518
--- /dev/null
+++ b/packages/content-handler/src/handlers/t-dot-co-handler.ts
@@ -0,0 +1,21 @@
+import { ContentHandler } from '../index'
+import axios from 'axios'
+
+class TDotCoHandler extends ContentHandler {
+  shouldResolve(url: string): boolean {
+    const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/
+    return T_DOT_CO_URL_MATCH.test(url)
+  }
+
+  async resolve(url: string) {
+    return await axios
+      .get(url, { maxRedirects: 0, validateStatus: null })
+      .then((res) => {
+        return new URL(res.headers.location).href
+      })
+      .catch((err) => {
+        console.log('err with t.co url', err)
+        return undefined
+      })
+  }
+}
diff --git a/packages/content-handler/src/handlers/twitter-handler.ts b/packages/content-handler/src/handlers/twitter-handler.ts
new file mode 100644
index 000000000..265bfe955
--- /dev/null
+++ b/packages/content-handler/src/handlers/twitter-handler.ts
@@ -0,0 +1,142 @@
+import { ContentHandler, PreHandleResult } from '../index'
+import axios from 'axios'
+import { DateTime } from 'luxon'
+import _ from 'underscore'
+
+const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN
+const TWITTER_URL_MATCH =
+  /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
+
+const getTweetFields = () => {
+  const TWEET_FIELDS =
+    '&tweet.fields=attachments,author_id,conversation_id,created_at,' +
+    'entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets,' +
+    'source,withheld'
+  const EXPANSIONS = '&expansions=author_id,attachments.media_keys'
+  const USER_FIELDS =
+    '&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld'
+  const MEDIA_FIELDS =
+    '&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width'
+
+  return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`
+}
+
+const getTweetById = async (id: string) => {
+  const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/'
+  const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
+
+  return await axios.get(apiUrl.toString(), {
+    headers: {
+      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
+      redirect: 'follow',
+    },
+  })
+}
+
+const titleForAuthor = (author: any) => {
+  return `${author.name} on Twitter`
+}
+
+const tweetIdFromStatusUrl = (url: string): string | undefined => {
+  const match = url.toString().match(TWITTER_URL_MATCH)
+  return match?.[2]
+}
+
+const formatTimestamp = (timestamp: string) => {
+  return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(
+    DateTime.DATETIME_FULL
+  )
+}
+
+class TwitterHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
+  }
+
+  // version of the handler that uses the oembed API
+  // This isn't great as it doesn't work well with our
+  // readability API. But could potentially give a more consistent
+  // look to the tweets
+  // prehandle: async (url, env) => {
+  //   const oeTweet = await embeddedTweet(url)
+  //   const dom = new JSDOM(oeTweet.data.html);
+  //   const bq = dom.window.document.querySelector('blockquote')
+  //   console.log('blockquote:', bq);
+
+  //   const title = titleForTweet(oeTweet)
+  //   return { title, content: '<div>' + bq.innerHTML + '</div>', url: oeTweet.data.url };
+  // }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    console.log('prehandling twitter url', url)
+
+    const tweetId = tweetIdFromStatusUrl(url)
+    if (!tweetId) {
+      throw new Error('could not find tweet id in url')
+    }
+    const tweetData = (await getTweetById(tweetId)).data
+    const authorId = tweetData.data.author_id
+    const author = tweetData.includes.users.filter(
+      (u: any) => (u.id = authorId)
+    )[0]
+    // escape html entities in title
+    const title = _.escape(titleForAuthor(author))
+    const authorImage = author.profile_image_url.replace('_normal', '_400x400')
+
+    let text = tweetData.data.text
+    if (tweetData.data.entities && tweetData.data.entities.urls) {
+      for (let urlObj of tweetData.data.entities.urls) {
+        text = text.replace(
+          urlObj.url,
+          `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
+        )
+      }
+    }
+
+    const front = `
+    <div>
+      <p>${text}</p>
+    `
+
+    var includesHtml = ''
+    if (tweetData.includes.media) {
+      includesHtml = tweetData.includes.media
+        .map((m: any) => {
+          const linkUrl = m.type == 'photo' ? m.url : url
+          const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
+          const mediaOpen = `<a class="media-link" href=${linkUrl}>
+          <picture>
+            <img class="tweet-img" src=${previewUrl} />
+          </picture>
+          </a>`
+          return mediaOpen
+        })
+        .join('\n')
+    }
+
+    const back = `
+       — <a href="https://twitter.com/${author.username}">${
+      author.username
+    }</a> ${author.name} <a href="${url}">${formatTimestamp(
+      tweetData.data.created_at
+    )}</a>
+    </div>
+    `
+    const content = `
+    <head>
+      <meta property="og:image" content="${authorImage}" />
+      <meta property="og:image:secure_url" content="${authorImage}" />
+      <meta property="og:title" content="${title}" />
+      <meta property="og:description" content="${_.escape(
+        tweetData.data.text
+      )}" />
+    </head>
+    <body>
+      ${front}
+      ${includesHtml}
+      ${back}
+    </body>`
+
+    return { content, url, title }
+  }
+}
diff --git a/packages/content-handler/src/handlers/youtube-handler.ts b/packages/content-handler/src/handlers/youtube-handler.ts
new file mode 100644
index 000000000..60ae22970
--- /dev/null
+++ b/packages/content-handler/src/handlers/youtube-handler.ts
@@ -0,0 +1,66 @@
+import { PreHandleResult } from '../index'
+import axios from 'axios'
+import _ from 'underscore'
+
+const { ContentHandler } = require('../index')
+
+const YOUTUBE_URL_MATCH =
+  /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/
+
+function getYoutubeVideoId(url: string) {
+  const u = new URL(url)
+  const videoId = u.searchParams.get('v')
+  if (!videoId) {
+    const match = url.toString().match(YOUTUBE_URL_MATCH)
+    if (match === null || match.length < 6 || !match[5]) {
+      return undefined
+    }
+    return match[5]
+  }
+  return videoId
+}
+
+class YoutubeHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    return YOUTUBE_URL_MATCH.test(url.toString())
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    const videoId = getYoutubeVideoId(url)
+    if (!videoId) {
+      return {}
+    }
+
+    const oembedUrl =
+      `https://www.youtube.com/oembed?format=json&url=` +
+      encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`)
+    const oembed = (await axios.get(oembedUrl.toString())).data
+    // escape html entities in title
+    const title = _.escape(oembed.title)
+    const ratio = oembed.width / oembed.height
+    const thumbnail = oembed.thumbnail_url
+    const height = 350
+    const width = height * ratio
+    const authorName = _.escape(oembed.author_name)
+
+    const content = `
+    <html>
+      <head><title>${title}</title>
+      <meta property="og:image" content="${thumbnail}" />
+      <meta property="og:image:secure_url" content="${thumbnail}" />
+      <meta property="og:title" content="${title}" />
+      <meta property="og:description" content="" />
+      <meta property="og:article:author" content="${authorName}" />
+      </head>
+      <body>
+      <iframe width="${width}" height="${height}" src="https://www.youtube.com/embed/${videoId}" title="${title}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+        <p><a href="${url}" target="_blank">${title}</a></p>
+        <p itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="${oembed.author_url}" target="_blank">${authorName}</a></p>
+      </body>
+    </html>`
+
+    console.log('got video id', videoId)
+
+    return { content, title: 'Youtube Content' }
+  }
+}
diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts
index 24f3e874e..8f3cfbd81 100644
--- a/packages/content-handler/src/index.ts
+++ b/packages/content-handler/src/index.ts
@@ -17,18 +17,26 @@ interface NewsletterMessage {
   unsubHttpUrl?: string
 }
 
+export interface PreHandleResult {
+  url?: string
+  title?: string
+  content?: string
+  contentType?: string
+  dom?: Document
+}
+
 export class ContentHandler {
   protected senderRegex = /NEWSLETTER_SENDER_REGEX/
   protected urlRegex = /NEWSLETTER_URL_REGEX/
   protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
   protected name = ''
 
-  shouldPrehandle(url: URL, dom: Document): boolean {
+  shouldPreHandle(url: string, dom: Document): boolean {
     return false
   }
 
-  prehandle(url: URL, document: Document): Promise<Document> {
-    return Promise.resolve(document)
+  preHandle(url: string, document: Document): Promise<PreHandleResult> {
+    return Promise.resolve({ url, dom: document })
   }
 
   isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
@@ -67,14 +75,14 @@ export class ContentHandler {
     }
   }
 
-  async handleNewsletter(
+  handleNewsletter(
     email: string,
     html: string,
     postHeader: string,
     title: string,
     from: string,
     unSubHeader: string
-  ): Promise<NewsletterMessage> {
+  ): NewsletterMessage {
     console.log('handleNewsletter', email, postHeader, title, from)
 
     if (!email || !html || !title || !from) {
diff --git a/yarn.lock b/yarn.lock
index 97a48bba4..3da81da9b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -17652,6 +17652,17 @@ linkedom@^0.14.12:
     htmlparser2 "^8.0.1"
     uhyphen "^0.1.0"
 
+linkedom@^0.14.16:
+  version "0.14.16"
+  resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.16.tgz#124eb006fad1dfe7ed8f96ec8ae74ab0fb0fd88e"
+  integrity sha512-a4QWl4W93P15/x+4d9k8K+C81nOzQeGOs3D37uG0TFqKZYGLEyZwXweSFrypK8yvUx5U2cuZKkdDIOjaouv3ag==
+  dependencies:
+    css-select "^5.1.0"
+    cssom "^0.5.0"
+    html-escaper "^3.0.3"
+    htmlparser2 "^8.0.1"
+    uhyphen "^0.1.0"
+
 linkedom@^0.14.9:
   version "0.14.9"
   resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.9.tgz#34c6f15eddc809406f42d8ee48cd30b0222eccb0"
@@ -18151,6 +18162,11 @@ luxon@^2.3.1:
   resolved "https://registry.yarnpkg.com/luxon/-/luxon-2.3.1.tgz#f276b1b53fd9a740a60e666a541a7f6dbed4155a"
   integrity sha512-I8vnjOmhXsMSlNMZlMkSOvgrxKJl0uOsEzdGgGNZuZPaS9KlefpE9KV95QFftlJSC+1UyCC9/I69R02cz/zcCA==
 
+luxon@^3.0.4:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.4.tgz#d179e4e9f05e092241e7044f64aaa54796b03929"
+  integrity sha512-aV48rGUwP/Vydn8HT+5cdr26YYQiUZ42NM6ToMoaGKwYfWbfLeRkEu1wXWMHBZT6+KyLfcbbtVcoQFCbbPjKlw==
+
 lz-string@^1.4.4:
   version "1.4.4"
   resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26"
@@ -24493,6 +24509,11 @@ underscore@^1.13.4, underscore@^1.9.1:
   resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.4.tgz#7886b46bbdf07f768e0052f1828e1dcab40c0dee"
   integrity sha512-BQFnUDuAQ4Yf/cYY5LNrK9NCJFKriaRbD9uR1fTeXnBeoa97W0i41qkZfGO9pSo8I5KzjAcSY2XYtdf0oKd7KQ==
 
+underscore@^1.13.6:
+  version "1.13.6"
+  resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.6.tgz#04786a1f589dc6c09f761fc5f45b89e935136441"
+  integrity sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==
+
 undici@^4.9.3:
   version "4.14.1"
   resolved "https://registry.yarnpkg.com/undici/-/undici-4.14.1.tgz#7633b143a8a10d6d63335e00511d071e8d52a1d9"

From 5fdb8b337dc4b02f9d0c567da79e4ecae0448929 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 29 Sep 2022 16:08:38 +0800
Subject: [PATCH 04/15] Lint

---
 .../src/handlers/apple-news-handler.ts        |  2 +-
 .../src/handlers/bloomberg-handler.ts         |  2 +-
 .../src/handlers/derstandard-handler.ts       |  2 +-
 .../src/handlers/image-handler.ts             |  4 +-
 .../src/handlers/medium-handler.ts            |  2 +-
 .../src/handlers/pdf-handler.ts               |  2 +-
 .../src/handlers/scrapingBee-handler.ts       |  2 +-
 .../src/handlers/t-dot-co-handler.ts          |  2 +-
 .../src/handlers/twitter-handler.ts           | 66 ++++++++++++-------
 .../src/handlers/youtube-handler.ts           | 13 ++--
 10 files changed, 61 insertions(+), 36 deletions(-)

diff --git a/packages/content-handler/src/handlers/apple-news-handler.ts b/packages/content-handler/src/handlers/apple-news-handler.ts
index a53c8ef02..ab11bdc49 100644
--- a/packages/content-handler/src/handlers/apple-news-handler.ts
+++ b/packages/content-handler/src/handlers/apple-news-handler.ts
@@ -14,7 +14,7 @@ export class AppleNewsHandler extends ContentHandler {
     const response = await axios.get(url, {
       headers: { 'User-Agent': MOBILE_USER_AGENT },
     })
-    const data = response.data
+    const data = response.data as string
     const dom = parseHTML(data).document
     // make sure it's a valid URL by wrapping in new URL
     const href = dom
diff --git a/packages/content-handler/src/handlers/bloomberg-handler.ts b/packages/content-handler/src/handlers/bloomberg-handler.ts
index 05100429c..8052e36e0 100644
--- a/packages/content-handler/src/handlers/bloomberg-handler.ts
+++ b/packages/content-handler/src/handlers/bloomberg-handler.ts
@@ -5,7 +5,7 @@ import { parseHTML } from 'linkedom'
 class BloombergHandler extends ContentHandler {
   shouldPreHandle(url: string, _dom: Document): boolean {
     const BLOOMBERG_URL_MATCH =
-      /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
+      /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)/
     return BLOOMBERG_URL_MATCH.test(url.toString())
   }
 
diff --git a/packages/content-handler/src/handlers/derstandard-handler.ts b/packages/content-handler/src/handlers/derstandard-handler.ts
index fd77ce108..1aae94213 100644
--- a/packages/content-handler/src/handlers/derstandard-handler.ts
+++ b/packages/content-handler/src/handlers/derstandard-handler.ts
@@ -15,7 +15,7 @@ class DerstandardHandler extends ContentHandler {
         cookie: `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`,
       },
     })
-    const content = response.data
+    const content = response.data as string
 
     const dom = parseHTML(content).document
     const titleElement = dom.querySelector('.article-title')
diff --git a/packages/content-handler/src/handlers/image-handler.ts b/packages/content-handler/src/handlers/image-handler.ts
index 3f994332b..78447fe33 100644
--- a/packages/content-handler/src/handlers/image-handler.ts
+++ b/packages/content-handler/src/handlers/image-handler.ts
@@ -7,7 +7,7 @@ class ImageHandler extends ContentHandler {
   }
 
   async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
-    const title = url.toString().split('/').pop()
+    const title = url.toString().split('/').pop() || 'Image'
     const content = `
       <html>
         <head>
@@ -22,6 +22,6 @@ class ImageHandler extends ContentHandler {
         </body>
       </html>`
 
-    return { title, content }
+    return Promise.resolve({ title, content })
   }
 }
diff --git a/packages/content-handler/src/handlers/medium-handler.ts b/packages/content-handler/src/handlers/medium-handler.ts
index d87d4338f..03cc1be8f 100644
--- a/packages/content-handler/src/handlers/medium-handler.ts
+++ b/packages/content-handler/src/handlers/medium-handler.ts
@@ -12,7 +12,7 @@ class MediumHandler extends ContentHandler {
     try {
       const res = new URL(url)
       res.searchParams.delete('source')
-      return { url: res.toString() }
+      return Promise.resolve({ url: res.toString() })
     } catch (error) {
       console.error('error prehandling medium url', error)
       throw error
diff --git a/packages/content-handler/src/handlers/pdf-handler.ts b/packages/content-handler/src/handlers/pdf-handler.ts
index 54c9e6bf4..fb9b7e045 100644
--- a/packages/content-handler/src/handlers/pdf-handler.ts
+++ b/packages/content-handler/src/handlers/pdf-handler.ts
@@ -8,6 +8,6 @@ class PdfHandler extends ContentHandler {
   }
 
   async preHandle(_url: string, _document: Document): Promise<PreHandleResult> {
-    return { contentType: 'application/pdf' }
+    return Promise.resolve({ contentType: 'application/pdf' })
   }
 }
diff --git a/packages/content-handler/src/handlers/scrapingBee-handler.ts b/packages/content-handler/src/handlers/scrapingBee-handler.ts
index 1c84ba06b..c9cbc8afa 100644
--- a/packages/content-handler/src/handlers/scrapingBee-handler.ts
+++ b/packages/content-handler/src/handlers/scrapingBee-handler.ts
@@ -24,7 +24,7 @@ class ScrapingBeeHandler extends ContentHandler {
         },
       })
       const dom = parseHTML(response.data).document
-      return { title: dom.title, content: response.data, url: url }
+      return { title: dom.title, content: response.data as string, url: url }
     } catch (error) {
       console.error('error prehandling url w/scrapingbee', error)
       throw error
diff --git a/packages/content-handler/src/handlers/t-dot-co-handler.ts b/packages/content-handler/src/handlers/t-dot-co-handler.ts
index e83443518..3a97849f3 100644
--- a/packages/content-handler/src/handlers/t-dot-co-handler.ts
+++ b/packages/content-handler/src/handlers/t-dot-co-handler.ts
@@ -8,7 +8,7 @@ class TDotCoHandler extends ContentHandler {
   }
 
   async resolve(url: string) {
-    return await axios
+    return axios
       .get(url, { maxRedirects: 0, validateStatus: null })
       .then((res) => {
         return new URL(res.headers.location).href
diff --git a/packages/content-handler/src/handlers/twitter-handler.ts b/packages/content-handler/src/handlers/twitter-handler.ts
index 265bfe955..e4234dcbc 100644
--- a/packages/content-handler/src/handlers/twitter-handler.ts
+++ b/packages/content-handler/src/handlers/twitter-handler.ts
@@ -25,7 +25,11 @@ const getTweetById = async (id: string) => {
   const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/'
   const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
 
-  return await axios.get(apiUrl.toString(), {
+  if (!TWITTER_BEARER_TOKEN) {
+    throw new Error('No Twitter bearer token found')
+  }
+
+  return axios.get(apiUrl.toString(), {
     headers: {
       Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
       redirect: 'follow',
@@ -33,7 +37,7 @@ const getTweetById = async (id: string) => {
   })
 }
 
-const titleForAuthor = (author: any) => {
+const titleForAuthor = (author: { name: string }) => {
   return `${author.name} on Twitter`
 }
 
@@ -53,20 +57,6 @@ class TwitterHandler extends ContentHandler {
     return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
   }
 
-  // version of the handler that uses the oembed API
-  // This isn't great as it doesn't work well with our
-  // readability API. But could potentially give a more consistent
-  // look to the tweets
-  // prehandle: async (url, env) => {
-  //   const oeTweet = await embeddedTweet(url)
-  //   const dom = new JSDOM(oeTweet.data.html);
-  //   const bq = dom.window.document.querySelector('blockquote')
-  //   console.log('blockquote:', bq);
-
-  //   const title = titleForTweet(oeTweet)
-  //   return { title, content: '<div>' + bq.innerHTML + '</div>', url: oeTweet.data.url };
-  // }
-
   async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
     console.log('prehandling twitter url', url)
 
@@ -74,18 +64,48 @@ class TwitterHandler extends ContentHandler {
     if (!tweetId) {
       throw new Error('could not find tweet id in url')
     }
-    const tweetData = (await getTweetById(tweetId)).data
+    const tweetData = (await getTweetById(tweetId)).data as {
+      data: {
+        author_id: string
+        text: string
+        entities: {
+          urls: [
+            {
+              url: string
+              expanded_url: string
+              display_url: string
+            }
+          ]
+        }
+        created_at: string
+      }
+      includes: {
+        users: [
+          {
+            id: string
+            name: string
+            profile_image_url: string
+            username: string
+          }
+        ]
+        media: [
+          {
+            preview_image_url: string
+            type: string
+            url: string
+          }
+        ]
+      }
+    }
     const authorId = tweetData.data.author_id
-    const author = tweetData.includes.users.filter(
-      (u: any) => (u.id = authorId)
-    )[0]
+    const author = tweetData.includes.users.filter((u) => (u.id = authorId))[0]
     // escape html entities in title
     const title = _.escape(titleForAuthor(author))
     const authorImage = author.profile_image_url.replace('_normal', '_400x400')
 
     let text = tweetData.data.text
     if (tweetData.data.entities && tweetData.data.entities.urls) {
-      for (let urlObj of tweetData.data.entities.urls) {
+      for (const urlObj of tweetData.data.entities.urls) {
         text = text.replace(
           urlObj.url,
           `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
@@ -98,10 +118,10 @@ class TwitterHandler extends ContentHandler {
       <p>${text}</p>
     `
 
-    var includesHtml = ''
+    let includesHtml = ''
     if (tweetData.includes.media) {
       includesHtml = tweetData.includes.media
-        .map((m: any) => {
+        .map((m) => {
           const linkUrl = m.type == 'photo' ? m.url : url
           const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
           const mediaOpen = `<a class="media-link" href=${linkUrl}>
diff --git a/packages/content-handler/src/handlers/youtube-handler.ts b/packages/content-handler/src/handlers/youtube-handler.ts
index 60ae22970..288f42aa0 100644
--- a/packages/content-handler/src/handlers/youtube-handler.ts
+++ b/packages/content-handler/src/handlers/youtube-handler.ts
@@ -1,9 +1,7 @@
-import { PreHandleResult } from '../index'
+import { ContentHandler, PreHandleResult } from '../index'
 import axios from 'axios'
 import _ from 'underscore'
 
-const { ContentHandler } = require('../index')
-
 const YOUTUBE_URL_MATCH =
   /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/
 
@@ -34,7 +32,14 @@ class YoutubeHandler extends ContentHandler {
     const oembedUrl =
       `https://www.youtube.com/oembed?format=json&url=` +
       encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`)
-    const oembed = (await axios.get(oembedUrl.toString())).data
+    const oembed = (await axios.get(oembedUrl.toString())).data as {
+      title: string
+      width: number
+      height: number
+      thumbnail_url: string
+      author_name: string
+      author_url: string
+    }
     // escape html entities in title
     const title = _.escape(oembed.title)
     const ratio = oembed.width / oembed.height

From 8fb398eae4b4a71901a598c8a254c7ad39e7e93d Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 29 Sep 2022 16:22:51 +0800
Subject: [PATCH 05/15] Add tests

---
 .../src/{handlers => }/apple-news-handler.ts  |  6 ++---
 .../src/{handlers => }/bloomberg-handler.ts   |  6 ++---
 .../src/{handlers => }/derstandard-handler.ts |  6 ++---
 .../src/{handlers => }/image-handler.ts       |  6 ++---
 packages/content-handler/src/index.ts         |  4 +--
 .../src/{handlers => }/medium-handler.ts      |  6 ++---
 .../src/{handlers => }/pdf-handler.ts         |  6 ++---
 .../src/{handlers => }/scrapingBee-handler.ts |  6 ++---
 .../src/{handlers => }/t-dot-co-handler.ts    |  2 +-
 .../src/{handlers => }/twitter-handler.ts     |  6 ++---
 .../src/{handlers => }/youtube-handler.ts     |  8 +++---
 .../test/apple-news-handler.test.ts           | 10 ++++++++
 packages/content-handler/test/stub.test.ts    | 13 ----------
 .../test/youtube-handler.test.ts              | 25 +++++++++++++++++++
 yarn.lock                                     | 20 +++++++++++++++
 15 files changed, 86 insertions(+), 44 deletions(-)
 rename packages/content-handler/src/{handlers => }/apple-news-handler.ts (81%)
 rename packages/content-handler/src/{handlers => }/bloomberg-handler.ts (83%)
 rename packages/content-handler/src/{handlers => }/derstandard-handler.ts (80%)
 rename packages/content-handler/src/{handlers => }/image-handler.ts (78%)
 rename packages/content-handler/src/{handlers => }/medium-handler.ts (70%)
 rename packages/content-handler/src/{handlers => }/pdf-handler.ts (57%)
 rename packages/content-handler/src/{handlers => }/scrapingBee-handler.ts (82%)
 rename packages/content-handler/src/{handlers => }/t-dot-co-handler.ts (92%)
 rename packages/content-handler/src/{handlers => }/twitter-handler.ts (96%)
 rename packages/content-handler/src/{handlers => }/youtube-handler.ts (90%)
 create mode 100644 packages/content-handler/test/apple-news-handler.test.ts
 delete mode 100644 packages/content-handler/test/stub.test.ts
 create mode 100644 packages/content-handler/test/youtube-handler.test.ts

diff --git a/packages/content-handler/src/handlers/apple-news-handler.ts b/packages/content-handler/src/apple-news-handler.ts
similarity index 81%
rename from packages/content-handler/src/handlers/apple-news-handler.ts
rename to packages/content-handler/src/apple-news-handler.ts
index ab11bdc49..1958b3b8c 100644
--- a/packages/content-handler/src/handlers/apple-news-handler.ts
+++ b/packages/content-handler/src/apple-news-handler.ts
@@ -1,14 +1,14 @@
-import { ContentHandler, PreHandleResult } from '../index'
+import { ContentHandler, PreHandleResult } from './index'
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
 
 export class AppleNewsHandler extends ContentHandler {
-  shouldPreHandle(url: string, _dom: Document): boolean {
+  shouldPreHandle(url: string, dom?: Document): boolean {
     const u = new URL(url)
     return u.hostname === 'apple.news'
   }
 
-  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
     const MOBILE_USER_AGENT =
       'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
     const response = await axios.get(url, {
diff --git a/packages/content-handler/src/handlers/bloomberg-handler.ts b/packages/content-handler/src/bloomberg-handler.ts
similarity index 83%
rename from packages/content-handler/src/handlers/bloomberg-handler.ts
rename to packages/content-handler/src/bloomberg-handler.ts
index 8052e36e0..ac4fb23ed 100644
--- a/packages/content-handler/src/handlers/bloomberg-handler.ts
+++ b/packages/content-handler/src/bloomberg-handler.ts
@@ -1,15 +1,15 @@
-import { ContentHandler, PreHandleResult } from '../index'
+import { ContentHandler, PreHandleResult } from './index'
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
 
 class BloombergHandler extends ContentHandler {
-  shouldPreHandle(url: string, _dom: Document): boolean {
+  shouldPreHandle(url: string, dom?: Document): boolean {
     const BLOOMBERG_URL_MATCH =
       /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)/
     return BLOOMBERG_URL_MATCH.test(url.toString())
   }
 
-  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
     console.log('prehandling bloomberg url', url)
 
     try {
diff --git a/packages/content-handler/src/handlers/derstandard-handler.ts b/packages/content-handler/src/derstandard-handler.ts
similarity index 80%
rename from packages/content-handler/src/handlers/derstandard-handler.ts
rename to packages/content-handler/src/derstandard-handler.ts
index 1aae94213..1a1f50778 100644
--- a/packages/content-handler/src/handlers/derstandard-handler.ts
+++ b/packages/content-handler/src/derstandard-handler.ts
@@ -1,14 +1,14 @@
-import { ContentHandler, PreHandleResult } from '../index'
+import { ContentHandler, PreHandleResult } from './index'
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
 
 class DerstandardHandler extends ContentHandler {
-  shouldPreHandle(url: string, _dom: Document): boolean {
+  shouldPreHandle(url: string, dom?: Document): boolean {
     const u = new URL(url)
     return u.hostname === 'www.derstandard.at'
   }
 
-  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
     const response = await axios.get(url, {
       // set cookie to give consent to get the article
       headers: {
diff --git a/packages/content-handler/src/handlers/image-handler.ts b/packages/content-handler/src/image-handler.ts
similarity index 78%
rename from packages/content-handler/src/handlers/image-handler.ts
rename to packages/content-handler/src/image-handler.ts
index 78447fe33..cb5e462c7 100644
--- a/packages/content-handler/src/handlers/image-handler.ts
+++ b/packages/content-handler/src/image-handler.ts
@@ -1,12 +1,12 @@
-import { ContentHandler, PreHandleResult } from '../index'
+import { ContentHandler, PreHandleResult } from './index'
 
 class ImageHandler extends ContentHandler {
-  shouldPreHandle(url: string, _dom: Document): boolean {
+  shouldPreHandle(url: string, dom?: Document): boolean {
     const IMAGE_URL_PATTERN = /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
     return IMAGE_URL_PATTERN.test(url.toString())
   }
 
-  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
     const title = url.toString().split('/').pop() || 'Image'
     const content = `
       <html>
diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts
index 8f3cfbd81..bfb0214e4 100644
--- a/packages/content-handler/src/index.ts
+++ b/packages/content-handler/src/index.ts
@@ -31,11 +31,11 @@ export class ContentHandler {
   protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
   protected name = ''
 
-  shouldPreHandle(url: string, dom: Document): boolean {
+  shouldPreHandle(url: string, dom?: Document): boolean {
     return false
   }
 
-  preHandle(url: string, document: Document): Promise<PreHandleResult> {
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
     return Promise.resolve({ url, dom: document })
   }
 
diff --git a/packages/content-handler/src/handlers/medium-handler.ts b/packages/content-handler/src/medium-handler.ts
similarity index 70%
rename from packages/content-handler/src/handlers/medium-handler.ts
rename to packages/content-handler/src/medium-handler.ts
index 03cc1be8f..0b9d2fcb5 100644
--- a/packages/content-handler/src/handlers/medium-handler.ts
+++ b/packages/content-handler/src/medium-handler.ts
@@ -1,12 +1,12 @@
-import { ContentHandler, PreHandleResult } from '../index'
+import { ContentHandler, PreHandleResult } from './index'
 
 class MediumHandler extends ContentHandler {
-  shouldPreHandle(url: string, _dom: Document): boolean {
+  shouldPreHandle(url: string, dom?: Document): boolean {
     const u = new URL(url)
     return u.hostname.endsWith('medium.com')
   }
 
-  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
     console.log('prehandling medium url', url)
 
     try {
diff --git a/packages/content-handler/src/handlers/pdf-handler.ts b/packages/content-handler/src/pdf-handler.ts
similarity index 57%
rename from packages/content-handler/src/handlers/pdf-handler.ts
rename to packages/content-handler/src/pdf-handler.ts
index fb9b7e045..54df72bc2 100644
--- a/packages/content-handler/src/handlers/pdf-handler.ts
+++ b/packages/content-handler/src/pdf-handler.ts
@@ -1,13 +1,13 @@
-import { ContentHandler, PreHandleResult } from '../index'
+import { ContentHandler, PreHandleResult } from './index'
 
 class PdfHandler extends ContentHandler {
-  shouldPreHandle(url: string, _dom: Document): boolean {
+  shouldPreHandle(url: string, dom?: Document): boolean {
     const u = new URL(url)
     const path = u.pathname.replace(u.search, '')
     return path.endsWith('.pdf')
   }
 
-  async preHandle(_url: string, _document: Document): Promise<PreHandleResult> {
+  async preHandle(_url: string, document?: Document): Promise<PreHandleResult> {
     return Promise.resolve({ contentType: 'application/pdf' })
   }
 }
diff --git a/packages/content-handler/src/handlers/scrapingBee-handler.ts b/packages/content-handler/src/scrapingBee-handler.ts
similarity index 82%
rename from packages/content-handler/src/handlers/scrapingBee-handler.ts
rename to packages/content-handler/src/scrapingBee-handler.ts
index c9cbc8afa..0b1b5984b 100644
--- a/packages/content-handler/src/handlers/scrapingBee-handler.ts
+++ b/packages/content-handler/src/scrapingBee-handler.ts
@@ -1,16 +1,16 @@
-import { ContentHandler, PreHandleResult } from '../index'
+import { ContentHandler, PreHandleResult } from './index'
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
 
 class ScrapingBeeHandler extends ContentHandler {
-  shouldPreHandle(url: string, _dom: Document): boolean {
+  shouldPreHandle(url: string, dom?: Document): boolean {
     const u = new URL(url)
     const hostnames = ['nytimes.com', 'news.google.com']
 
     return hostnames.some((h) => u.hostname.endsWith(h))
   }
 
-  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
     console.log('prehandling url with scrapingbee', url)
 
     try {
diff --git a/packages/content-handler/src/handlers/t-dot-co-handler.ts b/packages/content-handler/src/t-dot-co-handler.ts
similarity index 92%
rename from packages/content-handler/src/handlers/t-dot-co-handler.ts
rename to packages/content-handler/src/t-dot-co-handler.ts
index 3a97849f3..41c90e4a5 100644
--- a/packages/content-handler/src/handlers/t-dot-co-handler.ts
+++ b/packages/content-handler/src/t-dot-co-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler } from '../index'
+import { ContentHandler } from './index'
 import axios from 'axios'
 
 class TDotCoHandler extends ContentHandler {
diff --git a/packages/content-handler/src/handlers/twitter-handler.ts b/packages/content-handler/src/twitter-handler.ts
similarity index 96%
rename from packages/content-handler/src/handlers/twitter-handler.ts
rename to packages/content-handler/src/twitter-handler.ts
index e4234dcbc..10d9ead03 100644
--- a/packages/content-handler/src/handlers/twitter-handler.ts
+++ b/packages/content-handler/src/twitter-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from '../index'
+import { ContentHandler, PreHandleResult } from './index'
 import axios from 'axios'
 import { DateTime } from 'luxon'
 import _ from 'underscore'
@@ -53,11 +53,11 @@ const formatTimestamp = (timestamp: string) => {
 }
 
 class TwitterHandler extends ContentHandler {
-  shouldPreHandle(url: string, _dom: Document): boolean {
+  shouldPreHandle(url: string, dom?: Document): boolean {
     return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
   }
 
-  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
     console.log('prehandling twitter url', url)
 
     const tweetId = tweetIdFromStatusUrl(url)
diff --git a/packages/content-handler/src/handlers/youtube-handler.ts b/packages/content-handler/src/youtube-handler.ts
similarity index 90%
rename from packages/content-handler/src/handlers/youtube-handler.ts
rename to packages/content-handler/src/youtube-handler.ts
index 288f42aa0..0cbe0df57 100644
--- a/packages/content-handler/src/handlers/youtube-handler.ts
+++ b/packages/content-handler/src/youtube-handler.ts
@@ -1,11 +1,11 @@
-import { ContentHandler, PreHandleResult } from '../index'
+import { ContentHandler, PreHandleResult } from './index'
 import axios from 'axios'
 import _ from 'underscore'
 
 const YOUTUBE_URL_MATCH =
   /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/
 
-function getYoutubeVideoId(url: string) {
+export const getYoutubeVideoId = (url: string) => {
   const u = new URL(url)
   const videoId = u.searchParams.get('v')
   if (!videoId) {
@@ -19,11 +19,11 @@ function getYoutubeVideoId(url: string) {
 }
 
 class YoutubeHandler extends ContentHandler {
-  shouldPreHandle(url: string, _dom: Document): boolean {
+  shouldPreHandle(url: string, dom?: Document): boolean {
     return YOUTUBE_URL_MATCH.test(url.toString())
   }
 
-  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
     const videoId = getYoutubeVideoId(url)
     if (!videoId) {
       return {}
diff --git a/packages/content-handler/test/apple-news-handler.test.ts b/packages/content-handler/test/apple-news-handler.test.ts
new file mode 100644
index 000000000..9af3be580
--- /dev/null
+++ b/packages/content-handler/test/apple-news-handler.test.ts
@@ -0,0 +1,10 @@
+import { AppleNewsHandler } from '../src/apple-news-handler'
+
+describe('open a simple web page', () => {
+  it('should return a response', async () => {
+    const response = await new AppleNewsHandler().preHandle(
+      'https://apple.news/AxjzaZaPvSn23b67LhXI5EQ'
+    )
+    console.log('response', response)
+  })
+})
diff --git a/packages/content-handler/test/stub.test.ts b/packages/content-handler/test/stub.test.ts
deleted file mode 100644
index 173ca4917..000000000
--- a/packages/content-handler/test/stub.test.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import 'mocha'
-import * as chai from 'chai'
-import { expect } from 'chai'
-import 'chai/register-should'
-import chaiString from 'chai-string'
-
-chai.use(chaiString)
-
-describe('Stub test', () => {
-  it('should pass', () => {
-    expect(true).to.be.true
-  })
-})
diff --git a/packages/content-handler/test/youtube-handler.test.ts b/packages/content-handler/test/youtube-handler.test.ts
new file mode 100644
index 000000000..f8c3a51ff
--- /dev/null
+++ b/packages/content-handler/test/youtube-handler.test.ts
@@ -0,0 +1,25 @@
+import { expect } from 'chai'
+import 'mocha'
+import { getYoutubeVideoId } from '../src/youtube-handler'
+
+describe('getYoutubeVideoId', () => {
+  it('should parse video id out of a URL', async () => {
+    expect('BnSUk0je6oo').to.eq(
+      getYoutubeVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s')
+    )
+    expect('vFD2gu007dc').to.eq(
+      getYoutubeVideoId(
+        'https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1'
+      )
+    )
+    expect('vFD2gu007dc').to.eq(
+      getYoutubeVideoId('https://youtu.be/vFD2gu007dc')
+    )
+    expect('BMFVCnbRaV4').to.eq(
+      getYoutubeVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share')
+    )
+    expect('cg9b4RC87LI').to.eq(
+      getYoutubeVideoId('https://youtu.be/cg9b4RC87LI?t=116')
+    )
+  })
+})
diff --git a/yarn.lock b/yarn.lock
index 3da81da9b..d1c9a65dd 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -10579,6 +10579,19 @@ chai@^4.3.4:
     pathval "^1.1.1"
     type-detect "^4.0.5"
 
+chai@^4.3.6:
+  version "4.3.6"
+  resolved "https://registry.yarnpkg.com/chai/-/chai-4.3.6.tgz#ffe4ba2d9fa9d6680cc0b370adae709ec9011e9c"
+  integrity sha512-bbcp3YfHCUzMOvKqsztczerVgBKSsEijCySNlHHbX3VG1nskvqjz5Rfso1gGwD6w6oOV3eI60pKuMOV5MV7p3Q==
+  dependencies:
+    assertion-error "^1.1.0"
+    check-error "^1.0.2"
+    deep-eql "^3.0.1"
+    get-func-name "^2.0.0"
+    loupe "^2.3.1"
+    pathval "^1.1.1"
+    type-detect "^4.0.5"
+
 chalk@^1.0.0, chalk@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98"
@@ -18065,6 +18078,13 @@ loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0:
   dependencies:
     js-tokens "^3.0.0 || ^4.0.0"
 
+loupe@^2.3.1:
+  version "2.3.4"
+  resolved "https://registry.yarnpkg.com/loupe/-/loupe-2.3.4.tgz#7e0b9bffc76f148f9be769cb1321d3dcf3cb25f3"
+  integrity sha512-OvKfgCC2Ndby6aSTREl5aCCPTNIzlDfQZvZxNUrBrihDhL3xcrYegTblhmEiCrg2kKQz4XsFIaemE5BF4ybSaQ==
+  dependencies:
+    get-func-name "^2.0.0"
+
 lower-case-first@^1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/lower-case-first/-/lower-case-first-1.0.2.tgz#e5da7c26f29a7073be02d52bac9980e5922adfa1"

From 8c61832c77b39b78ba61b5f04c6be5ca2c092b28 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 29 Sep 2022 18:05:35 +0800
Subject: [PATCH 06/15] Import content-handler in content-fetch

---
 packages/content-fetch/apple-news-handler.js  |  36 ----
 packages/content-fetch/bloomberg-handler.js   |  39 ----
 packages/content-fetch/derstandard-handler.js |  35 ----
 packages/content-fetch/fetch-content.js       |  92 ++--------
 packages/content-fetch/image-handler.js       |  34 ----
 packages/content-fetch/medium-handler.js      |  29 ---
 packages/content-fetch/package.json           |   3 +-
 packages/content-fetch/pdf-handler.js         |  21 ---
 packages/content-fetch/scrapingBee-handler.js |  44 -----
 packages/content-fetch/t-dot-co-handler.js    |  31 ----
 .../test/apple-news-handler.test.js           |   9 -
 .../test/youtube-handler.test.js              |  12 --
 packages/content-fetch/twitter-handler.js     | 172 ------------------
 packages/content-fetch/youtube-handler.js     |  68 -------
 .../content-handler/src/apple-news-handler.ts |   7 +-
 .../content-handler/src/bloomberg-handler.ts  |   9 +-
 .../content-handler/src/content-handler.ts    | 126 +++++++++++++
 .../src/derstandard-handler.ts                |   9 +-
 packages/content-handler/src/image-handler.ts |   9 +-
 packages/content-handler/src/index.ts         | 171 +++++++----------
 .../content-handler/src/medium-handler.ts     |   9 +-
 packages/content-handler/src/pdf-handler.ts   |   9 +-
 .../src/scrapingBee-handler.ts                |   9 +-
 .../content-handler/src/t-dot-co-handler.ts   |   9 +-
 .../content-handler/src/twitter-handler.ts    |   9 +-
 .../content-handler/src/youtube-handler.ts    |   9 +-
 yarn.lock                                     |  20 --
 27 files changed, 279 insertions(+), 751 deletions(-)
 delete mode 100644 packages/content-fetch/apple-news-handler.js
 delete mode 100644 packages/content-fetch/bloomberg-handler.js
 delete mode 100644 packages/content-fetch/derstandard-handler.js
 delete mode 100644 packages/content-fetch/image-handler.js
 delete mode 100644 packages/content-fetch/medium-handler.js
 delete mode 100644 packages/content-fetch/pdf-handler.js
 delete mode 100644 packages/content-fetch/scrapingBee-handler.js
 delete mode 100644 packages/content-fetch/t-dot-co-handler.js
 delete mode 100644 packages/content-fetch/test/apple-news-handler.test.js
 delete mode 100644 packages/content-fetch/test/youtube-handler.test.js
 delete mode 100644 packages/content-fetch/twitter-handler.js
 delete mode 100644 packages/content-fetch/youtube-handler.js
 create mode 100644 packages/content-handler/src/content-handler.ts

diff --git a/packages/content-fetch/apple-news-handler.js b/packages/content-fetch/apple-news-handler.js
deleted file mode 100644
index 0759dec23..000000000
--- a/packages/content-fetch/apple-news-handler.js
+++ /dev/null
@@ -1,36 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const Url = require('url');
-const axios = require('axios');
-const { promisify } = require('util');
-const { DateTime } = require('luxon');
-const os = require('os');
-const { Cipher } = require('crypto');
-const { parseHTML } = require('linkedom');
-
-exports.appleNewsHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    if (u.hostname === 'apple.news') {
-      return true;
-    }
-    return false
-  },
-
-  prehandle: async (url, env) => {
-    const MOBILE_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
-    const response = await axios.get(url, { headers: { 'User-Agent': MOBILE_USER_AGENT }  } );
-    const data = response.data;
-
-    const dom = parseHTML(data).document;
-
-    // make sure its a valid URL by wrapping in new URL
-    const u = new URL(dom.querySelector('span.click-here').parentNode.href);
-    return { url: u.href };
-  }
-}
diff --git a/packages/content-fetch/bloomberg-handler.js b/packages/content-fetch/bloomberg-handler.js
deleted file mode 100644
index d79a568bb..000000000
--- a/packages/content-fetch/bloomberg-handler.js
+++ /dev/null
@@ -1,39 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const os = require('os');
-const { parseHTML } = require('linkedom');
-
-exports.bloombergHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const BLOOMBERG_URL_MATCH =
-      /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
-    return BLOOMBERG_URL_MATCH.test(url.toString())
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling bloomberg url', url)
-
-    try {
-      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
-        params: {
-          'api_key': process.env.SCRAPINGBEE_API_KEY,
-          'url': url,
-          'return_page_source': true,
-          'block_ads': true,
-          'block_resources': false,
-        }
-      })
-      const dom = parseHTML(response.data).document;
-      return { title: dom.title, content: dom.querySelector('body').innerHTML, url: url }
-    } catch (error) {
-      console.error('error prehandling bloomberg url', error)
-      throw error
-    }
-  }
-}
diff --git a/packages/content-fetch/derstandard-handler.js b/packages/content-fetch/derstandard-handler.js
deleted file mode 100644
index a44db6f2a..000000000
--- a/packages/content-fetch/derstandard-handler.js
+++ /dev/null
@@ -1,35 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { parseHTML } = require('linkedom');
-
-exports.derstandardHandler = {
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    return u.hostname === 'www.derstandard.at';
-  },
-
-  prehandle: async (url, env) => {
-    const response = await axios.get(url, {
-      // set cookie to give consent to get the article
-      headers: {
-        'cookie': `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`
-      },
-    });
-    const content = response.data;
-
-    var title = undefined;
-    const dom = parseHTML(content).document;
-    const titleElement = dom.querySelector('.article-title')
-    if (!titleElement) {
-      title = titleElement.textContent
-      titleElement.remove()
-    }
-
-    return { content: dom.body.outerHTML, title: title };
-  }
-}
diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js
index 01dbe199c..658229e3d 100644
--- a/packages/content-fetch/fetch-content.js
+++ b/packages/content-fetch/fetch-content.js
@@ -9,16 +9,10 @@ const puppeteer = require('puppeteer-core');
 const axios = require('axios');
 const jwt = require('jsonwebtoken');
 const { promisify } = require('util');
+const { parseHTML } = require('linkedom');
+const { preHandleContent } = require('@omnivore/content-handler');
+
 const signToken = promisify(jwt.sign);
-const { appleNewsHandler } = require('./apple-news-handler');
-const { twitterHandler } = require('./twitter-handler');
-const { youtubeHandler } = require('./youtube-handler');
-const { tDotCoHandler } = require('./t-dot-co-handler');
-const { pdfHandler } = require('./pdf-handler');
-const { mediumHandler } = require('./medium-handler');
-const { derstandardHandler } = require('./derstandard-handler');
-const { imageHandler } = require('./image-handler');
-const { scrapingBeeHandler } = require('./scrapingBee-handler')
 
 const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
 const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
@@ -29,8 +23,6 @@ const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com'];
 
 const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf'];
 
-const { parseHTML } = require('linkedom');
-
 // Add stealth plugin to hide puppeteer usage
 // const StealthPlugin = require('puppeteer-extra-plugin-stealth');
 // puppeteer.use(StealthPlugin());
@@ -207,19 +199,6 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId
   );
 };
 
-const handlers = {
-  'pdf': pdfHandler,
-  'apple-news': appleNewsHandler,
-  'twitter': twitterHandler,
-  'youtube': youtubeHandler,
-  't-dot-co': tDotCoHandler,
-  'medium': mediumHandler,
-  'derstandard': derstandardHandler,
-  'image': imageHandler,
-  'scrapingBee': scrapingBeeHandler,
-};
-
-
 async function fetchContent(req, res) {
   functionStartTime = Date.now();
 
@@ -246,61 +225,18 @@ async function fetchContent(req, res) {
     return res.sendStatus(400);
   }
 
-  // if (!userId || !articleSavingRequestId) {
-  //   Object.assign(logRecord, { invalidParams: true, body: req.body, query: req.query });
-  //   console.log(`Invalid parameters`, logRecord);
-  //   return res.sendStatus(400);
-  // }
-
-  // Before we run the regular handlers we check to see if we need tp
-  // pre-resolve the URL. TODO: This should probably happen recursively,
-  // so URLs can be pre-resolved, handled, pre-resolved, handled, etc.
-  for (const [key, handler] of Object.entries(handlers)) {
-    if (handler.shouldResolve && handler.shouldResolve(url)) {
-      try {
-        url = await handler.resolve(url);
-        validateUrlString(url);
-      } catch (err) {
-        console.log('error resolving url with handler', key, err);
-      }
-      break;
-    }
-  }
-
-  // Before we fetch the page we check the handlers, to see if they want
-  // to perform a prefetch action that can modify our requests.
-  // enumerate the handlers and see if any of them want to handle the request
-  const handler = Object.keys(handlers).find(key => {
-    try {
-      return handlers[key].shouldPrehandle(url)
-    } catch (e) {
-      console.log('error with handler: ', key, e);
-    }
-    return false;
-  });
-
-  var title = undefined;
-  var content = undefined;
-  var contentType = undefined;
-
-  if (handler) {
-    try {
-      // The only handler we have now can modify the URL, but in the
-      // future maybe we let it modify content. In that case
-      // we might exit the request early.
-      console.log('pre-handling url with handler: ', handler);
-
-      const result = await handlers[handler].prehandle(url);
-      if (result && result.url) {
-        url = result.url
-        validateUrlString(url);
-      }
-      if (result && result.title) { title = result.title }
-      if (result && result.content) { content = result.content }
-      if (result && result.contentType) { contentType = result.contentType }
-    } catch (e) {
-      console.log('error with handler: ', handler, e);
+  let title, content, contentType;
+  try {
+    const result = await preHandleContent(url);
+    if (result && result.url) {
+      url = result.url
+      validateUrlString(url);
     }
+    if (result && result.title) { title = result.title }
+    if (result && result.content) { content = result.content }
+    if (result && result.contentType) { contentType = result.contentType }
+  } catch (e) {
+    console.log('error with handler: ', e);
   }
 
   let context, page, finalUrl;
diff --git a/packages/content-fetch/image-handler.js b/packages/content-fetch/image-handler.js
deleted file mode 100644
index 59f132afc..000000000
--- a/packages/content-fetch/image-handler.js
+++ /dev/null
@@ -1,34 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-
-
-exports.imageHandler = {
-  shouldPrehandle: (url, env) => {
-    const IMAGE_URL_PATTERN =
-      /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
-    return IMAGE_URL_PATTERN.test(url.toString())
-  },
-
-  prehandle: async (url, env) => {
-    const title = url.toString().split('/').pop();
-    const content = `
-      <html>
-        <head>
-          <title>${title}</title>
-          <meta property="og:image" content="${url}" />
-          <meta property="og:title" content="${title}" />
-        </head>
-        <body>
-          <div>
-            <img src="${url}" alt="${title}">
-          </div>
-        </body>
-      </html>`
-
-    return { title, content };
-  }
-}
diff --git a/packages/content-fetch/medium-handler.js b/packages/content-fetch/medium-handler.js
deleted file mode 100644
index e6a605a0e..000000000
--- a/packages/content-fetch/medium-handler.js
+++ /dev/null
@@ -1,29 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const os = require('os');
-
-exports.mediumHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    return u.hostname.endsWith('medium.com')
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling medium url', url)
-
-    try {
-      const res = new URL(url);
-      res.searchParams.delete('source');
-      return { url: res.toString() }
-    } catch (error) {
-      console.error('error prehandling medium url', error)
-      throw error
-    }
-  }
-}
diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json
index f58675a74..3df85a237 100644
--- a/packages/content-fetch/package.json
+++ b/packages/content-fetch/package.json
@@ -11,7 +11,8 @@
     "linkedom": "^0.14.9",
     "luxon": "^2.3.1",
     "puppeteer-core": "^16.1.0",
-    "underscore": "^1.13.4"
+    "underscore": "^1.13.4",
+    "@omnivore/content-handler": "1.0.0"
   },
   "scripts": {
     "start": "node app.js",
diff --git a/packages/content-fetch/pdf-handler.js b/packages/content-fetch/pdf-handler.js
deleted file mode 100644
index 1260db287..000000000
--- a/packages/content-fetch/pdf-handler.js
+++ /dev/null
@@ -1,21 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const Url = require('url');
-
-
-exports.pdfHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = Url.parse(url)
-    const path = u.path.replace(u.search, '')
-    return path.endsWith('.pdf')
-  },
-
-  prehandle: async (url, env) => {
-    return { contentType: 'application/pdf' };
-  }
-}
diff --git a/packages/content-fetch/scrapingBee-handler.js b/packages/content-fetch/scrapingBee-handler.js
deleted file mode 100644
index 6563fca44..000000000
--- a/packages/content-fetch/scrapingBee-handler.js
+++ /dev/null
@@ -1,44 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { parseHTML } = require('linkedom');
-
-const os = require('os');
-
-exports.scrapingBeeHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    const hostnames = [
-      'nytimes.com',
-      'news.google.com',
-    ]
-
-    return hostnames.some((h) => u.hostname.endsWith(h))
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling url with scrapingbee', url)
-
-    try {
-      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
-        params: {
-          'api_key': process.env.SCRAPINGBEE_API_KEY,
-          'url': url,
-          'return_page_source': true,
-          'block_ads': true,
-          'block_resources': false,
-        }
-      })
-      const dom = parseHTML(response.data).document;
-      return { title: dom.title, content: response.data, url: url }
-    } catch (error) {
-      console.error('error prehandling url w/scrapingbee', error)
-      throw error
-    }
-  }
-}
diff --git a/packages/content-fetch/t-dot-co-handler.js b/packages/content-fetch/t-dot-co-handler.js
deleted file mode 100644
index 170f97fb7..000000000
--- a/packages/content-fetch/t-dot-co-handler.js
+++ /dev/null
@@ -1,31 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const Url = require('url');
-
-
-exports.tDotCoHandler = {
-
-  shouldResolve: function (url, env) {
-    const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/;
-    return T_DOT_CO_URL_MATCH.test(url);
-  },
-
-  resolve: async function(url, env) {
-    return await axios.get(url, { maxRedirects: 0, validateStatus: null })
-      .then(res => {
-        return Url.parse(res.headers.location).href;
-      }).catch((err) => {
-        console.log('err with t.co url', err);
-        return undefined;
-      });
-  },
-
-  shouldPrehandle: (url, env) => {
-    return false
-  },
-}
diff --git a/packages/content-fetch/test/apple-news-handler.test.js b/packages/content-fetch/test/apple-news-handler.test.js
deleted file mode 100644
index 4531d720e..000000000
--- a/packages/content-fetch/test/apple-news-handler.test.js
+++ /dev/null
@@ -1,9 +0,0 @@
-const { expect } = require('chai')
-const { appleNewsHandler } = require('../apple-news-handler')
-
-describe('open a simple web page', () => {
-  it('should return a response', async () => {
-    const response = await appleNewsHandler.prehandle('https://apple.news/AxjzaZaPvSn23b67LhXI5EQ')
-    console.log('response', response)
-  })
-})
diff --git a/packages/content-fetch/test/youtube-handler.test.js b/packages/content-fetch/test/youtube-handler.test.js
deleted file mode 100644
index d34643773..000000000
--- a/packages/content-fetch/test/youtube-handler.test.js
+++ /dev/null
@@ -1,12 +0,0 @@
-const { expect } = require('chai')
-const { getYoutubeVideoId } = require('../youtube-handler')
-
-describe('getYoutubeVideoId', () => {
-  it('should parse video id out of a URL', async () => {
-    expect('BnSUk0je6oo').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s'));
-    expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1'));
-    expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://youtu.be/vFD2gu007dc'));
-    expect('BMFVCnbRaV4').to.eq(getYoutubeVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share'));
-    expect('cg9b4RC87LI').to.eq(getYoutubeVideoId('https://youtu.be/cg9b4RC87LI?t=116'));
-  })
-})
diff --git a/packages/content-fetch/twitter-handler.js b/packages/content-fetch/twitter-handler.js
deleted file mode 100644
index 7ae93072c..000000000
--- a/packages/content-fetch/twitter-handler.js
+++ /dev/null
@@ -1,172 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { DateTime } = require('luxon');
-const _ = require('underscore');
-
-const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN;
-const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
-
-const embeddedTweet = async (url) => {
-
-  const BASE_ENDPOINT = 'https://publish.twitter.com/oembed'
-
-  const apiUrl = new URL(BASE_ENDPOINT)
-  apiUrl.searchParams.append('url', url);
-  apiUrl.searchParams.append('omit_script', true);
-  apiUrl.searchParams.append('dnt', true);
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const getTweetFields = () => {
-  const TWEET_FIELDS =
-  "&tweet.fields=attachments,author_id,conversation_id,created_at," +
-  "entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets," +
-  "source,withheld";
-  const EXPANSIONS = "&expansions=author_id,attachments.media_keys";
-  const USER_FIELDS =
-  "&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld";
-  const MEDIA_FIELDS =
-  "&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width";
-
-  return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`;
-}
-
-const getTweetById = async (id) => {
-  const BASE_ENDPOINT = "https://api.twitter.com/2/tweets/";
-  const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const getUserByUsername = async (username) => {
-  const BASE_ENDPOINT = "https://api.twitter.com/2/users/by/username/";
-
-  const apiUrl = new URL(BASE_ENDPOINT + username)
-  apiUrl.searchParams.append('user.fields', 'profile_image_url');
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const titleForTweet = (tweet) => {
-  return `${tweet.data.author_name} on Twitter`
-};
-
-const titleForAuthor = (author) => {
-  return `${author.name} on Twitter`
-};
-
-const usernameFromStatusUrl = (url) => {
-  const match = url.toString().match(TWITTER_URL_MATCH)
-  return match[1]
-};
-
-const tweetIdFromStatusUrl = (url) => {
-  const match = url.toString().match(TWITTER_URL_MATCH)
-  return match[2]
-};
-
-const formatTimestamp = (timestamp) => {
-  return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(DateTime.DATETIME_FULL);
-};
-
-exports.twitterHandler = {
-
-  shouldPrehandle: (url, env) => {
-    return TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
-  },
-
-  // version of the handler that uses the oembed API
-  // This isn't great as it doesn't work well with our
-  // readability API. But could potentially give a more consistent
-  // look to the tweets
-  // prehandle: async (url, env) => {
-  //   const oeTweet = await embeddedTweet(url)
-  //   const dom = new JSDOM(oeTweet.data.html);
-  //   const bq = dom.window.document.querySelector('blockquote')
-  //   console.log('blockquote:', bq);
-
-  //   const title = titleForTweet(oeTweet)
-  //   return { title, content: '<div>' + bq.innerHTML + '</div>', url: oeTweet.data.url };
-  // }
-
-  prehandle: async (url, env) => {
-    console.log('prehandling twitter url', url)
-
-    const tweetId = tweetIdFromStatusUrl(url)
-    const tweetData = (await getTweetById(tweetId)).data;
-    const authorId = tweetData.data.author_id;
-    const author = tweetData.includes.users.filter(u => u.id = authorId)[0];
-    // escape html entities in title
-    const title = _.escape(titleForAuthor(author))
-    const authorImage = author.profile_image_url.replace('_normal', '_400x400')
-
-    let text = tweetData.data.text;
-    if (tweetData.data.entities && tweetData.data.entities.urls) {
-      for (let urlObj of tweetData.data.entities.urls) {
-        text = text.replace(
-          urlObj.url,
-          `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
-        );
-      }
-    }
-
-    const front = `
-    <div>
-      <p>${text}</p>
-    `
-
-    var includesHtml =  '';
-    if (tweetData.includes.media) {
-      includesHtml = tweetData.includes.media.map(m => {
-        const linkUrl = m.type == 'photo' ? m.url : url;
-        const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url;
-        const mediaOpen = `<a class="media-link" href=${linkUrl}>
-          <picture>
-            <img class="tweet-img" src=${previewUrl} />
-          </picture>
-          </a>`
-        return mediaOpen
-      }).join('\n');
-   }
-
-   const back = `
-       — <a href="https://twitter.com/${author.username}">${author.username}</a> ${author.name} <a href="${url}">${formatTimestamp(tweetData.data.created_at)}</a>
-    </div>
-    `
-    const content = `
-    <head>
-      <meta property="og:image" content="${authorImage}" />
-      <meta property="og:image:secure_url" content="${authorImage}" />
-      <meta property="og:title" content="${title}" />
-      <meta property="og:description" content="${_.escape(tweetData.data.text)}" />
-    </head>
-    <body>
-      ${front}
-      ${includesHtml}
-      ${back}
-    </body>`
-
-    return { content, url, title };
-  }
-}
diff --git a/packages/content-fetch/youtube-handler.js b/packages/content-fetch/youtube-handler.js
deleted file mode 100644
index e1866428a..000000000
--- a/packages/content-fetch/youtube-handler.js
+++ /dev/null
@@ -1,68 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const _ = require('underscore');
-
-const YOUTUBE_URL_MATCH =
-  /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/
-
-function getYoutubeVideoId(url) {
-  const u = new URL(url);
-  const videoId = u.searchParams.get('v');
-  if (!videoId) {
-    const match = url.toString().match(YOUTUBE_URL_MATCH)
-    if (match === null || match.length < 6 || !match[5]) {
-      return undefined
-    }
-    return match[5]
-  }
-  return videoId
-}
-exports.getYoutubeVideoId = getYoutubeVideoId
-
-exports.youtubeHandler = {
-  shouldPrehandle: (url, env) => {
-    return YOUTUBE_URL_MATCH.test(url.toString())
-  },
-
-  prehandle: async (url, env) => {
-    const videoId = getYoutubeVideoId(url)
-    if (!videoId) {
-      return {}
-    }
-
-    const oembedUrl = `https://www.youtube.com/oembed?format=json&url=` + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`)
-    const oembed = (await axios.get(oembedUrl.toString())).data;
-    // escape html entities in title
-    const title = _.escape(oembed.title);
-    const ratio = oembed.width / oembed.height;
-    const thumbnail = oembed.thumbnail_url;
-    const height = 350;
-    const width = height * ratio;
-    const authorName = _.escape(oembed.author_name);
-
-    const content = `
-    <html>
-      <head><title>${title}</title>
-      <meta property="og:image" content="${thumbnail}" />
-      <meta property="og:image:secure_url" content="${thumbnail}" />
-      <meta property="og:title" content="${title}" />
-      <meta property="og:description" content="" />
-      <meta property="og:article:author" content="${authorName}" />
-      </head>
-      <body>
-      <iframe width="${width}" height="${height}" src="https://www.youtube.com/embed/${videoId}" title="${title}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
-        <p><a href="${url}" target="_blank">${title}</a></p>
-        <p itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="${oembed.author_url}" target="_blank">${authorName}</a></p>
-      </body>
-    </html>`
-
-    console.log('got video id', videoId)
-
-    return { content, title: 'Youtube Content' };
-  }
-}
diff --git a/packages/content-handler/src/apple-news-handler.ts b/packages/content-handler/src/apple-news-handler.ts
index 1958b3b8c..a4239565c 100644
--- a/packages/content-handler/src/apple-news-handler.ts
+++ b/packages/content-handler/src/apple-news-handler.ts
@@ -1,8 +1,13 @@
-import { ContentHandler, PreHandleResult } from './index'
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
+import { ContentHandler, PreHandleResult } from './content-handler'
 
 export class AppleNewsHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Apple News'
+  }
+
   shouldPreHandle(url: string, dom?: Document): boolean {
     const u = new URL(url)
     return u.hostname === 'apple.news'
diff --git a/packages/content-handler/src/bloomberg-handler.ts b/packages/content-handler/src/bloomberg-handler.ts
index ac4fb23ed..95670630e 100644
--- a/packages/content-handler/src/bloomberg-handler.ts
+++ b/packages/content-handler/src/bloomberg-handler.ts
@@ -1,8 +1,13 @@
-import { ContentHandler, PreHandleResult } from './index'
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
+import { ContentHandler, PreHandleResult } from './content-handler'
+
+export class BloombergHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Bloomberg'
+  }
 
-class BloombergHandler extends ContentHandler {
   shouldPreHandle(url: string, dom?: Document): boolean {
     const BLOOMBERG_URL_MATCH =
       /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)/
diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts
new file mode 100644
index 000000000..c49987368
--- /dev/null
+++ b/packages/content-handler/src/content-handler.ts
@@ -0,0 +1,126 @@
+import addressparser from 'addressparser'
+import rfc2047 from 'rfc2047'
+import { v4 as uuidv4 } from 'uuid'
+
+interface Unsubscribe {
+  mailTo?: string
+  httpUrl?: string
+}
+
+interface NewsletterMessage {
+  email: string
+  content: string
+  url: string
+  title: string
+  author: string
+  unsubMailTo?: string
+  unsubHttpUrl?: string
+}
+
+export interface PreHandleResult {
+  url?: string
+  title?: string
+  content?: string
+  contentType?: string
+  dom?: Document
+}
+
+export abstract class ContentHandler {
+  protected senderRegex: RegExp
+  protected urlRegex: RegExp
+  protected defaultUrl: string
+  public name: string
+
+  protected constructor() {
+    this.senderRegex = new RegExp(/NEWSLETTER_SENDER_REGEX/)
+    this.urlRegex = new RegExp(/NEWSLETTER_URL_REGEX/)
+    this.defaultUrl = 'NEWSLETTER_DEFAULT_URL'
+    this.name = 'Handler name'
+  }
+
+  shouldResolve(url: string): boolean {
+    return false
+  }
+
+  async resolve(url: string): Promise<string | undefined> {
+    return Promise.resolve(url)
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    return false
+  }
+
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+    return Promise.resolve({ url, dom: document })
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+
+  parseNewsletterUrl(_postHeader: string, html: string): string | undefined {
+    // get newsletter url from html
+    const matches = html.match(this.urlRegex)
+    if (matches) {
+      return matches[1]
+    }
+    return undefined
+  }
+
+  parseAuthor(from: string): string {
+    // get author name from email
+    // e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
+    // or 'Mike Allen <mike@axios.com>'
+    const parsed = addressparser(from)
+    if (parsed.length > 0) {
+      return parsed[0].name
+    }
+    return from
+  }
+
+  parseUnsubscribe(unSubHeader: string): Unsubscribe {
+    // parse list-unsubscribe header
+    // e.g. List-Unsubscribe: <https://omnivore.com/unsub>, <mailto:unsub@omnivore.com>
+    const decoded = rfc2047.decode(unSubHeader)
+    return {
+      mailTo: decoded.match(/<(https?:\/\/[^>]*)>/)?.[1],
+      httpUrl: decoded.match(/<mailto:([^>]*)>/)?.[1],
+    }
+  }
+
+  handleNewsletter(
+    email: string,
+    html: string,
+    postHeader: string,
+    title: string,
+    from: string,
+    unSubHeader: string
+  ): NewsletterMessage {
+    console.log('handleNewsletter', email, postHeader, title, from)
+
+    if (!email || !html || !title || !from) {
+      console.log('invalid newsletter email')
+      throw new Error('invalid newsletter email')
+    }
+
+    // fallback to default url if newsletter url does not exist
+    // assign a random uuid to the default url to avoid duplicate url
+    const url =
+      this.parseNewsletterUrl(postHeader, html) ||
+      `${this.defaultUrl}?source=newsletters&id=${uuidv4()}`
+    const author = this.parseAuthor(from)
+    const unsubscribe = this.parseUnsubscribe(unSubHeader)
+
+    return {
+      email,
+      content: html,
+      url,
+      title,
+      author,
+      unsubMailTo: unsubscribe.mailTo || '',
+      unsubHttpUrl: unsubscribe.httpUrl || '',
+    }
+  }
+}
diff --git a/packages/content-handler/src/derstandard-handler.ts b/packages/content-handler/src/derstandard-handler.ts
index 1a1f50778..2ac01ac86 100644
--- a/packages/content-handler/src/derstandard-handler.ts
+++ b/packages/content-handler/src/derstandard-handler.ts
@@ -1,8 +1,13 @@
-import { ContentHandler, PreHandleResult } from './index'
+import { ContentHandler, PreHandleResult } from './content-handler'
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
 
-class DerstandardHandler extends ContentHandler {
+export class DerstandardHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Derstandard'
+  }
+
   shouldPreHandle(url: string, dom?: Document): boolean {
     const u = new URL(url)
     return u.hostname === 'www.derstandard.at'
diff --git a/packages/content-handler/src/image-handler.ts b/packages/content-handler/src/image-handler.ts
index cb5e462c7..652756c51 100644
--- a/packages/content-handler/src/image-handler.ts
+++ b/packages/content-handler/src/image-handler.ts
@@ -1,6 +1,11 @@
-import { ContentHandler, PreHandleResult } from './index'
+import { ContentHandler, PreHandleResult } from './content-handler'
+
+export class ImageHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Image'
+  }
 
-class ImageHandler extends ContentHandler {
   shouldPreHandle(url: string, dom?: Document): boolean {
     const IMAGE_URL_PATTERN = /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
     return IMAGE_URL_PATTERN.test(url.toString())
diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts
index bfb0214e4..3275a3c89 100644
--- a/packages/content-handler/src/index.ts
+++ b/packages/content-handler/src/index.ts
@@ -1,111 +1,76 @@
-import addressparser from 'addressparser'
-import { v4 as uuidv4 } from 'uuid'
-import rfc2047 from 'rfc2047'
+import { AppleNewsHandler } from './apple-news-handler'
+import { BloombergHandler } from './bloomberg-handler'
+import { DerstandardHandler } from './derstandard-handler'
+import { ImageHandler } from './image-handler'
+import { MediumHandler } from './medium-handler'
+import { PdfHandler } from './pdf-handler'
+import { ScrapingBeeHandler } from './scrapingBee-handler'
+import { TDotCoHandler } from './t-dot-co-handler'
+import { TwitterHandler } from './twitter-handler'
+import { YoutubeHandler } from './youtube-handler'
+import { ContentHandler, PreHandleResult } from './content-handler'
 
-interface Unsubscribe {
-  mailTo?: string
-  httpUrl?: string
+const validateUrlString = (url: string) => {
+  const u = new URL(url)
+  // Make sure the URL is http or https
+  if (u.protocol !== 'http:' && u.protocol !== 'https:') {
+    throw new Error('Invalid URL protocol check failed')
+  }
+  // Make sure the domain is not localhost
+  if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') {
+    throw new Error('Invalid URL is localhost')
+  }
+  // Make sure the domain is not a private IP
+  if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) {
+    throw new Error('Invalid URL is private ip')
+  }
 }
 
-interface NewsletterMessage {
-  email: string
-  content: string
-  url: string
-  title: string
-  author: string
-  unsubMailTo?: string
-  unsubHttpUrl?: string
-}
+const contentHandlers: ContentHandler[] = [
+  new AppleNewsHandler(),
+  new BloombergHandler(),
+  new DerstandardHandler(),
+  new ImageHandler(),
+  new MediumHandler(),
+  new PdfHandler(),
+  new ScrapingBeeHandler(),
+  new TDotCoHandler(),
+  new TwitterHandler(),
+  new YoutubeHandler(),
+]
 
-export interface PreHandleResult {
-  url?: string
-  title?: string
-  content?: string
-  contentType?: string
+export const preHandleContent = async (
+  url: string,
   dom?: Document
+): Promise<PreHandleResult | undefined> => {
+  // Before we run the regular handlers we check to see if we need tp
+  // pre-resolve the URL. TODO: This should probably happen recursively,
+  // so URLs can be pre-resolved, handled, pre-resolved, handled, etc.
+  for (const handler of contentHandlers) {
+    if (handler.shouldResolve(url)) {
+      try {
+        const resolvedUrl = await handler.resolve(url)
+        if (resolvedUrl && validateUrlString(resolvedUrl)) {
+          url = resolvedUrl
+        }
+      } catch (err) {
+        console.log('error resolving url with handler', handler.name, err)
+      }
+      break
+    }
+  }
+  // Before we fetch the page we check the handlers, to see if they want
+  // to perform a prefetch action that can modify our requests.
+  // enumerate the handlers and see if any of them want to handle the request
+  for (const handler of contentHandlers) {
+    if (handler.shouldPreHandle(url, dom)) {
+      console.log('preHandleContent', handler.name, url)
+      return handler.preHandle(url, dom)
+    }
+  }
+  return undefined
 }
 
-export class ContentHandler {
-  protected senderRegex = /NEWSLETTER_SENDER_REGEX/
-  protected urlRegex = /NEWSLETTER_URL_REGEX/
-  protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
-  protected name = ''
-
-  shouldPreHandle(url: string, dom?: Document): boolean {
-    return false
-  }
-
-  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
-    return Promise.resolve({ url, dom: document })
-  }
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
-
-  parseNewsletterUrl(_postHeader: string, html: string): string | undefined {
-    // get newsletter url from html
-    const matches = html.match(this.urlRegex)
-    if (matches) {
-      return matches[1]
-    }
-    return undefined
-  }
-
-  parseAuthor(from: string): string {
-    // get author name from email
-    // e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
-    // or 'Mike Allen <mike@axios.com>'
-    const parsed = addressparser(from)
-    if (parsed.length > 0) {
-      return parsed[0].name
-    }
-    return from
-  }
-
-  parseUnsubscribe(unSubHeader: string): Unsubscribe {
-    // parse list-unsubscribe header
-    // e.g. List-Unsubscribe: <https://omnivore.com/unsub>, <mailto:unsub@omnivore.com>
-    const decoded = rfc2047.decode(unSubHeader)
-    return {
-      mailTo: decoded.match(/<(https?:\/\/[^>]*)>/)?.[1],
-      httpUrl: decoded.match(/<mailto:([^>]*)>/)?.[1],
-    }
-  }
-
-  handleNewsletter(
-    email: string,
-    html: string,
-    postHeader: string,
-    title: string,
-    from: string,
-    unSubHeader: string
-  ): NewsletterMessage {
-    console.log('handleNewsletter', email, postHeader, title, from)
-
-    if (!email || !html || !title || !from) {
-      console.log('invalid newsletter email')
-      throw new Error('invalid newsletter email')
-    }
-
-    // fallback to default url if newsletter url does not exist
-    // assign a random uuid to the default url to avoid duplicate url
-    const url =
-      this.parseNewsletterUrl(postHeader, html) ||
-      `${this.defaultUrl}?source=newsletters&id=${uuidv4()}`
-    const author = this.parseAuthor(from)
-    const unsubscribe = this.parseUnsubscribe(unSubHeader)
-
-    return {
-      email,
-      content: html,
-      url,
-      title,
-      author,
-      unsubMailTo: unsubscribe.mailTo || '',
-      unsubHttpUrl: unsubscribe.httpUrl || '',
-    }
-  }
+module.exports = {
+  preHandleContent,
 }
diff --git a/packages/content-handler/src/medium-handler.ts b/packages/content-handler/src/medium-handler.ts
index 0b9d2fcb5..8e14cebfe 100644
--- a/packages/content-handler/src/medium-handler.ts
+++ b/packages/content-handler/src/medium-handler.ts
@@ -1,6 +1,11 @@
-import { ContentHandler, PreHandleResult } from './index'
+import { ContentHandler, PreHandleResult } from './content-handler'
+
+export class MediumHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Medium'
+  }
 
-class MediumHandler extends ContentHandler {
   shouldPreHandle(url: string, dom?: Document): boolean {
     const u = new URL(url)
     return u.hostname.endsWith('medium.com')
diff --git a/packages/content-handler/src/pdf-handler.ts b/packages/content-handler/src/pdf-handler.ts
index 54df72bc2..245f9fc1b 100644
--- a/packages/content-handler/src/pdf-handler.ts
+++ b/packages/content-handler/src/pdf-handler.ts
@@ -1,6 +1,11 @@
-import { ContentHandler, PreHandleResult } from './index'
+import { ContentHandler, PreHandleResult } from './content-handler'
+
+export class PdfHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'PDF'
+  }
 
-class PdfHandler extends ContentHandler {
   shouldPreHandle(url: string, dom?: Document): boolean {
     const u = new URL(url)
     const path = u.pathname.replace(u.search, '')
diff --git a/packages/content-handler/src/scrapingBee-handler.ts b/packages/content-handler/src/scrapingBee-handler.ts
index 0b1b5984b..792c5a75c 100644
--- a/packages/content-handler/src/scrapingBee-handler.ts
+++ b/packages/content-handler/src/scrapingBee-handler.ts
@@ -1,8 +1,13 @@
-import { ContentHandler, PreHandleResult } from './index'
+import { ContentHandler, PreHandleResult } from './content-handler'
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
 
-class ScrapingBeeHandler extends ContentHandler {
+export class ScrapingBeeHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'ScrapingBee'
+  }
+
   shouldPreHandle(url: string, dom?: Document): boolean {
     const u = new URL(url)
     const hostnames = ['nytimes.com', 'news.google.com']
diff --git a/packages/content-handler/src/t-dot-co-handler.ts b/packages/content-handler/src/t-dot-co-handler.ts
index 41c90e4a5..b4b461c2f 100644
--- a/packages/content-handler/src/t-dot-co-handler.ts
+++ b/packages/content-handler/src/t-dot-co-handler.ts
@@ -1,7 +1,12 @@
-import { ContentHandler } from './index'
+import { ContentHandler } from './content-handler'
 import axios from 'axios'
 
-class TDotCoHandler extends ContentHandler {
+export class TDotCoHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 't.co'
+  }
+
   shouldResolve(url: string): boolean {
     const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/
     return T_DOT_CO_URL_MATCH.test(url)
diff --git a/packages/content-handler/src/twitter-handler.ts b/packages/content-handler/src/twitter-handler.ts
index 10d9ead03..b7345efe5 100644
--- a/packages/content-handler/src/twitter-handler.ts
+++ b/packages/content-handler/src/twitter-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from './index'
+import { ContentHandler, PreHandleResult } from './content-handler'
 import axios from 'axios'
 import { DateTime } from 'luxon'
 import _ from 'underscore'
@@ -52,7 +52,12 @@ const formatTimestamp = (timestamp: string) => {
   )
 }
 
-class TwitterHandler extends ContentHandler {
+export class TwitterHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Twitter'
+  }
+
   shouldPreHandle(url: string, dom?: Document): boolean {
     return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
   }
diff --git a/packages/content-handler/src/youtube-handler.ts b/packages/content-handler/src/youtube-handler.ts
index 0cbe0df57..898810fe9 100644
--- a/packages/content-handler/src/youtube-handler.ts
+++ b/packages/content-handler/src/youtube-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from './index'
+import { ContentHandler, PreHandleResult } from './content-handler'
 import axios from 'axios'
 import _ from 'underscore'
 
@@ -18,7 +18,12 @@ export const getYoutubeVideoId = (url: string) => {
   return videoId
 }
 
-class YoutubeHandler extends ContentHandler {
+export class YoutubeHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Youtube'
+  }
+
   shouldPreHandle(url: string, dom?: Document): boolean {
     return YOUTUBE_URL_MATCH.test(url.toString())
   }
diff --git a/yarn.lock b/yarn.lock
index d1c9a65dd..3da81da9b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -10579,19 +10579,6 @@ chai@^4.3.4:
     pathval "^1.1.1"
     type-detect "^4.0.5"
 
-chai@^4.3.6:
-  version "4.3.6"
-  resolved "https://registry.yarnpkg.com/chai/-/chai-4.3.6.tgz#ffe4ba2d9fa9d6680cc0b370adae709ec9011e9c"
-  integrity sha512-bbcp3YfHCUzMOvKqsztczerVgBKSsEijCySNlHHbX3VG1nskvqjz5Rfso1gGwD6w6oOV3eI60pKuMOV5MV7p3Q==
-  dependencies:
-    assertion-error "^1.1.0"
-    check-error "^1.0.2"
-    deep-eql "^3.0.1"
-    get-func-name "^2.0.0"
-    loupe "^2.3.1"
-    pathval "^1.1.1"
-    type-detect "^4.0.5"
-
 chalk@^1.0.0, chalk@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98"
@@ -18078,13 +18065,6 @@ loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0:
   dependencies:
     js-tokens "^3.0.0 || ^4.0.0"
 
-loupe@^2.3.1:
-  version "2.3.4"
-  resolved "https://registry.yarnpkg.com/loupe/-/loupe-2.3.4.tgz#7e0b9bffc76f148f9be769cb1321d3dcf3cb25f3"
-  integrity sha512-OvKfgCC2Ndby6aSTREl5aCCPTNIzlDfQZvZxNUrBrihDhL3xcrYegTblhmEiCrg2kKQz4XsFIaemE5BF4ybSaQ==
-  dependencies:
-    get-func-name "^2.0.0"
-
 lower-case-first@^1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/lower-case-first/-/lower-case-first-1.0.2.tgz#e5da7c26f29a7073be02d52bac9980e5922adfa1"

From 206d795c5475d31312547c44571f7548f7f86c16 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 29 Sep 2022 18:08:37 +0800
Subject: [PATCH 07/15] Import content-handler in puppeteer

---
 packages/content-fetch/fetch-content.js       |   1 +
 .../puppeteer-parse/apple-news-handler.js     |  36 ----
 packages/puppeteer-parse/bloomberg-handler.js |  39 ----
 .../puppeteer-parse/derstandard-handler.js    |  35 ----
 packages/puppeteer-parse/image-handler.js     |  34 ----
 packages/puppeteer-parse/index.js             |  88 ++-------
 packages/puppeteer-parse/medium-handler.js    |  31 ----
 packages/puppeteer-parse/pdf-handler.js       |  21 ---
 .../puppeteer-parse/scrapingBee-handler.js    |  44 -----
 packages/puppeteer-parse/t-dot-co-handler.js  |  32 ----
 .../test/apple-news-handler.test.js           |   9 -
 .../test/youtube-handler.test.js              |  12 --
 packages/puppeteer-parse/twitter-handler.js   | 171 ------------------
 packages/puppeteer-parse/youtube-handler.js   |  67 -------
 14 files changed, 14 insertions(+), 606 deletions(-)
 delete mode 100644 packages/puppeteer-parse/apple-news-handler.js
 delete mode 100644 packages/puppeteer-parse/bloomberg-handler.js
 delete mode 100644 packages/puppeteer-parse/derstandard-handler.js
 delete mode 100644 packages/puppeteer-parse/image-handler.js
 delete mode 100644 packages/puppeteer-parse/medium-handler.js
 delete mode 100644 packages/puppeteer-parse/pdf-handler.js
 delete mode 100644 packages/puppeteer-parse/scrapingBee-handler.js
 delete mode 100644 packages/puppeteer-parse/t-dot-co-handler.js
 delete mode 100644 packages/puppeteer-parse/test/apple-news-handler.test.js
 delete mode 100644 packages/puppeteer-parse/test/youtube-handler.test.js
 delete mode 100644 packages/puppeteer-parse/twitter-handler.js
 delete mode 100644 packages/puppeteer-parse/youtube-handler.js

diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js
index 658229e3d..0c3118544 100644
--- a/packages/content-fetch/fetch-content.js
+++ b/packages/content-fetch/fetch-content.js
@@ -225,6 +225,7 @@ async function fetchContent(req, res) {
     return res.sendStatus(400);
   }
 
+  // pre handle url with custom handlers
   let title, content, contentType;
   try {
     const result = await preHandleContent(url);
diff --git a/packages/puppeteer-parse/apple-news-handler.js b/packages/puppeteer-parse/apple-news-handler.js
deleted file mode 100644
index 0759dec23..000000000
--- a/packages/puppeteer-parse/apple-news-handler.js
+++ /dev/null
@@ -1,36 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const Url = require('url');
-const axios = require('axios');
-const { promisify } = require('util');
-const { DateTime } = require('luxon');
-const os = require('os');
-const { Cipher } = require('crypto');
-const { parseHTML } = require('linkedom');
-
-exports.appleNewsHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    if (u.hostname === 'apple.news') {
-      return true;
-    }
-    return false
-  },
-
-  prehandle: async (url, env) => {
-    const MOBILE_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
-    const response = await axios.get(url, { headers: { 'User-Agent': MOBILE_USER_AGENT }  } );
-    const data = response.data;
-
-    const dom = parseHTML(data).document;
-
-    // make sure its a valid URL by wrapping in new URL
-    const u = new URL(dom.querySelector('span.click-here').parentNode.href);
-    return { url: u.href };
-  }
-}
diff --git a/packages/puppeteer-parse/bloomberg-handler.js b/packages/puppeteer-parse/bloomberg-handler.js
deleted file mode 100644
index d79a568bb..000000000
--- a/packages/puppeteer-parse/bloomberg-handler.js
+++ /dev/null
@@ -1,39 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const os = require('os');
-const { parseHTML } = require('linkedom');
-
-exports.bloombergHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const BLOOMBERG_URL_MATCH =
-      /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
-    return BLOOMBERG_URL_MATCH.test(url.toString())
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling bloomberg url', url)
-
-    try {
-      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
-        params: {
-          'api_key': process.env.SCRAPINGBEE_API_KEY,
-          'url': url,
-          'return_page_source': true,
-          'block_ads': true,
-          'block_resources': false,
-        }
-      })
-      const dom = parseHTML(response.data).document;
-      return { title: dom.title, content: dom.querySelector('body').innerHTML, url: url }
-    } catch (error) {
-      console.error('error prehandling bloomberg url', error)
-      throw error
-    }
-  }
-}
diff --git a/packages/puppeteer-parse/derstandard-handler.js b/packages/puppeteer-parse/derstandard-handler.js
deleted file mode 100644
index a44db6f2a..000000000
--- a/packages/puppeteer-parse/derstandard-handler.js
+++ /dev/null
@@ -1,35 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { parseHTML } = require('linkedom');
-
-exports.derstandardHandler = {
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    return u.hostname === 'www.derstandard.at';
-  },
-
-  prehandle: async (url, env) => {
-    const response = await axios.get(url, {
-      // set cookie to give consent to get the article
-      headers: {
-        'cookie': `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`
-      },
-    });
-    const content = response.data;
-
-    var title = undefined;
-    const dom = parseHTML(content).document;
-    const titleElement = dom.querySelector('.article-title')
-    if (!titleElement) {
-      title = titleElement.textContent
-      titleElement.remove()
-    }
-
-    return { content: dom.body.outerHTML, title: title };
-  }
-}
diff --git a/packages/puppeteer-parse/image-handler.js b/packages/puppeteer-parse/image-handler.js
deleted file mode 100644
index 59f132afc..000000000
--- a/packages/puppeteer-parse/image-handler.js
+++ /dev/null
@@ -1,34 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-
-
-exports.imageHandler = {
-  shouldPrehandle: (url, env) => {
-    const IMAGE_URL_PATTERN =
-      /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
-    return IMAGE_URL_PATTERN.test(url.toString())
-  },
-
-  prehandle: async (url, env) => {
-    const title = url.toString().split('/').pop();
-    const content = `
-      <html>
-        <head>
-          <title>${title}</title>
-          <meta property="og:image" content="${url}" />
-          <meta property="og:title" content="${title}" />
-        </head>
-        <body>
-          <div>
-            <img src="${url}" alt="${title}">
-          </div>
-        </body>
-      </html>`
-
-    return { title, content };
-  }
-}
diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js
index 23e743f57..e158f06d7 100644
--- a/packages/puppeteer-parse/index.js
+++ b/packages/puppeteer-parse/index.js
@@ -15,18 +15,10 @@ const { DateTime } = require('luxon');
 const os = require('os');
 const Sentry = require('@sentry/serverless');
 const { Storage } = require('@google-cloud/storage');
-const { appleNewsHandler } = require('./apple-news-handler');
-const { twitterHandler } = require('./twitter-handler');
-const { youtubeHandler } = require('./youtube-handler');
-const { tDotCoHandler } = require('./t-dot-co-handler');
-const { pdfHandler } = require('./pdf-handler');
-const { mediumHandler } = require('./medium-handler');
-const { derstandardHandler } = require('./derstandard-handler');
-const { imageHandler } = require('./image-handler');
-const { scrappingBeeHandler } = require('./scrapingBee-handler');
 
 const chromium = require('chrome-aws-lambda');
 const puppeteer = require('puppeteer-core');
+const { preHandleContent } = require("@omnivore/content-handler");
 
 // Add stealth plugin to hide puppeteer usage
 // const StealthPlugin = require('puppeteer-extra-plugin-stealth');
@@ -257,18 +249,6 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId
   );
 };
 
-const handlers = {
-  'pdf': pdfHandler,
-  'apple-news': appleNewsHandler,
-  'twitter': twitterHandler,
-  'youtube': youtubeHandler,
-  't-dot-co': tDotCoHandler,
-  'medium': mediumHandler,
-  'derstandard': derstandardHandler,
-  'image': imageHandler,
-  'scrappingBee': scrappingBeeHandler,
-};
-
 /**
  * Cloud Function entry point, HTTP trigger.
  * Loads the requested URL via Puppeteer, captures page content and sends it to backend
@@ -309,61 +289,19 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
     return res.sendStatus(400);
   }
 
-  // if (!userId || !articleSavingRequestId) {
-  //   Object.assign(logRecord, { invalidParams: true, body: req.body, query: req.query });
-  //   logger.error(`Invalid parameters`, logRecord);
-  //   return res.sendStatus(400);
-  // }
-
-  // Before we run the regular handlers we check to see if we need tp
-  // pre-resolve the URL. TODO: This should probably happen recursively,
-  // so URLs can be pre-resolved, handled, pre-resolved, handled, etc.
-  for (const [key, handler] of Object.entries(handlers)) {
-    if (handler.shouldResolve && handler.shouldResolve(url)) {
-      try {
-        url = await handler.resolve(url);
-        validateUrlString(url);
-      } catch (err) {
-        console.log('error resolving url with handler', key, err);
-      }
-      break;
-    }
-  }
-
-  // Before we fetch the page we check the handlers, to see if they want
-  // to perform a prefetch action that can modify our requests.
-  // enumerate the handlers and see if any of them want to handle the request
-  const handler = Object.keys(handlers).find(key => {
-    try {
-      return handlers[key].shouldPrehandle(url)
-    } catch (e) {
-      console.log('error with handler: ', key, e);
-    }
-    return false;
-  });
-
-  var title = undefined;
-  var content = undefined;
-  var contentType = undefined;
-
-  if (handler) {
-    try {
-      // The only handler we have now can modify the URL, but in the
-      // future maybe we let it modify content. In that case
-      // we might exit the request early.
-      console.log('pre-handling url with handler: ', handler);
-
-      const result = await handlers[handler].prehandle(url);
-      if (result && result.url) { 
-        url = result.url
-        validateUrlString(url);
-      }
-      if (result && result.title) { title = result.title }
-      if (result && result.content) { content = result.content }
-      if (result && result.contentType) { contentType = result.contentType }
-    } catch (e) {
-      console.log('error with handler: ', handler, e);
+  // pre handle url with custom handlers
+  let title, content, contentType;
+  try {
+    const result = await preHandleContent(url);
+    if (result && result.url) {
+      url = result.url
+      validateUrlString(url);
     }
+    if (result && result.title) { title = result.title }
+    if (result && result.content) { content = result.content }
+    if (result && result.contentType) { contentType = result.contentType }
+  } catch (e) {
+    console.log('error with handler: ', e);
   }
 
   var context, page, finalUrl;
diff --git a/packages/puppeteer-parse/medium-handler.js b/packages/puppeteer-parse/medium-handler.js
deleted file mode 100644
index 8d0443447..000000000
--- a/packages/puppeteer-parse/medium-handler.js
+++ /dev/null
@@ -1,31 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const os = require('os');
-
-exports.mediumHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const MEDIUM_URL_MATCH =
-      /https?:\/\/(www\.)?medium.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
-    const res =  MEDIUM_URL_MATCH.test(url.toString())
-    return res
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling medium url', url)
-
-    try {
-      const res = new URL(url);
-      res.searchParams.delete('source');
-      return { url: res }
-    } catch (error) {
-      console.error('error prehandling medium url', error)
-      throw error
-    }
-  }
-}
diff --git a/packages/puppeteer-parse/pdf-handler.js b/packages/puppeteer-parse/pdf-handler.js
deleted file mode 100644
index 1260db287..000000000
--- a/packages/puppeteer-parse/pdf-handler.js
+++ /dev/null
@@ -1,21 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const Url = require('url');
-
-
-exports.pdfHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = Url.parse(url)
-    const path = u.path.replace(u.search, '')
-    return path.endsWith('.pdf')
-  },
-
-  prehandle: async (url, env) => {
-    return { contentType: 'application/pdf' };
-  }
-}
diff --git a/packages/puppeteer-parse/scrapingBee-handler.js b/packages/puppeteer-parse/scrapingBee-handler.js
deleted file mode 100644
index 6563fca44..000000000
--- a/packages/puppeteer-parse/scrapingBee-handler.js
+++ /dev/null
@@ -1,44 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { parseHTML } = require('linkedom');
-
-const os = require('os');
-
-exports.scrapingBeeHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    const hostnames = [
-      'nytimes.com',
-      'news.google.com',
-    ]
-
-    return hostnames.some((h) => u.hostname.endsWith(h))
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling url with scrapingbee', url)
-
-    try {
-      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
-        params: {
-          'api_key': process.env.SCRAPINGBEE_API_KEY,
-          'url': url,
-          'return_page_source': true,
-          'block_ads': true,
-          'block_resources': false,
-        }
-      })
-      const dom = parseHTML(response.data).document;
-      return { title: dom.title, content: response.data, url: url }
-    } catch (error) {
-      console.error('error prehandling url w/scrapingbee', error)
-      throw error
-    }
-  }
-}
diff --git a/packages/puppeteer-parse/t-dot-co-handler.js b/packages/puppeteer-parse/t-dot-co-handler.js
deleted file mode 100644
index cbbfb304a..000000000
--- a/packages/puppeteer-parse/t-dot-co-handler.js
+++ /dev/null
@@ -1,32 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const Url = require('url');
-
-
-exports.tDotCoHandler = {
-
-  shouldResolve: function (url, env) {
-    const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/;
-    console.log('should preresolve?', T_DOT_CO_URL_MATCH.test(url), url)
-    return T_DOT_CO_URL_MATCH.test(url);
-  },
-
-  resolve: async function(url, env) {
-    return await axios.get(url, { maxRedirects: 0, validateStatus: null })
-      .then(res => {
-        return Url.parse(res.headers.location).href;
-      }).catch((err) => {
-        console.log('err with t.co url', err);
-        return undefined;
-      });
-  },
-
-  shouldPrehandle: (url, env) => {
-    return false
-  },
-}
diff --git a/packages/puppeteer-parse/test/apple-news-handler.test.js b/packages/puppeteer-parse/test/apple-news-handler.test.js
deleted file mode 100644
index 4531d720e..000000000
--- a/packages/puppeteer-parse/test/apple-news-handler.test.js
+++ /dev/null
@@ -1,9 +0,0 @@
-const { expect } = require('chai')
-const { appleNewsHandler } = require('../apple-news-handler')
-
-describe('open a simple web page', () => {
-  it('should return a response', async () => {
-    const response = await appleNewsHandler.prehandle('https://apple.news/AxjzaZaPvSn23b67LhXI5EQ')
-    console.log('response', response)
-  })
-})
diff --git a/packages/puppeteer-parse/test/youtube-handler.test.js b/packages/puppeteer-parse/test/youtube-handler.test.js
deleted file mode 100644
index d34643773..000000000
--- a/packages/puppeteer-parse/test/youtube-handler.test.js
+++ /dev/null
@@ -1,12 +0,0 @@
-const { expect } = require('chai')
-const { getYoutubeVideoId } = require('../youtube-handler')
-
-describe('getYoutubeVideoId', () => {
-  it('should parse video id out of a URL', async () => {
-    expect('BnSUk0je6oo').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s'));
-    expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1'));
-    expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://youtu.be/vFD2gu007dc'));
-    expect('BMFVCnbRaV4').to.eq(getYoutubeVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share'));
-    expect('cg9b4RC87LI').to.eq(getYoutubeVideoId('https://youtu.be/cg9b4RC87LI?t=116'));
-  })
-})
diff --git a/packages/puppeteer-parse/twitter-handler.js b/packages/puppeteer-parse/twitter-handler.js
deleted file mode 100644
index fe68e4782..000000000
--- a/packages/puppeteer-parse/twitter-handler.js
+++ /dev/null
@@ -1,171 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { DateTime } = require('luxon');
-const _ = require("underscore");
-
-const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN;
-const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
-
-const embeddedTweet = async (url) => {
-
-  const BASE_ENDPOINT = 'https://publish.twitter.com/oembed'
-
-  const apiUrl = new URL(BASE_ENDPOINT)
-  apiUrl.searchParams.append('url', url);
-  apiUrl.searchParams.append('omit_script', true);
-  apiUrl.searchParams.append('dnt', true);
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const getTweetFields = () => {
-  const TWEET_FIELDS =
-  "&tweet.fields=attachments,author_id,conversation_id,created_at," +
-  "entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets," +
-  "source,withheld";
-  const EXPANSIONS = "&expansions=author_id,attachments.media_keys";
-  const USER_FIELDS =
-  "&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld";
-  const MEDIA_FIELDS =
-  "&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width";
-
-  return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`;
-}
-
-const getTweetById = async (id) => {
-  const BASE_ENDPOINT = "https://api.twitter.com/2/tweets/";
-  const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const getUserByUsername = async (username) => {
-  const BASE_ENDPOINT = "https://api.twitter.com/2/users/by/username/";
-
-  const apiUrl = new URL(BASE_ENDPOINT + username)
-  apiUrl.searchParams.append('user.fields', 'profile_image_url');
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const titleForTweet = (tweet) => {
-  return `${tweet.data.author_name} on Twitter`
-};
-
-const titleForAuthor = (author) => {
-  return `${author.name} on Twitter`
-};
-
-const usernameFromStatusUrl = (url) => {
-  const match = url.toString().match(TWITTER_URL_MATCH)
-  return match[1]
-};
-
-const tweetIdFromStatusUrl = (url) => {
-  const match = url.toString().match(TWITTER_URL_MATCH)
-  return match[2]
-};
-
-const formatTimestamp = (timestamp) => {
-  return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(DateTime.DATETIME_FULL);
-};
-
-exports.twitterHandler = {
-
-  shouldPrehandle: (url, env) => {
-    return TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
-  },
-
-  // version of the handler that uses the oembed API
-  // This isn't great as it doesn't work well with our
-  // readability API. But could potentially give a more consistent
-  // look to the tweets
-  // prehandle: async (url, env) => {
-  //   const oeTweet = await embeddedTweet(url)
-  //   const dom = new JSDOM(oeTweet.data.html);
-  //   const bq = dom.window.document.querySelector('blockquote')
-  //   console.log('blockquote:', bq);
-
-  //   const title = titleForTweet(oeTweet)
-  //   return { title, content: '<div>' + bq.innerHTML + '</div>', url: oeTweet.data.url };
-  // }
-
-  prehandle: async (url, env) => {
-    console.log('prehandling twitter url', url)
-
-    const tweetId = tweetIdFromStatusUrl(url)
-    const tweetData = (await getTweetById(tweetId)).data;
-    const authorId = tweetData.data.author_id;
-    const author = tweetData.includes.users.filter(u => u.id = authorId)[0];
-    const title = _.escape(titleForAuthor(author))
-    const authorImage = author.profile_image_url.replace('_normal', '_400x400')
-
-    let text = tweetData.data.text;
-    if (tweetData.data.entities && tweetData.data.entities.urls) {
-      for (let urlObj of tweetData.data.entities.urls) {
-        text = text.replace(
-          urlObj.url,
-          `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
-        );
-      }
-    }
-
-    const front = `
-    <div>
-      <p>${text}</p>
-    `
-
-    var includesHtml =  '';
-    if (tweetData.includes.media) {
-      includesHtml = tweetData.includes.media.map(m => {
-        const linkUrl = m.type == 'photo' ? m.url : url;
-        const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url;
-        const mediaOpen = `<a class="media-link" href=${linkUrl}>
-          <picture>
-            <img class="tweet-img" src=${previewUrl} />
-          </picture>
-          </a>`
-        return mediaOpen
-      }).join('\n');
-   }
-
-   const back = `
-       — <a href="https://twitter.com/${author.username}">${author.username}</a> ${author.name} <a href="${url}">${formatTimestamp(tweetData.data.created_at)}</a>
-    </div>
-    `
-    const content = `
-    <head>
-      <meta property="og:image" content="${authorImage}" />
-      <meta property="og:image:secure_url" content="${authorImage}" />
-      <meta property="og:title" content="${title}" />
-      <meta property="og:description" content="${_.escape(tweetData.data.text)}" />
-    </head>
-    <body>
-      ${front}
-      ${includesHtml}
-      ${back}
-    </body>`
-
-    return { content, url, title };
-  }
-}
diff --git a/packages/puppeteer-parse/youtube-handler.js b/packages/puppeteer-parse/youtube-handler.js
deleted file mode 100644
index 68dfc5af6..000000000
--- a/packages/puppeteer-parse/youtube-handler.js
+++ /dev/null
@@ -1,67 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const _ = require("underscore");
-
-const YOUTUBE_URL_MATCH =
-  /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/
-
-function getYoutubeVideoId(url) {
-  const u = new URL(url);
-  const videoId = u.searchParams.get('v');
-  if (!videoId) {
-    const match = url.toString().match(YOUTUBE_URL_MATCH)
-    if (match === null || match.length < 6 || !match[5]) {
-      return undefined
-    }
-    return match[5]
-  }
-  return videoId
-}
-exports.getYoutubeVideoId = getYoutubeVideoId
-
-exports.youtubeHandler = {
-  shouldPrehandle: (url, env) => {
-    return YOUTUBE_URL_MATCH.test(url.toString())
-  },
-
-  prehandle: async (url, env) => {
-    const videoId = getYoutubeVideoId(url)
-    if (!videoId) {
-      return {}
-    }
-
-    const oembedUrl = `https://www.youtube.com/oembed?format=json&url=` + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`)
-    const oembed = (await axios.get(oembedUrl.toString())).data;
-    const title = _.escape(oembed.title);
-    const ratio = oembed.width / oembed.height;
-    const thumbnail = oembed.thumbnail_url;
-    const height = 350;
-    const width = height * ratio;
-    const authorName = _.escape(oembed.author_name);
-
-    const content = `
-    <html>
-      <head><title>${title}</title>
-      <meta property="og:image" content="${thumbnail}" />
-      <meta property="og:image:secure_url" content="${thumbnail}" />
-      <meta property="og:title" content="${title}" />
-      <meta property="og:description" content="" />
-      <meta property="og:article:author" content="${authorName}" />
-      </head>
-      <body>
-      <iframe width="${width}" height="${height}" src="https://www.youtube.com/embed/${videoId}" title="${title}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
-        <p><a href="${url}" target="_blank">${title}</a></p>
-        <p itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="${oembed.author_url}" target="_blank">${authorName}</a></p>
-      </body>
-    </html>`
-
-    console.log('got video id', videoId)
-
-    return { content, title: 'Youtube Content' };
-  }
-}

From 99956539a02f517c541f0912fa754de45bf4c377 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 29 Sep 2022 18:40:44 +0800
Subject: [PATCH 08/15] Handle newsletter in content-handlers

---
 packages/content-fetch/test/babel-register.js |  3 +
 packages/content-fetch/test/stub.test.ts      | 13 ++++
 packages/content-handler/package.json         |  5 +-
 packages/content-handler/src/axios-handler.ts | 17 +++++
 .../content-handler/src/bloomberg-handler.ts  |  9 +++
 .../content-handler/src/content-handler.ts    | 31 ++++----
 .../content-handler/src/golang-handler.ts     | 17 +++++
 packages/content-handler/src/index.ts         | 28 +++++++-
 .../src/morning-brew-handler.ts               | 17 +++++
 .../content-handler/src/substack-handler.ts   | 22 ++++++
 .../content-handler/test/newsletter.test.ts   | 70 +++++++++++++++++++
 .../puppeteer-parse/test/babel-register.js    |  3 +
 packages/puppeteer-parse/test/stub.test.ts    | 13 ++++
 yarn.lock                                     |  5 ++
 14 files changed, 239 insertions(+), 14 deletions(-)
 create mode 100644 packages/content-fetch/test/babel-register.js
 create mode 100644 packages/content-fetch/test/stub.test.ts
 create mode 100644 packages/content-handler/src/axios-handler.ts
 create mode 100644 packages/content-handler/src/golang-handler.ts
 create mode 100644 packages/content-handler/src/morning-brew-handler.ts
 create mode 100644 packages/content-handler/src/substack-handler.ts
 create mode 100644 packages/content-handler/test/newsletter.test.ts
 create mode 100644 packages/puppeteer-parse/test/babel-register.js
 create mode 100644 packages/puppeteer-parse/test/stub.test.ts

diff --git a/packages/content-fetch/test/babel-register.js b/packages/content-fetch/test/babel-register.js
new file mode 100644
index 000000000..a6f65f60a
--- /dev/null
+++ b/packages/content-fetch/test/babel-register.js
@@ -0,0 +1,3 @@
+const register = require('@babel/register').default
+
+register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/content-fetch/test/stub.test.ts b/packages/content-fetch/test/stub.test.ts
new file mode 100644
index 000000000..173ca4917
--- /dev/null
+++ b/packages/content-fetch/test/stub.test.ts
@@ -0,0 +1,13 @@
+import 'mocha'
+import * as chai from 'chai'
+import { expect } from 'chai'
+import 'chai/register-should'
+import chaiString from 'chai-string'
+
+chai.use(chaiString)
+
+describe('Stub test', () => {
+  it('should pass', () => {
+    expect(true).to.be.true
+  })
+})
diff --git a/packages/content-handler/package.json b/packages/content-handler/package.json
index bae0e5953..c82d40c5d 100644
--- a/packages/content-handler/package.json
+++ b/packages/content-handler/package.json
@@ -18,9 +18,12 @@
     "eslint-plugin-prettier": "^4.0.0"
   },
   "dependencies": {
+    "addressparser": "^1.0.1",
     "axios": "^0.27.2",
     "linkedom": "^0.14.16",
     "luxon": "^3.0.4",
-    "underscore": "^1.13.6"
+    "rfc2047": "^4.0.1",
+    "underscore": "^1.13.6",
+    "uuid": "^9.0.0"
   }
 }
diff --git a/packages/content-handler/src/axios-handler.ts b/packages/content-handler/src/axios-handler.ts
new file mode 100644
index 000000000..73b4b3e8b
--- /dev/null
+++ b/packages/content-handler/src/axios-handler.ts
@@ -0,0 +1,17 @@
+import { ContentHandler } from './content-handler'
+
+export class AxiosHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@axios.com>/
+    this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
+    this.defaultUrl = 'https://axios.com'
+    this.name = 'Axios'
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+}
diff --git a/packages/content-handler/src/bloomberg-handler.ts b/packages/content-handler/src/bloomberg-handler.ts
index 95670630e..d82941e5a 100644
--- a/packages/content-handler/src/bloomberg-handler.ts
+++ b/packages/content-handler/src/bloomberg-handler.ts
@@ -5,6 +5,9 @@ import { ContentHandler, PreHandleResult } from './content-handler'
 export class BloombergHandler extends ContentHandler {
   constructor() {
     super()
+    this.senderRegex = /<.+@mail.bloomberg.*.com>/
+    this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
+    this.defaultUrl = 'https://www.bloomberg.com'
     this.name = 'Bloomberg'
   }
 
@@ -38,4 +41,10 @@ export class BloombergHandler extends ContentHandler {
       throw error
     }
   }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
 }
diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts
index c49987368..b69c3cbfe 100644
--- a/packages/content-handler/src/content-handler.ts
+++ b/packages/content-handler/src/content-handler.ts
@@ -7,7 +7,16 @@ interface Unsubscribe {
   httpUrl?: string
 }
 
-interface NewsletterMessage {
+export interface NewsletterInput {
+  postHeader: string
+  from: string
+  unSubHeader: string
+  email: string
+  html: string
+  title: string
+}
+
+export interface NewsletterResult {
   email: string
   content: string
   url: string
@@ -55,9 +64,7 @@ export abstract class ContentHandler {
   }
 
   isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
+    return false
   }
 
   parseNewsletterUrl(_postHeader: string, html: string): string | undefined {
@@ -90,14 +97,14 @@ export abstract class ContentHandler {
     }
   }
 
-  handleNewsletter(
-    email: string,
-    html: string,
-    postHeader: string,
-    title: string,
-    from: string,
-    unSubHeader: string
-  ): NewsletterMessage {
+  handleNewsletter({
+    email,
+    html,
+    postHeader,
+    title,
+    from,
+    unSubHeader,
+  }: NewsletterInput): NewsletterResult {
     console.log('handleNewsletter', email, postHeader, title, from)
 
     if (!email || !html || !title || !from) {
diff --git a/packages/content-handler/src/golang-handler.ts b/packages/content-handler/src/golang-handler.ts
new file mode 100644
index 000000000..d7d618f2e
--- /dev/null
+++ b/packages/content-handler/src/golang-handler.ts
@@ -0,0 +1,17 @@
+import { ContentHandler } from './content-handler'
+
+export class GolangHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@golangweekly.com>/
+    this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
+    this.defaultUrl = 'https://golangweekly.com'
+    this.name = 'Golang Weekly'
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+}
diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts
index 3275a3c89..88509c3e8 100644
--- a/packages/content-handler/src/index.ts
+++ b/packages/content-handler/src/index.ts
@@ -8,7 +8,16 @@ import { ScrapingBeeHandler } from './scrapingBee-handler'
 import { TDotCoHandler } from './t-dot-co-handler'
 import { TwitterHandler } from './twitter-handler'
 import { YoutubeHandler } from './youtube-handler'
-import { ContentHandler, PreHandleResult } from './content-handler'
+import {
+  ContentHandler,
+  NewsletterInput,
+  NewsletterResult,
+  PreHandleResult,
+} from './content-handler'
+import { SubstackHandler } from './substack-handler'
+import { AxiosHandler } from './axios-handler'
+import { GolangHandler } from './golang-handler'
+import { MorningBrewHandler } from './morning-brew-handler'
 
 const validateUrlString = (url: string) => {
   const u = new URL(url)
@@ -37,6 +46,10 @@ const contentHandlers: ContentHandler[] = [
   new TDotCoHandler(),
   new TwitterHandler(),
   new YoutubeHandler(),
+  new SubstackHandler(),
+  new AxiosHandler(),
+  new GolangHandler(),
+  new MorningBrewHandler(),
 ]
 
 export const preHandleContent = async (
@@ -71,6 +84,19 @@ export const preHandleContent = async (
   return undefined
 }
 
+export const handlerNewsletter = (
+  input: NewsletterInput
+): NewsletterResult | undefined => {
+  for (const handler of contentHandlers) {
+    if (handler.isNewsletter(input.postHeader, input.from, input.unSubHeader)) {
+      return handler.handleNewsletter(input)
+    }
+  }
+
+  return undefined
+}
+
 module.exports = {
   preHandleContent,
+  handlerNewsletter,
 }
diff --git a/packages/content-handler/src/morning-brew-handler.ts b/packages/content-handler/src/morning-brew-handler.ts
new file mode 100644
index 000000000..4d46e14e4
--- /dev/null
+++ b/packages/content-handler/src/morning-brew-handler.ts
@@ -0,0 +1,17 @@
+import { ContentHandler } from './content-handler'
+
+export class MorningBrewHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /Morning Brew <crew@morningbrew.com>/
+    this.urlRegex = /<a.* href=["']([^"']*)["'].*>View Online<\/a>/
+    this.defaultUrl = 'https://www.morningbrew.com'
+    this.name = 'Morning Brew'
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+}
diff --git a/packages/content-handler/src/substack-handler.ts b/packages/content-handler/src/substack-handler.ts
new file mode 100644
index 000000000..973ed59ea
--- /dev/null
+++ b/packages/content-handler/src/substack-handler.ts
@@ -0,0 +1,22 @@
+import addressparser from 'addressparser'
+import { ContentHandler } from './content-handler'
+
+export class SubstackHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.defaultUrl = 'https://www.substack.com'
+    this.name = 'Substack'
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    return !!postHeader
+  }
+
+  parseNewsletterUrl(postHeader: string, html: string): string | undefined {
+    // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
+    // we need to get the real url from the raw url
+    return addressparser(postHeader).length > 0
+      ? addressparser(postHeader)[0].name
+      : undefined
+  }
+}
diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts
new file mode 100644
index 000000000..75a91b438
--- /dev/null
+++ b/packages/content-handler/test/newsletter.test.ts
@@ -0,0 +1,70 @@
+import { expect } from 'chai'
+import { SubstackHandler } from '../src/substack-handler'
+import { AxiosHandler } from '../src/axios-handler'
+import { BloombergHandler } from '../src/bloomberg-handler'
+import { GolangHandler } from '../src/golang-handler'
+import { MorningBrewHandler } from '../src/morning-brew-handler'
+
+describe('Newsletter email test', () => {
+  describe('#getNewsletterUrl()', () => {
+    it('returns url when email is from SubStack', () => {
+      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
+
+      expect(new SubstackHandler().parseNewsletterUrl(rawUrl, '')).to.equal(
+        'https://hongbo130.substack.com/p/tldr'
+      )
+    })
+
+    it('returns url when email is from Axios', () => {
+      const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app'
+      const html = `View in browser at <a>${url}</a>`
+
+      expect(new AxiosHandler().parseNewsletterUrl('', html)).to.equal(url)
+    })
+
+    it('returns url when email is from Bloomberg', () => {
+      const url = 'https://www.bloomberg.com/news/google-is-now-a-partner'
+      const html = `
+        <a class="view-in-browser__url" href="${url}">
+        View in browser
+        </a>
+      `
+
+      expect(new BloombergHandler().parseNewsletterUrl('', html)).to.equal(url)
+    })
+
+    it('returns url when email is from Golang Weekly', () => {
+      const url = 'https://www.golangweekly.com/first'
+      const html = `
+        <a href="${url}" style="text-decoration: none">Read on the Web</a>
+      `
+
+      expect(new GolangHandler().parseNewsletterUrl('', html)).to.equal(url)
+    })
+
+    it('returns url when email is from Morning Brew', () => {
+      const url = 'https://www.morningbrew.com/daily/issues/first'
+      const html = `
+        <a style="color: #000000; text-decoration: none;" target="_blank" rel="noopener" href="${url}">View Online</a>
+      `
+
+      expect(new MorningBrewHandler().parseNewsletterUrl('', html)).to.equal(
+        url
+      )
+    })
+  })
+
+  describe('get author from email address', () => {
+    it('returns author when email is from Substack', () => {
+      const from = 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
+      expect(new AxiosHandler().parseAuthor(from)).to.equal(
+        'Jackson Harper from Omnivore App'
+      )
+    })
+
+    it('returns author when email is from Axios', () => {
+      const from = 'Mike Allen <mike@axios.com>'
+      expect(new AxiosHandler().parseAuthor(from)).to.equal('Mike Allen')
+    })
+  })
+})
diff --git a/packages/puppeteer-parse/test/babel-register.js b/packages/puppeteer-parse/test/babel-register.js
new file mode 100644
index 000000000..a6f65f60a
--- /dev/null
+++ b/packages/puppeteer-parse/test/babel-register.js
@@ -0,0 +1,3 @@
+const register = require('@babel/register').default
+
+register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/puppeteer-parse/test/stub.test.ts b/packages/puppeteer-parse/test/stub.test.ts
new file mode 100644
index 000000000..173ca4917
--- /dev/null
+++ b/packages/puppeteer-parse/test/stub.test.ts
@@ -0,0 +1,13 @@
+import 'mocha'
+import * as chai from 'chai'
+import { expect } from 'chai'
+import 'chai/register-should'
+import chaiString from 'chai-string'
+
+chai.use(chaiString)
+
+describe('Stub test', () => {
+  it('should pass', () => {
+    expect(true).to.be.true
+  })
+})
diff --git a/yarn.lock b/yarn.lock
index 3da81da9b..32066d1bb 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -24933,6 +24933,11 @@ uuid@^8.0.0, uuid@^8.3.0, uuid@^8.3.1, uuid@^8.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
 
+uuid@^9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5"
+  integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==
+
 v8-compile-cache-lib@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"

From dfbd317f993ec93d6d4417fd8ccfe63d88aefc38 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 29 Sep 2022 19:21:04 +0800
Subject: [PATCH 09/15] Import content-handler in inbound-email-handler

---
 packages/content-handler/src/index.ts         |   4 +-
 packages/inbound-email-handler/package.json   |   1 +
 .../src/axios-handler.ts                      |  10 --
 .../src/bloomberg-handler.ts                  |  10 --
 .../src/golang-handler.ts                     |  10 --
 packages/inbound-email-handler/src/index.ts   |  61 ++++-----
 .../src/morning-brew-handler.ts               |  10 --
 .../inbound-email-handler/src/newsletter.ts   |  85 +-----------
 .../src/substack-handler.ts                   |  25 ----
 .../test/newsletter.test.ts                   | 127 ------------------
 10 files changed, 29 insertions(+), 314 deletions(-)
 delete mode 100644 packages/inbound-email-handler/src/axios-handler.ts
 delete mode 100644 packages/inbound-email-handler/src/bloomberg-handler.ts
 delete mode 100644 packages/inbound-email-handler/src/golang-handler.ts
 delete mode 100644 packages/inbound-email-handler/src/morning-brew-handler.ts
 delete mode 100644 packages/inbound-email-handler/src/substack-handler.ts

diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts
index 88509c3e8..baa1a9ddd 100644
--- a/packages/content-handler/src/index.ts
+++ b/packages/content-handler/src/index.ts
@@ -84,7 +84,7 @@ export const preHandleContent = async (
   return undefined
 }
 
-export const handlerNewsletter = (
+export const handleNewsletter = (
   input: NewsletterInput
 ): NewsletterResult | undefined => {
   for (const handler of contentHandlers) {
@@ -98,5 +98,5 @@ export const handlerNewsletter = (
 
 module.exports = {
   preHandleContent,
-  handlerNewsletter,
+  handleNewsletter,
 }
diff --git a/packages/inbound-email-handler/package.json b/packages/inbound-email-handler/package.json
index f3048a3b2..ed22a170c 100644
--- a/packages/inbound-email-handler/package.json
+++ b/packages/inbound-email-handler/package.json
@@ -31,6 +31,7 @@
     "@google-cloud/pubsub": "^2.18.4",
     "@sendgrid/client": "^7.6.0",
     "@sentry/serverless": "^6.16.1",
+    "@omnivore/content-handler": "1.0.0",
     "addressparser": "^1.0.1",
     "axios": "^0.27.2",
     "jsonwebtoken": "^8.5.1",
diff --git a/packages/inbound-email-handler/src/axios-handler.ts b/packages/inbound-email-handler/src/axios-handler.ts
deleted file mode 100644
index 706d11047..000000000
--- a/packages/inbound-email-handler/src/axios-handler.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class AxiosHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /<.+@axios.com>/
-    this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
-    this.defaultUrl = 'https://axios.com'
-  }
-}
diff --git a/packages/inbound-email-handler/src/bloomberg-handler.ts b/packages/inbound-email-handler/src/bloomberg-handler.ts
deleted file mode 100644
index 3239ab176..000000000
--- a/packages/inbound-email-handler/src/bloomberg-handler.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class BloombergHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /<.+@mail.bloomberg.*.com>/
-    this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
-    this.defaultUrl = 'https://www.bloomberg.com'
-  }
-}
diff --git a/packages/inbound-email-handler/src/golang-handler.ts b/packages/inbound-email-handler/src/golang-handler.ts
deleted file mode 100644
index 1d93f005b..000000000
--- a/packages/inbound-email-handler/src/golang-handler.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class GolangHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /<.+@golangweekly.com>/
-    this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
-    this.defaultUrl = 'https://golangweekly.com'
-  }
-}
diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts
index ee11511dc..8e3a69874 100644
--- a/packages/inbound-email-handler/src/index.ts
+++ b/packages/inbound-email-handler/src/index.ts
@@ -9,35 +9,27 @@ import * as multipart from 'parse-multipart-data'
 import {
   handleConfirmation,
   isConfirmationEmail,
-  NewsletterHandler,
   parseUnsubscribe,
 } from './newsletter'
 import { PubSub } from '@google-cloud/pubsub'
 import { handlePdfAttachment } from './pdf'
-import { SubstackHandler } from './substack-handler'
-import { AxiosHandler } from './axios-handler'
-import { BloombergHandler } from './bloomberg-handler'
-import { GolangHandler } from './golang-handler'
-import { MorningBrewHandler } from './morning-brew-handler'
+import { handleNewsletter } from '@omnivore/content-handler'
 
+const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
 const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived'
 const pubsub = new PubSub()
-const NEWSLETTER_HANDLERS = [
-  new SubstackHandler(),
-  new AxiosHandler(),
-  new BloombergHandler(),
-  new GolangHandler(),
-  new MorningBrewHandler(),
-]
 
-export const getNewsletterHandler = (
-  postHeader: string,
-  from: string,
-  unSubHeader: string
-): NewsletterHandler | undefined => {
-  return NEWSLETTER_HANDLERS.find((h) => {
-    return h.isNewsletter(postHeader, from, unSubHeader)
-  })
+export const publishMessage = async (
+  topic: string,
+  message: any
+): Promise<string | undefined> => {
+  return pubsub
+    .topic(topic)
+    .publishMessage({ json: message })
+    .catch((err) => {
+      console.log('error publishing message:', err)
+      return undefined
+    })
 }
 
 export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
@@ -86,23 +78,20 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       try {
         // check if it is a confirmation email or forwarding newsletter
-        const newsletterHandler = getNewsletterHandler(
-          postHeader,
+        const newsletterMessage = handleNewsletter({
           from,
-          unSubHeader
-        )
-
-        if (newsletterHandler) {
-          console.log('handleNewsletter', from, to)
-          await newsletterHandler.handleNewsletter(
-            to,
-            html,
-            postHeader,
-            subject,
-            from,
-            unSubHeader
+          html,
+          postHeader,
+          unSubHeader,
+          email: to,
+          title: subject,
+        })
+        if (newsletterMessage) {
+          await publishMessage(
+            NEWSLETTER_EMAIL_RECEIVED_TOPIC,
+            newsletterMessage
           )
-          return res.send('ok')
+          return res.status(200).send('newsletter received')
         }
 
         console.log('non-newsletter email from', from, 'to', to)
diff --git a/packages/inbound-email-handler/src/morning-brew-handler.ts b/packages/inbound-email-handler/src/morning-brew-handler.ts
deleted file mode 100644
index 6f5478c63..000000000
--- a/packages/inbound-email-handler/src/morning-brew-handler.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class MorningBrewHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /Morning Brew <crew@morningbrew.com>/
-    this.urlRegex = /<a.* href=["']([^"']*)["'].*>View Online<\/a>/
-    this.defaultUrl = 'https://www.morningbrew.com'
-  }
-}
diff --git a/packages/inbound-email-handler/src/newsletter.ts b/packages/inbound-email-handler/src/newsletter.ts
index cf031e905..1f3294541 100644
--- a/packages/inbound-email-handler/src/newsletter.ts
+++ b/packages/inbound-email-handler/src/newsletter.ts
@@ -1,15 +1,12 @@
-import { PubSub } from '@google-cloud/pubsub'
-import { v4 as uuidv4 } from 'uuid'
 import addressparser from 'addressparser'
 import rfc2047 from 'rfc2047'
+import { publishMessage } from './index'
 
 interface Unsubscribe {
   mailTo?: string
   httpUrl?: string
 }
 
-const pubsub = new PubSub()
-const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
 const EMAIL_CONFIRMATION_CODE_RECEIVED_TOPIC = 'emailConfirmationCodeReceived'
 const CONFIRMATION_EMAIL_SENDER_ADDRESS = 'forwarding-noreply@google.com'
 // check unicode parentheses too
@@ -35,73 +32,6 @@ const parseAddress = (address: string): string => {
   return ''
 }
 
-export class NewsletterHandler {
-  protected senderRegex = /NEWSLETTER_SENDER_REGEX/
-  protected urlRegex = /NEWSLETTER_URL_REGEX/
-  protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
-
-  parseNewsletterUrl(_postHeader: string, html: string): string | undefined {
-    // get newsletter url from html
-    const matches = html.match(this.urlRegex)
-    if (matches) {
-      return matches[1]
-    }
-    return undefined
-  }
-
-  parseAuthor(from: string): string {
-    // get author name from email
-    // e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
-    // or 'Mike Allen <mike@axios.com>'
-    const parsed = addressparser(from)
-    if (parsed.length > 0) {
-      return parsed[0].name
-    }
-    return from
-  }
-
-  async handleNewsletter(
-    email: string,
-    html: string,
-    postHeader: string,
-    title: string,
-    from: string,
-    unSubHeader: string
-  ): Promise<string | undefined> {
-    console.log('handleNewsletter', email, postHeader, title, from)
-
-    if (!email || !html || !title || !from) {
-      console.log('invalid newsletter email')
-      throw new Error('invalid newsletter email')
-    }
-
-    // fallback to default url if newsletter url does not exist
-    // assign a random uuid to the default url to avoid duplicate url
-    const url =
-      this.parseNewsletterUrl(postHeader, html) ||
-      `${this.defaultUrl}?source=newsletters&id=${uuidv4()}`
-    const author = this.parseAuthor(from)
-    const unsubscribe = parseUnsubscribe(unSubHeader)
-    const message = {
-      email,
-      content: html,
-      url,
-      title,
-      author,
-      unsubMailTo: unsubscribe.mailTo || '',
-      unsubHttpUrl: unsubscribe.httpUrl || '',
-    }
-
-    return publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, message)
-  }
-}
-
 export const handleConfirmation = async (email: string, subject: string) => {
   console.log('confirmation email', email, subject)
 
@@ -136,16 +66,3 @@ export const isConfirmationEmail = (from: string, subject: string): boolean => {
     CONFIRMATION_CODE_PATTERN.test(subject)
   )
 }
-
-const publishMessage = async (
-  topic: string,
-  message: Record<string, string>
-): Promise<string | undefined> => {
-  return pubsub
-    .topic(topic)
-    .publishMessage({ json: message })
-    .catch((err) => {
-      console.log('error publishing message:', err)
-      return undefined
-    })
-}
diff --git a/packages/inbound-email-handler/src/substack-handler.ts b/packages/inbound-email-handler/src/substack-handler.ts
deleted file mode 100644
index 10160b76e..000000000
--- a/packages/inbound-email-handler/src/substack-handler.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-import addressparser from 'addressparser'
-
-export class SubstackHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.defaultUrl = 'https://www.substack.com'
-  }
-
-  parseNewsletterUrl(postHeader: string, _html: string): string | undefined {
-    // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
-    // we need to get the real url from the raw url
-    return addressparser(postHeader).length > 0
-      ? addressparser(postHeader)[0].name
-      : undefined
-  }
-
-  isNewsletter(
-    postHeader: string,
-    _from: string,
-    _unSubHeader: string
-  ): boolean {
-    return !!postHeader
-  }
-}
diff --git a/packages/inbound-email-handler/test/newsletter.test.ts b/packages/inbound-email-handler/test/newsletter.test.ts
index 8c20d9747..253c294ab 100644
--- a/packages/inbound-email-handler/test/newsletter.test.ts
+++ b/packages/inbound-email-handler/test/newsletter.test.ts
@@ -2,15 +2,8 @@ import { expect } from 'chai'
 import {
   getConfirmationCode,
   isConfirmationEmail,
-  NewsletterHandler,
   parseUnsubscribe,
 } from '../src/newsletter'
-import { SubstackHandler } from '../src/substack-handler'
-import { AxiosHandler } from '../src/axios-handler'
-import { BloombergHandler } from '../src/bloomberg-handler'
-import { GolangHandler } from '../src/golang-handler'
-import { getNewsletterHandler } from '../src'
-import { MorningBrewHandler } from '../src/morning-brew-handler'
 
 describe('Confirmation email test', () => {
   describe('#isConfirmationEmail()', () => {
@@ -54,126 +47,6 @@ describe('Confirmation email test', () => {
 })
 
 describe('Newsletter email test', () => {
-  describe('#getNewsletterHandler()', () => {
-    it('returns SubstackHandler when email is from SubStack', () => {
-      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
-
-      expect(getNewsletterHandler(rawUrl, '', '')).to.be.instanceof(
-        SubstackHandler
-      )
-    })
-
-    it('returns AxiosHandler when email is from Axios', () => {
-      const from = 'Mike Allen <mike@axios.com>'
-      const unSubRawUrl =
-        '<https://axios.com/unsubscribe?email=mike%40axios.com&code=593781109>'
-
-      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-        AxiosHandler
-      )
-    })
-
-    context('when email is from Bloomberg', () => {
-      it('should return BloombergHandler when email is from Bloomberg Business', () => {
-        const from = 'From: Bloomberg <noreply@mail.bloombergbusiness.com>'
-        const unSubRawUrl = '<https://bloomberg.com/unsubscribe>'
-
-        expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-          BloombergHandler
-        )
-      })
-
-      it('should return BloombergHandler when email is from Bloomberg View', () => {
-        const from = 'From: Bloomberg <noreply@mail.bloombergview.com>'
-        const unSubRawUrl = '<https://bloomberg.com/unsubscribe>'
-
-        expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-          BloombergHandler
-        )
-      })
-    })
-
-    it('should return GolangHandler when email is from Golang Weekly', () => {
-      const from = 'Golang Weekly <peter@golangweekly.com>'
-      const unSubRawUrl = '<https://golangweekly.com/unsubscribe>'
-
-      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-        GolangHandler
-      )
-    })
-
-    it('should return MorningBrewHandler when email is from Morning Brew', () => {
-      const from = 'Morning Brew <crew@morningbrew.com>'
-      const unSubRawUrl = '<https://morningbrew.com/unsubscribe>'
-
-      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-        MorningBrewHandler
-      )
-    })
-  })
-
-  describe('#getNewsletterUrl()', () => {
-    it('returns url when email is from SubStack', () => {
-      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
-
-      expect(new SubstackHandler().parseNewsletterUrl(rawUrl, '')).to.equal(
-        'https://hongbo130.substack.com/p/tldr'
-      )
-    })
-
-    it('returns url when email is from Axios', () => {
-      const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app'
-      const html = `View in browser at <a>${url}</a>`
-
-      expect(new AxiosHandler().parseNewsletterUrl('', html)).to.equal(url)
-    })
-
-    it('returns url when email is from Bloomberg', () => {
-      const url = 'https://www.bloomberg.com/news/google-is-now-a-partner'
-      const html = `
-        <a class="view-in-browser__url" href="${url}">
-        View in browser
-        </a>
-      `
-
-      expect(new BloombergHandler().parseNewsletterUrl('', html)).to.equal(url)
-    })
-
-    it('returns url when email is from Golang Weekly', () => {
-      const url = 'https://www.golangweekly.com/first'
-      const html = `
-        <a href="${url}" style="text-decoration: none">Read on the Web</a>
-      `
-
-      expect(new GolangHandler().parseNewsletterUrl('', html)).to.equal(url)
-    })
-
-    it('returns url when email is from Morning Brew', () => {
-      const url = 'https://www.morningbrew.com/daily/issues/first'
-      const html = `
-        <a style="color: #000000; text-decoration: none;" target="_blank" rel="noopener" href="${url}">View Online</a>
-      `
-
-      expect(new MorningBrewHandler().parseNewsletterUrl('', html)).to.equal(
-        url
-      )
-    })
-  })
-
-  describe('get author from email address', () => {
-    it('returns author when email is from Substack', () => {
-      const from = 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
-      expect(new NewsletterHandler().parseAuthor(from)).to.equal(
-        'Jackson Harper from Omnivore App'
-      )
-    })
-
-    it('returns author when email is from Axios', () => {
-      const from = 'Mike Allen <mike@axios.com>'
-      expect(new NewsletterHandler().parseAuthor(from)).to.equal('Mike Allen')
-    })
-  })
-
   describe('get unsubscribe from header', () => {
     const mailTo = 'unsub@omnivore.com'
     const httpUrl = 'https://omnivore.com/unsubscribe'

From f7b20f318635c15c585f608c240c0b980747561b Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 30 Sep 2022 10:59:52 +0800
Subject: [PATCH 10/15] Add newsletter handlers

---
 packages/content-handler/src/axios-handler.ts | 17 ------
 .../content-handler/src/content-handler.ts    |  4 +-
 .../src/{ => content}/apple-news-handler.ts   |  2 +-
 .../src/{ => content}/bloomberg-handler.ts    | 11 +---
 .../src/{ => content}/derstandard-handler.ts  |  2 +-
 .../src/{ => content}/image-handler.ts        |  2 +-
 .../src/{ => content}/medium-handler.ts       |  2 +-
 .../src/{ => content}/pdf-handler.ts          |  2 +-
 .../src/{ => content}/scrapingBee-handler.ts  |  2 +-
 .../src/{ => content}/t-dot-co-handler.ts     |  2 +-
 .../src/{ => content}/twitter-handler.ts      |  2 +-
 .../src/content/wikipedia-handler.ts          | 20 +++++++
 .../src/{ => content}/youtube-handler.ts      |  2 +-
 .../content-handler/src/golang-handler.ts     | 17 ------
 packages/content-handler/src/index.ts         | 39 ++++++++------
 .../src/morning-brew-handler.ts               | 17 ------
 .../src/newsletters/axios-handler.ts          | 53 +++++++++++++++++++
 .../bloomberg-newsletter-handler.ts           | 44 +++++++++++++++
 .../src/newsletters/golang-handler.ts         | 34 ++++++++++++
 .../src/newsletters/morning-brew-handler.ts   | 42 +++++++++++++++
 .../src/newsletters/substack-handler.ts       | 52 ++++++++++++++++++
 .../content-handler/src/substack-handler.ts   | 22 --------
 .../test/apple-news-handler.test.ts           |  2 +-
 .../content-handler/test/newsletter.test.ts   | 14 ++---
 .../test/youtube-handler.test.ts              |  2 +-
 25 files changed, 290 insertions(+), 118 deletions(-)
 delete mode 100644 packages/content-handler/src/axios-handler.ts
 rename packages/content-handler/src/{ => content}/apple-news-handler.ts (93%)
 rename packages/content-handler/src/{ => content}/bloomberg-handler.ts (70%)
 rename packages/content-handler/src/{ => content}/derstandard-handler.ts (93%)
 rename packages/content-handler/src/{ => content}/image-handler.ts (92%)
 rename packages/content-handler/src/{ => content}/medium-handler.ts (90%)
 rename packages/content-handler/src/{ => content}/pdf-handler.ts (86%)
 rename packages/content-handler/src/{ => content}/scrapingBee-handler.ts (93%)
 rename packages/content-handler/src/{ => content}/t-dot-co-handler.ts (91%)
 rename packages/content-handler/src/{ => content}/twitter-handler.ts (98%)
 create mode 100644 packages/content-handler/src/content/wikipedia-handler.ts
 rename packages/content-handler/src/{ => content}/youtube-handler.ts (97%)
 delete mode 100644 packages/content-handler/src/golang-handler.ts
 delete mode 100644 packages/content-handler/src/morning-brew-handler.ts
 create mode 100644 packages/content-handler/src/newsletters/axios-handler.ts
 create mode 100644 packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts
 create mode 100644 packages/content-handler/src/newsletters/golang-handler.ts
 create mode 100644 packages/content-handler/src/newsletters/morning-brew-handler.ts
 create mode 100644 packages/content-handler/src/newsletters/substack-handler.ts
 delete mode 100644 packages/content-handler/src/substack-handler.ts

diff --git a/packages/content-handler/src/axios-handler.ts b/packages/content-handler/src/axios-handler.ts
deleted file mode 100644
index 73b4b3e8b..000000000
--- a/packages/content-handler/src/axios-handler.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-import { ContentHandler } from './content-handler'
-
-export class AxiosHandler extends ContentHandler {
-  constructor() {
-    super()
-    this.senderRegex = /<.+@axios.com>/
-    this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
-    this.defaultUrl = 'https://axios.com'
-    this.name = 'Axios'
-  }
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
-}
diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts
index b69c3cbfe..98ab1ac29 100644
--- a/packages/content-handler/src/content-handler.ts
+++ b/packages/content-handler/src/content-handler.ts
@@ -59,8 +59,8 @@ export abstract class ContentHandler {
     return false
   }
 
-  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
-    return Promise.resolve({ url, dom: document })
+  async preHandle(url: string, dom?: Document): Promise<PreHandleResult> {
+    return Promise.resolve({ url, dom })
   }
 
   isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
diff --git a/packages/content-handler/src/apple-news-handler.ts b/packages/content-handler/src/content/apple-news-handler.ts
similarity index 93%
rename from packages/content-handler/src/apple-news-handler.ts
rename to packages/content-handler/src/content/apple-news-handler.ts
index a4239565c..0b4026fb6 100644
--- a/packages/content-handler/src/apple-news-handler.ts
+++ b/packages/content-handler/src/content/apple-news-handler.ts
@@ -1,6 +1,6 @@
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
-import { ContentHandler, PreHandleResult } from './content-handler'
+import { ContentHandler, PreHandleResult } from '../content-handler'
 
 export class AppleNewsHandler extends ContentHandler {
   constructor() {
diff --git a/packages/content-handler/src/bloomberg-handler.ts b/packages/content-handler/src/content/bloomberg-handler.ts
similarity index 70%
rename from packages/content-handler/src/bloomberg-handler.ts
rename to packages/content-handler/src/content/bloomberg-handler.ts
index d82941e5a..a867a3503 100644
--- a/packages/content-handler/src/bloomberg-handler.ts
+++ b/packages/content-handler/src/content/bloomberg-handler.ts
@@ -1,13 +1,10 @@
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
-import { ContentHandler, PreHandleResult } from './content-handler'
+import { ContentHandler, PreHandleResult } from '../content-handler'
 
 export class BloombergHandler extends ContentHandler {
   constructor() {
     super()
-    this.senderRegex = /<.+@mail.bloomberg.*.com>/
-    this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
-    this.defaultUrl = 'https://www.bloomberg.com'
     this.name = 'Bloomberg'
   }
 
@@ -41,10 +38,4 @@ export class BloombergHandler extends ContentHandler {
       throw error
     }
   }
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
 }
diff --git a/packages/content-handler/src/derstandard-handler.ts b/packages/content-handler/src/content/derstandard-handler.ts
similarity index 93%
rename from packages/content-handler/src/derstandard-handler.ts
rename to packages/content-handler/src/content/derstandard-handler.ts
index 2ac01ac86..28742a3e5 100644
--- a/packages/content-handler/src/derstandard-handler.ts
+++ b/packages/content-handler/src/content/derstandard-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from './content-handler'
+import { ContentHandler, PreHandleResult } from '../content-handler'
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
 
diff --git a/packages/content-handler/src/image-handler.ts b/packages/content-handler/src/content/image-handler.ts
similarity index 92%
rename from packages/content-handler/src/image-handler.ts
rename to packages/content-handler/src/content/image-handler.ts
index 652756c51..068a1cc66 100644
--- a/packages/content-handler/src/image-handler.ts
+++ b/packages/content-handler/src/content/image-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from './content-handler'
+import { ContentHandler, PreHandleResult } from '../content-handler'
 
 export class ImageHandler extends ContentHandler {
   constructor() {
diff --git a/packages/content-handler/src/medium-handler.ts b/packages/content-handler/src/content/medium-handler.ts
similarity index 90%
rename from packages/content-handler/src/medium-handler.ts
rename to packages/content-handler/src/content/medium-handler.ts
index 8e14cebfe..211a30c37 100644
--- a/packages/content-handler/src/medium-handler.ts
+++ b/packages/content-handler/src/content/medium-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from './content-handler'
+import { ContentHandler, PreHandleResult } from '../content-handler'
 
 export class MediumHandler extends ContentHandler {
   constructor() {
diff --git a/packages/content-handler/src/pdf-handler.ts b/packages/content-handler/src/content/pdf-handler.ts
similarity index 86%
rename from packages/content-handler/src/pdf-handler.ts
rename to packages/content-handler/src/content/pdf-handler.ts
index 245f9fc1b..4c4ef748d 100644
--- a/packages/content-handler/src/pdf-handler.ts
+++ b/packages/content-handler/src/content/pdf-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from './content-handler'
+import { ContentHandler, PreHandleResult } from '../content-handler'
 
 export class PdfHandler extends ContentHandler {
   constructor() {
diff --git a/packages/content-handler/src/scrapingBee-handler.ts b/packages/content-handler/src/content/scrapingBee-handler.ts
similarity index 93%
rename from packages/content-handler/src/scrapingBee-handler.ts
rename to packages/content-handler/src/content/scrapingBee-handler.ts
index 792c5a75c..4c04d00e8 100644
--- a/packages/content-handler/src/scrapingBee-handler.ts
+++ b/packages/content-handler/src/content/scrapingBee-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from './content-handler'
+import { ContentHandler, PreHandleResult } from '../content-handler'
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
 
diff --git a/packages/content-handler/src/t-dot-co-handler.ts b/packages/content-handler/src/content/t-dot-co-handler.ts
similarity index 91%
rename from packages/content-handler/src/t-dot-co-handler.ts
rename to packages/content-handler/src/content/t-dot-co-handler.ts
index b4b461c2f..277a8c087 100644
--- a/packages/content-handler/src/t-dot-co-handler.ts
+++ b/packages/content-handler/src/content/t-dot-co-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler } from './content-handler'
+import { ContentHandler } from '../content-handler'
 import axios from 'axios'
 
 export class TDotCoHandler extends ContentHandler {
diff --git a/packages/content-handler/src/twitter-handler.ts b/packages/content-handler/src/content/twitter-handler.ts
similarity index 98%
rename from packages/content-handler/src/twitter-handler.ts
rename to packages/content-handler/src/content/twitter-handler.ts
index b7345efe5..ddd37e45c 100644
--- a/packages/content-handler/src/twitter-handler.ts
+++ b/packages/content-handler/src/content/twitter-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from './content-handler'
+import { ContentHandler, PreHandleResult } from '../content-handler'
 import axios from 'axios'
 import { DateTime } from 'luxon'
 import _ from 'underscore'
diff --git a/packages/content-handler/src/content/wikipedia-handler.ts b/packages/content-handler/src/content/wikipedia-handler.ts
new file mode 100644
index 000000000..8c3a176fd
--- /dev/null
+++ b/packages/content-handler/src/content/wikipedia-handler.ts
@@ -0,0 +1,20 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class WikipediaHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'wikipedia'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    return new URL(url).hostname.endsWith('wikipedia.org')
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    // This removes the [edit] anchors from wikipedia pages
+    dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove())
+    // this removes the sidebar
+    dom.querySelector('.infobox')?.remove()
+    return Promise.resolve({ dom })
+  }
+}
diff --git a/packages/content-handler/src/youtube-handler.ts b/packages/content-handler/src/content/youtube-handler.ts
similarity index 97%
rename from packages/content-handler/src/youtube-handler.ts
rename to packages/content-handler/src/content/youtube-handler.ts
index 898810fe9..4cdb7ee98 100644
--- a/packages/content-handler/src/youtube-handler.ts
+++ b/packages/content-handler/src/content/youtube-handler.ts
@@ -1,4 +1,4 @@
-import { ContentHandler, PreHandleResult } from './content-handler'
+import { ContentHandler, PreHandleResult } from '../content-handler'
 import axios from 'axios'
 import _ from 'underscore'
 
diff --git a/packages/content-handler/src/golang-handler.ts b/packages/content-handler/src/golang-handler.ts
deleted file mode 100644
index d7d618f2e..000000000
--- a/packages/content-handler/src/golang-handler.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-import { ContentHandler } from './content-handler'
-
-export class GolangHandler extends ContentHandler {
-  constructor() {
-    super()
-    this.senderRegex = /<.+@golangweekly.com>/
-    this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
-    this.defaultUrl = 'https://golangweekly.com'
-    this.name = 'Golang Weekly'
-  }
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
-}
diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts
index baa1a9ddd..95522d699 100644
--- a/packages/content-handler/src/index.ts
+++ b/packages/content-handler/src/index.ts
@@ -1,23 +1,25 @@
-import { AppleNewsHandler } from './apple-news-handler'
-import { BloombergHandler } from './bloomberg-handler'
-import { DerstandardHandler } from './derstandard-handler'
-import { ImageHandler } from './image-handler'
-import { MediumHandler } from './medium-handler'
-import { PdfHandler } from './pdf-handler'
-import { ScrapingBeeHandler } from './scrapingBee-handler'
-import { TDotCoHandler } from './t-dot-co-handler'
-import { TwitterHandler } from './twitter-handler'
-import { YoutubeHandler } from './youtube-handler'
+import { AppleNewsHandler } from './content/apple-news-handler'
+import { BloombergHandler } from './content/bloomberg-handler'
+import { DerstandardHandler } from './content/derstandard-handler'
+import { ImageHandler } from './content/image-handler'
+import { MediumHandler } from './content/medium-handler'
+import { PdfHandler } from './content/pdf-handler'
+import { ScrapingBeeHandler } from './content/scrapingBee-handler'
+import { TDotCoHandler } from './content/t-dot-co-handler'
+import { TwitterHandler } from './content/twitter-handler'
+import { YoutubeHandler } from './content/youtube-handler'
 import {
   ContentHandler,
   NewsletterInput,
   NewsletterResult,
   PreHandleResult,
 } from './content-handler'
-import { SubstackHandler } from './substack-handler'
-import { AxiosHandler } from './axios-handler'
-import { GolangHandler } from './golang-handler'
-import { MorningBrewHandler } from './morning-brew-handler'
+import { SubstackHandler } from './newsletters/substack-handler'
+import { AxiosHandler } from './newsletters/axios-handler'
+import { GolangHandler } from './newsletters/golang-handler'
+import { MorningBrewHandler } from './newsletters/morning-brew-handler'
+import { BloombergNewsletterHandler } from './newsletters/bloomberg-newsletter-handler'
+import { WikipediaHandler } from './content/wikipedia-handler'
 
 const validateUrlString = (url: string) => {
   const u = new URL(url)
@@ -46,9 +48,14 @@ const contentHandlers: ContentHandler[] = [
   new TDotCoHandler(),
   new TwitterHandler(),
   new YoutubeHandler(),
-  new SubstackHandler(),
+  new WikipediaHandler(),
+]
+
+const newsletterHandlers: ContentHandler[] = [
   new AxiosHandler(),
+  new BloombergNewsletterHandler(),
   new GolangHandler(),
+  new SubstackHandler(),
   new MorningBrewHandler(),
 ]
 
@@ -87,7 +94,7 @@ export const preHandleContent = async (
 export const handleNewsletter = (
   input: NewsletterInput
 ): NewsletterResult | undefined => {
-  for (const handler of contentHandlers) {
+  for (const handler of newsletterHandlers) {
     if (handler.isNewsletter(input.postHeader, input.from, input.unSubHeader)) {
       return handler.handleNewsletter(input)
     }
diff --git a/packages/content-handler/src/morning-brew-handler.ts b/packages/content-handler/src/morning-brew-handler.ts
deleted file mode 100644
index 4d46e14e4..000000000
--- a/packages/content-handler/src/morning-brew-handler.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-import { ContentHandler } from './content-handler'
-
-export class MorningBrewHandler extends ContentHandler {
-  constructor() {
-    super()
-    this.senderRegex = /Morning Brew <crew@morningbrew.com>/
-    this.urlRegex = /<a.* href=["']([^"']*)["'].*>View Online<\/a>/
-    this.defaultUrl = 'https://www.morningbrew.com'
-    this.name = 'Morning Brew'
-  }
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
-}
diff --git a/packages/content-handler/src/newsletters/axios-handler.ts b/packages/content-handler/src/newsletters/axios-handler.ts
new file mode 100644
index 000000000..afbdd9e52
--- /dev/null
+++ b/packages/content-handler/src/newsletters/axios-handler.ts
@@ -0,0 +1,53 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class AxiosHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@axios.com>/
+    this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
+    this.defaultUrl = 'https://axios.com'
+    this.name = 'axios'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const host = this.name + '.com'
+    // check if url ends with axios.com
+    return new URL(url).hostname.endsWith(host)
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    const body = dom.querySelector('table')
+
+    let isFooter = false
+    // this removes ads and replaces table with a div
+    body?.querySelectorAll('table').forEach((el) => {
+      // remove the footer and the ads
+      if (!el.textContent || el.textContent.length < 20 || isFooter) {
+        el.remove()
+      } else {
+        // removes the first few rows of the table (the header)
+        // remove the last two rows of the table (they are ads)
+        el.querySelectorAll('tr').forEach((tr, i) => {
+          if (i <= 7 || i >= el.querySelectorAll('tr').length - 2) {
+            console.log('removing', tr)
+            tr.remove()
+          }
+        })
+        // replace the table with a div
+        const div = dom.createElement('div')
+        div.innerHTML = el.innerHTML
+        el.parentNode?.replaceChild(div, el)
+        // set the isFooter flag to true because the next table is the footer
+        isFooter = true
+      }
+    })
+
+    return Promise.resolve({ dom })
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+}
diff --git a/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts b/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts
new file mode 100644
index 000000000..d7c71f6f8
--- /dev/null
+++ b/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts
@@ -0,0 +1,44 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class BloombergNewsletterHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@mail.bloomberg.*.com>/
+    this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
+    this.defaultUrl = 'https://www.bloomberg.com'
+    this.name = 'bloomberg'
+  }
+
+  shouldPreHandle(url: string, dom: Document): boolean {
+    const host = this.name + '.com'
+    // check if url ends with bloomberg.com
+    return (
+      new URL(url).hostname.endsWith(host) ||
+      dom.querySelector('.logo-image')?.getAttribute('alt')?.toLowerCase() ===
+        this.name
+    )
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    const body = dom.querySelector('.wrapper')
+
+    // this removes header
+    body?.querySelector('.sailthru-variables')?.remove()
+    body?.querySelector('.preview-text')?.remove()
+    body?.querySelector('.logo-wrapper')?.remove()
+    body?.querySelector('.by-the-number-wrapper')?.remove()
+    // this removes footer
+    body?.querySelector('.quote-box-wrapper')?.remove()
+    body?.querySelector('.header-wrapper')?.remove()
+    body?.querySelector('.component-wrapper')?.remove()
+    body?.querySelector('.footer')?.remove()
+
+    return Promise.resolve({ dom })
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+}
diff --git a/packages/content-handler/src/newsletters/golang-handler.ts b/packages/content-handler/src/newsletters/golang-handler.ts
new file mode 100644
index 000000000..1672e4b28
--- /dev/null
+++ b/packages/content-handler/src/newsletters/golang-handler.ts
@@ -0,0 +1,34 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class GolangHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@golangweekly.com>/
+    this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
+    this.defaultUrl = 'https://golangweekly.com'
+    this.name = 'golangweekly'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const host = this.name + '.com'
+    // check if url ends with golangweekly.com
+    return new URL(url).hostname.endsWith(host)
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    const body = dom.querySelector('body')
+
+    // this removes the "Subscribe" button
+    body?.querySelector('.el-splitbar')?.remove()
+    // this removes the title
+    body?.querySelector('.el-masthead')?.remove()
+
+    return Promise.resolve({ dom })
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+}
diff --git a/packages/content-handler/src/newsletters/morning-brew-handler.ts b/packages/content-handler/src/newsletters/morning-brew-handler.ts
new file mode 100644
index 000000000..95bfe4b57
--- /dev/null
+++ b/packages/content-handler/src/newsletters/morning-brew-handler.ts
@@ -0,0 +1,42 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class MorningBrewHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /Morning Brew <crew@morningbrew.com>/
+    this.urlRegex = /<a.* href=["']([^"']*)["'].*>View Online<\/a>/
+    this.defaultUrl = 'https://www.morningbrew.com'
+    this.name = 'morningbrew'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const host = this.name + '.com'
+    // check if url ends with morningbrew.com
+    return new URL(url).hostname.endsWith(host)
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    // retain the width of the cells in the table of market info
+    dom.querySelectorAll('.markets-arrow-cell').forEach((td) => {
+      const table = td.closest('table')
+      if (table) {
+        const bubbleTable = table.querySelector('.markets-bubble')
+        if (bubbleTable) {
+          // replace the nested table with the text
+          const e = bubbleTable.querySelector('.markets-table-text')
+          e && bubbleTable.parentNode?.replaceChild(e, bubbleTable)
+        }
+        // set custom class for the table
+        table.className = 'morning-brew-markets'
+      }
+    })
+
+    return Promise.resolve({ dom })
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+}
diff --git a/packages/content-handler/src/newsletters/substack-handler.ts b/packages/content-handler/src/newsletters/substack-handler.ts
new file mode 100644
index 000000000..675582096
--- /dev/null
+++ b/packages/content-handler/src/newsletters/substack-handler.ts
@@ -0,0 +1,52 @@
+import addressparser from 'addressparser'
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class SubstackHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.defaultUrl = 'https://www.substack.com'
+    this.name = 'substack'
+  }
+
+  shouldPreHandle(url: string, dom: Document): boolean {
+    const host = this.name + '.com'
+    // check if url ends with substack.com
+    // or has a profile image hosted at substack.com
+    return (
+      new URL(url).hostname.endsWith(host) ||
+      !!dom
+        .querySelector('.email-body img')
+        ?.getAttribute('src')
+        ?.includes(host)
+    )
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    const body = dom.querySelector('.email-body-container')
+
+    // this removes header and profile avatar
+    body?.querySelector('.header')?.remove()
+    body?.querySelector('.preamble')?.remove()
+    body?.querySelector('.meta-author-wrap')?.remove()
+    // this removes meta button
+    body?.querySelector('.post-meta')?.remove()
+    // this removes footer
+    body?.querySelector('.post-cta')?.remove()
+    body?.querySelector('.container-border')?.remove()
+    body?.querySelector('.footer')?.remove()
+
+    return Promise.resolve(dom)
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    return !!postHeader
+  }
+
+  parseNewsletterUrl(postHeader: string, html: string): string | undefined {
+    // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
+    // we need to get the real url from the raw url
+    return addressparser(postHeader).length > 0
+      ? addressparser(postHeader)[0].name
+      : undefined
+  }
+}
diff --git a/packages/content-handler/src/substack-handler.ts b/packages/content-handler/src/substack-handler.ts
deleted file mode 100644
index 973ed59ea..000000000
--- a/packages/content-handler/src/substack-handler.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import addressparser from 'addressparser'
-import { ContentHandler } from './content-handler'
-
-export class SubstackHandler extends ContentHandler {
-  constructor() {
-    super()
-    this.defaultUrl = 'https://www.substack.com'
-    this.name = 'Substack'
-  }
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    return !!postHeader
-  }
-
-  parseNewsletterUrl(postHeader: string, html: string): string | undefined {
-    // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
-    // we need to get the real url from the raw url
-    return addressparser(postHeader).length > 0
-      ? addressparser(postHeader)[0].name
-      : undefined
-  }
-}
diff --git a/packages/content-handler/test/apple-news-handler.test.ts b/packages/content-handler/test/apple-news-handler.test.ts
index 9af3be580..5a19542f5 100644
--- a/packages/content-handler/test/apple-news-handler.test.ts
+++ b/packages/content-handler/test/apple-news-handler.test.ts
@@ -1,4 +1,4 @@
-import { AppleNewsHandler } from '../src/apple-news-handler'
+import { AppleNewsHandler } from '../src/content/apple-news-handler'
 
 describe('open a simple web page', () => {
   it('should return a response', async () => {
diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts
index 75a91b438..50d94f646 100644
--- a/packages/content-handler/test/newsletter.test.ts
+++ b/packages/content-handler/test/newsletter.test.ts
@@ -1,9 +1,9 @@
 import { expect } from 'chai'
-import { SubstackHandler } from '../src/substack-handler'
-import { AxiosHandler } from '../src/axios-handler'
-import { BloombergHandler } from '../src/bloomberg-handler'
-import { GolangHandler } from '../src/golang-handler'
-import { MorningBrewHandler } from '../src/morning-brew-handler'
+import { SubstackHandler } from '../src/newsletters/substack-handler'
+import { AxiosHandler } from '../src/newsletters/axios-handler'
+import { BloombergNewsletterHandler } from '../src/newsletters/bloomberg-newsletter-handler'
+import { GolangHandler } from '../src/newsletters/golang-handler'
+import { MorningBrewHandler } from '../src/newsletters/morning-brew-handler'
 
 describe('Newsletter email test', () => {
   describe('#getNewsletterUrl()', () => {
@@ -30,7 +30,9 @@ describe('Newsletter email test', () => {
         </a>
       `
 
-      expect(new BloombergHandler().parseNewsletterUrl('', html)).to.equal(url)
+      expect(
+        new BloombergNewsletterHandler().parseNewsletterUrl('', html)
+      ).to.equal(url)
     })
 
     it('returns url when email is from Golang Weekly', () => {
diff --git a/packages/content-handler/test/youtube-handler.test.ts b/packages/content-handler/test/youtube-handler.test.ts
index f8c3a51ff..4e7fcb913 100644
--- a/packages/content-handler/test/youtube-handler.test.ts
+++ b/packages/content-handler/test/youtube-handler.test.ts
@@ -1,6 +1,6 @@
 import { expect } from 'chai'
 import 'mocha'
-import { getYoutubeVideoId } from '../src/youtube-handler'
+import { getYoutubeVideoId } from '../src/content/youtube-handler'
 
 describe('getYoutubeVideoId', () => {
   it('should parse video id out of a URL', async () => {

From 9b209314a6cc9bc166a7527a8f8f0fb463073019 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 30 Sep 2022 11:00:22 +0800
Subject: [PATCH 11/15] Rename content to websites

---
 .../src/{content => websites}/apple-news-handler.ts               | 0
 .../src/{content => websites}/bloomberg-handler.ts                | 0
 .../src/{content => websites}/derstandard-handler.ts              | 0
 .../content-handler/src/{content => websites}/image-handler.ts    | 0
 .../content-handler/src/{content => websites}/medium-handler.ts   | 0
 packages/content-handler/src/{content => websites}/pdf-handler.ts | 0
 .../src/{content => websites}/scrapingBee-handler.ts              | 0
 .../content-handler/src/{content => websites}/t-dot-co-handler.ts | 0
 .../content-handler/src/{content => websites}/twitter-handler.ts  | 0
 .../src/{content => websites}/wikipedia-handler.ts                | 0
 .../content-handler/src/{content => websites}/youtube-handler.ts  | 0
 11 files changed, 0 insertions(+), 0 deletions(-)
 rename packages/content-handler/src/{content => websites}/apple-news-handler.ts (100%)
 rename packages/content-handler/src/{content => websites}/bloomberg-handler.ts (100%)
 rename packages/content-handler/src/{content => websites}/derstandard-handler.ts (100%)
 rename packages/content-handler/src/{content => websites}/image-handler.ts (100%)
 rename packages/content-handler/src/{content => websites}/medium-handler.ts (100%)
 rename packages/content-handler/src/{content => websites}/pdf-handler.ts (100%)
 rename packages/content-handler/src/{content => websites}/scrapingBee-handler.ts (100%)
 rename packages/content-handler/src/{content => websites}/t-dot-co-handler.ts (100%)
 rename packages/content-handler/src/{content => websites}/twitter-handler.ts (100%)
 rename packages/content-handler/src/{content => websites}/wikipedia-handler.ts (100%)
 rename packages/content-handler/src/{content => websites}/youtube-handler.ts (100%)

diff --git a/packages/content-handler/src/content/apple-news-handler.ts b/packages/content-handler/src/websites/apple-news-handler.ts
similarity index 100%
rename from packages/content-handler/src/content/apple-news-handler.ts
rename to packages/content-handler/src/websites/apple-news-handler.ts
diff --git a/packages/content-handler/src/content/bloomberg-handler.ts b/packages/content-handler/src/websites/bloomberg-handler.ts
similarity index 100%
rename from packages/content-handler/src/content/bloomberg-handler.ts
rename to packages/content-handler/src/websites/bloomberg-handler.ts
diff --git a/packages/content-handler/src/content/derstandard-handler.ts b/packages/content-handler/src/websites/derstandard-handler.ts
similarity index 100%
rename from packages/content-handler/src/content/derstandard-handler.ts
rename to packages/content-handler/src/websites/derstandard-handler.ts
diff --git a/packages/content-handler/src/content/image-handler.ts b/packages/content-handler/src/websites/image-handler.ts
similarity index 100%
rename from packages/content-handler/src/content/image-handler.ts
rename to packages/content-handler/src/websites/image-handler.ts
diff --git a/packages/content-handler/src/content/medium-handler.ts b/packages/content-handler/src/websites/medium-handler.ts
similarity index 100%
rename from packages/content-handler/src/content/medium-handler.ts
rename to packages/content-handler/src/websites/medium-handler.ts
diff --git a/packages/content-handler/src/content/pdf-handler.ts b/packages/content-handler/src/websites/pdf-handler.ts
similarity index 100%
rename from packages/content-handler/src/content/pdf-handler.ts
rename to packages/content-handler/src/websites/pdf-handler.ts
diff --git a/packages/content-handler/src/content/scrapingBee-handler.ts b/packages/content-handler/src/websites/scrapingBee-handler.ts
similarity index 100%
rename from packages/content-handler/src/content/scrapingBee-handler.ts
rename to packages/content-handler/src/websites/scrapingBee-handler.ts
diff --git a/packages/content-handler/src/content/t-dot-co-handler.ts b/packages/content-handler/src/websites/t-dot-co-handler.ts
similarity index 100%
rename from packages/content-handler/src/content/t-dot-co-handler.ts
rename to packages/content-handler/src/websites/t-dot-co-handler.ts
diff --git a/packages/content-handler/src/content/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts
similarity index 100%
rename from packages/content-handler/src/content/twitter-handler.ts
rename to packages/content-handler/src/websites/twitter-handler.ts
diff --git a/packages/content-handler/src/content/wikipedia-handler.ts b/packages/content-handler/src/websites/wikipedia-handler.ts
similarity index 100%
rename from packages/content-handler/src/content/wikipedia-handler.ts
rename to packages/content-handler/src/websites/wikipedia-handler.ts
diff --git a/packages/content-handler/src/content/youtube-handler.ts b/packages/content-handler/src/websites/youtube-handler.ts
similarity index 100%
rename from packages/content-handler/src/content/youtube-handler.ts
rename to packages/content-handler/src/websites/youtube-handler.ts

From b00a516737c17ef59fc7e431450ff6a4e0dd555d Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 30 Sep 2022 12:42:41 +0800
Subject: [PATCH 12/15] Add other newsletter handlers

---
 packages/api/src/routers/svc/emails.ts        |  22 ---
 packages/api/test/routers/emails.test.ts      |  32 ----
 packages/api/test/utils/parser.test.ts        |  75 ---------
 packages/content-handler/package.json         |   7 +-
 .../content-handler/src/content-handler.ts    |  68 ++++++--
 packages/content-handler/src/index.ts         |  35 ++--
 .../src/newsletters/axios-handler.ts          |   7 -
 .../src/newsletters/beehiiv-handler.ts        |  43 +++++
 .../bloomberg-newsletter-handler.ts           |   7 -
 .../src/newsletters/convertkit-handler.ts     |  41 +++++
 .../src/newsletters/golang-handler.ts         |   7 -
 .../src/newsletters/morning-brew-handler.ts   |   7 -
 .../src/newsletters/revue-handler.ts          |  46 ++++++
 .../src/newsletters/substack-handler.ts       |  52 +++++-
 .../test/apple-news-handler.test.ts           |   2 +-
 .../test}/data/beehiiv-newsletter.html        |   0
 .../data/substack-forwarded-newsletter.html   |   0
 .../substack-forwarded-welcome-email.html     |   0
 ...substack-private-forwarded-newsletter.html |   0
 .../content-handler/test/newsletter.test.ts   | 149 ++++++++++++++++--
 .../test/youtube-handler.test.ts              |   2 +-
 packages/inbound-email-handler/src/index.ts   |   2 +-
 yarn.lock                                     | 111 +++++++++++--
 23 files changed, 488 insertions(+), 227 deletions(-)
 create mode 100644 packages/content-handler/src/newsletters/beehiiv-handler.ts
 create mode 100644 packages/content-handler/src/newsletters/convertkit-handler.ts
 create mode 100644 packages/content-handler/src/newsletters/revue-handler.ts
 rename packages/{api/test/utils => content-handler/test}/data/beehiiv-newsletter.html (100%)
 rename packages/{api/test/utils => content-handler/test}/data/substack-forwarded-newsletter.html (100%)
 rename packages/{api/test/utils => content-handler/test}/data/substack-forwarded-welcome-email.html (100%)
 rename packages/{api/test/utils => content-handler/test}/data/substack-private-forwarded-newsletter.html (100%)

diff --git a/packages/api/src/routers/svc/emails.ts b/packages/api/src/routers/svc/emails.ts
index e8ccc8c05..6aae07d74 100644
--- a/packages/api/src/routers/svc/emails.ts
+++ b/packages/api/src/routers/svc/emails.ts
@@ -8,14 +8,11 @@ import { analytics } from '../../utils/analytics'
 import { getNewsletterEmail } from '../../services/newsletters'
 import { env } from '../../env'
 import {
-  findNewsletterUrl,
   generateUniqueUrl,
   getTitleFromEmailSubject,
   isProbablyArticle,
-  isProbablyNewsletter,
   parseEmailAddress,
 } from '../../utils/parser'
-import { saveNewsletterEmail } from '../../services/save_newsletter_email'
 import { saveEmail } from '../../services/save_email'
 import { buildLogger } from '../../utils/logger'
 
@@ -80,25 +77,6 @@ export function emailsServiceRouter() {
       const ctx = { pubsub: createPubSubClient(), uid: user.id }
       const parsedFrom = parseEmailAddress(data.from)
 
-      if (await isProbablyNewsletter(data.html)) {
-        logger.info('handling as newsletter', data)
-        await saveNewsletterEmail(
-          {
-            email: data.to,
-            title: data.subject,
-            content: data.html,
-            author: parsedFrom.name,
-            url: (await findNewsletterUrl(data.html)) || generateUniqueUrl(),
-            unsubMailTo: data.unsubMailTo,
-            unsubHttpUrl: data.unsubHttpUrl,
-            newsletterEmail,
-          },
-          ctx
-        )
-        res.status(200).send('Newsletter')
-        return
-      }
-
       if (
         await isProbablyArticle(
           data.forwardedFrom || parsedFrom.address,
diff --git a/packages/api/test/routers/emails.test.ts b/packages/api/test/routers/emails.test.ts
index 26220d656..2f78483a7 100644
--- a/packages/api/test/routers/emails.test.ts
+++ b/packages/api/test/routers/emails.test.ts
@@ -52,35 +52,8 @@ describe('Emails Router', () => {
       sinon.restore()
     })
 
-    context('when email is a newsletter', () => {
-      before(() => {
-        sinon.replace(parser, 'isProbablyNewsletter', sinon.fake.resolves(true))
-      })
-
-      it('saves the email as a newsletter', async () => {
-        const data = {
-          message: {
-            data: Buffer.from(
-              JSON.stringify({ from, to, subject, html })
-            ).toString('base64'),
-            publishTime: new Date().toISOString(),
-          },
-        }
-        const res = await request
-          .post(`/svc/pubsub/emails/forward?token=${token}`)
-          .send(data)
-          .expect(200)
-        expect(res.text).to.eql('Newsletter')
-      })
-    })
-
     context('when email is an article', () => {
       before(() => {
-        sinon.replace(
-          parser,
-          'isProbablyNewsletter',
-          sinon.fake.resolves(false)
-        )
         sinon.replace(parser, 'isProbablyArticle', sinon.fake.resolves(true))
       })
 
@@ -103,11 +76,6 @@ describe('Emails Router', () => {
 
     context('when email is a regular email', () => {
       before(() => {
-        sinon.replace(
-          parser,
-          'isProbablyNewsletter',
-          sinon.fake.resolves(false)
-        )
         sinon.replace(parser, 'isProbablyArticle', sinon.fake.resolves(false))
       })
 
diff --git a/packages/api/test/utils/parser.test.ts b/packages/api/test/utils/parser.test.ts
index 83bd23294..356dfde21 100644
--- a/packages/api/test/utils/parser.test.ts
+++ b/packages/api/test/utils/parser.test.ts
@@ -4,11 +4,8 @@ import { expect } from 'chai'
 import 'chai/register-should'
 import fs from 'fs'
 import {
-  findNewsletterUrl,
-  generateUniqueUrl,
   getTitleFromEmailSubject,
   isProbablyArticle,
-  isProbablyNewsletter,
   parseEmailAddress,
   parsePageMetadata,
   parsePreparedContent,
@@ -24,69 +21,6 @@ const load = (path: string): string => {
   return fs.readFileSync(path, 'utf8')
 }
 
-describe('isProbablyNewsletter', () => {
-  it('returns true for substack newsletter', async () => {
-    const html = load('./test/utils/data/substack-forwarded-newsletter.html')
-    await expect(isProbablyNewsletter(html)).to.eventually.be.true
-  })
-  it('returns true for private forwarded substack newsletter', async () => {
-    const html = load(
-      './test/utils/data/substack-private-forwarded-newsletter.html'
-    )
-    await expect(isProbablyNewsletter(html)).to.eventually.be.true
-  })
-  it('returns false for substack welcome email', async () => {
-    const html = load('./test/utils/data/substack-forwarded-welcome-email.html')
-    await expect(isProbablyNewsletter(html)).to.eventually.be.false
-  })
-  it('returns true for beehiiv.com newsletter', async () => {
-    const html = load('./test/utils/data/beehiiv-newsletter.html')
-    await expect(isProbablyNewsletter(html)).to.eventually.be.true
-  })
-})
-
-describe('findNewsletterUrl', async () => {
-  it('gets the URL from the header if it is a substack newsletter', async () => {
-    nock('https://email.mg2.substack.com')
-      .head(
-        '/c/eJxNkk2TojAQhn-N3KTyQfg4cGDGchdnYcsZx9K5UCE0EMVAkTiKv36iHnarupNUd7rfVJ4W3EDTj1M89No496Uw0wCxgovuwBgYnbOGsZBVjDHzKPWYU8VehUMWOlIX9Qhw4rKLzXgGZziXnRTcyF7dK0iIGMVOG_OS1aTmKPRDilgVhTQUPCQIcE0x-MFTmJ8rCUpA3KtuenR2urg1ZtAzmszI0tq_Z7m66y-ilQo0uAqMTQ7WRX8auJKg56blZg7WB-iHDuYEBzO6NP0R1IwuYFphQbbTjnTH9NBfs80nym4Zyj8uUvyKbtUyGr5eUz9fNDQ7JCxfJDo9dW1lY9lmj_JNivPbGmf2Pt_lN9tDit9b-WeTetni85Z9pDpVOd7L1E_Vy7egayNO23ZP34eSeLJeux1b0rer_xaZ7ykS78nuSjMY-nL98rparNZNcv07JCjN06_EkTFBxBqOUMACErnELUNMSxTUjLDQZwzcqa4bRjCfeejUEFefS224OLr2S5wxPtij7lVrs80d2CNseRV2P52VNFMBipcdVE-U5jkRD7hFAwpGOylVwU2Mfc9qBh7DoR89yVnWXhgQFHnIsbpVb6tU_B-hH_2yzWY'
-      )
-      .reply(302, undefined, {
-        Location:
-          'https://newsletter.slowchinese.net/p/companies-that-eat-people-217',
-      })
-      .get('/p/companies-that-eat-people-217')
-      .reply(200, '')
-    const html = load('./test/utils/data/substack-forwarded-newsletter.html')
-    const url = await findNewsletterUrl(html)
-    // Not sure if the redirects from substack expire, this test could eventually fail
-    expect(url).to.startWith(
-      'https://newsletter.slowchinese.net/p/companies-that-eat-people-217'
-    )
-  })
-  it('gets the URL from the header if it is a beehiiv newsletter', async () => {
-    nock('https://u23463625.ct.sendgrid.net')
-      .head(
-        '/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno'
-      )
-      .reply(302, undefined, {
-        Location: 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple',
-      })
-      .get('/p/talked-guy-spent-30m-beeple')
-      .reply(200, '')
-    const html = load('./test/utils/data/beehiiv-newsletter.html')
-    const url = await findNewsletterUrl(html)
-    expect(url).to.startWith(
-      'https://www.milkroad.com/p/talked-guy-spent-30m-beeple'
-    )
-  })
-  it('returns undefined if it is not a newsletter', async () => {
-    const html = load('./test/utils/data/substack-forwarded-welcome-email.html')
-    const url = await findNewsletterUrl(html)
-    expect(url).to.be.undefined
-  })
-})
-
 describe('parseMetadata', async () => {
   it('gets author, title, image, description', async () => {
     const html = load('./test/utils/data/substack-post.html')
@@ -164,15 +98,6 @@ describe('isProbablyArticle', () => {
   })
 })
 
-describe('generateUniqueUrl', () => {
-  it('generates a unique URL', () => {
-    const url1 = generateUniqueUrl()
-    const url2 = generateUniqueUrl()
-
-    expect(url1).to.not.eql(url2)
-  })
-})
-
 describe('getTitleFromEmailSubject', () => {
   it('returns the title from the email subject', () => {
     const title = 'test subject'
diff --git a/packages/content-handler/package.json b/packages/content-handler/package.json
index c82d40c5d..e4021b3e4 100644
--- a/packages/content-handler/package.json
+++ b/packages/content-handler/package.json
@@ -15,7 +15,12 @@
     "build": "tsc"
   },
   "devDependencies": {
-    "eslint-plugin-prettier": "^4.0.0"
+    "chai": "^4.3.6",
+    "chai-as-promised": "^7.1.1",
+    "chai-string": "^1.5.0",
+    "eslint-plugin-prettier": "^4.0.0",
+    "mocha": "^10.0.0",
+    "nock": "^13.2.9"
   },
   "dependencies": {
     "addressparser": "^1.0.1",
diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts
index 98ab1ac29..22216fabe 100644
--- a/packages/content-handler/src/content-handler.ts
+++ b/packages/content-handler/src/content-handler.ts
@@ -1,6 +1,8 @@
 import addressparser from 'addressparser'
 import rfc2047 from 'rfc2047'
-import { v4 as uuidv4 } from 'uuid'
+import { v4 as uuid } from 'uuid'
+import { parseHTML } from 'linkedom'
+import axios from 'axios'
 
 interface Unsubscribe {
   mailTo?: string
@@ -34,16 +36,17 @@ export interface PreHandleResult {
   dom?: Document
 }
 
+export const FAKE_URL_PREFIX = 'https://omnivore.app/no_url?q='
+export const generateUniqueUrl = () => FAKE_URL_PREFIX + uuid()
+
 export abstract class ContentHandler {
   protected senderRegex: RegExp
   protected urlRegex: RegExp
-  protected defaultUrl: string
-  public name: string
+  name: string
 
   protected constructor() {
     this.senderRegex = new RegExp(/NEWSLETTER_SENDER_REGEX/)
     this.urlRegex = new RegExp(/NEWSLETTER_URL_REGEX/)
-    this.defaultUrl = 'NEWSLETTER_DEFAULT_URL'
     this.name = 'Handler name'
   }
 
@@ -63,17 +66,57 @@ export abstract class ContentHandler {
     return Promise.resolve({ url, dom })
   }
 
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    return false
+  async isNewsletter(input: {
+    postHeader: string
+    from: string
+    unSubHeader: string
+    html?: string
+  }): Promise<boolean> {
+    const re = new RegExp(this.senderRegex)
+    return Promise.resolve(
+      re.test(input.from) && (!!input.postHeader || !!input.unSubHeader)
+    )
   }
 
-  parseNewsletterUrl(_postHeader: string, html: string): string | undefined {
+  findNewsletterHeaderHref(dom: Document): string | undefined {
+    return undefined
+  }
+
+  // Given an HTML blob tries to find a URL to use for
+  // a canonical URL.
+  async findNewsletterUrl(html: string): Promise<string | undefined> {
+    const dom = parseHTML(html).document
+
+    // Check if this is a substack newsletter
+    const href = this.findNewsletterHeaderHref(dom)
+    if (href) {
+      // Try to make a HEAD request, so we get the redirected URL, since these
+      // will usually be behind tracking url redirects
+      try {
+        const response = await axios.head(href, { timeout: 5000 })
+        return Promise.resolve(
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+          response.request.res.responseUrl as string | undefined
+        )
+      } catch (e) {
+        console.log('error making HEAD request', e)
+        return Promise.resolve(href)
+      }
+    }
+
+    return Promise.resolve(undefined)
+  }
+
+  async parseNewsletterUrl(
+    _postHeader: string,
+    html: string
+  ): Promise<string | undefined> {
     // get newsletter url from html
     const matches = html.match(this.urlRegex)
     if (matches) {
-      return matches[1]
+      return Promise.resolve(matches[1])
     }
-    return undefined
+    return Promise.resolve(undefined)
   }
 
   parseAuthor(from: string): string {
@@ -97,14 +140,14 @@ export abstract class ContentHandler {
     }
   }
 
-  handleNewsletter({
+  async handleNewsletter({
     email,
     html,
     postHeader,
     title,
     from,
     unSubHeader,
-  }: NewsletterInput): NewsletterResult {
+  }: NewsletterInput): Promise<NewsletterResult> {
     console.log('handleNewsletter', email, postHeader, title, from)
 
     if (!email || !html || !title || !from) {
@@ -115,8 +158,7 @@ export abstract class ContentHandler {
     // fallback to default url if newsletter url does not exist
     // assign a random uuid to the default url to avoid duplicate url
     const url =
-      this.parseNewsletterUrl(postHeader, html) ||
-      `${this.defaultUrl}?source=newsletters&id=${uuidv4()}`
+      (await this.parseNewsletterUrl(postHeader, html)) || generateUniqueUrl()
     const author = this.parseAuthor(from)
     const unsubscribe = this.parseUnsubscribe(unSubHeader)
 
diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts
index 95522d699..e41c811c4 100644
--- a/packages/content-handler/src/index.ts
+++ b/packages/content-handler/src/index.ts
@@ -1,13 +1,14 @@
-import { AppleNewsHandler } from './content/apple-news-handler'
-import { BloombergHandler } from './content/bloomberg-handler'
-import { DerstandardHandler } from './content/derstandard-handler'
-import { ImageHandler } from './content/image-handler'
-import { MediumHandler } from './content/medium-handler'
-import { PdfHandler } from './content/pdf-handler'
-import { ScrapingBeeHandler } from './content/scrapingBee-handler'
-import { TDotCoHandler } from './content/t-dot-co-handler'
-import { TwitterHandler } from './content/twitter-handler'
-import { YoutubeHandler } from './content/youtube-handler'
+import { AppleNewsHandler } from './websites/apple-news-handler'
+import { BloombergHandler } from './websites/bloomberg-handler'
+import { DerstandardHandler } from './websites/derstandard-handler'
+import { ImageHandler } from './websites/image-handler'
+import { MediumHandler } from './websites/medium-handler'
+import { PdfHandler } from './websites/pdf-handler'
+import { ScrapingBeeHandler } from './websites/scrapingBee-handler'
+import { TDotCoHandler } from './websites/t-dot-co-handler'
+import { TwitterHandler } from './websites/twitter-handler'
+import { YoutubeHandler } from './websites/youtube-handler'
+import { WikipediaHandler } from './websites/wikipedia-handler'
 import {
   ContentHandler,
   NewsletterInput,
@@ -19,7 +20,9 @@ import { AxiosHandler } from './newsletters/axios-handler'
 import { GolangHandler } from './newsletters/golang-handler'
 import { MorningBrewHandler } from './newsletters/morning-brew-handler'
 import { BloombergNewsletterHandler } from './newsletters/bloomberg-newsletter-handler'
-import { WikipediaHandler } from './content/wikipedia-handler'
+import { BeehiivHandler } from './newsletters/beehiiv-handler'
+import { ConvertkitHandler } from './newsletters/convertkit-handler'
+import { RevueHandler } from './newsletters/revue-handler'
 
 const validateUrlString = (url: string) => {
   const u = new URL(url)
@@ -57,6 +60,10 @@ const newsletterHandlers: ContentHandler[] = [
   new GolangHandler(),
   new SubstackHandler(),
   new MorningBrewHandler(),
+  new SubstackHandler(),
+  new BeehiivHandler(),
+  new ConvertkitHandler(),
+  new RevueHandler(),
 ]
 
 export const preHandleContent = async (
@@ -91,11 +98,11 @@ export const preHandleContent = async (
   return undefined
 }
 
-export const handleNewsletter = (
+export const handleNewsletter = async (
   input: NewsletterInput
-): NewsletterResult | undefined => {
+): Promise<NewsletterResult | undefined> => {
   for (const handler of newsletterHandlers) {
-    if (handler.isNewsletter(input.postHeader, input.from, input.unSubHeader)) {
+    if (await handler.isNewsletter(input)) {
       return handler.handleNewsletter(input)
     }
   }
diff --git a/packages/content-handler/src/newsletters/axios-handler.ts b/packages/content-handler/src/newsletters/axios-handler.ts
index afbdd9e52..cd783c30e 100644
--- a/packages/content-handler/src/newsletters/axios-handler.ts
+++ b/packages/content-handler/src/newsletters/axios-handler.ts
@@ -5,7 +5,6 @@ export class AxiosHandler extends ContentHandler {
     super()
     this.senderRegex = /<.+@axios.com>/
     this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
-    this.defaultUrl = 'https://axios.com'
     this.name = 'axios'
   }
 
@@ -44,10 +43,4 @@ export class AxiosHandler extends ContentHandler {
 
     return Promise.resolve({ dom })
   }
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
 }
diff --git a/packages/content-handler/src/newsletters/beehiiv-handler.ts b/packages/content-handler/src/newsletters/beehiiv-handler.ts
new file mode 100644
index 000000000..0a50c1920
--- /dev/null
+++ b/packages/content-handler/src/newsletters/beehiiv-handler.ts
@@ -0,0 +1,43 @@
+import { ContentHandler } from '../content-handler'
+import { parseHTML } from 'linkedom'
+
+export class BeehiivHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'beehiiv'
+  }
+
+  findNewsletterHeaderHref(dom: Document): string | undefined {
+    const readOnline = dom.querySelectorAll('table tr td div a[class*="link"]')
+    let res: string | undefined = undefined
+    readOnline.forEach((e) => {
+      if (e.textContent === 'Read Online') {
+        res = e.getAttribute('href') || undefined
+      }
+    })
+    return res
+  }
+
+  async isNewsletter(input: {
+    postHeader: string
+    from: string
+    unSubHeader: string
+    html: string
+  }): Promise<boolean> {
+    const dom = parseHTML(input.html).document
+    if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) {
+      const beehiivUrl = this.findNewsletterHeaderHref(dom)
+      if (beehiivUrl) {
+        return Promise.resolve(true)
+      }
+    }
+    return false
+  }
+
+  async parseNewsletterUrl(
+    postHeader: string,
+    html: string
+  ): Promise<string | undefined> {
+    return this.findNewsletterUrl(html)
+  }
+}
diff --git a/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts b/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts
index d7c71f6f8..a5f84f076 100644
--- a/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts
+++ b/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts
@@ -5,7 +5,6 @@ export class BloombergNewsletterHandler extends ContentHandler {
     super()
     this.senderRegex = /<.+@mail.bloomberg.*.com>/
     this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
-    this.defaultUrl = 'https://www.bloomberg.com'
     this.name = 'bloomberg'
   }
 
@@ -35,10 +34,4 @@ export class BloombergNewsletterHandler extends ContentHandler {
 
     return Promise.resolve({ dom })
   }
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
 }
diff --git a/packages/content-handler/src/newsletters/convertkit-handler.ts b/packages/content-handler/src/newsletters/convertkit-handler.ts
new file mode 100644
index 000000000..72e65f5da
--- /dev/null
+++ b/packages/content-handler/src/newsletters/convertkit-handler.ts
@@ -0,0 +1,41 @@
+import { ContentHandler } from '../content-handler'
+import { parseHTML } from 'linkedom'
+
+export class ConvertkitHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'convertkit'
+  }
+
+  findNewsletterHeaderHref(dom: Document): string | undefined {
+    const readOnline = dom.querySelectorAll('table tr td a')
+    let res: string | undefined = undefined
+    readOnline.forEach((e) => {
+      if (e.textContent === 'View this email in your browser') {
+        res = e.getAttribute('href') || undefined
+      }
+    })
+    return res
+  }
+
+  async isNewsletter(input: {
+    postHeader: string
+    from: string
+    unSubHeader: string
+    html: string
+  }): Promise<boolean> {
+    const dom = parseHTML(input.html).document
+    return Promise.resolve(
+      dom.querySelectorAll(
+        'img[src*="convertkit.com"], img[src*="convertkit-mail.com"]'
+      ).length > 0
+    )
+  }
+
+  async parseNewsletterUrl(
+    postHeader: string,
+    html: string
+  ): Promise<string | undefined> {
+    return this.findNewsletterUrl(html)
+  }
+}
diff --git a/packages/content-handler/src/newsletters/golang-handler.ts b/packages/content-handler/src/newsletters/golang-handler.ts
index 1672e4b28..7d4724004 100644
--- a/packages/content-handler/src/newsletters/golang-handler.ts
+++ b/packages/content-handler/src/newsletters/golang-handler.ts
@@ -5,7 +5,6 @@ export class GolangHandler extends ContentHandler {
     super()
     this.senderRegex = /<.+@golangweekly.com>/
     this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
-    this.defaultUrl = 'https://golangweekly.com'
     this.name = 'golangweekly'
   }
 
@@ -25,10 +24,4 @@ export class GolangHandler extends ContentHandler {
 
     return Promise.resolve({ dom })
   }
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
 }
diff --git a/packages/content-handler/src/newsletters/morning-brew-handler.ts b/packages/content-handler/src/newsletters/morning-brew-handler.ts
index 95bfe4b57..f187ac0dc 100644
--- a/packages/content-handler/src/newsletters/morning-brew-handler.ts
+++ b/packages/content-handler/src/newsletters/morning-brew-handler.ts
@@ -5,7 +5,6 @@ export class MorningBrewHandler extends ContentHandler {
     super()
     this.senderRegex = /Morning Brew <crew@morningbrew.com>/
     this.urlRegex = /<a.* href=["']([^"']*)["'].*>View Online<\/a>/
-    this.defaultUrl = 'https://www.morningbrew.com'
     this.name = 'morningbrew'
   }
 
@@ -33,10 +32,4 @@ export class MorningBrewHandler extends ContentHandler {
 
     return Promise.resolve({ dom })
   }
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
 }
diff --git a/packages/content-handler/src/newsletters/revue-handler.ts b/packages/content-handler/src/newsletters/revue-handler.ts
new file mode 100644
index 000000000..d8c8f911c
--- /dev/null
+++ b/packages/content-handler/src/newsletters/revue-handler.ts
@@ -0,0 +1,46 @@
+import { ContentHandler } from '../content-handler'
+import { parseHTML } from 'linkedom'
+
+export class RevueHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'revue'
+  }
+
+  findNewsletterHeaderHref(dom: Document): string | undefined {
+    const viewOnline = dom.querySelectorAll('table tr td a[target="_blank"]')
+    let res: string | undefined = undefined
+    viewOnline.forEach((e) => {
+      if (e.textContent === 'View online') {
+        res = e.getAttribute('href') || undefined
+      }
+    })
+    return res
+  }
+
+  async isNewsletter(input: {
+    postHeader: string
+    from: string
+    unSubHeader: string
+    html: string
+  }): Promise<boolean> {
+    const dom = parseHTML(input.html).document
+    if (
+      dom.querySelectorAll('img[src*="getrevue.co"], img[src*="revue.email"]')
+        .length > 0
+    ) {
+      const getrevueUrl = this.findNewsletterHeaderHref(dom)
+      if (getrevueUrl) {
+        return Promise.resolve(true)
+      }
+    }
+    return false
+  }
+
+  async parseNewsletterUrl(
+    postHeader: string,
+    html: string
+  ): Promise<string | undefined> {
+    return this.findNewsletterUrl(html)
+  }
+}
diff --git a/packages/content-handler/src/newsletters/substack-handler.ts b/packages/content-handler/src/newsletters/substack-handler.ts
index 675582096..164068623 100644
--- a/packages/content-handler/src/newsletters/substack-handler.ts
+++ b/packages/content-handler/src/newsletters/substack-handler.ts
@@ -1,10 +1,10 @@
 import addressparser from 'addressparser'
 import { ContentHandler, PreHandleResult } from '../content-handler'
+import { parseHTML } from 'linkedom'
 
 export class SubstackHandler extends ContentHandler {
   constructor() {
     super()
-    this.defaultUrl = 'https://www.substack.com'
     this.name = 'substack'
   }
 
@@ -38,15 +38,53 @@ export class SubstackHandler extends ContentHandler {
     return Promise.resolve(dom)
   }
 
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    return !!postHeader
+  findNewsletterHeaderHref(dom: Document): string | undefined {
+    // Substack header links
+    const postLink = dom.querySelector('h1 a ')
+    if (postLink) {
+      return postLink.getAttribute('href') || undefined
+    }
+
+    return undefined
   }
 
-  parseNewsletterUrl(postHeader: string, html: string): string | undefined {
+  async isNewsletter({
+    postHeader,
+    html,
+  }: {
+    postHeader: string
+    from: string
+    unSubHeader: string
+    html: string
+  }): Promise<boolean> {
+    if (postHeader) {
+      return Promise.resolve(true)
+    }
+    const dom = parseHTML(html).document
+    // substack newsletter emails have tables with a *post-meta class
+    if (dom.querySelector('table[class$="post-meta"]')) {
+      return true
+    }
+    // If the article has a header link, and substack icons its probably a newsletter
+    const href = this.findNewsletterHeaderHref(dom)
+    const heartIcon = dom.querySelector(
+      'table tbody td span a img[src*="HeartIcon"]'
+    )
+    const recommendIcon = dom.querySelector(
+      'table tbody td span a img[src*="RecommendIconRounded"]'
+    )
+    return Promise.resolve(!!(href && (heartIcon || recommendIcon)))
+  }
+
+  async parseNewsletterUrl(
+    postHeader: string,
+    html: string
+  ): Promise<string | undefined> {
     // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
     // we need to get the real url from the raw url
-    return addressparser(postHeader).length > 0
-      ? addressparser(postHeader)[0].name
-      : undefined
+    if (postHeader && addressparser(postHeader).length > 0) {
+      return Promise.resolve(addressparser(postHeader)[0].name)
+    }
+    return this.findNewsletterUrl(html)
   }
 }
diff --git a/packages/content-handler/test/apple-news-handler.test.ts b/packages/content-handler/test/apple-news-handler.test.ts
index 5a19542f5..1584f9e28 100644
--- a/packages/content-handler/test/apple-news-handler.test.ts
+++ b/packages/content-handler/test/apple-news-handler.test.ts
@@ -1,4 +1,4 @@
-import { AppleNewsHandler } from '../src/content/apple-news-handler'
+import { AppleNewsHandler } from '../src/websites/apple-news-handler'
 
 describe('open a simple web page', () => {
   it('should return a response', async () => {
diff --git a/packages/api/test/utils/data/beehiiv-newsletter.html b/packages/content-handler/test/data/beehiiv-newsletter.html
similarity index 100%
rename from packages/api/test/utils/data/beehiiv-newsletter.html
rename to packages/content-handler/test/data/beehiiv-newsletter.html
diff --git a/packages/api/test/utils/data/substack-forwarded-newsletter.html b/packages/content-handler/test/data/substack-forwarded-newsletter.html
similarity index 100%
rename from packages/api/test/utils/data/substack-forwarded-newsletter.html
rename to packages/content-handler/test/data/substack-forwarded-newsletter.html
diff --git a/packages/api/test/utils/data/substack-forwarded-welcome-email.html b/packages/content-handler/test/data/substack-forwarded-welcome-email.html
similarity index 100%
rename from packages/api/test/utils/data/substack-forwarded-welcome-email.html
rename to packages/content-handler/test/data/substack-forwarded-welcome-email.html
diff --git a/packages/api/test/utils/data/substack-private-forwarded-newsletter.html b/packages/content-handler/test/data/substack-private-forwarded-newsletter.html
similarity index 100%
rename from packages/api/test/utils/data/substack-private-forwarded-newsletter.html
rename to packages/content-handler/test/data/substack-private-forwarded-newsletter.html
diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts
index 50d94f646..1d36aefe4 100644
--- a/packages/content-handler/test/newsletter.test.ts
+++ b/packages/content-handler/test/newsletter.test.ts
@@ -1,28 +1,45 @@
+import 'mocha'
+import * as chai from 'chai'
 import { expect } from 'chai'
+import chaiAsPromised from 'chai-as-promised'
+import chaiString from 'chai-string'
 import { SubstackHandler } from '../src/newsletters/substack-handler'
 import { AxiosHandler } from '../src/newsletters/axios-handler'
 import { BloombergNewsletterHandler } from '../src/newsletters/bloomberg-newsletter-handler'
 import { GolangHandler } from '../src/newsletters/golang-handler'
 import { MorningBrewHandler } from '../src/newsletters/morning-brew-handler'
+import nock from 'nock'
+import { generateUniqueUrl } from '../src/content-handler'
+import fs from 'fs'
+import { BeehiivHandler } from '../src/newsletters/beehiiv-handler'
+
+chai.use(chaiAsPromised)
+chai.use(chaiString)
+
+const load = (path: string): string => {
+  return fs.readFileSync(path, 'utf8')
+}
 
 describe('Newsletter email test', () => {
   describe('#getNewsletterUrl()', () => {
-    it('returns url when email is from SubStack', () => {
+    it('returns url when email is from SubStack', async () => {
       const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
 
-      expect(new SubstackHandler().parseNewsletterUrl(rawUrl, '')).to.equal(
-        'https://hongbo130.substack.com/p/tldr'
-      )
+      await expect(
+        new SubstackHandler().parseNewsletterUrl(rawUrl, '')
+      ).to.eventually.equal('https://hongbo130.substack.com/p/tldr')
     })
 
-    it('returns url when email is from Axios', () => {
+    it('returns url when email is from Axios', async () => {
       const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app'
       const html = `View in browser at <a>${url}</a>`
 
-      expect(new AxiosHandler().parseNewsletterUrl('', html)).to.equal(url)
+      await expect(
+        new AxiosHandler().parseNewsletterUrl('', html)
+      ).to.eventually.equal(url)
     })
 
-    it('returns url when email is from Bloomberg', () => {
+    it('returns url when email is from Bloomberg', async () => {
       const url = 'https://www.bloomberg.com/news/google-is-now-a-partner'
       const html = `
         <a class="view-in-browser__url" href="${url}">
@@ -30,29 +47,31 @@ describe('Newsletter email test', () => {
         </a>
       `
 
-      expect(
+      await expect(
         new BloombergNewsletterHandler().parseNewsletterUrl('', html)
-      ).to.equal(url)
+      ).to.eventually.equal(url)
     })
 
-    it('returns url when email is from Golang Weekly', () => {
+    it('returns url when email is from Golang Weekly', async () => {
       const url = 'https://www.golangweekly.com/first'
       const html = `
         <a href="${url}" style="text-decoration: none">Read on the Web</a>
       `
 
-      expect(new GolangHandler().parseNewsletterUrl('', html)).to.equal(url)
+      await expect(
+        new GolangHandler().parseNewsletterUrl('', html)
+      ).to.eventually.equal(url)
     })
 
-    it('returns url when email is from Morning Brew', () => {
+    it('returns url when email is from Morning Brew', async () => {
       const url = 'https://www.morningbrew.com/daily/issues/first'
       const html = `
         <a style="color: #000000; text-decoration: none;" target="_blank" rel="noopener" href="${url}">View Online</a>
       `
 
-      expect(new MorningBrewHandler().parseNewsletterUrl('', html)).to.equal(
-        url
-      )
+      await expect(
+        new MorningBrewHandler().parseNewsletterUrl('', html)
+      ).to.eventually.equal(url)
     })
   })
 
@@ -69,4 +88,104 @@ describe('Newsletter email test', () => {
       expect(new AxiosHandler().parseAuthor(from)).to.equal('Mike Allen')
     })
   })
+
+  describe('isProbablyNewsletter', () => {
+    it('returns true for substack newsletter', async () => {
+      const html = load('./test/data/substack-forwarded-newsletter.html')
+      await expect(
+        new SubstackHandler().isNewsletter({
+          html,
+          postHeader: '',
+          from: '',
+          unSubHeader: '',
+        })
+      ).to.eventually.be.true
+    })
+    it('returns true for private forwarded substack newsletter', async () => {
+      const html = load(
+        './test/data/substack-private-forwarded-newsletter.html'
+      )
+      await expect(
+        new SubstackHandler().isNewsletter({
+          html,
+          postHeader: '',
+          from: '',
+          unSubHeader: '',
+        })
+      ).to.eventually.be.true
+    })
+    it('returns false for substack welcome email', async () => {
+      const html = load('./test/data/substack-forwarded-welcome-email.html')
+      await expect(
+        new SubstackHandler().isNewsletter({
+          html,
+          postHeader: '',
+          from: '',
+          unSubHeader: '',
+        })
+      ).to.eventually.be.false
+    })
+    it('returns true for beehiiv.com newsletter', async () => {
+      const html = load('./test/data/beehiiv-newsletter.html')
+      await expect(
+        new BeehiivHandler().isNewsletter({
+          html,
+          postHeader: '',
+          from: '',
+          unSubHeader: '',
+        })
+      ).to.eventually.be.true
+    })
+  })
+
+  describe('findNewsletterUrl', async () => {
+    it('gets the URL from the header if it is a substack newsletter', async () => {
+      nock('https://email.mg2.substack.com')
+        .head(
+          '/c/eJxNkk2TojAQhn-N3KTyQfg4cGDGchdnYcsZx9K5UCE0EMVAkTiKv36iHnarupNUd7rfVJ4W3EDTj1M89No496Uw0wCxgovuwBgYnbOGsZBVjDHzKPWYU8VehUMWOlIX9Qhw4rKLzXgGZziXnRTcyF7dK0iIGMVOG_OS1aTmKPRDilgVhTQUPCQIcE0x-MFTmJ8rCUpA3KtuenR2urg1ZtAzmszI0tq_Z7m66y-ilQo0uAqMTQ7WRX8auJKg56blZg7WB-iHDuYEBzO6NP0R1IwuYFphQbbTjnTH9NBfs80nym4Zyj8uUvyKbtUyGr5eUz9fNDQ7JCxfJDo9dW1lY9lmj_JNivPbGmf2Pt_lN9tDit9b-WeTetni85Z9pDpVOd7L1E_Vy7egayNO23ZP34eSeLJeux1b0rer_xaZ7ykS78nuSjMY-nL98rparNZNcv07JCjN06_EkTFBxBqOUMACErnELUNMSxTUjLDQZwzcqa4bRjCfeejUEFefS224OLr2S5wxPtij7lVrs80d2CNseRV2P52VNFMBipcdVE-U5jkRD7hFAwpGOylVwU2Mfc9qBh7DoR89yVnWXhgQFHnIsbpVb6tU_B-hH_2yzWY'
+        )
+        .reply(302, undefined, {
+          Location:
+            'https://newsletter.slowchinese.net/p/companies-that-eat-people-217',
+        })
+        .get('/p/companies-that-eat-people-217')
+        .reply(200, '')
+      const html = load('./test/data/substack-forwarded-newsletter.html')
+      const url = await new SubstackHandler().findNewsletterUrl(html)
+      // Not sure if the redirects from substack expire, this test could eventually fail
+      expect(url).to.startWith(
+        'https://newsletter.slowchinese.net/p/companies-that-eat-people-217'
+      )
+    })
+    it('gets the URL from the header if it is a beehiiv newsletter', async () => {
+      nock('https://u23463625.ct.sendgrid.net')
+        .head(
+          '/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno'
+        )
+        .reply(302, undefined, {
+          Location: 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple',
+        })
+        .get('/p/talked-guy-spent-30m-beeple')
+        .reply(200, '')
+      const html = load('./test/data/beehiiv-newsletter.html')
+      const url = await new BeehiivHandler().findNewsletterUrl(html)
+      expect(url).to.startWith(
+        'https://www.milkroad.com/p/talked-guy-spent-30m-beeple'
+      )
+    })
+    it('returns undefined if it is not a newsletter', async () => {
+      const html = load('./test/data/substack-forwarded-welcome-email.html')
+      const url = await new SubstackHandler().findNewsletterUrl(html)
+      expect(url).to.be.undefined
+    })
+  })
+
+  describe('generateUniqueUrl', () => {
+    it('generates a unique URL', () => {
+      const url1 = generateUniqueUrl()
+      const url2 = generateUniqueUrl()
+
+      expect(url1).to.not.eql(url2)
+    })
+  })
 })
diff --git a/packages/content-handler/test/youtube-handler.test.ts b/packages/content-handler/test/youtube-handler.test.ts
index 4e7fcb913..beb4d3a66 100644
--- a/packages/content-handler/test/youtube-handler.test.ts
+++ b/packages/content-handler/test/youtube-handler.test.ts
@@ -1,6 +1,6 @@
 import { expect } from 'chai'
 import 'mocha'
-import { getYoutubeVideoId } from '../src/content/youtube-handler'
+import { getYoutubeVideoId } from '../src/websites/youtube-handler'
 
 describe('getYoutubeVideoId', () => {
   it('should parse video id out of a URL', async () => {
diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts
index 8e3a69874..67d38694d 100644
--- a/packages/inbound-email-handler/src/index.ts
+++ b/packages/inbound-email-handler/src/index.ts
@@ -78,7 +78,7 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       try {
         // check if it is a confirmation email or forwarding newsletter
-        const newsletterMessage = handleNewsletter({
+        const newsletterMessage = await handleNewsletter({
           from,
           html,
           postHeader,
diff --git a/yarn.lock b/yarn.lock
index 32066d1bb..b76eae9d8 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -10104,6 +10104,13 @@ brace-expansion@^1.1.7:
     balanced-match "^1.0.0"
     concat-map "0.0.1"
 
+brace-expansion@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
+  integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
+  dependencies:
+    balanced-match "^1.0.0"
+
 braces@^2.3.1, braces@^2.3.2:
   version "2.3.2"
   resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729"
@@ -10579,6 +10586,19 @@ chai@^4.3.4:
     pathval "^1.1.1"
     type-detect "^4.0.5"
 
+chai@^4.3.6:
+  version "4.3.6"
+  resolved "https://registry.yarnpkg.com/chai/-/chai-4.3.6.tgz#ffe4ba2d9fa9d6680cc0b370adae709ec9011e9c"
+  integrity sha512-bbcp3YfHCUzMOvKqsztczerVgBKSsEijCySNlHHbX3VG1nskvqjz5Rfso1gGwD6w6oOV3eI60pKuMOV5MV7p3Q==
+  dependencies:
+    assertion-error "^1.1.0"
+    check-error "^1.0.2"
+    deep-eql "^3.0.1"
+    get-func-name "^2.0.0"
+    loupe "^2.3.1"
+    pathval "^1.1.1"
+    type-detect "^4.0.5"
+
 chalk@^1.0.0, chalk@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98"
@@ -10748,6 +10768,21 @@ chokidar@3.5.2:
   optionalDependencies:
     fsevents "~2.3.2"
 
+chokidar@3.5.3, chokidar@^3.4.1, chokidar@^3.4.2, chokidar@^3.5.1, chokidar@^3.5.2, chokidar@^3.5.3:
+  version "3.5.3"
+  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
+  integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==
+  dependencies:
+    anymatch "~3.1.2"
+    braces "~3.0.2"
+    glob-parent "~5.1.2"
+    is-binary-path "~2.1.0"
+    is-glob "~4.0.1"
+    normalize-path "~3.0.0"
+    readdirp "~3.6.0"
+  optionalDependencies:
+    fsevents "~2.3.2"
+
 chokidar@^2.1.8:
   version "2.1.8"
   resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917"
@@ -10767,21 +10802,6 @@ chokidar@^2.1.8:
   optionalDependencies:
     fsevents "^1.2.7"
 
-chokidar@^3.4.1, chokidar@^3.4.2, chokidar@^3.5.1, chokidar@^3.5.2, chokidar@^3.5.3:
-  version "3.5.3"
-  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
-  integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==
-  dependencies:
-    anymatch "~3.1.2"
-    braces "~3.0.2"
-    glob-parent "~5.1.2"
-    is-binary-path "~2.1.0"
-    is-glob "~4.0.1"
-    normalize-path "~3.0.0"
-    readdirp "~3.6.0"
-  optionalDependencies:
-    fsevents "~2.3.2"
-
 chownr@^1.1.1, chownr@^1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
@@ -14489,7 +14509,7 @@ glob@7.1.7:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.0:
+glob@7.2.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.0:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023"
   integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==
@@ -18065,6 +18085,13 @@ loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0:
   dependencies:
     js-tokens "^3.0.0 || ^4.0.0"
 
+loupe@^2.3.1:
+  version "2.3.4"
+  resolved "https://registry.yarnpkg.com/loupe/-/loupe-2.3.4.tgz#7e0b9bffc76f148f9be769cb1321d3dcf3cb25f3"
+  integrity sha512-OvKfgCC2Ndby6aSTREl5aCCPTNIzlDfQZvZxNUrBrihDhL3xcrYegTblhmEiCrg2kKQz4XsFIaemE5BF4ybSaQ==
+  dependencies:
+    get-func-name "^2.0.0"
+
 lower-case-first@^1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/lower-case-first/-/lower-case-first-1.0.2.tgz#e5da7c26f29a7073be02d52bac9980e5922adfa1"
@@ -18621,6 +18648,13 @@ minimatch@3.0.4:
   dependencies:
     brace-expansion "^1.1.7"
 
+minimatch@5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.0.1.tgz#fb9022f7528125187c92bd9e9b6366be1cf3415b"
+  integrity sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==
+  dependencies:
+    brace-expansion "^2.0.1"
+
 minimatch@^3.0.2, minimatch@^3.0.4:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
@@ -18803,6 +18837,34 @@ mocha-unfunk-reporter@^0.4.0:
     miniwrite "~0.1.3"
     unfunk-diff "~0.0.1"
 
+mocha@^10.0.0:
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.0.0.tgz#205447d8993ec755335c4b13deba3d3a13c4def9"
+  integrity sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA==
+  dependencies:
+    "@ungap/promise-all-settled" "1.1.2"
+    ansi-colors "4.1.1"
+    browser-stdout "1.3.1"
+    chokidar "3.5.3"
+    debug "4.3.4"
+    diff "5.0.0"
+    escape-string-regexp "4.0.0"
+    find-up "5.0.0"
+    glob "7.2.0"
+    he "1.2.0"
+    js-yaml "4.1.0"
+    log-symbols "4.1.0"
+    minimatch "5.0.1"
+    ms "2.1.3"
+    nanoid "3.3.3"
+    serialize-javascript "6.0.0"
+    strip-json-comments "3.1.1"
+    supports-color "8.1.1"
+    workerpool "6.2.1"
+    yargs "16.2.0"
+    yargs-parser "20.2.4"
+    yargs-unparser "2.0.0"
+
 mocha@^8.2.0:
   version "8.4.0"
   resolved "https://registry.yarnpkg.com/mocha/-/mocha-8.4.0.tgz#677be88bf15980a3cae03a73e10a0fc3997f0cff"
@@ -18965,7 +19027,7 @@ nan@^2.12.1:
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.15.0.tgz#3f34a473ff18e15c1b5626b62903b5ad6e665fee"
   integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==
 
-nanoid@*, nanoid@^3.1.23, nanoid@^3.1.25, nanoid@^3.1.29, nanoid@^3.1.30, nanoid@^3.3.1:
+nanoid@*, nanoid@3.3.3, nanoid@^3.1.23, nanoid@^3.1.25, nanoid@^3.1.29, nanoid@^3.1.30, nanoid@^3.3.1:
   version "3.3.3"
   resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.3.tgz#fd8e8b7aa761fe807dba2d1b98fb7241bb724a25"
   integrity sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==
@@ -19136,6 +19198,16 @@ nock@^13.2.4:
     lodash.set "^4.3.2"
     propagate "^2.0.0"
 
+nock@^13.2.9:
+  version "13.2.9"
+  resolved "https://registry.yarnpkg.com/nock/-/nock-13.2.9.tgz#4faf6c28175d36044da4cfa68e33e5a15086ad4c"
+  integrity sha512-1+XfJNYF1cjGB+TKMWi29eZ0b82QOvQs2YoLNzbpWGqFMtRQHTa57osqdGj4FrFPgkO4D4AZinzUJR9VvW3QUA==
+  dependencies:
+    debug "^4.1.0"
+    json-stringify-safe "^5.0.1"
+    lodash "^4.17.21"
+    propagate "^2.0.0"
+
 node-addon-api@^1.2.0:
   version "1.7.2"
   resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-1.7.2.tgz#3df30b95720b53c24e59948b49532b662444f54d"
@@ -25608,6 +25680,11 @@ workerpool@6.1.5:
   resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.1.5.tgz#0f7cf076b6215fd7e1da903ff6f22ddd1886b581"
   integrity sha512-XdKkCK0Zqc6w3iTxLckiuJ81tiD/o5rBE/m+nXpRCB+/Sq4DqkfXZ/x0jW02DG1tGsfUGXbTJyZDP+eu67haSw==
 
+workerpool@6.2.1:
+  version "6.2.1"
+  resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343"
+  integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==
+
 wrap-ansi@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba"

From 9841ce7f8efc19c22eeecd2422a947b2c83835e3 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 30 Sep 2022 12:51:00 +0800
Subject: [PATCH 13/15] Remove parsing newsletter emails from forwarded emails

---
 packages/api/src/utils/parser.ts | 144 -------------------------------
 1 file changed, 144 deletions(-)

diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts
index ee62d1050..b071f2478 100644
--- a/packages/api/src/utils/parser.ts
+++ b/packages/api/src/utils/parser.ts
@@ -450,150 +450,6 @@ export const parseUrlMetadata = async (
   }
 }
 
-// Attempt to determine if an HTML blob is a newsletter
-// based on it's contents.
-// TODO: when we consolidate the handlers we could include this
-// as a utility method on each one.
-export const isProbablyNewsletter = async (html: string): Promise<boolean> => {
-  const dom = parseHTML(html).document
-  const domCopy = parseHTML(dom.documentElement.outerHTML).document
-  const article = await new Readability(domCopy, {
-    debug: false,
-    keepTables: true,
-  }).parse()
-
-  if (!article || !article.content) {
-    return false
-  }
-
-  // substack newsletter emails have tables with a *post-meta class
-  if (dom.querySelector('table[class$="post-meta"]')) {
-    return true
-  }
-
-  // If the article has a header link, and substack icons its probably a newsletter
-  const href = findNewsletterHeaderHref(dom)
-  const heartIcon = dom.querySelector(
-    'table tbody td span a img[src*="HeartIcon"]'
-  )
-  const recommendIcon = dom.querySelector(
-    'table tbody td span a img[src*="RecommendIconRounded"]'
-  )
-  if (href && (heartIcon || recommendIcon)) {
-    return true
-  }
-
-  // Check if this is a beehiiv.net newsletter
-  if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) {
-    const beehiivUrl = beehiivNewsletterHref(dom)
-    if (beehiivUrl) {
-      return true
-    }
-  }
-
-  // Check if this is a newsletter from revue
-  if (
-    dom.querySelectorAll('img[src*="getrevue.co"], img[src*="revue.email"]')
-      .length > 0
-  ) {
-    const getrevueUrl = revueNewsletterHref(dom)
-    if (getrevueUrl) {
-      return true
-    }
-  }
-
-  // Check if this is a convertkit.com newsletter
-  return (
-    dom.querySelectorAll(
-      'img[src*="convertkit.com"], img[src*="convertkit-mail.com"]'
-    ).length > 0
-  )
-}
-
-const beehiivNewsletterHref = (dom: Document): string | undefined => {
-  const readOnline = dom.querySelectorAll('table tr td div a[class*="link"]')
-  let res: string | undefined = undefined
-  readOnline.forEach((e) => {
-    if (e.textContent === 'Read Online') {
-      res = e.getAttribute('href') || undefined
-    }
-  })
-  return res
-}
-
-const convertkitNewsletterHref = (dom: Document): string | undefined => {
-  const readOnline = dom.querySelectorAll('table tr td a')
-  let res: string | undefined = undefined
-  readOnline.forEach((e) => {
-    if (e.textContent === 'View this email in your browser') {
-      res = e.getAttribute('href') || undefined
-    }
-  })
-  return res
-}
-
-const revueNewsletterHref = (dom: Document): string | undefined => {
-  const viewOnline = dom.querySelectorAll('table tr td a[target="_blank"]')
-  let res: string | undefined = undefined
-  viewOnline.forEach((e) => {
-    if (e.textContent === 'View online') {
-      res = e.getAttribute('href') || undefined
-    }
-  })
-  return res
-}
-
-const findNewsletterHeaderHref = (dom: Document): string | undefined => {
-  // Substack header links
-  const postLink = dom.querySelector('h1 a ')
-  if (postLink) {
-    return postLink.getAttribute('href') || undefined
-  }
-
-  // Check if this is a beehiiv.net newsletter
-  const beehiiv = beehiivNewsletterHref(dom)
-  if (beehiiv) {
-    return beehiiv
-  }
-
-  // Check if this is a revue newsletter
-  const revue = revueNewsletterHref(dom)
-  if (revue) {
-    return revue
-  }
-
-  // Check if this is a convertkit.com newsletter
-  const convertkitUrl = convertkitNewsletterHref(dom)
-  if (convertkitUrl) {
-    return convertkitUrl
-  }
-
-  return undefined
-}
-
-// Given an HTML blob tries to find a URL to use for
-// a canonical URL.
-export const findNewsletterUrl = async (
-  html: string
-): Promise<string | undefined> => {
-  const dom = parseHTML(html).document
-
-  // Check if this is a substack newsletter
-  const href = findNewsletterHeaderHref(dom)
-  if (href) {
-    // Try to make a HEAD request so we get the redirected URL, since these
-    // will usually be behind tracking url redirects
-    return axios({
-      method: 'HEAD',
-      url: href,
-    })
-      .then((res) => res.request.res.responseUrl as string | undefined)
-      .catch((e) => href)
-  }
-
-  return undefined
-}
-
 export const isProbablyArticle = async (
   email: string,
   subject: string

From 89423980924af526e30f40a1564eec0b914a120c Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 30 Sep 2022 12:52:09 +0800
Subject: [PATCH 14/15] Change GET to HEAD

---
 packages/api/src/utils/parser.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts
index b071f2478..3d6242264 100644
--- a/packages/api/src/utils/parser.ts
+++ b/packages/api/src/utils/parser.ts
@@ -483,7 +483,7 @@ export const fetchFavicon = async (
 ): Promise<string | undefined> => {
   try {
     // get the correct url if it's a redirect
-    const response = await axios.get(url, { timeout: 5000 })
+    const response = await axios.head(url, { timeout: 5000 })
     const realUrl = response.request.res.responseUrl
     const domain = new URL(realUrl).hostname
     return `https://api.faviconkit.com/${domain}/32`

From 25740465f23d16fcaf72c44c48688a92c7cc7594 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 30 Sep 2022 13:10:58 +0800
Subject: [PATCH 15/15] Add timeout

---
 packages/api/test/resolvers/article.test.ts      | 12 ++++++++++--
 packages/content-handler/test/newsletter.test.ts |  2 +-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts
index 298adeb97..d64a23d4e 100644
--- a/packages/api/test/resolvers/article.test.ts
+++ b/packages/api/test/resolvers/article.test.ts
@@ -10,7 +10,11 @@ import { expect } from 'chai'
 import 'mocha'
 import { User } from '../../src/entity/user'
 import chaiString from 'chai-string'
-import { UpdateReason, UploadFileStatus } from '../../src/generated/graphql'
+import {
+  SyncUpdatedItemEdge,
+  UpdateReason,
+  UploadFileStatus,
+} from '../../src/generated/graphql'
 import {
   ArticleSavingRequestStatus,
   Highlight,
@@ -1033,7 +1037,11 @@ describe('Article API', () => {
         authToken
       ).expect(200)
 
-      expect(res.body.data.updatesSince.edges.length).to.eql(3)
+      expect(
+        res.body.data.updatesSince.edges.filter(
+          (e: SyncUpdatedItemEdge) => e.updateReason === UpdateReason.Deleted
+        ).length
+      ).to.eql(3)
       expect(res.body.data.updatesSince.edges[0].itemID).to.eq(
         deletedPages[0].id
       )
diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts
index 1d36aefe4..dd3b7941c 100644
--- a/packages/content-handler/test/newsletter.test.ts
+++ b/packages/content-handler/test/newsletter.test.ts
@@ -156,7 +156,7 @@ describe('Newsletter email test', () => {
       expect(url).to.startWith(
         'https://newsletter.slowchinese.net/p/companies-that-eat-people-217'
       )
-    })
+    }).timeout(10000)
     it('gets the URL from the header if it is a beehiiv newsletter', async () => {
       nock('https://u23463625.ct.sendgrid.net')
         .head(