Merge pull request #1257 from omnivore-app/consolidate-newsletter-handler

Consolidate newsletters and other handlers
2026-03-11 08:54:26 +00:00 · 2022-09-30 16:30:20 +08:00 · 2022-09-30 16:30:20 +08:00 · 33355cb208
commit 33355cb208
parent f6cb7a279d 25740465f2
81 changed files with 1636 additions and 1777 deletions
--- a/packages/api/src/routers/svc/emails.ts
+++ b/packages/api/src/routers/svc/emails.ts
@ -8,14 +8,11 @@ import { analytics } from '../../utils/analytics'
 import { getNewsletterEmail } from '../../services/newsletters'
 import { env } from '../../env'
 import {
-  findNewsletterUrl,
  generateUniqueUrl,
  getTitleFromEmailSubject,
  isProbablyArticle,
-  isProbablyNewsletter,
  parseEmailAddress,
 } from '../../utils/parser'
-import { saveNewsletterEmail } from '../../services/save_newsletter_email'
 import { saveEmail } from '../../services/save_email'
 import { buildLogger } from '../../utils/logger'

@ -80,25 +77,6 @@ export function emailsServiceRouter() {
      const ctx = { pubsub: createPubSubClient(), uid: user.id }
      const parsedFrom = parseEmailAddress(data.from)

-      if (await isProbablyNewsletter(data.html)) {
-        logger.info('handling as newsletter', data)
-        await saveNewsletterEmail(
-          {
-            email: data.to,
-            title: data.subject,
-            content: data.html,
-            author: parsedFrom.name,
-            url: (await findNewsletterUrl(data.html)) || generateUniqueUrl(),
-            unsubMailTo: data.unsubMailTo,
-            unsubHttpUrl: data.unsubHttpUrl,
-            newsletterEmail,
-          },
-          ctx
-        )
-        res.status(200).send('Newsletter')
-        return
-      }
-
      if (
        await isProbablyArticle(
          data.forwardedFrom || parsedFrom.address,
--- a/packages/api/src/utils/parser.ts
+++ b/packages/api/src/utils/parser.ts
@ -450,150 +450,6 @@ export const parseUrlMetadata = async (
  }
 }

-// Attempt to determine if an HTML blob is a newsletter
-// based on it's contents.
-// TODO: when we consolidate the handlers we could include this
-// as a utility method on each one.
-export const isProbablyNewsletter = async (html: string): Promise<boolean> => {
-  const dom = parseHTML(html).document
-  const domCopy = parseHTML(dom.documentElement.outerHTML).document
-  const article = await new Readability(domCopy, {
-    debug: false,
-    keepTables: true,
-  }).parse()
-
-  if (!article || !article.content) {
-    return false
-  }
-
-  // substack newsletter emails have tables with a *post-meta class
-  if (dom.querySelector('table[class$="post-meta"]')) {
-    return true
-  }
-
-  // If the article has a header link, and substack icons its probably a newsletter
-  const href = findNewsletterHeaderHref(dom)
-  const heartIcon = dom.querySelector(
-    'table tbody td span a img[src*="HeartIcon"]'
-  )
-  const recommendIcon = dom.querySelector(
-    'table tbody td span a img[src*="RecommendIconRounded"]'
-  )
-  if (href && (heartIcon || recommendIcon)) {
-    return true
-  }
-
-  // Check if this is a beehiiv.net newsletter
-  if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) {
-    const beehiivUrl = beehiivNewsletterHref(dom)
-    if (beehiivUrl) {
-      return true
-    }
-  }
-
-  // Check if this is a newsletter from revue
-  if (
-    dom.querySelectorAll('img[src*="getrevue.co"], img[src*="revue.email"]')
-      .length > 0
-  ) {
-    const getrevueUrl = revueNewsletterHref(dom)
-    if (getrevueUrl) {
-      return true
-    }
-  }
-
-  // Check if this is a convertkit.com newsletter
-  return (
-    dom.querySelectorAll(
-      'img[src*="convertkit.com"], img[src*="convertkit-mail.com"]'
-    ).length > 0
-  )
-}
-
-const beehiivNewsletterHref = (dom: Document): string | undefined => {
-  const readOnline = dom.querySelectorAll('table tr td div a[class*="link"]')
-  let res: string | undefined = undefined
-  readOnline.forEach((e) => {
-    if (e.textContent === 'Read Online') {
-      res = e.getAttribute('href') || undefined
-    }
-  })
-  return res
-}
-
-const convertkitNewsletterHref = (dom: Document): string | undefined => {
-  const readOnline = dom.querySelectorAll('table tr td a')
-  let res: string | undefined = undefined
-  readOnline.forEach((e) => {
-    if (e.textContent === 'View this email in your browser') {
-      res = e.getAttribute('href') || undefined
-    }
-  })
-  return res
-}
-
-const revueNewsletterHref = (dom: Document): string | undefined => {
-  const viewOnline = dom.querySelectorAll('table tr td a[target="_blank"]')
-  let res: string | undefined = undefined
-  viewOnline.forEach((e) => {
-    if (e.textContent === 'View online') {
-      res = e.getAttribute('href') || undefined
-    }
-  })
-  return res
-}
-
-const findNewsletterHeaderHref = (dom: Document): string | undefined => {
-  // Substack header links
-  const postLink = dom.querySelector('h1 a ')
-  if (postLink) {
-    return postLink.getAttribute('href') || undefined
-  }
-
-  // Check if this is a beehiiv.net newsletter
-  const beehiiv = beehiivNewsletterHref(dom)
-  if (beehiiv) {
-    return beehiiv
-  }
-
-  // Check if this is a revue newsletter
-  const revue = revueNewsletterHref(dom)
-  if (revue) {
-    return revue
-  }
-
-  // Check if this is a convertkit.com newsletter
-  const convertkitUrl = convertkitNewsletterHref(dom)
-  if (convertkitUrl) {
-    return convertkitUrl
-  }
-
-  return undefined
-}
-
-// Given an HTML blob tries to find a URL to use for
-// a canonical URL.
-export const findNewsletterUrl = async (
-  html: string
-): Promise<string | undefined> => {
-  const dom = parseHTML(html).document
-
-  // Check if this is a substack newsletter
-  const href = findNewsletterHeaderHref(dom)
-  if (href) {
-    // Try to make a HEAD request so we get the redirected URL, since these
-    // will usually be behind tracking url redirects
-    return axios({
-      method: 'HEAD',
-      url: href,
-    })
-      .then((res) => res.request.res.responseUrl as string | undefined)
-      .catch((e) => href)
-  }
-
-  return undefined
-}
-
 export const isProbablyArticle = async (
  email: string,
  subject: string
@ -627,7 +483,7 @@ export const fetchFavicon = async (
 ): Promise<string | undefined> => {
  try {
    // get the correct url if it's a redirect
-    const response = await axios.get(url, { timeout: 5000 })
+    const response = await axios.head(url, { timeout: 5000 })
    const realUrl = response.request.res.responseUrl
    const domain = new URL(realUrl).hostname
    return `https://api.faviconkit.com/${domain}/32`
--- a/packages/api/test/resolvers/article.test.ts
+++ b/packages/api/test/resolvers/article.test.ts
@ -10,7 +10,11 @@ import { expect } from 'chai'
 import 'mocha'
 import { User } from '../../src/entity/user'
 import chaiString from 'chai-string'
-import { UpdateReason, UploadFileStatus } from '../../src/generated/graphql'
+import {
+  SyncUpdatedItemEdge,
+  UpdateReason,
+  UploadFileStatus,
+} from '../../src/generated/graphql'
 import {
  ArticleSavingRequestStatus,
  Highlight,
@ -1033,7 +1037,11 @@ describe('Article API', () => {
        authToken
      ).expect(200)

-      expect(res.body.data.updatesSince.edges.length).to.eql(3)
+      expect(
+        res.body.data.updatesSince.edges.filter(
+          (e: SyncUpdatedItemEdge) => e.updateReason === UpdateReason.Deleted
+        ).length
+      ).to.eql(3)
      expect(res.body.data.updatesSince.edges[0].itemID).to.eq(
        deletedPages[0].id
      )
--- a/packages/api/test/routers/emails.test.ts
+++ b/packages/api/test/routers/emails.test.ts
@ -52,35 +52,8 @@ describe('Emails Router', () => {
      sinon.restore()
    })

-    context('when email is a newsletter', () => {
-      before(() => {
-        sinon.replace(parser, 'isProbablyNewsletter', sinon.fake.resolves(true))
-      })
-
-      it('saves the email as a newsletter', async () => {
-        const data = {
-          message: {
-            data: Buffer.from(
-              JSON.stringify({ from, to, subject, html })
-            ).toString('base64'),
-            publishTime: new Date().toISOString(),
-          },
-        }
-        const res = await request
-          .post(`/svc/pubsub/emails/forward?token=${token}`)
-          .send(data)
-          .expect(200)
-        expect(res.text).to.eql('Newsletter')
-      })
-    })
-
    context('when email is an article', () => {
      before(() => {
-        sinon.replace(
-          parser,
-          'isProbablyNewsletter',
-          sinon.fake.resolves(false)
-        )
        sinon.replace(parser, 'isProbablyArticle', sinon.fake.resolves(true))
      })

@ -103,11 +76,6 @@ describe('Emails Router', () => {

    context('when email is a regular email', () => {
      before(() => {
-        sinon.replace(
-          parser,
-          'isProbablyNewsletter',
-          sinon.fake.resolves(false)
-        )
        sinon.replace(parser, 'isProbablyArticle', sinon.fake.resolves(false))
      })

--- a/packages/api/test/utils/parser.test.ts
+++ b/packages/api/test/utils/parser.test.ts
@ -4,11 +4,8 @@ import { expect } from 'chai'
 import 'chai/register-should'
 import fs from 'fs'
 import {
-  findNewsletterUrl,
-  generateUniqueUrl,
  getTitleFromEmailSubject,
  isProbablyArticle,
-  isProbablyNewsletter,
  parseEmailAddress,
  parsePageMetadata,
  parsePreparedContent,
@ -24,69 +21,6 @@ const load = (path: string): string => {
  return fs.readFileSync(path, 'utf8')
 }

-describe('isProbablyNewsletter', () => {
-  it('returns true for substack newsletter', async () => {
-    const html = load('./test/utils/data/substack-forwarded-newsletter.html')
-    await expect(isProbablyNewsletter(html)).to.eventually.be.true
-  })
-  it('returns true for private forwarded substack newsletter', async () => {
-    const html = load(
-      './test/utils/data/substack-private-forwarded-newsletter.html'
-    )
-    await expect(isProbablyNewsletter(html)).to.eventually.be.true
-  })
-  it('returns false for substack welcome email', async () => {
-    const html = load('./test/utils/data/substack-forwarded-welcome-email.html')
-    await expect(isProbablyNewsletter(html)).to.eventually.be.false
-  })
-  it('returns true for beehiiv.com newsletter', async () => {
-    const html = load('./test/utils/data/beehiiv-newsletter.html')
-    await expect(isProbablyNewsletter(html)).to.eventually.be.true
-  })
-})
-
-describe('findNewsletterUrl', async () => {
-  it('gets the URL from the header if it is a substack newsletter', async () => {
-    nock('https://email.mg2.substack.com')
-      .head(
-        '/c/eJxNkk2TojAQhn-N3KTyQfg4cGDGchdnYcsZx9K5UCE0EMVAkTiKv36iHnarupNUd7rfVJ4W3EDTj1M89No496Uw0wCxgovuwBgYnbOGsZBVjDHzKPWYU8VehUMWOlIX9Qhw4rKLzXgGZziXnRTcyF7dK0iIGMVOG_OS1aTmKPRDilgVhTQUPCQIcE0x-MFTmJ8rCUpA3KtuenR2urg1ZtAzmszI0tq_Z7m66y-ilQo0uAqMTQ7WRX8auJKg56blZg7WB-iHDuYEBzO6NP0R1IwuYFphQbbTjnTH9NBfs80nym4Zyj8uUvyKbtUyGr5eUz9fNDQ7JCxfJDo9dW1lY9lmj_JNivPbGmf2Pt_lN9tDit9b-WeTetni85Z9pDpVOd7L1E_Vy7egayNO23ZP34eSeLJeux1b0rer_xaZ7ykS78nuSjMY-nL98rparNZNcv07JCjN06_EkTFBxBqOUMACErnELUNMSxTUjLDQZwzcqa4bRjCfeejUEFefS224OLr2S5wxPtij7lVrs80d2CNseRV2P52VNFMBipcdVE-U5jkRD7hFAwpGOylVwU2Mfc9qBh7DoR89yVnWXhgQFHnIsbpVb6tU_B-hH_2yzWY'
-      )
-      .reply(302, undefined, {
-        Location:
-          'https://newsletter.slowchinese.net/p/companies-that-eat-people-217',
-      })
-      .get('/p/companies-that-eat-people-217')
-      .reply(200, '')
-    const html = load('./test/utils/data/substack-forwarded-newsletter.html')
-    const url = await findNewsletterUrl(html)
-    // Not sure if the redirects from substack expire, this test could eventually fail
-    expect(url).to.startWith(
-      'https://newsletter.slowchinese.net/p/companies-that-eat-people-217'
-    )
-  })
-  it('gets the URL from the header if it is a beehiiv newsletter', async () => {
-    nock('https://u23463625.ct.sendgrid.net')
-      .head(
-        '/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno'
-      )
-      .reply(302, undefined, {
-        Location: 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple',
-      })
-      .get('/p/talked-guy-spent-30m-beeple')
-      .reply(200, '')
-    const html = load('./test/utils/data/beehiiv-newsletter.html')
-    const url = await findNewsletterUrl(html)
-    expect(url).to.startWith(
-      'https://www.milkroad.com/p/talked-guy-spent-30m-beeple'
-    )
-  })
-  it('returns undefined if it is not a newsletter', async () => {
-    const html = load('./test/utils/data/substack-forwarded-welcome-email.html')
-    const url = await findNewsletterUrl(html)
-    expect(url).to.be.undefined
-  })
-})
-
 describe('parseMetadata', async () => {
  it('gets author, title, image, description', async () => {
    const html = load('./test/utils/data/substack-post.html')
@ -164,15 +98,6 @@ describe('isProbablyArticle', () => {
  })
 })

-describe('generateUniqueUrl', () => {
-  it('generates a unique URL', () => {
-    const url1 = generateUniqueUrl()
-    const url2 = generateUniqueUrl()
-
-    expect(url1).to.not.eql(url2)
-  })
-})
-
 describe('getTitleFromEmailSubject', () => {
  it('returns the title from the email subject', () => {
    const title = 'test subject'
--- a/packages/content-fetch/apple-news-handler.js
+++ b/packages/content-fetch/apple-news-handler.js
@ -1,36 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const Url = require('url');
-const axios = require('axios');
-const { promisify } = require('util');
-const { DateTime } = require('luxon');
-const os = require('os');
-const { Cipher } = require('crypto');
-const { parseHTML } = require('linkedom');
-
-exports.appleNewsHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    if (u.hostname === 'apple.news') {
-      return true;
-    }
-    return false
-  },
-
-  prehandle: async (url, env) => {
-    const MOBILE_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
-    const response = await axios.get(url, { headers: { 'User-Agent': MOBILE_USER_AGENT }  } );
-    const data = response.data;
-
-    const dom = parseHTML(data).document;
-
-    // make sure its a valid URL by wrapping in new URL
-    const u = new URL(dom.querySelector('span.click-here').parentNode.href);
-    return { url: u.href };
-  }
-}
--- a/packages/content-fetch/bloomberg-handler.js
+++ b/packages/content-fetch/bloomberg-handler.js
@ -1,39 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const os = require('os');
-const { parseHTML } = require('linkedom');
-
-exports.bloombergHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const BLOOMBERG_URL_MATCH =
-      /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
-    return BLOOMBERG_URL_MATCH.test(url.toString())
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling bloomberg url', url)
-
-    try {
-      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
-        params: {
-          'api_key': process.env.SCRAPINGBEE_API_KEY,
-          'url': url,
-          'return_page_source': true,
-          'block_ads': true,
-          'block_resources': false,
-        }
-      })
-      const dom = parseHTML(response.data).document;
-      return { title: dom.title, content: dom.querySelector('body').innerHTML, url: url }
-    } catch (error) {
-      console.error('error prehandling bloomberg url', error)
-      throw error
-    }
-  }
-}
--- a/packages/content-fetch/derstandard-handler.js
+++ b/packages/content-fetch/derstandard-handler.js
@ -1,35 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { parseHTML } = require('linkedom');
-
-exports.derstandardHandler = {
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    return u.hostname === 'www.derstandard.at';
-  },
-
-  prehandle: async (url, env) => {
-    const response = await axios.get(url, {
-      // set cookie to give consent to get the article
-      headers: {
-        'cookie': `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`
-      },
-    });
-    const content = response.data;
-
-    var title = undefined;
-    const dom = parseHTML(content).document;
-    const titleElement = dom.querySelector('.article-title')
-    if (!titleElement) {
-      title = titleElement.textContent
-      titleElement.remove()
-    }
-
-    return { content: dom.body.outerHTML, title: title };
-  }
-}
--- a/packages/content-fetch/fetch-content.js
+++ b/packages/content-fetch/fetch-content.js
@ -9,16 +9,10 @@ const puppeteer = require('puppeteer-core');
 const axios = require('axios');
 const jwt = require('jsonwebtoken');
 const { promisify } = require('util');
+const { parseHTML } = require('linkedom');
+const { preHandleContent } = require('@omnivore/content-handler');
+
 const signToken = promisify(jwt.sign);
-const { appleNewsHandler } = require('./apple-news-handler');
-const { twitterHandler } = require('./twitter-handler');
-const { youtubeHandler } = require('./youtube-handler');
-const { tDotCoHandler } = require('./t-dot-co-handler');
-const { pdfHandler } = require('./pdf-handler');
-const { mediumHandler } = require('./medium-handler');
-const { derstandardHandler } = require('./derstandard-handler');
-const { imageHandler } = require('./image-handler');
-const { scrapingBeeHandler } = require('./scrapingBee-handler')

 const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
 const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
@ -29,8 +23,6 @@ const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com'];

 const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf'];

-const { parseHTML } = require('linkedom');
-
 // Add stealth plugin to hide puppeteer usage
 // const StealthPlugin = require('puppeteer-extra-plugin-stealth');
 // puppeteer.use(StealthPlugin());
@ -207,19 +199,6 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId
  );
 };

-const handlers = {
-  'pdf': pdfHandler,
-  'apple-news': appleNewsHandler,
-  'twitter': twitterHandler,
-  'youtube': youtubeHandler,
-  't-dot-co': tDotCoHandler,
-  'medium': mediumHandler,
-  'derstandard': derstandardHandler,
-  'image': imageHandler,
-  'scrapingBee': scrapingBeeHandler,
-};
-
-
 async function fetchContent(req, res) {
  functionStartTime = Date.now();

@ -246,61 +225,19 @@ async function fetchContent(req, res) {
    return res.sendStatus(400);
  }

-  // if (!userId || !articleSavingRequestId) {
-  //   Object.assign(logRecord, { invalidParams: true, body: req.body, query: req.query });
-  //   console.log(`Invalid parameters`, logRecord);
-  //   return res.sendStatus(400);
-  // }
-
-  // Before we run the regular handlers we check to see if we need tp
-  // pre-resolve the URL. TODO: This should probably happen recursively,
-  // so URLs can be pre-resolved, handled, pre-resolved, handled, etc.
-  for (const [key, handler] of Object.entries(handlers)) {
-    if (handler.shouldResolve && handler.shouldResolve(url)) {
-      try {
-        url = await handler.resolve(url);
-        validateUrlString(url);
-      } catch (err) {
-        console.log('error resolving url with handler', key, err);
-      }
-      break;
-    }
-  }
-
-  // Before we fetch the page we check the handlers, to see if they want
-  // to perform a prefetch action that can modify our requests.
-  // enumerate the handlers and see if any of them want to handle the request
-  const handler = Object.keys(handlers).find(key => {
-    try {
-      return handlers[key].shouldPrehandle(url)
-    } catch (e) {
-      console.log('error with handler: ', key, e);
-    }
-    return false;
-  });
-
-  var title = undefined;
-  var content = undefined;
-  var contentType = undefined;
-
-  if (handler) {
-    try {
-      // The only handler we have now can modify the URL, but in the
-      // future maybe we let it modify content. In that case
-      // we might exit the request early.
-      console.log('pre-handling url with handler: ', handler);
-
-      const result = await handlers[handler].prehandle(url);
-      if (result && result.url) {
-        url = result.url
-        validateUrlString(url);
-      }
-      if (result && result.title) { title = result.title }
-      if (result && result.content) { content = result.content }
-      if (result && result.contentType) { contentType = result.contentType }
-    } catch (e) {
-      console.log('error with handler: ', handler, e);
+  // pre handle url with custom handlers
+  let title, content, contentType;
+  try {
+    const result = await preHandleContent(url);
+    if (result && result.url) {
+      url = result.url
+      validateUrlString(url);
    }
+    if (result && result.title) { title = result.title }
+    if (result && result.content) { content = result.content }
+    if (result && result.contentType) { contentType = result.contentType }
+  } catch (e) {
+    console.log('error with handler: ', e);
  }

  let context, page, finalUrl;
--- a/packages/content-fetch/image-handler.js
+++ b/packages/content-fetch/image-handler.js
@ -1,34 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-
-
-exports.imageHandler = {
-  shouldPrehandle: (url, env) => {
-    const IMAGE_URL_PATTERN =
-      /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
-    return IMAGE_URL_PATTERN.test(url.toString())
-  },
-
-  prehandle: async (url, env) => {
-    const title = url.toString().split('/').pop();
-    const content = `
-      <html>
-        <head>
-          <title>${title}</title>
-          <meta property="og:image" content="${url}" />
-          <meta property="og:title" content="${title}" />
-        </head>
-        <body>
-          <div>
-            <img src="${url}" alt="${title}">
-          </div>
-        </body>
-      </html>`
-
-    return { title, content };
-  }
-}
--- a/packages/content-fetch/medium-handler.js
+++ b/packages/content-fetch/medium-handler.js
@ -1,29 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const os = require('os');
-
-exports.mediumHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    return u.hostname.endsWith('medium.com')
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling medium url', url)
-
-    try {
-      const res = new URL(url);
-      res.searchParams.delete('source');
-      return { url: res.toString() }
-    } catch (error) {
-      console.error('error prehandling medium url', error)
-      throw error
-    }
-  }
-}
--- a/packages/content-fetch/package.json
+++ b/packages/content-fetch/package.json
@ -11,7 +11,8 @@
    "linkedom": "^0.14.9",
    "luxon": "^2.3.1",
    "puppeteer-core": "^16.1.0",
-    "underscore": "^1.13.4"
+    "underscore": "^1.13.4",
+    "@omnivore/content-handler": "1.0.0"
  },
  "scripts": {
    "start": "node app.js",
--- a/packages/content-fetch/pdf-handler.js
+++ b/packages/content-fetch/pdf-handler.js
@ -1,21 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const Url = require('url');
-
-
-exports.pdfHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = Url.parse(url)
-    const path = u.path.replace(u.search, '')
-    return path.endsWith('.pdf')
-  },
-
-  prehandle: async (url, env) => {
-    return { contentType: 'application/pdf' };
-  }
-}
--- a/packages/content-fetch/scrapingBee-handler.js
+++ b/packages/content-fetch/scrapingBee-handler.js
@ -1,44 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { parseHTML } = require('linkedom');
-
-const os = require('os');
-
-exports.scrapingBeeHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    const hostnames = [
-      'nytimes.com',
-      'news.google.com',
-    ]
-
-    return hostnames.some((h) => u.hostname.endsWith(h))
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling url with scrapingbee', url)
-
-    try {
-      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
-        params: {
-          'api_key': process.env.SCRAPINGBEE_API_KEY,
-          'url': url,
-          'return_page_source': true,
-          'block_ads': true,
-          'block_resources': false,
-        }
-      })
-      const dom = parseHTML(response.data).document;
-      return { title: dom.title, content: response.data, url: url }
-    } catch (error) {
-      console.error('error prehandling url w/scrapingbee', error)
-      throw error
-    }
-  }
-}
--- a/packages/content-fetch/t-dot-co-handler.js
+++ b/packages/content-fetch/t-dot-co-handler.js
@ -1,31 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const Url = require('url');
-
-
-exports.tDotCoHandler = {
-
-  shouldResolve: function (url, env) {
-    const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/;
-    return T_DOT_CO_URL_MATCH.test(url);
-  },
-
-  resolve: async function(url, env) {
-    return await axios.get(url, { maxRedirects: 0, validateStatus: null })
-      .then(res => {
-        return Url.parse(res.headers.location).href;
-      }).catch((err) => {
-        console.log('err with t.co url', err);
-        return undefined;
-      });
-  },
-
-  shouldPrehandle: (url, env) => {
-    return false
-  },
-}
--- a/packages/content-fetch/test/apple-news-handler.test.js
+++ b/packages/content-fetch/test/apple-news-handler.test.js
@ -1,9 +0,0 @@
-const { expect } = require('chai')
-const { appleNewsHandler } = require('../apple-news-handler')
-
-describe('open a simple web page', () => {
-  it('should return a response', async () => {
-    const response = await appleNewsHandler.prehandle('https://apple.news/AxjzaZaPvSn23b67LhXI5EQ')
-    console.log('response', response)
-  })
-})
--- a/packages/content-fetch/test/babel-register.js
+++ b/packages/content-fetch/test/babel-register.js
@ -0,0 +1,3 @@
+const register = require('@babel/register').default
+
+register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
--- a/packages/content-fetch/test/stub.test.ts
+++ b/packages/content-fetch/test/stub.test.ts
@ -0,0 +1,13 @@
+import 'mocha'
+import * as chai from 'chai'
+import { expect } from 'chai'
+import 'chai/register-should'
+import chaiString from 'chai-string'
+
+chai.use(chaiString)
+
+describe('Stub test', () => {
+  it('should pass', () => {
+    expect(true).to.be.true
+  })
+})
--- a/packages/content-fetch/test/youtube-handler.test.js
+++ b/packages/content-fetch/test/youtube-handler.test.js
@ -1,12 +0,0 @@
-const { expect } = require('chai')
-const { getYoutubeVideoId } = require('../youtube-handler')
-
-describe('getYoutubeVideoId', () => {
-  it('should parse video id out of a URL', async () => {
-    expect('BnSUk0je6oo').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s'));
-    expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1'));
-    expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://youtu.be/vFD2gu007dc'));
-    expect('BMFVCnbRaV4').to.eq(getYoutubeVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share'));
-    expect('cg9b4RC87LI').to.eq(getYoutubeVideoId('https://youtu.be/cg9b4RC87LI?t=116'));
-  })
-})
--- a/packages/content-fetch/twitter-handler.js
+++ b/packages/content-fetch/twitter-handler.js
@ -1,172 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { DateTime } = require('luxon');
-const _ = require('underscore');
-
-const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN;
-const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
-
-const embeddedTweet = async (url) => {
-
-  const BASE_ENDPOINT = 'https://publish.twitter.com/oembed'
-
-  const apiUrl = new URL(BASE_ENDPOINT)
-  apiUrl.searchParams.append('url', url);
-  apiUrl.searchParams.append('omit_script', true);
-  apiUrl.searchParams.append('dnt', true);
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const getTweetFields = () => {
-  const TWEET_FIELDS =
-  "&tweet.fields=attachments,author_id,conversation_id,created_at," +
-  "entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets," +
-  "source,withheld";
-  const EXPANSIONS = "&expansions=author_id,attachments.media_keys";
-  const USER_FIELDS =
-  "&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld";
-  const MEDIA_FIELDS =
-  "&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width";
-
-  return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`;
-}
-
-const getTweetById = async (id) => {
-  const BASE_ENDPOINT = "https://api.twitter.com/2/tweets/";
-  const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const getUserByUsername = async (username) => {
-  const BASE_ENDPOINT = "https://api.twitter.com/2/users/by/username/";
-
-  const apiUrl = new URL(BASE_ENDPOINT + username)
-  apiUrl.searchParams.append('user.fields', 'profile_image_url');
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const titleForTweet = (tweet) => {
-  return `${tweet.data.author_name} on Twitter`
-};
-
-const titleForAuthor = (author) => {
-  return `${author.name} on Twitter`
-};
-
-const usernameFromStatusUrl = (url) => {
-  const match = url.toString().match(TWITTER_URL_MATCH)
-  return match[1]
-};
-
-const tweetIdFromStatusUrl = (url) => {
-  const match = url.toString().match(TWITTER_URL_MATCH)
-  return match[2]
-};
-
-const formatTimestamp = (timestamp) => {
-  return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(DateTime.DATETIME_FULL);
-};
-
-exports.twitterHandler = {
-
-  shouldPrehandle: (url, env) => {
-    return TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
-  },
-
-  // version of the handler that uses the oembed API
-  // This isn't great as it doesn't work well with our
-  // readability API. But could potentially give a more consistent
-  // look to the tweets
-  // prehandle: async (url, env) => {
-  //   const oeTweet = await embeddedTweet(url)
-  //   const dom = new JSDOM(oeTweet.data.html);
-  //   const bq = dom.window.document.querySelector('blockquote')
-  //   console.log('blockquote:', bq);
-
-  //   const title = titleForTweet(oeTweet)
-  //   return { title, content: '<div>' + bq.innerHTML + '</div>', url: oeTweet.data.url };
-  // }
-
-  prehandle: async (url, env) => {
-    console.log('prehandling twitter url', url)
-
-    const tweetId = tweetIdFromStatusUrl(url)
-    const tweetData = (await getTweetById(tweetId)).data;
-    const authorId = tweetData.data.author_id;
-    const author = tweetData.includes.users.filter(u => u.id = authorId)[0];
-    // escape html entities in title
-    const title = _.escape(titleForAuthor(author))
-    const authorImage = author.profile_image_url.replace('_normal', '_400x400')
-
-    let text = tweetData.data.text;
-    if (tweetData.data.entities && tweetData.data.entities.urls) {
-      for (let urlObj of tweetData.data.entities.urls) {
-        text = text.replace(
-          urlObj.url,
-          `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
-        );
-      }
-    }
-
-    const front = `
-    <div>
-      <p>${text}</p>
-    `
-
-    var includesHtml =  '';
-    if (tweetData.includes.media) {
-      includesHtml = tweetData.includes.media.map(m => {
-        const linkUrl = m.type == 'photo' ? m.url : url;
-        const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url;
-        const mediaOpen = `<a class="media-link" href=${linkUrl}>
-          <picture>
-            <img class="tweet-img" src=${previewUrl} />
-          </picture>
-          </a>`
-        return mediaOpen
-      }).join('\n');
-   }
-
-   const back = `
-       — <a href="https://twitter.com/${author.username}">${author.username}</a> ${author.name} <a href="${url}">${formatTimestamp(tweetData.data.created_at)}</a>
-    </div>
-    `
-    const content = `
-    <head>
-      <meta property="og:image" content="${authorImage}" />
-      <meta property="og:image:secure_url" content="${authorImage}" />
-      <meta property="og:title" content="${title}" />
-      <meta property="og:description" content="${_.escape(tweetData.data.text)}" />
-    </head>
-    <body>
-      ${front}
-      ${includesHtml}
-      ${back}
-    </body>`
-
-    return { content, url, title };
-  }
-}
--- a/packages/content-handler/.eslintignore
+++ b/packages/content-handler/.eslintignore
@ -0,0 +1 @@
+node_modules/
--- a/packages/content-handler/.eslintrc
+++ b/packages/content-handler/.eslintrc
@ -0,0 +1,6 @@
+{
+  "extends": "../../.eslintrc",
+  "parserOptions": {
+    "project": "tsconfig.json"
+  }
+}
--- a/packages/content-handler/.gitignore
+++ b/packages/content-handler/.gitignore
@ -0,0 +1,2 @@
+node_modules
+/lib
--- a/packages/content-handler/.npmignore
+++ b/packages/content-handler/.npmignore
@ -0,0 +1,7 @@
+/test/
+src
+tsconfig.json
+.eslintrc
+.eslintignore
+.gitignore
+mocha-config.json
--- a/packages/content-handler/mocha-config.json
+++ b/packages/content-handler/mocha-config.json
@ -0,0 +1,5 @@
+{
+    "extension": ["ts"],
+    "spec": "test/**/*.test.ts",
+    "require": "test/babel-register.js"
+  }
--- a/packages/content-handler/package.json
+++ b/packages/content-handler/package.json
@ -0,0 +1,34 @@
+{
+  "name": "@omnivore/content-handler",
+  "version": "1.0.0",
+  "description": "A standalone version of content handler to parse and format each type of content",
+  "main": "build/src/index.js",
+  "types": "build/src/index.d.ts",
+  "files": [
+    "build/src"
+  ],
+  "license": "Apache-2.0",
+  "scripts": {
+    "test": "yarn mocha -r ts-node/register --config mocha-config.json",
+    "lint": "eslint src --ext ts,js,tsx,jsx",
+    "compile": "tsc",
+    "build": "tsc"
+  },
+  "devDependencies": {
+    "chai": "^4.3.6",
+    "chai-as-promised": "^7.1.1",
+    "chai-string": "^1.5.0",
+    "eslint-plugin-prettier": "^4.0.0",
+    "mocha": "^10.0.0",
+    "nock": "^13.2.9"
+  },
+  "dependencies": {
+    "addressparser": "^1.0.1",
+    "axios": "^0.27.2",
+    "linkedom": "^0.14.16",
+    "luxon": "^3.0.4",
+    "rfc2047": "^4.0.1",
+    "underscore": "^1.13.6",
+    "uuid": "^9.0.0"
+  }
+}
--- a/packages/content-handler/src/content-handler.ts
+++ b/packages/content-handler/src/content-handler.ts
@ -0,0 +1,175 @@
+import addressparser from 'addressparser'
+import rfc2047 from 'rfc2047'
+import { v4 as uuid } from 'uuid'
+import { parseHTML } from 'linkedom'
+import axios from 'axios'
+
+interface Unsubscribe {
+  mailTo?: string
+  httpUrl?: string
+}
+
+export interface NewsletterInput {
+  postHeader: string
+  from: string
+  unSubHeader: string
+  email: string
+  html: string
+  title: string
+}
+
+export interface NewsletterResult {
+  email: string
+  content: string
+  url: string
+  title: string
+  author: string
+  unsubMailTo?: string
+  unsubHttpUrl?: string
+}
+
+export interface PreHandleResult {
+  url?: string
+  title?: string
+  content?: string
+  contentType?: string
+  dom?: Document
+}
+
+export const FAKE_URL_PREFIX = 'https://omnivore.app/no_url?q='
+export const generateUniqueUrl = () => FAKE_URL_PREFIX + uuid()
+
+export abstract class ContentHandler {
+  protected senderRegex: RegExp
+  protected urlRegex: RegExp
+  name: string
+
+  protected constructor() {
+    this.senderRegex = new RegExp(/NEWSLETTER_SENDER_REGEX/)
+    this.urlRegex = new RegExp(/NEWSLETTER_URL_REGEX/)
+    this.name = 'Handler name'
+  }
+
+  shouldResolve(url: string): boolean {
+    return false
+  }
+
+  async resolve(url: string): Promise<string | undefined> {
+    return Promise.resolve(url)
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    return false
+  }
+
+  async preHandle(url: string, dom?: Document): Promise<PreHandleResult> {
+    return Promise.resolve({ url, dom })
+  }
+
+  async isNewsletter(input: {
+    postHeader: string
+    from: string
+    unSubHeader: string
+    html?: string
+  }): Promise<boolean> {
+    const re = new RegExp(this.senderRegex)
+    return Promise.resolve(
+      re.test(input.from) && (!!input.postHeader || !!input.unSubHeader)
+    )
+  }
+
+  findNewsletterHeaderHref(dom: Document): string | undefined {
+    return undefined
+  }
+
+  // Given an HTML blob tries to find a URL to use for
+  // a canonical URL.
+  async findNewsletterUrl(html: string): Promise<string | undefined> {
+    const dom = parseHTML(html).document
+
+    // Check if this is a substack newsletter
+    const href = this.findNewsletterHeaderHref(dom)
+    if (href) {
+      // Try to make a HEAD request, so we get the redirected URL, since these
+      // will usually be behind tracking url redirects
+      try {
+        const response = await axios.head(href, { timeout: 5000 })
+        return Promise.resolve(
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+          response.request.res.responseUrl as string | undefined
+        )
+      } catch (e) {
+        console.log('error making HEAD request', e)
+        return Promise.resolve(href)
+      }
+    }
+
+    return Promise.resolve(undefined)
+  }
+
+  async parseNewsletterUrl(
+    _postHeader: string,
+    html: string
+  ): Promise<string | undefined> {
+    // get newsletter url from html
+    const matches = html.match(this.urlRegex)
+    if (matches) {
+      return Promise.resolve(matches[1])
+    }
+    return Promise.resolve(undefined)
+  }
+
+  parseAuthor(from: string): string {
+    // get author name from email
+    // e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
+    // or 'Mike Allen <mike@axios.com>'
+    const parsed = addressparser(from)
+    if (parsed.length > 0) {
+      return parsed[0].name
+    }
+    return from
+  }
+
+  parseUnsubscribe(unSubHeader: string): Unsubscribe {
+    // parse list-unsubscribe header
+    // e.g. List-Unsubscribe: <https://omnivore.com/unsub>, <mailto:unsub@omnivore.com>
+    const decoded = rfc2047.decode(unSubHeader)
+    return {
+      mailTo: decoded.match(/<(https?:\/\/[^>]*)>/)?.[1],
+      httpUrl: decoded.match(/<mailto:([^>]*)>/)?.[1],
+    }
+  }
+
+  async handleNewsletter({
+    email,
+    html,
+    postHeader,
+    title,
+    from,
+    unSubHeader,
+  }: NewsletterInput): Promise<NewsletterResult> {
+    console.log('handleNewsletter', email, postHeader, title, from)
+
+    if (!email || !html || !title || !from) {
+      console.log('invalid newsletter email')
+      throw new Error('invalid newsletter email')
+    }
+
+    // fallback to default url if newsletter url does not exist
+    // assign a random uuid to the default url to avoid duplicate url
+    const url =
+      (await this.parseNewsletterUrl(postHeader, html)) || generateUniqueUrl()
+    const author = this.parseAuthor(from)
+    const unsubscribe = this.parseUnsubscribe(unSubHeader)
+
+    return {
+      email,
+      content: html,
+      url,
+      title,
+      author,
+      unsubMailTo: unsubscribe.mailTo || '',
+      unsubHttpUrl: unsubscribe.httpUrl || '',
+    }
+  }
+}
--- a/packages/content-handler/src/index.ts
+++ b/packages/content-handler/src/index.ts
@ -0,0 +1,116 @@
+import { AppleNewsHandler } from './websites/apple-news-handler'
+import { BloombergHandler } from './websites/bloomberg-handler'
+import { DerstandardHandler } from './websites/derstandard-handler'
+import { ImageHandler } from './websites/image-handler'
+import { MediumHandler } from './websites/medium-handler'
+import { PdfHandler } from './websites/pdf-handler'
+import { ScrapingBeeHandler } from './websites/scrapingBee-handler'
+import { TDotCoHandler } from './websites/t-dot-co-handler'
+import { TwitterHandler } from './websites/twitter-handler'
+import { YoutubeHandler } from './websites/youtube-handler'
+import { WikipediaHandler } from './websites/wikipedia-handler'
+import {
+  ContentHandler,
+  NewsletterInput,
+  NewsletterResult,
+  PreHandleResult,
+} from './content-handler'
+import { SubstackHandler } from './newsletters/substack-handler'
+import { AxiosHandler } from './newsletters/axios-handler'
+import { GolangHandler } from './newsletters/golang-handler'
+import { MorningBrewHandler } from './newsletters/morning-brew-handler'
+import { BloombergNewsletterHandler } from './newsletters/bloomberg-newsletter-handler'
+import { BeehiivHandler } from './newsletters/beehiiv-handler'
+import { ConvertkitHandler } from './newsletters/convertkit-handler'
+import { RevueHandler } from './newsletters/revue-handler'
+
+const validateUrlString = (url: string) => {
+  const u = new URL(url)
+  // Make sure the URL is http or https
+  if (u.protocol !== 'http:' && u.protocol !== 'https:') {
+    throw new Error('Invalid URL protocol check failed')
+  }
+  // Make sure the domain is not localhost
+  if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') {
+    throw new Error('Invalid URL is localhost')
+  }
+  // Make sure the domain is not a private IP
+  if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) {
+    throw new Error('Invalid URL is private ip')
+  }
+}
+
+const contentHandlers: ContentHandler[] = [
+  new AppleNewsHandler(),
+  new BloombergHandler(),
+  new DerstandardHandler(),
+  new ImageHandler(),
+  new MediumHandler(),
+  new PdfHandler(),
+  new ScrapingBeeHandler(),
+  new TDotCoHandler(),
+  new TwitterHandler(),
+  new YoutubeHandler(),
+  new WikipediaHandler(),
+]
+
+const newsletterHandlers: ContentHandler[] = [
+  new AxiosHandler(),
+  new BloombergNewsletterHandler(),
+  new GolangHandler(),
+  new SubstackHandler(),
+  new MorningBrewHandler(),
+  new SubstackHandler(),
+  new BeehiivHandler(),
+  new ConvertkitHandler(),
+  new RevueHandler(),
+]
+
+export const preHandleContent = async (
+  url: string,
+  dom?: Document
+): Promise<PreHandleResult | undefined> => {
+  // Before we run the regular handlers we check to see if we need tp
+  // pre-resolve the URL. TODO: This should probably happen recursively,
+  // so URLs can be pre-resolved, handled, pre-resolved, handled, etc.
+  for (const handler of contentHandlers) {
+    if (handler.shouldResolve(url)) {
+      try {
+        const resolvedUrl = await handler.resolve(url)
+        if (resolvedUrl && validateUrlString(resolvedUrl)) {
+          url = resolvedUrl
+        }
+      } catch (err) {
+        console.log('error resolving url with handler', handler.name, err)
+      }
+      break
+    }
+  }
+  // Before we fetch the page we check the handlers, to see if they want
+  // to perform a prefetch action that can modify our requests.
+  // enumerate the handlers and see if any of them want to handle the request
+  for (const handler of contentHandlers) {
+    if (handler.shouldPreHandle(url, dom)) {
+      console.log('preHandleContent', handler.name, url)
+      return handler.preHandle(url, dom)
+    }
+  }
+  return undefined
+}
+
+export const handleNewsletter = async (
+  input: NewsletterInput
+): Promise<NewsletterResult | undefined> => {
+  for (const handler of newsletterHandlers) {
+    if (await handler.isNewsletter(input)) {
+      return handler.handleNewsletter(input)
+    }
+  }
+
+  return undefined
+}
+
+module.exports = {
+  preHandleContent,
+  handleNewsletter,
+}
--- a/packages/content-handler/src/newsletters/axios-handler.ts
+++ b/packages/content-handler/src/newsletters/axios-handler.ts
@ -0,0 +1,46 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class AxiosHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@axios.com>/
+    this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
+    this.name = 'axios'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const host = this.name + '.com'
+    // check if url ends with axios.com
+    return new URL(url).hostname.endsWith(host)
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    const body = dom.querySelector('table')
+
+    let isFooter = false
+    // this removes ads and replaces table with a div
+    body?.querySelectorAll('table').forEach((el) => {
+      // remove the footer and the ads
+      if (!el.textContent || el.textContent.length < 20 || isFooter) {
+        el.remove()
+      } else {
+        // removes the first few rows of the table (the header)
+        // remove the last two rows of the table (they are ads)
+        el.querySelectorAll('tr').forEach((tr, i) => {
+          if (i <= 7 || i >= el.querySelectorAll('tr').length - 2) {
+            console.log('removing', tr)
+            tr.remove()
+          }
+        })
+        // replace the table with a div
+        const div = dom.createElement('div')
+        div.innerHTML = el.innerHTML
+        el.parentNode?.replaceChild(div, el)
+        // set the isFooter flag to true because the next table is the footer
+        isFooter = true
+      }
+    })
+
+    return Promise.resolve({ dom })
+  }
+}
--- a/packages/content-handler/src/newsletters/beehiiv-handler.ts
+++ b/packages/content-handler/src/newsletters/beehiiv-handler.ts
@ -0,0 +1,43 @@
+import { ContentHandler } from '../content-handler'
+import { parseHTML } from 'linkedom'
+
+export class BeehiivHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'beehiiv'
+  }
+
+  findNewsletterHeaderHref(dom: Document): string | undefined {
+    const readOnline = dom.querySelectorAll('table tr td div a[class*="link"]')
+    let res: string | undefined = undefined
+    readOnline.forEach((e) => {
+      if (e.textContent === 'Read Online') {
+        res = e.getAttribute('href') || undefined
+      }
+    })
+    return res
+  }
+
+  async isNewsletter(input: {
+    postHeader: string
+    from: string
+    unSubHeader: string
+    html: string
+  }): Promise<boolean> {
+    const dom = parseHTML(input.html).document
+    if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) {
+      const beehiivUrl = this.findNewsletterHeaderHref(dom)
+      if (beehiivUrl) {
+        return Promise.resolve(true)
+      }
+    }
+    return false
+  }
+
+  async parseNewsletterUrl(
+    postHeader: string,
+    html: string
+  ): Promise<string | undefined> {
+    return this.findNewsletterUrl(html)
+  }
+}
--- a/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts
+++ b/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts
@ -0,0 +1,37 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class BloombergNewsletterHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@mail.bloomberg.*.com>/
+    this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
+    this.name = 'bloomberg'
+  }
+
+  shouldPreHandle(url: string, dom: Document): boolean {
+    const host = this.name + '.com'
+    // check if url ends with bloomberg.com
+    return (
+      new URL(url).hostname.endsWith(host) ||
+      dom.querySelector('.logo-image')?.getAttribute('alt')?.toLowerCase() ===
+        this.name
+    )
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    const body = dom.querySelector('.wrapper')
+
+    // this removes header
+    body?.querySelector('.sailthru-variables')?.remove()
+    body?.querySelector('.preview-text')?.remove()
+    body?.querySelector('.logo-wrapper')?.remove()
+    body?.querySelector('.by-the-number-wrapper')?.remove()
+    // this removes footer
+    body?.querySelector('.quote-box-wrapper')?.remove()
+    body?.querySelector('.header-wrapper')?.remove()
+    body?.querySelector('.component-wrapper')?.remove()
+    body?.querySelector('.footer')?.remove()
+
+    return Promise.resolve({ dom })
+  }
+}
--- a/packages/content-handler/src/newsletters/convertkit-handler.ts
+++ b/packages/content-handler/src/newsletters/convertkit-handler.ts
@ -0,0 +1,41 @@
+import { ContentHandler } from '../content-handler'
+import { parseHTML } from 'linkedom'
+
+export class ConvertkitHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'convertkit'
+  }
+
+  findNewsletterHeaderHref(dom: Document): string | undefined {
+    const readOnline = dom.querySelectorAll('table tr td a')
+    let res: string | undefined = undefined
+    readOnline.forEach((e) => {
+      if (e.textContent === 'View this email in your browser') {
+        res = e.getAttribute('href') || undefined
+      }
+    })
+    return res
+  }
+
+  async isNewsletter(input: {
+    postHeader: string
+    from: string
+    unSubHeader: string
+    html: string
+  }): Promise<boolean> {
+    const dom = parseHTML(input.html).document
+    return Promise.resolve(
+      dom.querySelectorAll(
+        'img[src*="convertkit.com"], img[src*="convertkit-mail.com"]'
+      ).length > 0
+    )
+  }
+
+  async parseNewsletterUrl(
+    postHeader: string,
+    html: string
+  ): Promise<string | undefined> {
+    return this.findNewsletterUrl(html)
+  }
+}
--- a/packages/content-handler/src/newsletters/golang-handler.ts
+++ b/packages/content-handler/src/newsletters/golang-handler.ts
@ -0,0 +1,27 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class GolangHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@golangweekly.com>/
+    this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
+    this.name = 'golangweekly'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const host = this.name + '.com'
+    // check if url ends with golangweekly.com
+    return new URL(url).hostname.endsWith(host)
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    const body = dom.querySelector('body')
+
+    // this removes the "Subscribe" button
+    body?.querySelector('.el-splitbar')?.remove()
+    // this removes the title
+    body?.querySelector('.el-masthead')?.remove()
+
+    return Promise.resolve({ dom })
+  }
+}
--- a/packages/content-handler/src/newsletters/morning-brew-handler.ts
+++ b/packages/content-handler/src/newsletters/morning-brew-handler.ts
@ -0,0 +1,35 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class MorningBrewHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /Morning Brew <crew@morningbrew.com>/
+    this.urlRegex = /<a.* href=["']([^"']*)["'].*>View Online<\/a>/
+    this.name = 'morningbrew'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const host = this.name + '.com'
+    // check if url ends with morningbrew.com
+    return new URL(url).hostname.endsWith(host)
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    // retain the width of the cells in the table of market info
+    dom.querySelectorAll('.markets-arrow-cell').forEach((td) => {
+      const table = td.closest('table')
+      if (table) {
+        const bubbleTable = table.querySelector('.markets-bubble')
+        if (bubbleTable) {
+          // replace the nested table with the text
+          const e = bubbleTable.querySelector('.markets-table-text')
+          e && bubbleTable.parentNode?.replaceChild(e, bubbleTable)
+        }
+        // set custom class for the table
+        table.className = 'morning-brew-markets'
+      }
+    })
+
+    return Promise.resolve({ dom })
+  }
+}
--- a/packages/content-handler/src/newsletters/revue-handler.ts
+++ b/packages/content-handler/src/newsletters/revue-handler.ts
@ -0,0 +1,46 @@
+import { ContentHandler } from '../content-handler'
+import { parseHTML } from 'linkedom'
+
+export class RevueHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'revue'
+  }
+
+  findNewsletterHeaderHref(dom: Document): string | undefined {
+    const viewOnline = dom.querySelectorAll('table tr td a[target="_blank"]')
+    let res: string | undefined = undefined
+    viewOnline.forEach((e) => {
+      if (e.textContent === 'View online') {
+        res = e.getAttribute('href') || undefined
+      }
+    })
+    return res
+  }
+
+  async isNewsletter(input: {
+    postHeader: string
+    from: string
+    unSubHeader: string
+    html: string
+  }): Promise<boolean> {
+    const dom = parseHTML(input.html).document
+    if (
+      dom.querySelectorAll('img[src*="getrevue.co"], img[src*="revue.email"]')
+        .length > 0
+    ) {
+      const getrevueUrl = this.findNewsletterHeaderHref(dom)
+      if (getrevueUrl) {
+        return Promise.resolve(true)
+      }
+    }
+    return false
+  }
+
+  async parseNewsletterUrl(
+    postHeader: string,
+    html: string
+  ): Promise<string | undefined> {
+    return this.findNewsletterUrl(html)
+  }
+}
--- a/packages/content-handler/src/newsletters/substack-handler.ts
+++ b/packages/content-handler/src/newsletters/substack-handler.ts
@ -0,0 +1,90 @@
+import addressparser from 'addressparser'
+import { ContentHandler, PreHandleResult } from '../content-handler'
+import { parseHTML } from 'linkedom'
+
+export class SubstackHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'substack'
+  }
+
+  shouldPreHandle(url: string, dom: Document): boolean {
+    const host = this.name + '.com'
+    // check if url ends with substack.com
+    // or has a profile image hosted at substack.com
+    return (
+      new URL(url).hostname.endsWith(host) ||
+      !!dom
+        .querySelector('.email-body img')
+        ?.getAttribute('src')
+        ?.includes(host)
+    )
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    const body = dom.querySelector('.email-body-container')
+
+    // this removes header and profile avatar
+    body?.querySelector('.header')?.remove()
+    body?.querySelector('.preamble')?.remove()
+    body?.querySelector('.meta-author-wrap')?.remove()
+    // this removes meta button
+    body?.querySelector('.post-meta')?.remove()
+    // this removes footer
+    body?.querySelector('.post-cta')?.remove()
+    body?.querySelector('.container-border')?.remove()
+    body?.querySelector('.footer')?.remove()
+
+    return Promise.resolve(dom)
+  }
+
+  findNewsletterHeaderHref(dom: Document): string | undefined {
+    // Substack header links
+    const postLink = dom.querySelector('h1 a ')
+    if (postLink) {
+      return postLink.getAttribute('href') || undefined
+    }
+
+    return undefined
+  }
+
+  async isNewsletter({
+    postHeader,
+    html,
+  }: {
+    postHeader: string
+    from: string
+    unSubHeader: string
+    html: string
+  }): Promise<boolean> {
+    if (postHeader) {
+      return Promise.resolve(true)
+    }
+    const dom = parseHTML(html).document
+    // substack newsletter emails have tables with a *post-meta class
+    if (dom.querySelector('table[class$="post-meta"]')) {
+      return true
+    }
+    // If the article has a header link, and substack icons its probably a newsletter
+    const href = this.findNewsletterHeaderHref(dom)
+    const heartIcon = dom.querySelector(
+      'table tbody td span a img[src*="HeartIcon"]'
+    )
+    const recommendIcon = dom.querySelector(
+      'table tbody td span a img[src*="RecommendIconRounded"]'
+    )
+    return Promise.resolve(!!(href && (heartIcon || recommendIcon)))
+  }
+
+  async parseNewsletterUrl(
+    postHeader: string,
+    html: string
+  ): Promise<string | undefined> {
+    // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
+    // we need to get the real url from the raw url
+    if (postHeader && addressparser(postHeader).length > 0) {
+      return Promise.resolve(addressparser(postHeader)[0].name)
+    }
+    return this.findNewsletterUrl(html)
+  }
+}
--- a/packages/content-handler/src/websites/apple-news-handler.ts
+++ b/packages/content-handler/src/websites/apple-news-handler.ts
@ -0,0 +1,31 @@
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class AppleNewsHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Apple News'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const u = new URL(url)
+    return u.hostname === 'apple.news'
+  }
+
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+    const MOBILE_USER_AGENT =
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
+    const response = await axios.get(url, {
+      headers: { 'User-Agent': MOBILE_USER_AGENT },
+    })
+    const data = response.data as string
+    const dom = parseHTML(data).document
+    // make sure it's a valid URL by wrapping in new URL
+    const href = dom
+      .querySelector('span.click-here')
+      ?.parentElement?.getAttribute('href')
+    const u = href ? new URL(href) : undefined
+    return { url: u?.href }
+  }
+}
--- a/packages/content-handler/src/websites/bloomberg-handler.ts
+++ b/packages/content-handler/src/websites/bloomberg-handler.ts
@ -0,0 +1,41 @@
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class BloombergHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Bloomberg'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const BLOOMBERG_URL_MATCH =
+      /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)/
+    return BLOOMBERG_URL_MATCH.test(url.toString())
+  }
+
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+    console.log('prehandling bloomberg url', url)
+
+    try {
+      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+        params: {
+          api_key: process.env.SCRAPINGBEE_API_KEY,
+          url: url,
+          return_page_source: true,
+          block_ads: true,
+          block_resources: false,
+        },
+      })
+      const dom = parseHTML(response.data).document
+      return {
+        title: dom.title,
+        content: dom.querySelector('body')?.innerHTML,
+        url: url,
+      }
+    } catch (error) {
+      console.error('error prehandling bloomberg url', error)
+      throw error
+    }
+  }
+}
--- a/packages/content-handler/src/websites/derstandard-handler.ts
+++ b/packages/content-handler/src/websites/derstandard-handler.ts
@ -0,0 +1,34 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+export class DerstandardHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Derstandard'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const u = new URL(url)
+    return u.hostname === 'www.derstandard.at'
+  }
+
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+    const response = await axios.get(url, {
+      // set cookie to give consent to get the article
+      headers: {
+        cookie: `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`,
+      },
+    })
+    const content = response.data as string
+
+    const dom = parseHTML(content).document
+    const titleElement = dom.querySelector('.article-title')
+    titleElement && titleElement.remove()
+
+    return {
+      content: dom.body.outerHTML,
+      title: titleElement?.textContent || undefined,
+    }
+  }
+}
--- a/packages/content-handler/src/websites/image-handler.ts
+++ b/packages/content-handler/src/websites/image-handler.ts
@ -0,0 +1,32 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class ImageHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Image'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const IMAGE_URL_PATTERN = /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
+    return IMAGE_URL_PATTERN.test(url.toString())
+  }
+
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+    const title = url.toString().split('/').pop() || 'Image'
+    const content = `
+      <html>
+        <head>
+          <title>${title}</title>
+          <meta property="og:image" content="${url}" />
+          <meta property="og:title" content="${title}" />
+        </head>
+        <body>
+          <div>
+            <img src="${url}" alt="${title}">
+          </div>
+        </body>
+      </html>`
+
+    return Promise.resolve({ title, content })
+  }
+}
--- a/packages/content-handler/src/websites/medium-handler.ts
+++ b/packages/content-handler/src/websites/medium-handler.ts
@ -0,0 +1,26 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class MediumHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Medium'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const u = new URL(url)
+    return u.hostname.endsWith('medium.com')
+  }
+
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+    console.log('prehandling medium url', url)
+
+    try {
+      const res = new URL(url)
+      res.searchParams.delete('source')
+      return Promise.resolve({ url: res.toString() })
+    } catch (error) {
+      console.error('error prehandling medium url', error)
+      throw error
+    }
+  }
+}
--- a/packages/content-handler/src/websites/pdf-handler.ts
+++ b/packages/content-handler/src/websites/pdf-handler.ts
@ -0,0 +1,18 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class PdfHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'PDF'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const u = new URL(url)
+    const path = u.pathname.replace(u.search, '')
+    return path.endsWith('.pdf')
+  }
+
+  async preHandle(_url: string, document?: Document): Promise<PreHandleResult> {
+    return Promise.resolve({ contentType: 'application/pdf' })
+  }
+}
--- a/packages/content-handler/src/websites/scrapingBee-handler.ts
+++ b/packages/content-handler/src/websites/scrapingBee-handler.ts
@ -0,0 +1,38 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+export class ScrapingBeeHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'ScrapingBee'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    const u = new URL(url)
+    const hostnames = ['nytimes.com', 'news.google.com']
+
+    return hostnames.some((h) => u.hostname.endsWith(h))
+  }
+
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+    console.log('prehandling url with scrapingbee', url)
+
+    try {
+      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+        params: {
+          api_key: process.env.SCRAPINGBEE_API_KEY,
+          url: url,
+          return_page_source: true,
+          block_ads: true,
+          block_resources: false,
+        },
+      })
+      const dom = parseHTML(response.data).document
+      return { title: dom.title, content: response.data as string, url: url }
+    } catch (error) {
+      console.error('error prehandling url w/scrapingbee', error)
+      throw error
+    }
+  }
+}
--- a/packages/content-handler/src/websites/t-dot-co-handler.ts
+++ b/packages/content-handler/src/websites/t-dot-co-handler.ts
@ -0,0 +1,26 @@
+import { ContentHandler } from '../content-handler'
+import axios from 'axios'
+
+export class TDotCoHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 't.co'
+  }
+
+  shouldResolve(url: string): boolean {
+    const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/
+    return T_DOT_CO_URL_MATCH.test(url)
+  }
+
+  async resolve(url: string) {
+    return axios
+      .get(url, { maxRedirects: 0, validateStatus: null })
+      .then((res) => {
+        return new URL(res.headers.location).href
+      })
+      .catch((err) => {
+        console.log('err with t.co url', err)
+        return undefined
+      })
+  }
+}
--- a/packages/content-handler/src/websites/twitter-handler.ts
+++ b/packages/content-handler/src/websites/twitter-handler.ts
@ -0,0 +1,167 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+import axios from 'axios'
+import { DateTime } from 'luxon'
+import _ from 'underscore'
+
+const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN
+const TWITTER_URL_MATCH =
+  /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
+
+const getTweetFields = () => {
+  const TWEET_FIELDS =
+    '&tweet.fields=attachments,author_id,conversation_id,created_at,' +
+    'entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets,' +
+    'source,withheld'
+  const EXPANSIONS = '&expansions=author_id,attachments.media_keys'
+  const USER_FIELDS =
+    '&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld'
+  const MEDIA_FIELDS =
+    '&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width'
+
+  return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`
+}
+
+const getTweetById = async (id: string) => {
+  const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/'
+  const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
+
+  if (!TWITTER_BEARER_TOKEN) {
+    throw new Error('No Twitter bearer token found')
+  }
+
+  return axios.get(apiUrl.toString(), {
+    headers: {
+      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
+      redirect: 'follow',
+    },
+  })
+}
+
+const titleForAuthor = (author: { name: string }) => {
+  return `${author.name} on Twitter`
+}
+
+const tweetIdFromStatusUrl = (url: string): string | undefined => {
+  const match = url.toString().match(TWITTER_URL_MATCH)
+  return match?.[2]
+}
+
+const formatTimestamp = (timestamp: string) => {
+  return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(
+    DateTime.DATETIME_FULL
+  )
+}
+
+export class TwitterHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Twitter'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
+  }
+
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
+    console.log('prehandling twitter url', url)
+
+    const tweetId = tweetIdFromStatusUrl(url)
+    if (!tweetId) {
+      throw new Error('could not find tweet id in url')
+    }
+    const tweetData = (await getTweetById(tweetId)).data as {
+      data: {
+        author_id: string
+        text: string
+        entities: {
+          urls: [
+            {
+              url: string
+              expanded_url: string
+              display_url: string
+            }
+          ]
+        }
+        created_at: string
+      }
+      includes: {
+        users: [
+          {
+            id: string
+            name: string
+            profile_image_url: string
+            username: string
+          }
+        ]
+        media: [
+          {
+            preview_image_url: string
+            type: string
+            url: string
+          }
+        ]
+      }
+    }
+    const authorId = tweetData.data.author_id
+    const author = tweetData.includes.users.filter((u) => (u.id = authorId))[0]
+    // escape html entities in title
+    const title = _.escape(titleForAuthor(author))
+    const authorImage = author.profile_image_url.replace('_normal', '_400x400')
+
+    let text = tweetData.data.text
+    if (tweetData.data.entities && tweetData.data.entities.urls) {
+      for (const urlObj of tweetData.data.entities.urls) {
+        text = text.replace(
+          urlObj.url,
+          `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
+        )
+      }
+    }
+
+    const front = `
+    <div>
+      <p>${text}</p>
+    `
+
+    let includesHtml = ''
+    if (tweetData.includes.media) {
+      includesHtml = tweetData.includes.media
+        .map((m) => {
+          const linkUrl = m.type == 'photo' ? m.url : url
+          const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
+          const mediaOpen = `<a class="media-link" href=${linkUrl}>
+          <picture>
+            <img class="tweet-img" src=${previewUrl} />
+          </picture>
+          </a>`
+          return mediaOpen
+        })
+        .join('\n')
+    }
+
+    const back = `
+       — <a href="https://twitter.com/${author.username}">${
+      author.username
+    }</a> ${author.name} <a href="${url}">${formatTimestamp(
+      tweetData.data.created_at
+    )}</a>
+    </div>
+    `
+    const content = `
+    <head>
+      <meta property="og:image" content="${authorImage}" />
+      <meta property="og:image:secure_url" content="${authorImage}" />
+      <meta property="og:title" content="${title}" />
+      <meta property="og:description" content="${_.escape(
+        tweetData.data.text
+      )}" />
+    </head>
+    <body>
+      ${front}
+      ${includesHtml}
+      ${back}
+    </body>`
+
+    return { content, url, title }
+  }
+}
--- a/packages/content-handler/src/websites/wikipedia-handler.ts
+++ b/packages/content-handler/src/websites/wikipedia-handler.ts
@ -0,0 +1,20 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class WikipediaHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'wikipedia'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
+    return new URL(url).hostname.endsWith('wikipedia.org')
+  }
+
+  async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
+    // This removes the [edit] anchors from wikipedia pages
+    dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove())
+    // this removes the sidebar
+    dom.querySelector('.infobox')?.remove()
+    return Promise.resolve({ dom })
+  }
+}
--- a/packages/content-handler/src/websites/youtube-handler.ts
+++ b/packages/content-handler/src/websites/youtube-handler.ts
@ -1,18 +1,13 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const _ = require('underscore');
+import { ContentHandler, PreHandleResult } from '../content-handler'
+import axios from 'axios'
+import _ from 'underscore'

 const YOUTUBE_URL_MATCH =
  /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/

-function getYoutubeVideoId(url) {
-  const u = new URL(url);
-  const videoId = u.searchParams.get('v');
+export const getYoutubeVideoId = (url: string) => {
+  const u = new URL(url)
+  const videoId = u.searchParams.get('v')
  if (!videoId) {
    const match = url.toString().match(YOUTUBE_URL_MATCH)
    if (match === null || match.length < 6 || !match[5]) {
@ -22,28 +17,41 @@ function getYoutubeVideoId(url) {
  }
  return videoId
 }
-exports.getYoutubeVideoId = getYoutubeVideoId

-exports.youtubeHandler = {
-  shouldPrehandle: (url, env) => {
+export class YoutubeHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Youtube'
+  }
+
+  shouldPreHandle(url: string, dom?: Document): boolean {
    return YOUTUBE_URL_MATCH.test(url.toString())
-  },
+  }

-  prehandle: async (url, env) => {
+  async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
    const videoId = getYoutubeVideoId(url)
    if (!videoId) {
      return {}
    }

-    const oembedUrl = `https://www.youtube.com/oembed?format=json&url=` + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`)
-    const oembed = (await axios.get(oembedUrl.toString())).data;
+    const oembedUrl =
+      `https://www.youtube.com/oembed?format=json&url=` +
+      encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`)
+    const oembed = (await axios.get(oembedUrl.toString())).data as {
+      title: string
+      width: number
+      height: number
+      thumbnail_url: string
+      author_name: string
+      author_url: string
+    }
    // escape html entities in title
-    const title = _.escape(oembed.title);
-    const ratio = oembed.width / oembed.height;
-    const thumbnail = oembed.thumbnail_url;
-    const height = 350;
-    const width = height * ratio;
-    const authorName = _.escape(oembed.author_name);
+    const title = _.escape(oembed.title)
+    const ratio = oembed.width / oembed.height
+    const thumbnail = oembed.thumbnail_url
+    const height = 350
+    const width = height * ratio
+    const authorName = _.escape(oembed.author_name)

    const content = `
    <html>
@ -63,6 +71,6 @@ exports.youtubeHandler = {

    console.log('got video id', videoId)

-    return { content, title: 'Youtube Content' };
+    return { content, title: 'Youtube Content' }
  }
 }
--- a/packages/content-handler/test/apple-news-handler.test.ts
+++ b/packages/content-handler/test/apple-news-handler.test.ts
@ -0,0 +1,10 @@
+import { AppleNewsHandler } from '../src/websites/apple-news-handler'
+
+describe('open a simple web page', () => {
+  it('should return a response', async () => {
+    const response = await new AppleNewsHandler().preHandle(
+      'https://apple.news/AxjzaZaPvSn23b67LhXI5EQ'
+    )
+    console.log('response', response)
+  })
+})
--- a/packages/content-handler/test/babel-register.js
+++ b/packages/content-handler/test/babel-register.js
@ -0,0 +1,3 @@
+const register = require('@babel/register').default
+
+register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
--- a/packages/content-handler/test/data/beehiiv-newsletter.html
+++ b/packages/content-handler/test/data/beehiiv-newsletter.html
--- a/packages/content-handler/test/data/substack-forwarded-newsletter.html
+++ b/packages/content-handler/test/data/substack-forwarded-newsletter.html
--- a/packages/content-handler/test/data/substack-forwarded-welcome-email.html
+++ b/packages/content-handler/test/data/substack-forwarded-welcome-email.html
--- a/packages/content-handler/test/data/substack-private-forwarded-newsletter.html
+++ b/packages/content-handler/test/data/substack-private-forwarded-newsletter.html
--- a/packages/content-handler/test/newsletter.test.ts
+++ b/packages/content-handler/test/newsletter.test.ts
@ -0,0 +1,191 @@
+import 'mocha'
+import * as chai from 'chai'
+import { expect } from 'chai'
+import chaiAsPromised from 'chai-as-promised'
+import chaiString from 'chai-string'
+import { SubstackHandler } from '../src/newsletters/substack-handler'
+import { AxiosHandler } from '../src/newsletters/axios-handler'
+import { BloombergNewsletterHandler } from '../src/newsletters/bloomberg-newsletter-handler'
+import { GolangHandler } from '../src/newsletters/golang-handler'
+import { MorningBrewHandler } from '../src/newsletters/morning-brew-handler'
+import nock from 'nock'
+import { generateUniqueUrl } from '../src/content-handler'
+import fs from 'fs'
+import { BeehiivHandler } from '../src/newsletters/beehiiv-handler'
+
+chai.use(chaiAsPromised)
+chai.use(chaiString)
+
+const load = (path: string): string => {
+  return fs.readFileSync(path, 'utf8')
+}
+
+describe('Newsletter email test', () => {
+  describe('#getNewsletterUrl()', () => {
+    it('returns url when email is from SubStack', async () => {
+      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
+
+      await expect(
+        new SubstackHandler().parseNewsletterUrl(rawUrl, '')
+      ).to.eventually.equal('https://hongbo130.substack.com/p/tldr')
+    })
+
+    it('returns url when email is from Axios', async () => {
+      const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app'
+      const html = `View in browser at <a>${url}</a>`
+
+      await expect(
+        new AxiosHandler().parseNewsletterUrl('', html)
+      ).to.eventually.equal(url)
+    })
+
+    it('returns url when email is from Bloomberg', async () => {
+      const url = 'https://www.bloomberg.com/news/google-is-now-a-partner'
+      const html = `
+        <a class="view-in-browser__url" href="${url}">
+        View in browser
+        </a>
+      `
+
+      await expect(
+        new BloombergNewsletterHandler().parseNewsletterUrl('', html)
+      ).to.eventually.equal(url)
+    })
+
+    it('returns url when email is from Golang Weekly', async () => {
+      const url = 'https://www.golangweekly.com/first'
+      const html = `
+        <a href="${url}" style="text-decoration: none">Read on the Web</a>
+      `
+
+      await expect(
+        new GolangHandler().parseNewsletterUrl('', html)
+      ).to.eventually.equal(url)
+    })
+
+    it('returns url when email is from Morning Brew', async () => {
+      const url = 'https://www.morningbrew.com/daily/issues/first'
+      const html = `
+        <a style="color: #000000; text-decoration: none;" target="_blank" rel="noopener" href="${url}">View Online</a>
+      `
+
+      await expect(
+        new MorningBrewHandler().parseNewsletterUrl('', html)
+      ).to.eventually.equal(url)
+    })
+  })
+
+  describe('get author from email address', () => {
+    it('returns author when email is from Substack', () => {
+      const from = 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
+      expect(new AxiosHandler().parseAuthor(from)).to.equal(
+        'Jackson Harper from Omnivore App'
+      )
+    })
+
+    it('returns author when email is from Axios', () => {
+      const from = 'Mike Allen <mike@axios.com>'
+      expect(new AxiosHandler().parseAuthor(from)).to.equal('Mike Allen')
+    })
+  })
+
+  describe('isProbablyNewsletter', () => {
+    it('returns true for substack newsletter', async () => {
+      const html = load('./test/data/substack-forwarded-newsletter.html')
+      await expect(
+        new SubstackHandler().isNewsletter({
+          html,
+          postHeader: '',
+          from: '',
+          unSubHeader: '',
+        })
+      ).to.eventually.be.true
+    })
+    it('returns true for private forwarded substack newsletter', async () => {
+      const html = load(
+        './test/data/substack-private-forwarded-newsletter.html'
+      )
+      await expect(
+        new SubstackHandler().isNewsletter({
+          html,
+          postHeader: '',
+          from: '',
+          unSubHeader: '',
+        })
+      ).to.eventually.be.true
+    })
+    it('returns false for substack welcome email', async () => {
+      const html = load('./test/data/substack-forwarded-welcome-email.html')
+      await expect(
+        new SubstackHandler().isNewsletter({
+          html,
+          postHeader: '',
+          from: '',
+          unSubHeader: '',
+        })
+      ).to.eventually.be.false
+    })
+    it('returns true for beehiiv.com newsletter', async () => {
+      const html = load('./test/data/beehiiv-newsletter.html')
+      await expect(
+        new BeehiivHandler().isNewsletter({
+          html,
+          postHeader: '',
+          from: '',
+          unSubHeader: '',
+        })
+      ).to.eventually.be.true
+    })
+  })
+
+  describe('findNewsletterUrl', async () => {
+    it('gets the URL from the header if it is a substack newsletter', async () => {
+      nock('https://email.mg2.substack.com')
+        .head(
+          '/c/eJxNkk2TojAQhn-N3KTyQfg4cGDGchdnYcsZx9K5UCE0EMVAkTiKv36iHnarupNUd7rfVJ4W3EDTj1M89No496Uw0wCxgovuwBgYnbOGsZBVjDHzKPWYU8VehUMWOlIX9Qhw4rKLzXgGZziXnRTcyF7dK0iIGMVOG_OS1aTmKPRDilgVhTQUPCQIcE0x-MFTmJ8rCUpA3KtuenR2urg1ZtAzmszI0tq_Z7m66y-ilQo0uAqMTQ7WRX8auJKg56blZg7WB-iHDuYEBzO6NP0R1IwuYFphQbbTjnTH9NBfs80nym4Zyj8uUvyKbtUyGr5eUz9fNDQ7JCxfJDo9dW1lY9lmj_JNivPbGmf2Pt_lN9tDit9b-WeTetni85Z9pDpVOd7L1E_Vy7egayNO23ZP34eSeLJeux1b0rer_xaZ7ykS78nuSjMY-nL98rparNZNcv07JCjN06_EkTFBxBqOUMACErnELUNMSxTUjLDQZwzcqa4bRjCfeejUEFefS224OLr2S5wxPtij7lVrs80d2CNseRV2P52VNFMBipcdVE-U5jkRD7hFAwpGOylVwU2Mfc9qBh7DoR89yVnWXhgQFHnIsbpVb6tU_B-hH_2yzWY'
+        )
+        .reply(302, undefined, {
+          Location:
+            'https://newsletter.slowchinese.net/p/companies-that-eat-people-217',
+        })
+        .get('/p/companies-that-eat-people-217')
+        .reply(200, '')
+      const html = load('./test/data/substack-forwarded-newsletter.html')
+      const url = await new SubstackHandler().findNewsletterUrl(html)
+      // Not sure if the redirects from substack expire, this test could eventually fail
+      expect(url).to.startWith(
+        'https://newsletter.slowchinese.net/p/companies-that-eat-people-217'
+      )
+    }).timeout(10000)
+    it('gets the URL from the header if it is a beehiiv newsletter', async () => {
+      nock('https://u23463625.ct.sendgrid.net')
+        .head(
+          '/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno'
+        )
+        .reply(302, undefined, {
+          Location: 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple',
+        })
+        .get('/p/talked-guy-spent-30m-beeple')
+        .reply(200, '')
+      const html = load('./test/data/beehiiv-newsletter.html')
+      const url = await new BeehiivHandler().findNewsletterUrl(html)
+      expect(url).to.startWith(
+        'https://www.milkroad.com/p/talked-guy-spent-30m-beeple'
+      )
+    })
+    it('returns undefined if it is not a newsletter', async () => {
+      const html = load('./test/data/substack-forwarded-welcome-email.html')
+      const url = await new SubstackHandler().findNewsletterUrl(html)
+      expect(url).to.be.undefined
+    })
+  })
+
+  describe('generateUniqueUrl', () => {
+    it('generates a unique URL', () => {
+      const url1 = generateUniqueUrl()
+      const url2 = generateUniqueUrl()
+
+      expect(url1).to.not.eql(url2)
+    })
+  })
+})
--- a/packages/content-handler/test/youtube-handler.test.ts
+++ b/packages/content-handler/test/youtube-handler.test.ts
@ -0,0 +1,25 @@
+import { expect } from 'chai'
+import 'mocha'
+import { getYoutubeVideoId } from '../src/websites/youtube-handler'
+
+describe('getYoutubeVideoId', () => {
+  it('should parse video id out of a URL', async () => {
+    expect('BnSUk0je6oo').to.eq(
+      getYoutubeVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s')
+    )
+    expect('vFD2gu007dc').to.eq(
+      getYoutubeVideoId(
+        'https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1'
+      )
+    )
+    expect('vFD2gu007dc').to.eq(
+      getYoutubeVideoId('https://youtu.be/vFD2gu007dc')
+    )
+    expect('BMFVCnbRaV4').to.eq(
+      getYoutubeVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share')
+    )
+    expect('cg9b4RC87LI').to.eq(
+      getYoutubeVideoId('https://youtu.be/cg9b4RC87LI?t=116')
+    )
+  })
+})
--- a/packages/content-handler/tsconfig.json
+++ b/packages/content-handler/tsconfig.json
@ -0,0 +1,10 @@
+{
+  "extends": "@tsconfig/node14/tsconfig.json",
+  "compilerOptions": {
+    "rootDir": ".",
+    "declaration": true,
+    "outDir": "build",
+    "lib": ["dom"]
+  },
+  "include": ["src"]
+}
--- a/packages/inbound-email-handler/package.json
+++ b/packages/inbound-email-handler/package.json
@ -31,6 +31,7 @@
    "@google-cloud/pubsub": "^2.18.4",
    "@sendgrid/client": "^7.6.0",
    "@sentry/serverless": "^6.16.1",
+    "@omnivore/content-handler": "1.0.0",
    "addressparser": "^1.0.1",
    "axios": "^0.27.2",
    "jsonwebtoken": "^8.5.1",
--- a/packages/inbound-email-handler/src/axios-handler.ts
+++ b/packages/inbound-email-handler/src/axios-handler.ts
@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class AxiosHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /<.+@axios.com>/
-    this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
-    this.defaultUrl = 'https://axios.com'
-  }
-}
--- a/packages/inbound-email-handler/src/bloomberg-handler.ts
+++ b/packages/inbound-email-handler/src/bloomberg-handler.ts
@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class BloombergHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /<.+@mail.bloomberg.*.com>/
-    this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
-    this.defaultUrl = 'https://www.bloomberg.com'
-  }
-}
--- a/packages/inbound-email-handler/src/golang-handler.ts
+++ b/packages/inbound-email-handler/src/golang-handler.ts
@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class GolangHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /<.+@golangweekly.com>/
-    this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
-    this.defaultUrl = 'https://golangweekly.com'
-  }
-}
--- a/packages/inbound-email-handler/src/index.ts
+++ b/packages/inbound-email-handler/src/index.ts
@ -9,35 +9,27 @@ import * as multipart from 'parse-multipart-data'
 import {
  handleConfirmation,
  isConfirmationEmail,
-  NewsletterHandler,
  parseUnsubscribe,
 } from './newsletter'
 import { PubSub } from '@google-cloud/pubsub'
 import { handlePdfAttachment } from './pdf'
-import { SubstackHandler } from './substack-handler'
-import { AxiosHandler } from './axios-handler'
-import { BloombergHandler } from './bloomberg-handler'
-import { GolangHandler } from './golang-handler'
-import { MorningBrewHandler } from './morning-brew-handler'
+import { handleNewsletter } from '@omnivore/content-handler'

+const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
 const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived'
 const pubsub = new PubSub()
-const NEWSLETTER_HANDLERS = [
-  new SubstackHandler(),
-  new AxiosHandler(),
-  new BloombergHandler(),
-  new GolangHandler(),
-  new MorningBrewHandler(),
-]

-export const getNewsletterHandler = (
-  postHeader: string,
-  from: string,
-  unSubHeader: string
-): NewsletterHandler | undefined => {
-  return NEWSLETTER_HANDLERS.find((h) => {
-    return h.isNewsletter(postHeader, from, unSubHeader)
-  })
+export const publishMessage = async (
+  topic: string,
+  message: any
+): Promise<string | undefined> => {
+  return pubsub
+    .topic(topic)
+    .publishMessage({ json: message })
+    .catch((err) => {
+      console.log('error publishing message:', err)
+      return undefined
+    })
 }

 export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
@ -86,23 +78,20 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(

      try {
        // check if it is a confirmation email or forwarding newsletter
-        const newsletterHandler = getNewsletterHandler(
-          postHeader,
+        const newsletterMessage = await handleNewsletter({
          from,
-          unSubHeader
-        )
-
-        if (newsletterHandler) {
-          console.log('handleNewsletter', from, to)
-          await newsletterHandler.handleNewsletter(
-            to,
-            html,
-            postHeader,
-            subject,
-            from,
-            unSubHeader
+          html,
+          postHeader,
+          unSubHeader,
+          email: to,
+          title: subject,
+        })
+        if (newsletterMessage) {
+          await publishMessage(
+            NEWSLETTER_EMAIL_RECEIVED_TOPIC,
+            newsletterMessage
          )
-          return res.send('ok')
+          return res.status(200).send('newsletter received')
        }

        console.log('non-newsletter email from', from, 'to', to)
--- a/packages/inbound-email-handler/src/morning-brew-handler.ts
+++ b/packages/inbound-email-handler/src/morning-brew-handler.ts
@ -1,10 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-
-export class MorningBrewHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.senderRegex = /Morning Brew <crew@morningbrew.com>/
-    this.urlRegex = /<a.* href=["']([^"']*)["'].*>View Online<\/a>/
-    this.defaultUrl = 'https://www.morningbrew.com'
-  }
-}
--- a/packages/inbound-email-handler/src/newsletter.ts
+++ b/packages/inbound-email-handler/src/newsletter.ts
@ -1,15 +1,12 @@
-import { PubSub } from '@google-cloud/pubsub'
-import { v4 as uuidv4 } from 'uuid'
 import addressparser from 'addressparser'
 import rfc2047 from 'rfc2047'
+import { publishMessage } from './index'

 interface Unsubscribe {
  mailTo?: string
  httpUrl?: string
 }

-const pubsub = new PubSub()
-const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
 const EMAIL_CONFIRMATION_CODE_RECEIVED_TOPIC = 'emailConfirmationCodeReceived'
 const CONFIRMATION_EMAIL_SENDER_ADDRESS = 'forwarding-noreply@google.com'
 // check unicode parentheses too
@ -35,73 +32,6 @@ const parseAddress = (address: string): string => {
  return ''
 }

-export class NewsletterHandler {
-  protected senderRegex = /NEWSLETTER_SENDER_REGEX/
-  protected urlRegex = /NEWSLETTER_URL_REGEX/
-  protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
-
-  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
-  }
-
-  parseNewsletterUrl(_postHeader: string, html: string): string | undefined {
-    // get newsletter url from html
-    const matches = html.match(this.urlRegex)
-    if (matches) {
-      return matches[1]
-    }
-    return undefined
-  }
-
-  parseAuthor(from: string): string {
-    // get author name from email
-    // e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
-    // or 'Mike Allen <mike@axios.com>'
-    const parsed = addressparser(from)
-    if (parsed.length > 0) {
-      return parsed[0].name
-    }
-    return from
-  }
-
-  async handleNewsletter(
-    email: string,
-    html: string,
-    postHeader: string,
-    title: string,
-    from: string,
-    unSubHeader: string
-  ): Promise<string | undefined> {
-    console.log('handleNewsletter', email, postHeader, title, from)
-
-    if (!email || !html || !title || !from) {
-      console.log('invalid newsletter email')
-      throw new Error('invalid newsletter email')
-    }
-
-    // fallback to default url if newsletter url does not exist
-    // assign a random uuid to the default url to avoid duplicate url
-    const url =
-      this.parseNewsletterUrl(postHeader, html) ||
-      `${this.defaultUrl}?source=newsletters&id=${uuidv4()}`
-    const author = this.parseAuthor(from)
-    const unsubscribe = parseUnsubscribe(unSubHeader)
-    const message = {
-      email,
-      content: html,
-      url,
-      title,
-      author,
-      unsubMailTo: unsubscribe.mailTo || '',
-      unsubHttpUrl: unsubscribe.httpUrl || '',
-    }
-
-    return publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, message)
-  }
-}
-
 export const handleConfirmation = async (email: string, subject: string) => {
  console.log('confirmation email', email, subject)

@ -136,16 +66,3 @@ export const isConfirmationEmail = (from: string, subject: string): boolean => {
    CONFIRMATION_CODE_PATTERN.test(subject)
  )
 }
-
-const publishMessage = async (
-  topic: string,
-  message: Record<string, string>
-): Promise<string | undefined> => {
-  return pubsub
-    .topic(topic)
-    .publishMessage({ json: message })
-    .catch((err) => {
-      console.log('error publishing message:', err)
-      return undefined
-    })
-}
--- a/packages/inbound-email-handler/src/substack-handler.ts
+++ b/packages/inbound-email-handler/src/substack-handler.ts
@ -1,25 +0,0 @@
-import { NewsletterHandler } from './newsletter'
-import addressparser from 'addressparser'
-
-export class SubstackHandler extends NewsletterHandler {
-  constructor() {
-    super()
-    this.defaultUrl = 'https://www.substack.com'
-  }
-
-  parseNewsletterUrl(postHeader: string, _html: string): string | undefined {
-    // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
-    // we need to get the real url from the raw url
-    return addressparser(postHeader).length > 0
-      ? addressparser(postHeader)[0].name
-      : undefined
-  }
-
-  isNewsletter(
-    postHeader: string,
-    _from: string,
-    _unSubHeader: string
-  ): boolean {
-    return !!postHeader
-  }
-}
--- a/packages/inbound-email-handler/test/newsletter.test.ts
+++ b/packages/inbound-email-handler/test/newsletter.test.ts
@ -2,15 +2,8 @@ import { expect } from 'chai'
 import {
  getConfirmationCode,
  isConfirmationEmail,
-  NewsletterHandler,
  parseUnsubscribe,
 } from '../src/newsletter'
-import { SubstackHandler } from '../src/substack-handler'
-import { AxiosHandler } from '../src/axios-handler'
-import { BloombergHandler } from '../src/bloomberg-handler'
-import { GolangHandler } from '../src/golang-handler'
-import { getNewsletterHandler } from '../src'
-import { MorningBrewHandler } from '../src/morning-brew-handler'

 describe('Confirmation email test', () => {
  describe('#isConfirmationEmail()', () => {
@ -54,126 +47,6 @@ describe('Confirmation email test', () => {
 })

 describe('Newsletter email test', () => {
-  describe('#getNewsletterHandler()', () => {
-    it('returns SubstackHandler when email is from SubStack', () => {
-      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
-
-      expect(getNewsletterHandler(rawUrl, '', '')).to.be.instanceof(
-        SubstackHandler
-      )
-    })
-
-    it('returns AxiosHandler when email is from Axios', () => {
-      const from = 'Mike Allen <mike@axios.com>'
-      const unSubRawUrl =
-        '<https://axios.com/unsubscribe?email=mike%40axios.com&code=593781109>'
-
-      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-        AxiosHandler
-      )
-    })
-
-    context('when email is from Bloomberg', () => {
-      it('should return BloombergHandler when email is from Bloomberg Business', () => {
-        const from = 'From: Bloomberg <noreply@mail.bloombergbusiness.com>'
-        const unSubRawUrl = '<https://bloomberg.com/unsubscribe>'
-
-        expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-          BloombergHandler
-        )
-      })
-
-      it('should return BloombergHandler when email is from Bloomberg View', () => {
-        const from = 'From: Bloomberg <noreply@mail.bloombergview.com>'
-        const unSubRawUrl = '<https://bloomberg.com/unsubscribe>'
-
-        expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-          BloombergHandler
-        )
-      })
-    })
-
-    it('should return GolangHandler when email is from Golang Weekly', () => {
-      const from = 'Golang Weekly <peter@golangweekly.com>'
-      const unSubRawUrl = '<https://golangweekly.com/unsubscribe>'
-
-      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-        GolangHandler
-      )
-    })
-
-    it('should return MorningBrewHandler when email is from Morning Brew', () => {
-      const from = 'Morning Brew <crew@morningbrew.com>'
-      const unSubRawUrl = '<https://morningbrew.com/unsubscribe>'
-
-      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
-        MorningBrewHandler
-      )
-    })
-  })
-
-  describe('#getNewsletterUrl()', () => {
-    it('returns url when email is from SubStack', () => {
-      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
-
-      expect(new SubstackHandler().parseNewsletterUrl(rawUrl, '')).to.equal(
-        'https://hongbo130.substack.com/p/tldr'
-      )
-    })
-
-    it('returns url when email is from Axios', () => {
-      const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app'
-      const html = `View in browser at <a>${url}</a>`
-
-      expect(new AxiosHandler().parseNewsletterUrl('', html)).to.equal(url)
-    })
-
-    it('returns url when email is from Bloomberg', () => {
-      const url = 'https://www.bloomberg.com/news/google-is-now-a-partner'
-      const html = `
-        <a class="view-in-browser__url" href="${url}">
-        View in browser
-        </a>
-      `
-
-      expect(new BloombergHandler().parseNewsletterUrl('', html)).to.equal(url)
-    })
-
-    it('returns url when email is from Golang Weekly', () => {
-      const url = 'https://www.golangweekly.com/first'
-      const html = `
-        <a href="${url}" style="text-decoration: none">Read on the Web</a>
-      `
-
-      expect(new GolangHandler().parseNewsletterUrl('', html)).to.equal(url)
-    })
-
-    it('returns url when email is from Morning Brew', () => {
-      const url = 'https://www.morningbrew.com/daily/issues/first'
-      const html = `
-        <a style="color: #000000; text-decoration: none;" target="_blank" rel="noopener" href="${url}">View Online</a>
-      `
-
-      expect(new MorningBrewHandler().parseNewsletterUrl('', html)).to.equal(
-        url
-      )
-    })
-  })
-
-  describe('get author from email address', () => {
-    it('returns author when email is from Substack', () => {
-      const from = 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
-      expect(new NewsletterHandler().parseAuthor(from)).to.equal(
-        'Jackson Harper from Omnivore App'
-      )
-    })
-
-    it('returns author when email is from Axios', () => {
-      const from = 'Mike Allen <mike@axios.com>'
-      expect(new NewsletterHandler().parseAuthor(from)).to.equal('Mike Allen')
-    })
-  })
-
  describe('get unsubscribe from header', () => {
    const mailTo = 'unsub@omnivore.com'
    const httpUrl = 'https://omnivore.com/unsubscribe'
--- a/packages/puppeteer-parse/apple-news-handler.js
+++ b/packages/puppeteer-parse/apple-news-handler.js
@ -1,36 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const Url = require('url');
-const axios = require('axios');
-const { promisify } = require('util');
-const { DateTime } = require('luxon');
-const os = require('os');
-const { Cipher } = require('crypto');
-const { parseHTML } = require('linkedom');
-
-exports.appleNewsHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    if (u.hostname === 'apple.news') {
-      return true;
-    }
-    return false
-  },
-
-  prehandle: async (url, env) => {
-    const MOBILE_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
-    const response = await axios.get(url, { headers: { 'User-Agent': MOBILE_USER_AGENT }  } );
-    const data = response.data;
-
-    const dom = parseHTML(data).document;
-
-    // make sure its a valid URL by wrapping in new URL
-    const u = new URL(dom.querySelector('span.click-here').parentNode.href);
-    return { url: u.href };
-  }
-}
--- a/packages/puppeteer-parse/bloomberg-handler.js
+++ b/packages/puppeteer-parse/bloomberg-handler.js
@ -1,39 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const os = require('os');
-const { parseHTML } = require('linkedom');
-
-exports.bloombergHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const BLOOMBERG_URL_MATCH =
-      /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
-    return BLOOMBERG_URL_MATCH.test(url.toString())
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling bloomberg url', url)
-
-    try {
-      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
-        params: {
-          'api_key': process.env.SCRAPINGBEE_API_KEY,
-          'url': url,
-          'return_page_source': true,
-          'block_ads': true,
-          'block_resources': false,
-        }
-      })
-      const dom = parseHTML(response.data).document;
-      return { title: dom.title, content: dom.querySelector('body').innerHTML, url: url }
-    } catch (error) {
-      console.error('error prehandling bloomberg url', error)
-      throw error
-    }
-  }
-}
--- a/packages/puppeteer-parse/derstandard-handler.js
+++ b/packages/puppeteer-parse/derstandard-handler.js
@ -1,35 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { parseHTML } = require('linkedom');
-
-exports.derstandardHandler = {
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    return u.hostname === 'www.derstandard.at';
-  },
-
-  prehandle: async (url, env) => {
-    const response = await axios.get(url, {
-      // set cookie to give consent to get the article
-      headers: {
-        'cookie': `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`
-      },
-    });
-    const content = response.data;
-
-    var title = undefined;
-    const dom = parseHTML(content).document;
-    const titleElement = dom.querySelector('.article-title')
-    if (!titleElement) {
-      title = titleElement.textContent
-      titleElement.remove()
-    }
-
-    return { content: dom.body.outerHTML, title: title };
-  }
-}
--- a/packages/puppeteer-parse/image-handler.js
+++ b/packages/puppeteer-parse/image-handler.js
@ -1,34 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-
-
-exports.imageHandler = {
-  shouldPrehandle: (url, env) => {
-    const IMAGE_URL_PATTERN =
-      /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
-    return IMAGE_URL_PATTERN.test(url.toString())
-  },
-
-  prehandle: async (url, env) => {
-    const title = url.toString().split('/').pop();
-    const content = `
-      <html>
-        <head>
-          <title>${title}</title>
-          <meta property="og:image" content="${url}" />
-          <meta property="og:title" content="${title}" />
-        </head>
-        <body>
-          <div>
-            <img src="${url}" alt="${title}">
-          </div>
-        </body>
-      </html>`
-
-    return { title, content };
-  }
-}
--- a/packages/puppeteer-parse/index.js
+++ b/packages/puppeteer-parse/index.js
@ -15,18 +15,10 @@ const { DateTime } = require('luxon');
 const os = require('os');
 const Sentry = require('@sentry/serverless');
 const { Storage } = require('@google-cloud/storage');
-const { appleNewsHandler } = require('./apple-news-handler');
-const { twitterHandler } = require('./twitter-handler');
-const { youtubeHandler } = require('./youtube-handler');
-const { tDotCoHandler } = require('./t-dot-co-handler');
-const { pdfHandler } = require('./pdf-handler');
-const { mediumHandler } = require('./medium-handler');
-const { derstandardHandler } = require('./derstandard-handler');
-const { imageHandler } = require('./image-handler');
-const { scrappingBeeHandler } = require('./scrapingBee-handler');

 const chromium = require('chrome-aws-lambda');
 const puppeteer = require('puppeteer-core');
+const { preHandleContent } = require("@omnivore/content-handler");

 // Add stealth plugin to hide puppeteer usage
 // const StealthPlugin = require('puppeteer-extra-plugin-stealth');
@ -257,18 +249,6 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId
  );
 };

-const handlers = {
-  'pdf': pdfHandler,
-  'apple-news': appleNewsHandler,
-  'twitter': twitterHandler,
-  'youtube': youtubeHandler,
-  't-dot-co': tDotCoHandler,
-  'medium': mediumHandler,
-  'derstandard': derstandardHandler,
-  'image': imageHandler,
-  'scrappingBee': scrappingBeeHandler,
-};
-
 /**
 * Cloud Function entry point, HTTP trigger.
 * Loads the requested URL via Puppeteer, captures page content and sends it to backend
@ -309,61 +289,19 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
    return res.sendStatus(400);
  }

-  // if (!userId || !articleSavingRequestId) {
-  //   Object.assign(logRecord, { invalidParams: true, body: req.body, query: req.query });
-  //   logger.error(`Invalid parameters`, logRecord);
-  //   return res.sendStatus(400);
-  // }
-
-  // Before we run the regular handlers we check to see if we need tp
-  // pre-resolve the URL. TODO: This should probably happen recursively,
-  // so URLs can be pre-resolved, handled, pre-resolved, handled, etc.
-  for (const [key, handler] of Object.entries(handlers)) {
-    if (handler.shouldResolve && handler.shouldResolve(url)) {
-      try {
-        url = await handler.resolve(url);
-        validateUrlString(url);
-      } catch (err) {
-        console.log('error resolving url with handler', key, err);
-      }
-      break;
-    }
-  }
-
-  // Before we fetch the page we check the handlers, to see if they want
-  // to perform a prefetch action that can modify our requests.
-  // enumerate the handlers and see if any of them want to handle the request
-  const handler = Object.keys(handlers).find(key => {
-    try {
-      return handlers[key].shouldPrehandle(url)
-    } catch (e) {
-      console.log('error with handler: ', key, e);
-    }
-    return false;
-  });
-
-  var title = undefined;
-  var content = undefined;
-  var contentType = undefined;
-
-  if (handler) {
-    try {
-      // The only handler we have now can modify the URL, but in the
-      // future maybe we let it modify content. In that case
-      // we might exit the request early.
-      console.log('pre-handling url with handler: ', handler);
-
-      const result = await handlers[handler].prehandle(url);
-      if (result && result.url) { 
-        url = result.url
-        validateUrlString(url);
-      }
-      if (result && result.title) { title = result.title }
-      if (result && result.content) { content = result.content }
-      if (result && result.contentType) { contentType = result.contentType }
-    } catch (e) {
-      console.log('error with handler: ', handler, e);
+  // pre handle url with custom handlers
+  let title, content, contentType;
+  try {
+    const result = await preHandleContent(url);
+    if (result && result.url) {
+      url = result.url
+      validateUrlString(url);
    }
+    if (result && result.title) { title = result.title }
+    if (result && result.content) { content = result.content }
+    if (result && result.contentType) { contentType = result.contentType }
+  } catch (e) {
+    console.log('error with handler: ', e);
  }

  var context, page, finalUrl;
--- a/packages/puppeteer-parse/medium-handler.js
+++ b/packages/puppeteer-parse/medium-handler.js
@ -1,31 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const os = require('os');
-
-exports.mediumHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const MEDIUM_URL_MATCH =
-      /https?:\/\/(www\.)?medium.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
-    const res =  MEDIUM_URL_MATCH.test(url.toString())
-    return res
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling medium url', url)
-
-    try {
-      const res = new URL(url);
-      res.searchParams.delete('source');
-      return { url: res }
-    } catch (error) {
-      console.error('error prehandling medium url', error)
-      throw error
-    }
-  }
-}
--- a/packages/puppeteer-parse/pdf-handler.js
+++ b/packages/puppeteer-parse/pdf-handler.js
@ -1,21 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const Url = require('url');
-
-
-exports.pdfHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = Url.parse(url)
-    const path = u.path.replace(u.search, '')
-    return path.endsWith('.pdf')
-  },
-
-  prehandle: async (url, env) => {
-    return { contentType: 'application/pdf' };
-  }
-}
--- a/packages/puppeteer-parse/scrapingBee-handler.js
+++ b/packages/puppeteer-parse/scrapingBee-handler.js
@ -1,44 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { parseHTML } = require('linkedom');
-
-const os = require('os');
-
-exports.scrapingBeeHandler = {
-
-  shouldPrehandle: (url, env) => {
-    const u = new URL(url);
-    const hostnames = [
-      'nytimes.com',
-      'news.google.com',
-    ]
-
-    return hostnames.some((h) => u.hostname.endsWith(h))
-  },
-
-  prehandle: async (url, env) => {
-    console.log('prehandling url with scrapingbee', url)
-
-    try {
-      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
-        params: {
-          'api_key': process.env.SCRAPINGBEE_API_KEY,
-          'url': url,
-          'return_page_source': true,
-          'block_ads': true,
-          'block_resources': false,
-        }
-      })
-      const dom = parseHTML(response.data).document;
-      return { title: dom.title, content: response.data, url: url }
-    } catch (error) {
-      console.error('error prehandling url w/scrapingbee', error)
-      throw error
-    }
-  }
-}
--- a/packages/puppeteer-parse/t-dot-co-handler.js
+++ b/packages/puppeteer-parse/t-dot-co-handler.js
@ -1,32 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const Url = require('url');
-
-
-exports.tDotCoHandler = {
-
-  shouldResolve: function (url, env) {
-    const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/;
-    console.log('should preresolve?', T_DOT_CO_URL_MATCH.test(url), url)
-    return T_DOT_CO_URL_MATCH.test(url);
-  },
-
-  resolve: async function(url, env) {
-    return await axios.get(url, { maxRedirects: 0, validateStatus: null })
-      .then(res => {
-        return Url.parse(res.headers.location).href;
-      }).catch((err) => {
-        console.log('err with t.co url', err);
-        return undefined;
-      });
-  },
-
-  shouldPrehandle: (url, env) => {
-    return false
-  },
-}
--- a/packages/puppeteer-parse/test/apple-news-handler.test.js
+++ b/packages/puppeteer-parse/test/apple-news-handler.test.js
@ -1,9 +0,0 @@
-const { expect } = require('chai')
-const { appleNewsHandler } = require('../apple-news-handler')
-
-describe('open a simple web page', () => {
-  it('should return a response', async () => {
-    const response = await appleNewsHandler.prehandle('https://apple.news/AxjzaZaPvSn23b67LhXI5EQ')
-    console.log('response', response)
-  })
-})
--- a/packages/puppeteer-parse/test/babel-register.js
+++ b/packages/puppeteer-parse/test/babel-register.js
@ -0,0 +1,3 @@
+const register = require('@babel/register').default
+
+register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
--- a/packages/puppeteer-parse/test/stub.test.ts
+++ b/packages/puppeteer-parse/test/stub.test.ts
@ -0,0 +1,13 @@
+import 'mocha'
+import * as chai from 'chai'
+import { expect } from 'chai'
+import 'chai/register-should'
+import chaiString from 'chai-string'
+
+chai.use(chaiString)
+
+describe('Stub test', () => {
+  it('should pass', () => {
+    expect(true).to.be.true
+  })
+})
--- a/packages/puppeteer-parse/test/youtube-handler.test.js
+++ b/packages/puppeteer-parse/test/youtube-handler.test.js
@ -1,12 +0,0 @@
-const { expect } = require('chai')
-const { getYoutubeVideoId } = require('../youtube-handler')
-
-describe('getYoutubeVideoId', () => {
-  it('should parse video id out of a URL', async () => {
-    expect('BnSUk0je6oo').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s'));
-    expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1'));
-    expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://youtu.be/vFD2gu007dc'));
-    expect('BMFVCnbRaV4').to.eq(getYoutubeVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share'));
-    expect('cg9b4RC87LI').to.eq(getYoutubeVideoId('https://youtu.be/cg9b4RC87LI?t=116'));
-  })
-})
--- a/packages/puppeteer-parse/twitter-handler.js
+++ b/packages/puppeteer-parse/twitter-handler.js
@ -1,171 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const { DateTime } = require('luxon');
-const _ = require("underscore");
-
-const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN;
-const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
-
-const embeddedTweet = async (url) => {
-
-  const BASE_ENDPOINT = 'https://publish.twitter.com/oembed'
-
-  const apiUrl = new URL(BASE_ENDPOINT)
-  apiUrl.searchParams.append('url', url);
-  apiUrl.searchParams.append('omit_script', true);
-  apiUrl.searchParams.append('dnt', true);
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const getTweetFields = () => {
-  const TWEET_FIELDS =
-  "&tweet.fields=attachments,author_id,conversation_id,created_at," +
-  "entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets," +
-  "source,withheld";
-  const EXPANSIONS = "&expansions=author_id,attachments.media_keys";
-  const USER_FIELDS =
-  "&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld";
-  const MEDIA_FIELDS =
-  "&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width";
-
-  return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`;
-}
-
-const getTweetById = async (id) => {
-  const BASE_ENDPOINT = "https://api.twitter.com/2/tweets/";
-  const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const getUserByUsername = async (username) => {
-  const BASE_ENDPOINT = "https://api.twitter.com/2/users/by/username/";
-
-  const apiUrl = new URL(BASE_ENDPOINT + username)
-  apiUrl.searchParams.append('user.fields', 'profile_image_url');
-
-  return await axios.get(apiUrl.toString(), {
-    headers: {
-      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
-      redirect: "follow",
-    },
-  });
-};
-
-const titleForTweet = (tweet) => {
-  return `${tweet.data.author_name} on Twitter`
-};
-
-const titleForAuthor = (author) => {
-  return `${author.name} on Twitter`
-};
-
-const usernameFromStatusUrl = (url) => {
-  const match = url.toString().match(TWITTER_URL_MATCH)
-  return match[1]
-};
-
-const tweetIdFromStatusUrl = (url) => {
-  const match = url.toString().match(TWITTER_URL_MATCH)
-  return match[2]
-};
-
-const formatTimestamp = (timestamp) => {
-  return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(DateTime.DATETIME_FULL);
-};
-
-exports.twitterHandler = {
-
-  shouldPrehandle: (url, env) => {
-    return TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
-  },
-
-  // version of the handler that uses the oembed API
-  // This isn't great as it doesn't work well with our
-  // readability API. But could potentially give a more consistent
-  // look to the tweets
-  // prehandle: async (url, env) => {
-  //   const oeTweet = await embeddedTweet(url)
-  //   const dom = new JSDOM(oeTweet.data.html);
-  //   const bq = dom.window.document.querySelector('blockquote')
-  //   console.log('blockquote:', bq);
-
-  //   const title = titleForTweet(oeTweet)
-  //   return { title, content: '<div>' + bq.innerHTML + '</div>', url: oeTweet.data.url };
-  // }
-
-  prehandle: async (url, env) => {
-    console.log('prehandling twitter url', url)
-
-    const tweetId = tweetIdFromStatusUrl(url)
-    const tweetData = (await getTweetById(tweetId)).data;
-    const authorId = tweetData.data.author_id;
-    const author = tweetData.includes.users.filter(u => u.id = authorId)[0];
-    const title = _.escape(titleForAuthor(author))
-    const authorImage = author.profile_image_url.replace('_normal', '_400x400')
-
-    let text = tweetData.data.text;
-    if (tweetData.data.entities && tweetData.data.entities.urls) {
-      for (let urlObj of tweetData.data.entities.urls) {
-        text = text.replace(
-          urlObj.url,
-          `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
-        );
-      }
-    }
-
-    const front = `
-    <div>
-      <p>${text}</p>
-    `
-
-    var includesHtml =  '';
-    if (tweetData.includes.media) {
-      includesHtml = tweetData.includes.media.map(m => {
-        const linkUrl = m.type == 'photo' ? m.url : url;
-        const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url;
-        const mediaOpen = `<a class="media-link" href=${linkUrl}>
-          <picture>
-            <img class="tweet-img" src=${previewUrl} />
-          </picture>
-          </a>`
-        return mediaOpen
-      }).join('\n');
-   }
-
-   const back = `
-       — <a href="https://twitter.com/${author.username}">${author.username}</a> ${author.name} <a href="${url}">${formatTimestamp(tweetData.data.created_at)}</a>
-    </div>
-    `
-    const content = `
-    <head>
-      <meta property="og:image" content="${authorImage}" />
-      <meta property="og:image:secure_url" content="${authorImage}" />
-      <meta property="og:title" content="${title}" />
-      <meta property="og:description" content="${_.escape(tweetData.data.text)}" />
-    </head>
-    <body>
-      ${front}
-      ${includesHtml}
-      ${back}
-    </body>`
-
-    return { content, url, title };
-  }
-}
--- a/packages/puppeteer-parse/youtube-handler.js
+++ b/packages/puppeteer-parse/youtube-handler.js
@ -1,67 +0,0 @@
-/* eslint-disable no-undef */
-/* eslint-disable no-empty */
-/* eslint-disable @typescript-eslint/explicit-function-return-type */
-/* eslint-disable @typescript-eslint/no-var-requires */
-/* eslint-disable @typescript-eslint/no-require-imports */
-require('dotenv').config();
-const axios = require('axios');
-const _ = require("underscore");
-
-const YOUTUBE_URL_MATCH =
-  /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/
-
-function getYoutubeVideoId(url) {
-  const u = new URL(url);
-  const videoId = u.searchParams.get('v');
-  if (!videoId) {
-    const match = url.toString().match(YOUTUBE_URL_MATCH)
-    if (match === null || match.length < 6 || !match[5]) {
-      return undefined
-    }
-    return match[5]
-  }
-  return videoId
-}
-exports.getYoutubeVideoId = getYoutubeVideoId
-
-exports.youtubeHandler = {
-  shouldPrehandle: (url, env) => {
-    return YOUTUBE_URL_MATCH.test(url.toString())
-  },
-
-  prehandle: async (url, env) => {
-    const videoId = getYoutubeVideoId(url)
-    if (!videoId) {
-      return {}
-    }
-
-    const oembedUrl = `https://www.youtube.com/oembed?format=json&url=` + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`)
-    const oembed = (await axios.get(oembedUrl.toString())).data;
-    const title = _.escape(oembed.title);
-    const ratio = oembed.width / oembed.height;
-    const thumbnail = oembed.thumbnail_url;
-    const height = 350;
-    const width = height * ratio;
-    const authorName = _.escape(oembed.author_name);
-
-    const content = `
-    <html>
-      <head><title>${title}</title>
-      <meta property="og:image" content="${thumbnail}" />
-      <meta property="og:image:secure_url" content="${thumbnail}" />
-      <meta property="og:title" content="${title}" />
-      <meta property="og:description" content="" />
-      <meta property="og:article:author" content="${authorName}" />
-      </head>
-      <body>
-      <iframe width="${width}" height="${height}" src="https://www.youtube.com/embed/${videoId}" title="${title}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
-        <p><a href="${url}" target="_blank">${title}</a></p>
-        <p itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="${oembed.author_url}" target="_blank">${authorName}</a></p>
-      </body>
-    </html>`
-
-    console.log('got video id', videoId)
-
-    return { content, title: 'Youtube Content' };
-  }
-}
--- a/yarn.lock
+++ b/yarn.lock
@ -10104,6 +10104,13 @@ brace-expansion@^1.1.7:
    balanced-match "^1.0.0"
    concat-map "0.0.1"

+brace-expansion@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
+  integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
+  dependencies:
+    balanced-match "^1.0.0"
+
 braces@^2.3.1, braces@^2.3.2:
  version "2.3.2"
  resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729"
@ -10579,6 +10586,19 @@ chai@^4.3.4:
    pathval "^1.1.1"
    type-detect "^4.0.5"

+chai@^4.3.6:
+  version "4.3.6"
+  resolved "https://registry.yarnpkg.com/chai/-/chai-4.3.6.tgz#ffe4ba2d9fa9d6680cc0b370adae709ec9011e9c"
+  integrity sha512-bbcp3YfHCUzMOvKqsztczerVgBKSsEijCySNlHHbX3VG1nskvqjz5Rfso1gGwD6w6oOV3eI60pKuMOV5MV7p3Q==
+  dependencies:
+    assertion-error "^1.1.0"
+    check-error "^1.0.2"
+    deep-eql "^3.0.1"
+    get-func-name "^2.0.0"
+    loupe "^2.3.1"
+    pathval "^1.1.1"
+    type-detect "^4.0.5"
+
 chalk@^1.0.0, chalk@^1.1.3:
  version "1.1.3"
  resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98"
@ -10748,6 +10768,21 @@ chokidar@3.5.2:
  optionalDependencies:
    fsevents "~2.3.2"

+chokidar@3.5.3, chokidar@^3.4.1, chokidar@^3.4.2, chokidar@^3.5.1, chokidar@^3.5.2, chokidar@^3.5.3:
+  version "3.5.3"
+  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
+  integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==
+  dependencies:
+    anymatch "~3.1.2"
+    braces "~3.0.2"
+    glob-parent "~5.1.2"
+    is-binary-path "~2.1.0"
+    is-glob "~4.0.1"
+    normalize-path "~3.0.0"
+    readdirp "~3.6.0"
+  optionalDependencies:
+    fsevents "~2.3.2"
+
 chokidar@^2.1.8:
  version "2.1.8"
  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917"
@ -10767,21 +10802,6 @@ chokidar@^2.1.8:
  optionalDependencies:
    fsevents "^1.2.7"

-chokidar@^3.4.1, chokidar@^3.4.2, chokidar@^3.5.1, chokidar@^3.5.2, chokidar@^3.5.3:
-  version "3.5.3"
-  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
-  integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==
-  dependencies:
-    anymatch "~3.1.2"
-    braces "~3.0.2"
-    glob-parent "~5.1.2"
-    is-binary-path "~2.1.0"
-    is-glob "~4.0.1"
-    normalize-path "~3.0.0"
-    readdirp "~3.6.0"
-  optionalDependencies:
-    fsevents "~2.3.2"
-
 chownr@^1.1.1, chownr@^1.1.4:
  version "1.1.4"
  resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
@ -14489,7 +14509,7 @@ glob@7.1.7:
    once "^1.3.0"
    path-is-absolute "^1.0.0"

-glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.0:
+glob@7.2.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.0:
  version "7.2.0"
  resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023"
  integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==
@ -17652,6 +17672,17 @@ linkedom@^0.14.12:
    htmlparser2 "^8.0.1"
    uhyphen "^0.1.0"

+linkedom@^0.14.16:
+  version "0.14.16"
+  resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.16.tgz#124eb006fad1dfe7ed8f96ec8ae74ab0fb0fd88e"
+  integrity sha512-a4QWl4W93P15/x+4d9k8K+C81nOzQeGOs3D37uG0TFqKZYGLEyZwXweSFrypK8yvUx5U2cuZKkdDIOjaouv3ag==
+  dependencies:
+    css-select "^5.1.0"
+    cssom "^0.5.0"
+    html-escaper "^3.0.3"
+    htmlparser2 "^8.0.1"
+    uhyphen "^0.1.0"
+
 linkedom@^0.14.9:
  version "0.14.9"
  resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.9.tgz#34c6f15eddc809406f42d8ee48cd30b0222eccb0"
@ -18054,6 +18085,13 @@ loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0:
  dependencies:
    js-tokens "^3.0.0 || ^4.0.0"

+loupe@^2.3.1:
+  version "2.3.4"
+  resolved "https://registry.yarnpkg.com/loupe/-/loupe-2.3.4.tgz#7e0b9bffc76f148f9be769cb1321d3dcf3cb25f3"
+  integrity sha512-OvKfgCC2Ndby6aSTREl5aCCPTNIzlDfQZvZxNUrBrihDhL3xcrYegTblhmEiCrg2kKQz4XsFIaemE5BF4ybSaQ==
+  dependencies:
+    get-func-name "^2.0.0"
+
 lower-case-first@^1.0.0:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/lower-case-first/-/lower-case-first-1.0.2.tgz#e5da7c26f29a7073be02d52bac9980e5922adfa1"
@ -18151,6 +18189,11 @@ luxon@^2.3.1:
  resolved "https://registry.yarnpkg.com/luxon/-/luxon-2.3.1.tgz#f276b1b53fd9a740a60e666a541a7f6dbed4155a"
  integrity sha512-I8vnjOmhXsMSlNMZlMkSOvgrxKJl0uOsEzdGgGNZuZPaS9KlefpE9KV95QFftlJSC+1UyCC9/I69R02cz/zcCA==

+luxon@^3.0.4:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.4.tgz#d179e4e9f05e092241e7044f64aaa54796b03929"
+  integrity sha512-aV48rGUwP/Vydn8HT+5cdr26YYQiUZ42NM6ToMoaGKwYfWbfLeRkEu1wXWMHBZT6+KyLfcbbtVcoQFCbbPjKlw==
+
 lz-string@^1.4.4:
  version "1.4.4"
  resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26"
@ -18605,6 +18648,13 @@ minimatch@3.0.4:
  dependencies:
    brace-expansion "^1.1.7"

+minimatch@5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.0.1.tgz#fb9022f7528125187c92bd9e9b6366be1cf3415b"
+  integrity sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==
+  dependencies:
+    brace-expansion "^2.0.1"
+
 minimatch@^3.0.2, minimatch@^3.0.4:
  version "3.1.2"
  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
@ -18787,6 +18837,34 @@ mocha-unfunk-reporter@^0.4.0:
    miniwrite "~0.1.3"
    unfunk-diff "~0.0.1"

+mocha@^10.0.0:
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.0.0.tgz#205447d8993ec755335c4b13deba3d3a13c4def9"
+  integrity sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA==
+  dependencies:
+    "@ungap/promise-all-settled" "1.1.2"
+    ansi-colors "4.1.1"
+    browser-stdout "1.3.1"
+    chokidar "3.5.3"
+    debug "4.3.4"
+    diff "5.0.0"
+    escape-string-regexp "4.0.0"
+    find-up "5.0.0"
+    glob "7.2.0"
+    he "1.2.0"
+    js-yaml "4.1.0"
+    log-symbols "4.1.0"
+    minimatch "5.0.1"
+    ms "2.1.3"
+    nanoid "3.3.3"
+    serialize-javascript "6.0.0"
+    strip-json-comments "3.1.1"
+    supports-color "8.1.1"
+    workerpool "6.2.1"
+    yargs "16.2.0"
+    yargs-parser "20.2.4"
+    yargs-unparser "2.0.0"
+
 mocha@^8.2.0:
  version "8.4.0"
  resolved "https://registry.yarnpkg.com/mocha/-/mocha-8.4.0.tgz#677be88bf15980a3cae03a73e10a0fc3997f0cff"
@ -18949,7 +19027,7 @@ nan@^2.12.1:
  resolved "https://registry.yarnpkg.com/nan/-/nan-2.15.0.tgz#3f34a473ff18e15c1b5626b62903b5ad6e665fee"
  integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==

-nanoid@*, nanoid@^3.1.23, nanoid@^3.1.25, nanoid@^3.1.29, nanoid@^3.1.30, nanoid@^3.3.1:
+nanoid@*, nanoid@3.3.3, nanoid@^3.1.23, nanoid@^3.1.25, nanoid@^3.1.29, nanoid@^3.1.30, nanoid@^3.3.1:
  version "3.3.3"
  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.3.tgz#fd8e8b7aa761fe807dba2d1b98fb7241bb724a25"
  integrity sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==
@ -19120,6 +19198,16 @@ nock@^13.2.4:
    lodash.set "^4.3.2"
    propagate "^2.0.0"

+nock@^13.2.9:
+  version "13.2.9"
+  resolved "https://registry.yarnpkg.com/nock/-/nock-13.2.9.tgz#4faf6c28175d36044da4cfa68e33e5a15086ad4c"
+  integrity sha512-1+XfJNYF1cjGB+TKMWi29eZ0b82QOvQs2YoLNzbpWGqFMtRQHTa57osqdGj4FrFPgkO4D4AZinzUJR9VvW3QUA==
+  dependencies:
+    debug "^4.1.0"
+    json-stringify-safe "^5.0.1"
+    lodash "^4.17.21"
+    propagate "^2.0.0"
+
 node-addon-api@^1.2.0:
  version "1.7.2"
  resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-1.7.2.tgz#3df30b95720b53c24e59948b49532b662444f54d"
@ -24493,6 +24581,11 @@ underscore@^1.13.4, underscore@^1.9.1:
  resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.4.tgz#7886b46bbdf07f768e0052f1828e1dcab40c0dee"
  integrity sha512-BQFnUDuAQ4Yf/cYY5LNrK9NCJFKriaRbD9uR1fTeXnBeoa97W0i41qkZfGO9pSo8I5KzjAcSY2XYtdf0oKd7KQ==

+underscore@^1.13.6:
+  version "1.13.6"
+  resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.6.tgz#04786a1f589dc6c09f761fc5f45b89e935136441"
+  integrity sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==
+
 undici@^4.9.3:
  version "4.14.1"
  resolved "https://registry.yarnpkg.com/undici/-/undici-4.14.1.tgz#7633b143a8a10d6d63335e00511d071e8d52a1d9"
@ -24912,6 +25005,11 @@ uuid@^8.0.0, uuid@^8.3.0, uuid@^8.3.1, uuid@^8.3.2:
  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
  integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==

+uuid@^9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5"
+  integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==
+
 v8-compile-cache-lib@^3.0.1:
  version "3.0.1"
  resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"
@ -25582,6 +25680,11 @@ workerpool@6.1.5:
  resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.1.5.tgz#0f7cf076b6215fd7e1da903ff6f22ddd1886b581"
  integrity sha512-XdKkCK0Zqc6w3iTxLckiuJ81tiD/o5rBE/m+nXpRCB+/Sq4DqkfXZ/x0jW02DG1tGsfUGXbTJyZDP+eu67haSw==

+workerpool@6.2.1:
+  version "6.2.1"
+  resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343"
+  integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==
+
 wrap-ansi@^3.0.1:
  version "3.0.1"
  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba"