Share puppeteer browser instance between content handlers

packages/content-handler/src/content-handler.ts

@@ -3,6 +3,7 @@ import rfc2047 from 'rfc2047'
 import { v4 as uuid } from 'uuid'
 import { parseHTML } from 'linkedom'
 import axios from 'axios'
+import { Browser } from 'puppeteer-core'
 
 interface Unsubscribe {
   mailTo?: string
@@ -62,7 +63,7 @@ export abstract class ContentHandler {
     return false
   }
 
-  async preHandle(url: string): Promise<PreHandleResult> {
+  async preHandle(url: string, browser?: Browser): Promise<PreHandleResult> {
     return Promise.resolve({ url })
   }
 

packages/content-handler/src/index.ts

@@ -28,6 +28,7 @@ import { GhostHandler } from './newsletters/ghost-handler'
 import { parseHTML } from 'linkedom'
 import { CooperPressHandler } from './newsletters/cooper-press-handler'
 import { HeyWorldHandler } from './newsletters/hey-world-handler'
+import { Browser } from 'puppeteer-core'
 
 const validateUrlString = (url: string) => {
   const u = new URL(url)
@@ -80,7 +81,8 @@ const newsletterHandlers: ContentHandler[] = [
 ]
 
 export const preHandleContent = async (
-  url: string
+  url: string,
+  browser: Browser
 ): Promise<PreHandleResult | undefined> => {
   // Before we run the regular handlers we check to see if we need tp
   // pre-resolve the URL. TODO: This should probably happen recursively,
@@ -104,7 +106,7 @@ export const preHandleContent = async (
   for (const handler of contentHandlers) {
     if (handler.shouldPreHandle(url)) {
       console.log('preHandleContent', handler.name, url)
-      return handler.preHandle(url)
+      return handler.preHandle(url, browser)
     }
   }
   return undefined

packages/content-handler/src/websites/twitter-handler.ts

@@ -2,7 +2,7 @@ import { ContentHandler, PreHandleResult } from '../content-handler'
 import axios from 'axios'
 import { DateTime } from 'luxon'
 import _ from 'underscore'
-import puppeteer from 'puppeteer-core'
+import { Browser } from 'puppeteer-core'
 
 interface TweetIncludes {
   users: {
@@ -168,10 +168,11 @@ const getTweetsFromResponse = (response: Tweets): Tweet[] => {
 }
 
 const getOldTweets = async (
+  browser: Browser,
   conversationId: string,
   username: string
 ): Promise<Tweet[]> => {
-  const tweetIds = await getTweetIds(conversationId, username)
+  const tweetIds = await getTweetIds(browser, conversationId, username)
   if (tweetIds.length === 0) {
     return []
   }
@@ -197,37 +198,40 @@ const waitFor = (ms: number) =>
 
 /**
  * Get tweets(even older than 7 days) using puppeteer
+ * @param browser
  * @param {string} tweetId
  * @param {string} author
  */
 const getTweetIds = async (
+  browser: Browser,
   tweetId: string,
   author: string
 ): Promise<string[]> => {
   const pageURL = `https://twitter.com/${author}/status/${tweetId}`
 
-  // Modify this variable to control the size of viewport
-  const factor = 0.2
-  const height = Math.floor(2000 / factor)
-  const width = Math.floor(1700 / factor)
-
-  const browser = await puppeteer.launch({
-    executablePath: process.env.CHROMIUM_PATH,
-    headless: !!process.env.LAUNCH_HEADLESS,
-    defaultViewport: {
-      width,
-      height,
-    },
-    args: [
-      `--force-device-scale-factor=${factor}`,
-      `--window-size=${width},${height}`,
-      '--no-sandbox',
-      '--disable-setuid-sandbox',
-    ],
-  })
+  // // Modify this variable to control the size of viewport
+  // const factor = 0.2
+  // const height = Math.floor(2000 / factor)
+  // const width = Math.floor(1700 / factor)
+  //
+  // const browser = await puppeteer.launch({
+  //   executablePath: process.env.CHROMIUM_PATH,
+  //   headless: !!process.env.LAUNCH_HEADLESS,
+  //   defaultViewport: {
+  //     width,
+  //     height,
+  //   },
+  //   args: [
+  //     `--force-device-scale-factor=${factor}`,
+  //     `--window-size=${width},${height}`,
+  //     '--no-sandbox',
+  //     '--disable-setuid-sandbox',
+  //   ],
+  // })
 
+  const context = await browser.createIncognitoBrowserContext()
   try {
-    const page = await browser.newPage()
+    const page = await context.newPage()
 
     await page.goto(pageURL, {
       waitUntil: 'networkidle2',
@@ -291,7 +295,7 @@ const getTweetIds = async (
     console.log(error)
     return []
   } finally {
-    await browser.close()
+    await context.close()
   }
 }
 
@@ -305,7 +309,7 @@ export class TwitterHandler extends ContentHandler {
     return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
   }
 
-  async preHandle(url: string): Promise<PreHandleResult> {
+  async preHandle(url: string, browser: Browser): Promise<PreHandleResult> {
     const tweetId = tweetIdFromStatusUrl(url)
     if (!tweetId) {
       throw new Error('could not find tweet id in url')
@@ -326,7 +330,7 @@ export class TwitterHandler extends ContentHandler {
     const description = _.escape(tweetData.text)
 
     // use puppeteer to get all tweet replies in the thread
-    const tweets = await getOldTweets(conversationId, author.username)
+    const tweets = await getOldTweets(browser, conversationId, author.username)
 
     let tweetsContent = ''
     for (const tweet of tweets) {

packages/puppeteer-parse/index.js

@@ -237,7 +237,8 @@ async function fetchContent(req, res) {
   // pre handle url with custom handlers
   let title, content, contentType;
   try {
-    const result = await preHandleContent(url);
+    const browser = await getBrowserPromise;
+    const result = await preHandleContent(url, browser);
     if (result && result.url) {
       url = result.url
       validateUrlString(url);
@@ -682,16 +683,16 @@ async function preview(req, res) {
     return res.sendStatus(400);
   }
 
-  const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH);
+  const browser = await getBrowserPromise;
   logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime };
 
   const page = await browser.newPage();
   const pageLoadingStart = Date.now();
   const modifiedUrl = new URL(url);
-  modifiedUrl.searchParams.append('fontSize', 24);
-  modifiedUrl.searchParams.append('adjustAspectRatio', 1.91);
+  modifiedUrl.searchParams.append('fontSize', '24');
+  modifiedUrl.searchParams.append('adjustAspectRatio', '1.91');
   try {
-    await page.goto(modifiedUrl);
+    await page.goto(modifiedUrl.toString());
     logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart };
   } catch (error) {
     console.log('error going to page: ', modifiedUrl)