From d9237723c57d181ae0bfc977b2cfe5b1f77ad8a9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 17 Nov 2022 10:43:30 +0800 Subject: [PATCH] Share puppeteer browser instance between content handlers --- .../content-handler/src/content-handler.ts | 3 +- packages/content-handler/src/index.ts | 6 ++- .../src/websites/twitter-handler.ts | 54 ++++++++++--------- packages/puppeteer-parse/index.js | 11 ++-- 4 files changed, 41 insertions(+), 33 deletions(-) diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts index fd1fe5291..a05e2ac1a 100644 --- a/packages/content-handler/src/content-handler.ts +++ b/packages/content-handler/src/content-handler.ts @@ -3,6 +3,7 @@ import rfc2047 from 'rfc2047' import { v4 as uuid } from 'uuid' import { parseHTML } from 'linkedom' import axios from 'axios' +import { Browser } from 'puppeteer-core' interface Unsubscribe { mailTo?: string @@ -62,7 +63,7 @@ export abstract class ContentHandler { return false } - async preHandle(url: string): Promise { + async preHandle(url: string, browser?: Browser): Promise { return Promise.resolve({ url }) } diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index ce61c0a64..e210d025d 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -28,6 +28,7 @@ import { GhostHandler } from './newsletters/ghost-handler' import { parseHTML } from 'linkedom' import { CooperPressHandler } from './newsletters/cooper-press-handler' import { HeyWorldHandler } from './newsletters/hey-world-handler' +import { Browser } from 'puppeteer-core' const validateUrlString = (url: string) => { const u = new URL(url) @@ -80,7 +81,8 @@ const newsletterHandlers: ContentHandler[] = [ ] export const preHandleContent = async ( - url: string + url: string, + browser: Browser ): Promise => { // Before we run the regular handlers we check to see if we need tp // pre-resolve the URL. TODO: This should probably happen recursively, @@ -104,7 +106,7 @@ export const preHandleContent = async ( for (const handler of contentHandlers) { if (handler.shouldPreHandle(url)) { console.log('preHandleContent', handler.name, url) - return handler.preHandle(url) + return handler.preHandle(url, browser) } } return undefined diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 17bbdc4c2..788301b8e 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -2,7 +2,7 @@ import { ContentHandler, PreHandleResult } from '../content-handler' import axios from 'axios' import { DateTime } from 'luxon' import _ from 'underscore' -import puppeteer from 'puppeteer-core' +import { Browser } from 'puppeteer-core' interface TweetIncludes { users: { @@ -168,10 +168,11 @@ const getTweetsFromResponse = (response: Tweets): Tweet[] => { } const getOldTweets = async ( + browser: Browser, conversationId: string, username: string ): Promise => { - const tweetIds = await getTweetIds(conversationId, username) + const tweetIds = await getTweetIds(browser, conversationId, username) if (tweetIds.length === 0) { return [] } @@ -197,37 +198,40 @@ const waitFor = (ms: number) => /** * Get tweets(even older than 7 days) using puppeteer + * @param browser * @param {string} tweetId * @param {string} author */ const getTweetIds = async ( + browser: Browser, tweetId: string, author: string ): Promise => { const pageURL = `https://twitter.com/${author}/status/${tweetId}` - // Modify this variable to control the size of viewport - const factor = 0.2 - const height = Math.floor(2000 / factor) - const width = Math.floor(1700 / factor) - - const browser = await puppeteer.launch({ - executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, - defaultViewport: { - width, - height, - }, - args: [ - `--force-device-scale-factor=${factor}`, - `--window-size=${width},${height}`, - '--no-sandbox', - '--disable-setuid-sandbox', - ], - }) + // // Modify this variable to control the size of viewport + // const factor = 0.2 + // const height = Math.floor(2000 / factor) + // const width = Math.floor(1700 / factor) + // + // const browser = await puppeteer.launch({ + // executablePath: process.env.CHROMIUM_PATH, + // headless: !!process.env.LAUNCH_HEADLESS, + // defaultViewport: { + // width, + // height, + // }, + // args: [ + // `--force-device-scale-factor=${factor}`, + // `--window-size=${width},${height}`, + // '--no-sandbox', + // '--disable-setuid-sandbox', + // ], + // }) + const context = await browser.createIncognitoBrowserContext() try { - const page = await browser.newPage() + const page = await context.newPage() await page.goto(pageURL, { waitUntil: 'networkidle2', @@ -291,7 +295,7 @@ const getTweetIds = async ( console.log(error) return [] } finally { - await browser.close() + await context.close() } } @@ -305,7 +309,7 @@ export class TwitterHandler extends ContentHandler { return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) } - async preHandle(url: string): Promise { + async preHandle(url: string, browser: Browser): Promise { const tweetId = tweetIdFromStatusUrl(url) if (!tweetId) { throw new Error('could not find tweet id in url') @@ -326,7 +330,7 @@ export class TwitterHandler extends ContentHandler { const description = _.escape(tweetData.text) // use puppeteer to get all tweet replies in the thread - const tweets = await getOldTweets(conversationId, author.username) + const tweets = await getOldTweets(browser, conversationId, author.username) let tweetsContent = '' for (const tweet of tweets) { diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index ccf74d39e..fca2e292b 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -237,7 +237,8 @@ async function fetchContent(req, res) { // pre handle url with custom handlers let title, content, contentType; try { - const result = await preHandleContent(url); + const browser = await getBrowserPromise; + const result = await preHandleContent(url, browser); if (result && result.url) { url = result.url validateUrlString(url); @@ -682,16 +683,16 @@ async function preview(req, res) { return res.sendStatus(400); } - const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH); + const browser = await getBrowserPromise; logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; const page = await browser.newPage(); const pageLoadingStart = Date.now(); const modifiedUrl = new URL(url); - modifiedUrl.searchParams.append('fontSize', 24); - modifiedUrl.searchParams.append('adjustAspectRatio', 1.91); + modifiedUrl.searchParams.append('fontSize', '24'); + modifiedUrl.searchParams.append('adjustAspectRatio', '1.91'); try { - await page.goto(modifiedUrl); + await page.goto(modifiedUrl.toString()); logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart }; } catch (error) { console.log('error going to page: ', modifiedUrl)