From 6ca1e7dc25bbc6f904022ced52e64ffc1e7da22d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 27 Oct 2022 12:49:00 +0800 Subject: [PATCH 1/3] Add no-sandbox in the puppeteer args --- packages/content-handler/src/websites/twitter-handler.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 4267270ae..4389d0f81 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -211,6 +211,8 @@ const getTweetIds = async (tweetId: string): Promise => { args: [ `--force-device-scale-factor=${factor}`, `--window-size=${width},${height}`, + '--no-sandbox', + '--disable-setuid-sandbox', ], }) From b556bc17e8fe6561256b0630be2b1cd3550edc2c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 27 Oct 2022 13:05:35 +0800 Subject: [PATCH 2/3] Set navigation timeout to be 60 seconds --- packages/content-handler/src/websites/twitter-handler.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 4389d0f81..2d602c010 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -169,6 +169,9 @@ const getTweetsFromResponse = (response: Tweets): Tweet[] => { const getOldTweets = async (conversationId: string): Promise => { const tweetIds = await getTweetIds(conversationId) + if (tweetIds.length === 0) { + return [] + } const response = await getTweetsByIds(tweetIds) return getTweetsFromResponse(response) } @@ -203,7 +206,7 @@ const getTweetIds = async (tweetId: string): Promise => { const browser = await puppeteer.launch({ executablePath: process.env.CHROMIUM_PATH, - headless: true, + headless: !!process.env.LAUNCH_HEADLESS, defaultViewport: { width, height, @@ -221,6 +224,7 @@ const getTweetIds = async (tweetId: string): Promise => { await page.goto(pageURL, { waitUntil: 'networkidle2', + timeout: 60000, }) await waitFor(4000) From a536ed9b5e21569dab0e1a5b7b919c5b1c3e5251 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 27 Oct 2022 14:20:20 +0800 Subject: [PATCH 3/3] Save newer thread with puppeteer too --- .../content-handler/src/websites/twitter-handler.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 2d602c010..8a60adeb9 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -313,16 +313,8 @@ export class TwitterHandler extends ContentHandler { const authorImage = author.profile_image_url.replace('_normal', '_400x400') const description = _.escape(tweetData.text) - let tweets: Tweet[] - if ( - new Date(tweet.data.created_at).getTime() < - Date.now() - 7 * 24 * 60 * 60 * 1000 - ) { - // tweet is older than 7 days, so we need to use puppeteer to get the older tweets - tweets = await getOldTweets(conversationId) - } else { - tweets = [tweet, ...(await getRecentTweets(conversationId))] - } + // use puppeteer to get all tweet replies in the thread + const tweets = await getOldTweets(conversationId) let tweetsContent = '' for (const tweet of tweets) {