From b5926ccf1c86eefe5447cb9b28b8b0fa0a2b8aa0 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 26 Oct 2022 20:41:51 +0800 Subject: [PATCH] Get old tweet thread with puppeteer and new tweet with twitter api --- packages/content-fetch/fetch-content.js | 5 +- .../src/websites/twitter-handler.ts | 109 ++++++++++++------ 2 files changed, 77 insertions(+), 37 deletions(-) diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js index 869de8aa2..7a35e1eeb 100644 --- a/packages/content-fetch/fetch-content.js +++ b/packages/content-fetch/fetch-content.js @@ -268,7 +268,6 @@ async function fetchContent(req, res) { } } else { console.log('using prefetched content and title'); - console.log(content); } logRecord.fetchContentTime = Date.now() - functionStartTime; @@ -288,10 +287,8 @@ async function fetchContent(req, res) { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; - console.log(`parse-page`, logRecord); } } catch (e) { - console.log('error', e) logRecord.error = e.message; console.log(`Error while retrieving page`, logRecord); @@ -316,11 +313,11 @@ async function fetchContent(req, res) { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; - console.log(`parse-page`, logRecord); } finally { if (context) { await context.close(); } + console.log(`parse-page`, logRecord); } return res.sendStatus(200); diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 06549624f..e7291c61a 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -49,7 +49,7 @@ interface Tweet { includes: TweetIncludes } -interface TweetThread { +interface Tweets { data: TweetData[] includes: TweetIncludes meta: TweetMeta @@ -74,17 +74,12 @@ const getTweetFields = () => { } // unroll recent tweet thread -const getTweetThread = async ( - conversationId: string, - username: string -): Promise => { +const getTweetThread = async (conversationId: string): Promise => { const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/search/recent' const apiUrl = new URL( BASE_ENDPOINT + '?query=' + - encodeURIComponent( - `conversation_id:${conversationId} from:${username} to:${username}` - ) + + encodeURIComponent(`conversation_id:${conversationId}`) + getTweetFields() + '&max_results=100' ) @@ -93,7 +88,7 @@ const getTweetThread = async ( throw new Error('No Twitter bearer token found') } - const response = await axios.get(apiUrl.toString(), { + const response = await axios.get(apiUrl.toString(), { headers: { Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, redirect: 'follow', @@ -120,6 +115,24 @@ const getTweetById = async (id: string): Promise => { return response.data } +const getTweetsByIds = async (ids: string[]): Promise => { + const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets?ids=' + const apiUrl = new URL(BASE_ENDPOINT + ids.join() + getTweetFields()) + + if (!TWITTER_BEARER_TOKEN) { + throw new Error('No Twitter bearer token found') + } + + const response = await axios.get(apiUrl.toString(), { + headers: { + Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, + redirect: 'follow', + }, + }) + + return response.data +} + const titleForAuthor = (author: { name: string }) => { return `${author.name} on Twitter` } @@ -135,18 +148,51 @@ const formatTimestamp = (timestamp: string) => { ) } +const getTweetsFromResponse = (response: Tweets): Tweet[] => { + const tweets = [] + for (const t of response.data) { + const media = response.includes.media?.filter((m) => + t.attachments?.media_keys?.includes(m.media_key) + ) + const tweet: Tweet = { + data: t, + includes: { + users: response.includes.users, + media, + }, + } + tweets.push(tweet) + } + return tweets +} + +const getOldTweets = async (conversationId: string): Promise => { + const tweetIds = await getTweetIds(conversationId) + const response = await getTweetsByIds(tweetIds) + return getTweetsFromResponse(response) +} + +const getRecentTweets = async (conversationId: string): Promise => { + const thread = await getTweetThread(conversationId) + if (thread.meta.result_count === 0) { + return [] + } + // tweets are in reverse chronological order in the thread + return getTweetsFromResponse(thread).reverse() +} + /** * Wait for `ms` amount of milliseconds * @param {number} ms */ -const waitFor = async (ms: number) => +const waitFor = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) /** * Get tweets(even older than 7 days) using puppeteer * @param {string} tweetId */ -const getTweetIdsFromThreadId = async (tweetId: string): Promise => { +const getTweetIds = async (tweetId: string): Promise => { const pageURL = `https://twitter.com/anyone/status/${tweetId}` // Modify this variable to control the size of viewport @@ -155,6 +201,7 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise => { const width = Math.floor(1700 / factor) const browser = await puppeteer.launch({ + executablePath: process.env.CHROMIUM_PATH, headless: true, defaultViewport: { width, @@ -170,7 +217,6 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise => { await page.goto(pageURL, { waitUntil: 'networkidle2', - timeout: 5 * 60 * 1000, }) await waitFor(4000) @@ -179,6 +225,13 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise => { const tweetIds = (await page.evaluate(async () => { const ids: string[] = [] + /** + * Wait for `ms` amount of milliseconds + * @param {number} ms + */ + const waitFor = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)) + // Find the first Show thread button and click it const showRepliesButton = Array.from( document.querySelectorAll('div[dir="auto"]') @@ -199,7 +252,7 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise => { const timeContainerAnchor = timeNode.parentElement if (!timeContainerAnchor) continue - if (timeContainerAnchor?.tagName === 'SPAN') continue + if (timeContainerAnchor.tagName === 'SPAN') continue const id = (timeContainerAnchor as HTMLAnchorElement).href .split('/') @@ -213,7 +266,7 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise => { await browser.close() - return [tweetId, ...tweetIds] + return tweetIds } export class TwitterHandler extends ContentHandler { @@ -246,25 +299,15 @@ export class TwitterHandler extends ContentHandler { const authorImage = author.profile_image_url.replace('_normal', '_400x400') const description = _.escape(tweetData.text) - const tweets = [tweet] - // we want to get the full thread - const thread = await getTweetThread(conversationId, author.username) - if (thread.meta.result_count > 0) { - // tweets are in reverse chronological order in the thread - for (const t of thread.data.reverse()) { - // get the tweet media if it exists - const media = thread.includes.media?.filter((m) => - t.attachments?.media_keys?.includes(m.media_key) - ) - const tweet: Tweet = { - data: t, - includes: { - users: thread.includes.users, - media, - }, - } - tweets.push(tweet) - } + let tweets: Tweet[] + if ( + new Date(tweet.data.created_at).getTime() < + Date.now() - 7 * 24 * 60 * 60 * 1000 + ) { + // tweet is older than 7 days, so we need to use puppeteer to get the older tweets + tweets = await getOldTweets(conversationId) + } else { + tweets = [tweet, ...(await getRecentTweets(conversationId))] } let tweetsContent = ''