From 343c8fb5cca811dd31174dd0ca44ac36e6ef78f7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 25 Oct 2022 18:25:49 +0800 Subject: [PATCH 1/4] Get tweet reply ids from thread id --- .../src/websites/twitter-handler.ts | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 16138c1e1..3aa820468 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -140,6 +140,81 @@ export class TwitterHandler extends ContentHandler { this.name = 'Twitter' } + async getTweetIdsFromThreadId(tweetID: string): Promise { + const pageURL = `https://twitter.com/anyone/status/${tweetID}` + + // Modify this variable to control the size of viewport + const factor = 0.2 + const height = Math.floor(2000 / factor) + const width = Math.floor(1700 / factor) + + const browser = await puppeteer.launch({ + headless: true, + defaultViewport: { + width, + height, + }, + args: [ + `--force-device-scale-factor=${factor}`, + `--window-size=${width},${height}`, + ], + }) + + const page = await browser.newPage() + + await page.goto(pageURL, { + waitUntil: 'networkidle2', + timeout: 5 * 60 * 1000, + }) + + await waitFor(4000) + + /** @type {string[]} */ + const tweetIDs = await page.evaluate(async () => { + const ids = [] + + /** + * Wait for `ms` amount of milliseconds + * @param {number} ms + */ + const waitFor = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + + // Find the first Show thread button and click it + const showRepliesButton = [ + ...document.querySelectorAll('div[dir="auto"]'), + ] + .filter( + (node) => node.children[0] && node.children[0].tagName === 'SPAN' + ) + .find((node) => node.children[0].innerHTML === 'Show replies') + + if (showRepliesButton) { + showRepliesButton.click() + + await waitFor(2000) + } + + const timeNodes = Array.from(document.querySelectorAll('time')) + + for (const timeNode of timeNodes) { + /** @type {HTMLAnchorElement | HTMLSpanElement} */ + const timeContainerAnchor = timeNode.parentElement + + if (timeContainerAnchor.tagName === 'SPAN') continue + + const id = timeContainerAnchor.href.split('/').reverse()[0] + + ids.push(id) + } + + return ids + }) + + await browser.close() + + return [tweetID, ...tweetIDs] + } + shouldPreHandle(url: string): boolean { return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) } From 5f0d0ed69bb03e8766af79d57faf9d108195c1d7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 25 Oct 2022 22:09:16 +0800 Subject: [PATCH 2/4] Get tweet reply ids from thread id using puppeteer --- packages/content-handler/package.json | 1 + .../src/websites/twitter-handler.ts | 157 +++++++++--------- yarn.lock | 26 +++ 3 files changed, 109 insertions(+), 75 deletions(-) diff --git a/packages/content-handler/package.json b/packages/content-handler/package.json index dc4edb4b1..6f5c18197 100644 --- a/packages/content-handler/package.json +++ b/packages/content-handler/package.json @@ -32,6 +32,7 @@ "axios": "^0.27.2", "linkedom": "^0.14.16", "luxon": "^3.0.4", + "puppeteer-core": "^19.1.1", "rfc2047": "^4.0.1", "underscore": "^1.13.6", "uuid": "^9.0.0" diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 3aa820468..06549624f 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -2,6 +2,7 @@ import { ContentHandler, PreHandleResult } from '../content-handler' import axios from 'axios' import { DateTime } from 'luxon' import _ from 'underscore' +import puppeteer from 'puppeteer-core' interface TweetIncludes { users: { @@ -134,87 +135,93 @@ const formatTimestamp = (timestamp: string) => { ) } +/** + * Wait for `ms` amount of milliseconds + * @param {number} ms + */ +const waitFor = async (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)) + +/** + * Get tweets(even older than 7 days) using puppeteer + * @param {string} tweetId + */ +const getTweetIdsFromThreadId = async (tweetId: string): Promise => { + const pageURL = `https://twitter.com/anyone/status/${tweetId}` + + // Modify this variable to control the size of viewport + const factor = 0.2 + const height = Math.floor(2000 / factor) + const width = Math.floor(1700 / factor) + + const browser = await puppeteer.launch({ + headless: true, + defaultViewport: { + width, + height, + }, + args: [ + `--force-device-scale-factor=${factor}`, + `--window-size=${width},${height}`, + ], + }) + + const page = await browser.newPage() + + await page.goto(pageURL, { + waitUntil: 'networkidle2', + timeout: 5 * 60 * 1000, + }) + + await waitFor(4000) + + /** @type {string[]} */ + const tweetIds = (await page.evaluate(async () => { + const ids: string[] = [] + + // Find the first Show thread button and click it + const showRepliesButton = Array.from( + document.querySelectorAll('div[dir="auto"]') + ) + .filter((node) => node.children[0] && node.children[0].tagName === 'SPAN') + .find((node) => node.children[0].innerHTML === 'Show replies') + + if (showRepliesButton) { + ;(showRepliesButton as HTMLElement).click() + + await waitFor(2000) + } + + const timeNodes = Array.from(document.querySelectorAll('time')) + + for (const timeNode of timeNodes) { + /** @type {HTMLAnchorElement | HTMLSpanElement} */ + const timeContainerAnchor = timeNode.parentElement + if (!timeContainerAnchor) continue + + if (timeContainerAnchor?.tagName === 'SPAN') continue + + const id = (timeContainerAnchor as HTMLAnchorElement).href + .split('/') + .reverse()[0] + + ids.push(id) + } + + return ids + })) as string[] + + await browser.close() + + return [tweetId, ...tweetIds] +} + export class TwitterHandler extends ContentHandler { constructor() { super() this.name = 'Twitter' } - async getTweetIdsFromThreadId(tweetID: string): Promise { - const pageURL = `https://twitter.com/anyone/status/${tweetID}` - - // Modify this variable to control the size of viewport - const factor = 0.2 - const height = Math.floor(2000 / factor) - const width = Math.floor(1700 / factor) - - const browser = await puppeteer.launch({ - headless: true, - defaultViewport: { - width, - height, - }, - args: [ - `--force-device-scale-factor=${factor}`, - `--window-size=${width},${height}`, - ], - }) - - const page = await browser.newPage() - - await page.goto(pageURL, { - waitUntil: 'networkidle2', - timeout: 5 * 60 * 1000, - }) - - await waitFor(4000) - - /** @type {string[]} */ - const tweetIDs = await page.evaluate(async () => { - const ids = [] - - /** - * Wait for `ms` amount of milliseconds - * @param {number} ms - */ - const waitFor = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) - - // Find the first Show thread button and click it - const showRepliesButton = [ - ...document.querySelectorAll('div[dir="auto"]'), - ] - .filter( - (node) => node.children[0] && node.children[0].tagName === 'SPAN' - ) - .find((node) => node.children[0].innerHTML === 'Show replies') - - if (showRepliesButton) { - showRepliesButton.click() - - await waitFor(2000) - } - - const timeNodes = Array.from(document.querySelectorAll('time')) - - for (const timeNode of timeNodes) { - /** @type {HTMLAnchorElement | HTMLSpanElement} */ - const timeContainerAnchor = timeNode.parentElement - - if (timeContainerAnchor.tagName === 'SPAN') continue - - const id = timeContainerAnchor.href.split('/').reverse()[0] - - ids.push(id) - } - - return ids - }) - - await browser.close() - - return [tweetID, ...tweetIDs] - } - shouldPreHandle(url: string): boolean { return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) } diff --git a/yarn.lock b/yarn.lock index a90c7857b..27452d1b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12333,6 +12333,11 @@ devtools-protocol@0.0.1019158: resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1019158.tgz#4b08d06108a784a2134313149626ba55f030a86f" integrity sha512-wvq+KscQ7/6spEV7czhnZc9RM/woz1AY+/Vpd8/h2HFMwJSdTliu7f/yr1A6vDdJfKICZsShqsYpEQbdhg8AFQ== +devtools-protocol@0.0.1045489: + version "0.0.1045489" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1045489.tgz#f959ad560b05acd72d55644bc3fb8168a83abf28" + integrity sha512-D+PTmWulkuQW4D1NTiCRCFxF7pQPn0hgp4YyX4wAQ6xYXKOadSWPR3ENGDQ47MW/Ewc9v2rpC/UEEGahgBYpSQ== + devtools-protocol@0.0.901419: version "0.0.901419" resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.901419.tgz#79b5459c48fe7e1c5563c02bd72f8fec3e0cebcd" @@ -21372,6 +21377,22 @@ puppeteer-core@^16.1.0: unbzip2-stream "1.4.3" ws "8.8.1" +puppeteer-core@^19.1.1: + version "19.1.1" + resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-19.1.1.tgz#6416ff925a9cc78523c490482a17a2998f7c0626" + integrity sha512-jV26Ke0VFel4MoXLjqm50uAW2uwksTP6Md1tvtXqWqXM5FyboKI6E9YYJ1qEQilUAqlhgGq8xLN5+SL8bPz/kw== + dependencies: + cross-fetch "3.1.5" + debug "4.3.4" + devtools-protocol "0.0.1045489" + extract-zip "2.0.1" + https-proxy-agent "5.0.1" + proxy-from-env "1.1.0" + rimraf "3.0.2" + tar-fs "2.1.1" + unbzip2-stream "1.4.3" + ws "8.9.0" + puppeteer@^10.1.0: version "10.4.0" resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-10.4.0.tgz#a6465ff97fda0576c4ac29601406f67e6fea3dc7" @@ -25896,6 +25917,11 @@ ws@8.8.1, ws@^8.2.3, ws@^8.3.0, ws@^8.4.2: resolved "https://registry.yarnpkg.com/ws/-/ws-8.8.1.tgz#5dbad0feb7ade8ecc99b830c1d77c913d4955ff0" integrity sha512-bGy2JzvzkPowEJV++hF07hAD6niYSr0JzBNo/J29WsB57A2r7Wlc1UFcTR9IzrPvuNVO4B8LGqF8qcpsVOhJCA== +ws@8.9.0: + version "8.9.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.9.0.tgz#2a994bb67144be1b53fe2d23c53c028adeb7f45e" + integrity sha512-Ja7nszREasGaYUYCI2k4lCKIRTt+y7XuqVoHR44YpI49TtryyqbqvDMn5eqfW7e6HzTukDRIsXqzVHScqRcafg== + "ws@^5.2.0 || ^6.0.0 || ^7.0.0", ws@^7.3.1, ws@^7.4.6: version "7.5.7" resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.7.tgz#9e0ac77ee50af70d58326ecff7e85eb3fa375e67" From b5926ccf1c86eefe5447cb9b28b8b0fa0a2b8aa0 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 26 Oct 2022 20:41:51 +0800 Subject: [PATCH 3/4] Get old tweet thread with puppeteer and new tweet with twitter api --- packages/content-fetch/fetch-content.js | 5 +- .../src/websites/twitter-handler.ts | 109 ++++++++++++------ 2 files changed, 77 insertions(+), 37 deletions(-) diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js index 869de8aa2..7a35e1eeb 100644 --- a/packages/content-fetch/fetch-content.js +++ b/packages/content-fetch/fetch-content.js @@ -268,7 +268,6 @@ async function fetchContent(req, res) { } } else { console.log('using prefetched content and title'); - console.log(content); } logRecord.fetchContentTime = Date.now() - functionStartTime; @@ -288,10 +287,8 @@ async function fetchContent(req, res) { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; - console.log(`parse-page`, logRecord); } } catch (e) { - console.log('error', e) logRecord.error = e.message; console.log(`Error while retrieving page`, logRecord); @@ -316,11 +313,11 @@ async function fetchContent(req, res) { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; - console.log(`parse-page`, logRecord); } finally { if (context) { await context.close(); } + console.log(`parse-page`, logRecord); } return res.sendStatus(200); diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 06549624f..e7291c61a 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -49,7 +49,7 @@ interface Tweet { includes: TweetIncludes } -interface TweetThread { +interface Tweets { data: TweetData[] includes: TweetIncludes meta: TweetMeta @@ -74,17 +74,12 @@ const getTweetFields = () => { } // unroll recent tweet thread -const getTweetThread = async ( - conversationId: string, - username: string -): Promise => { +const getTweetThread = async (conversationId: string): Promise => { const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/search/recent' const apiUrl = new URL( BASE_ENDPOINT + '?query=' + - encodeURIComponent( - `conversation_id:${conversationId} from:${username} to:${username}` - ) + + encodeURIComponent(`conversation_id:${conversationId}`) + getTweetFields() + '&max_results=100' ) @@ -93,7 +88,7 @@ const getTweetThread = async ( throw new Error('No Twitter bearer token found') } - const response = await axios.get(apiUrl.toString(), { + const response = await axios.get(apiUrl.toString(), { headers: { Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, redirect: 'follow', @@ -120,6 +115,24 @@ const getTweetById = async (id: string): Promise => { return response.data } +const getTweetsByIds = async (ids: string[]): Promise => { + const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets?ids=' + const apiUrl = new URL(BASE_ENDPOINT + ids.join() + getTweetFields()) + + if (!TWITTER_BEARER_TOKEN) { + throw new Error('No Twitter bearer token found') + } + + const response = await axios.get(apiUrl.toString(), { + headers: { + Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, + redirect: 'follow', + }, + }) + + return response.data +} + const titleForAuthor = (author: { name: string }) => { return `${author.name} on Twitter` } @@ -135,18 +148,51 @@ const formatTimestamp = (timestamp: string) => { ) } +const getTweetsFromResponse = (response: Tweets): Tweet[] => { + const tweets = [] + for (const t of response.data) { + const media = response.includes.media?.filter((m) => + t.attachments?.media_keys?.includes(m.media_key) + ) + const tweet: Tweet = { + data: t, + includes: { + users: response.includes.users, + media, + }, + } + tweets.push(tweet) + } + return tweets +} + +const getOldTweets = async (conversationId: string): Promise => { + const tweetIds = await getTweetIds(conversationId) + const response = await getTweetsByIds(tweetIds) + return getTweetsFromResponse(response) +} + +const getRecentTweets = async (conversationId: string): Promise => { + const thread = await getTweetThread(conversationId) + if (thread.meta.result_count === 0) { + return [] + } + // tweets are in reverse chronological order in the thread + return getTweetsFromResponse(thread).reverse() +} + /** * Wait for `ms` amount of milliseconds * @param {number} ms */ -const waitFor = async (ms: number) => +const waitFor = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) /** * Get tweets(even older than 7 days) using puppeteer * @param {string} tweetId */ -const getTweetIdsFromThreadId = async (tweetId: string): Promise => { +const getTweetIds = async (tweetId: string): Promise => { const pageURL = `https://twitter.com/anyone/status/${tweetId}` // Modify this variable to control the size of viewport @@ -155,6 +201,7 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise => { const width = Math.floor(1700 / factor) const browser = await puppeteer.launch({ + executablePath: process.env.CHROMIUM_PATH, headless: true, defaultViewport: { width, @@ -170,7 +217,6 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise => { await page.goto(pageURL, { waitUntil: 'networkidle2', - timeout: 5 * 60 * 1000, }) await waitFor(4000) @@ -179,6 +225,13 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise => { const tweetIds = (await page.evaluate(async () => { const ids: string[] = [] + /** + * Wait for `ms` amount of milliseconds + * @param {number} ms + */ + const waitFor = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)) + // Find the first Show thread button and click it const showRepliesButton = Array.from( document.querySelectorAll('div[dir="auto"]') @@ -199,7 +252,7 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise => { const timeContainerAnchor = timeNode.parentElement if (!timeContainerAnchor) continue - if (timeContainerAnchor?.tagName === 'SPAN') continue + if (timeContainerAnchor.tagName === 'SPAN') continue const id = (timeContainerAnchor as HTMLAnchorElement).href .split('/') @@ -213,7 +266,7 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise => { await browser.close() - return [tweetId, ...tweetIds] + return tweetIds } export class TwitterHandler extends ContentHandler { @@ -246,25 +299,15 @@ export class TwitterHandler extends ContentHandler { const authorImage = author.profile_image_url.replace('_normal', '_400x400') const description = _.escape(tweetData.text) - const tweets = [tweet] - // we want to get the full thread - const thread = await getTweetThread(conversationId, author.username) - if (thread.meta.result_count > 0) { - // tweets are in reverse chronological order in the thread - for (const t of thread.data.reverse()) { - // get the tweet media if it exists - const media = thread.includes.media?.filter((m) => - t.attachments?.media_keys?.includes(m.media_key) - ) - const tweet: Tweet = { - data: t, - includes: { - users: thread.includes.users, - media, - }, - } - tweets.push(tweet) - } + let tweets: Tweet[] + if ( + new Date(tweet.data.created_at).getTime() < + Date.now() - 7 * 24 * 60 * 60 * 1000 + ) { + // tweet is older than 7 days, so we need to use puppeteer to get the older tweets + tweets = await getOldTweets(conversationId) + } else { + tweets = [tweet, ...(await getRecentTweets(conversationId))] } let tweetsContent = '' From 03777587f28c8bafdfc5b7bf8940c083235b3fa5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 26 Oct 2022 22:44:19 +0800 Subject: [PATCH 4/4] Fetch max 100 tweets in a thread --- .../src/websites/twitter-handler.ts | 90 ++++++++++--------- 1 file changed, 49 insertions(+), 41 deletions(-) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index e7291c61a..4267270ae 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -58,6 +58,7 @@ interface Tweets { const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ +const MAX_THREAD_DEPTH = 100 const getTweetFields = () => { const TWEET_FIELDS = @@ -81,7 +82,7 @@ const getTweetThread = async (conversationId: string): Promise => { '?query=' + encodeURIComponent(`conversation_id:${conversationId}`) + getTweetFields() + - '&max_results=100' + `&max_results=${MAX_THREAD_DEPTH}` ) if (!TWITTER_BEARER_TOKEN) { @@ -213,60 +214,67 @@ const getTweetIds = async (tweetId: string): Promise => { ], }) - const page = await browser.newPage() + try { + const page = await browser.newPage() - await page.goto(pageURL, { - waitUntil: 'networkidle2', - }) + await page.goto(pageURL, { + waitUntil: 'networkidle2', + }) - await waitFor(4000) + await waitFor(4000) - /** @type {string[]} */ - const tweetIds = (await page.evaluate(async () => { - const ids: string[] = [] + return (await page.evaluate(async () => { + const MAX_THREAD_DEPTH = 100 + const ids: string[] = [] - /** - * Wait for `ms` amount of milliseconds - * @param {number} ms - */ - const waitFor = (ms: number) => - new Promise((resolve) => setTimeout(resolve, ms)) + /** + * Wait for `ms` amount of milliseconds + * @param {number} ms + */ + const waitFor = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)) - // Find the first Show thread button and click it - const showRepliesButton = Array.from( - document.querySelectorAll('div[dir="auto"]') - ) - .filter((node) => node.children[0] && node.children[0].tagName === 'SPAN') - .find((node) => node.children[0].innerHTML === 'Show replies') + // Find the first Show thread button and click it + const showRepliesButton = Array.from( + document.querySelectorAll('div[dir="auto"]') + ) + .filter( + (node) => node.children[0] && node.children[0].tagName === 'SPAN' + ) + .find((node) => node.children[0].innerHTML === 'Show replies') - if (showRepliesButton) { - ;(showRepliesButton as HTMLElement).click() + if (showRepliesButton) { + ;(showRepliesButton as HTMLElement).click() - await waitFor(2000) - } + await waitFor(2000) + } - const timeNodes = Array.from(document.querySelectorAll('time')) + const timeNodes = Array.from(document.querySelectorAll('time')) - for (const timeNode of timeNodes) { - /** @type {HTMLAnchorElement | HTMLSpanElement} */ - const timeContainerAnchor = timeNode.parentElement - if (!timeContainerAnchor) continue + for (let i = 0; i < timeNodes.length && i < MAX_THREAD_DEPTH; i++) { + const timeContainerAnchor: HTMLAnchorElement | HTMLSpanElement | null = + timeNodes[i].parentElement + if (!timeContainerAnchor) continue - if (timeContainerAnchor.tagName === 'SPAN') continue + if (timeContainerAnchor.tagName === 'SPAN') continue - const id = (timeContainerAnchor as HTMLAnchorElement).href - .split('/') - .reverse()[0] + const href = timeContainerAnchor.getAttribute('href') + if (!href) continue - ids.push(id) - } + const id = href.split('/').reverse()[0] + if (!id) continue - return ids - })) as string[] + ids.push(id) + } - await browser.close() - - return tweetIds + return ids + })) as string[] + } catch (error) { + console.log(error) + return [] + } finally { + await browser.close() + } } export class TwitterHandler extends ContentHandler {