Get old tweet thread with puppeteer and new tweet with twitter api

packages/content-fetch/fetch-content.js

@@ -268,7 +268,6 @@ async function fetchContent(req, res) {
         }
       } else {
         console.log('using prefetched content and title');
-        console.log(content);
       }
 
       logRecord.fetchContentTime = Date.now() - functionStartTime;
@@ -288,10 +287,8 @@ async function fetchContent(req, res) {
 
       logRecord.totalTime = Date.now() - functionStartTime;
       logRecord.result = apiResponse.createArticle;
-      console.log(`parse-page`, logRecord);
     }
   } catch (e) {
-    console.log('error', e)
     logRecord.error = e.message;
     console.log(`Error while retrieving page`, logRecord);
 
@@ -316,11 +313,11 @@ async function fetchContent(req, res) {
 
     logRecord.totalTime = Date.now() - functionStartTime;
     logRecord.result = apiResponse.createArticle;
-    console.log(`parse-page`, logRecord);
   } finally {
     if (context) {
       await context.close();
     }
+    console.log(`parse-page`, logRecord);
   }
 
   return res.sendStatus(200);

packages/content-handler/src/websites/twitter-handler.ts

@@ -49,7 +49,7 @@ interface Tweet {
   includes: TweetIncludes
 }
 
-interface TweetThread {
+interface Tweets {
   data: TweetData[]
   includes: TweetIncludes
   meta: TweetMeta
@@ -74,17 +74,12 @@ const getTweetFields = () => {
 }
 
 // unroll recent tweet thread
-const getTweetThread = async (
-  conversationId: string,
-  username: string
-): Promise<TweetThread> => {
+const getTweetThread = async (conversationId: string): Promise<Tweets> => {
   const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/search/recent'
   const apiUrl = new URL(
     BASE_ENDPOINT +
       '?query=' +
-      encodeURIComponent(
-        `conversation_id:${conversationId} from:${username} to:${username}`
-      ) +
+      encodeURIComponent(`conversation_id:${conversationId}`) +
       getTweetFields() +
       '&max_results=100'
   )
@@ -93,7 +88,7 @@ const getTweetThread = async (
     throw new Error('No Twitter bearer token found')
   }
 
-  const response = await axios.get<TweetThread>(apiUrl.toString(), {
+  const response = await axios.get<Tweets>(apiUrl.toString(), {
     headers: {
       Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
       redirect: 'follow',
@@ -120,6 +115,24 @@ const getTweetById = async (id: string): Promise<Tweet> => {
   return response.data
 }
 
+const getTweetsByIds = async (ids: string[]): Promise<Tweets> => {
+  const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets?ids='
+  const apiUrl = new URL(BASE_ENDPOINT + ids.join() + getTweetFields())
+
+  if (!TWITTER_BEARER_TOKEN) {
+    throw new Error('No Twitter bearer token found')
+  }
+
+  const response = await axios.get<Tweets>(apiUrl.toString(), {
+    headers: {
+      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
+      redirect: 'follow',
+    },
+  })
+
+  return response.data
+}
+
 const titleForAuthor = (author: { name: string }) => {
   return `${author.name} on Twitter`
 }
@@ -135,18 +148,51 @@ const formatTimestamp = (timestamp: string) => {
   )
 }
 
+const getTweetsFromResponse = (response: Tweets): Tweet[] => {
+  const tweets = []
+  for (const t of response.data) {
+    const media = response.includes.media?.filter((m) =>
+      t.attachments?.media_keys?.includes(m.media_key)
+    )
+    const tweet: Tweet = {
+      data: t,
+      includes: {
+        users: response.includes.users,
+        media,
+      },
+    }
+    tweets.push(tweet)
+  }
+  return tweets
+}
+
+const getOldTweets = async (conversationId: string): Promise<Tweet[]> => {
+  const tweetIds = await getTweetIds(conversationId)
+  const response = await getTweetsByIds(tweetIds)
+  return getTweetsFromResponse(response)
+}
+
+const getRecentTweets = async (conversationId: string): Promise<Tweet[]> => {
+  const thread = await getTweetThread(conversationId)
+  if (thread.meta.result_count === 0) {
+    return []
+  }
+  // tweets are in reverse chronological order in the thread
+  return getTweetsFromResponse(thread).reverse()
+}
+
 /**
  * Wait for `ms` amount of milliseconds
  * @param {number} ms
  */
-const waitFor = async (ms: number) =>
+const waitFor = (ms: number) =>
   new Promise((resolve) => setTimeout(resolve, ms))
 
 /**
  * Get tweets(even older than 7 days) using puppeteer
  * @param {string} tweetId
  */
-const getTweetIdsFromThreadId = async (tweetId: string): Promise<string[]> => {
+const getTweetIds = async (tweetId: string): Promise<string[]> => {
   const pageURL = `https://twitter.com/anyone/status/${tweetId}`
 
   // Modify this variable to control the size of viewport
@@ -155,6 +201,7 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise<string[]> => {
   const width = Math.floor(1700 / factor)
 
   const browser = await puppeteer.launch({
+    executablePath: process.env.CHROMIUM_PATH,
     headless: true,
     defaultViewport: {
       width,
@@ -170,7 +217,6 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise<string[]> => {
 
   await page.goto(pageURL, {
     waitUntil: 'networkidle2',
-    timeout: 5 * 60 * 1000,
   })
 
   await waitFor(4000)
@@ -179,6 +225,13 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise<string[]> => {
   const tweetIds = (await page.evaluate(async () => {
     const ids: string[] = []
 
+    /**
+     * Wait for `ms` amount of milliseconds
+     * @param {number} ms
+     */
+    const waitFor = (ms: number) =>
+      new Promise((resolve) => setTimeout(resolve, ms))
+
     // Find the first Show thread button and click it
     const showRepliesButton = Array.from(
       document.querySelectorAll('div[dir="auto"]')
@@ -199,7 +252,7 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise<string[]> => {
       const timeContainerAnchor = timeNode.parentElement
       if (!timeContainerAnchor) continue
 
-      if (timeContainerAnchor?.tagName === 'SPAN') continue
+      if (timeContainerAnchor.tagName === 'SPAN') continue
 
       const id = (timeContainerAnchor as HTMLAnchorElement).href
         .split('/')
@@ -213,7 +266,7 @@ const getTweetIdsFromThreadId = async (tweetId: string): Promise<string[]> => {
 
   await browser.close()
 
-  return [tweetId, ...tweetIds]
+  return tweetIds
 }
 
 export class TwitterHandler extends ContentHandler {
@@ -246,25 +299,15 @@ export class TwitterHandler extends ContentHandler {
     const authorImage = author.profile_image_url.replace('_normal', '_400x400')
     const description = _.escape(tweetData.text)
 
-    const tweets = [tweet]
-    // we want to get the full thread
-    const thread = await getTweetThread(conversationId, author.username)
-    if (thread.meta.result_count > 0) {
-      // tweets are in reverse chronological order in the thread
-      for (const t of thread.data.reverse()) {
-        // get the tweet media if it exists
-        const media = thread.includes.media?.filter((m) =>
-          t.attachments?.media_keys?.includes(m.media_key)
-        )
-        const tweet: Tweet = {
-          data: t,
-          includes: {
-            users: thread.includes.users,
-            media,
-          },
-        }
-        tweets.push(tweet)
-      }
+    let tweets: Tweet[]
+    if (
+      new Date(tweet.data.created_at).getTime() <
+      Date.now() - 7 * 24 * 60 * 60 * 1000
+    ) {
+      // tweet is older than 7 days, so we need to use puppeteer to get the older tweets
+      tweets = await getOldTweets(conversationId)
+    } else {
+      tweets = [tweet, ...(await getRecentTweets(conversationId))]
     }
 
     let tweetsContent = ''