Merge pull request #1349 from omnivore-app/save-old-twitter-thread

Save twitter thread older than 7 days

packages/content-fetch/fetch-content.js

@@ -268,7 +268,6 @@ async function fetchContent(req, res) {
         }
       } else {
         console.log('using prefetched content and title');
-        console.log(content);
       }
 
       logRecord.fetchContentTime = Date.now() - functionStartTime;
@@ -288,10 +287,8 @@ async function fetchContent(req, res) {
 
       logRecord.totalTime = Date.now() - functionStartTime;
       logRecord.result = apiResponse.createArticle;
-      console.log(`parse-page`, logRecord);
     }
   } catch (e) {
-    console.log('error', e)
     logRecord.error = e.message;
     console.log(`Error while retrieving page`, logRecord);
 
@@ -316,11 +313,11 @@ async function fetchContent(req, res) {
 
     logRecord.totalTime = Date.now() - functionStartTime;
     logRecord.result = apiResponse.createArticle;
-    console.log(`parse-page`, logRecord);
   } finally {
     if (context) {
       await context.close();
     }
+    console.log(`parse-page`, logRecord);
   }
 
   return res.sendStatus(200);

packages/content-handler/package.json

@@ -32,6 +32,7 @@
     "axios": "^0.27.2",
     "linkedom": "^0.14.16",
     "luxon": "^3.0.4",
+    "puppeteer-core": "^19.1.1",
     "rfc2047": "^4.0.1",
     "underscore": "^1.13.6",
     "uuid": "^9.0.0"

packages/content-handler/src/websites/twitter-handler.ts

@@ -2,6 +2,7 @@ import { ContentHandler, PreHandleResult } from '../content-handler'
 import axios from 'axios'
 import { DateTime } from 'luxon'
 import _ from 'underscore'
+import puppeteer from 'puppeteer-core'
 
 interface TweetIncludes {
   users: {
@@ -48,7 +49,7 @@ interface Tweet {
   includes: TweetIncludes
 }
 
-interface TweetThread {
+interface Tweets {
   data: TweetData[]
   includes: TweetIncludes
   meta: TweetMeta
@@ -57,6 +58,7 @@ interface TweetThread {
 const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN
 const TWITTER_URL_MATCH =
   /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
+const MAX_THREAD_DEPTH = 100
 
 const getTweetFields = () => {
   const TWEET_FIELDS =
@@ -73,26 +75,21 @@ const getTweetFields = () => {
 }
 
 // unroll recent tweet thread
-const getTweetThread = async (
-  conversationId: string,
-  username: string
-): Promise<TweetThread> => {
+const getTweetThread = async (conversationId: string): Promise<Tweets> => {
   const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/search/recent'
   const apiUrl = new URL(
     BASE_ENDPOINT +
       '?query=' +
-      encodeURIComponent(
-        `conversation_id:${conversationId} from:${username} to:${username}`
-      ) +
+      encodeURIComponent(`conversation_id:${conversationId}`) +
       getTweetFields() +
-      '&max_results=100'
+      `&max_results=${MAX_THREAD_DEPTH}`
   )
 
   if (!TWITTER_BEARER_TOKEN) {
     throw new Error('No Twitter bearer token found')
   }
 
-  const response = await axios.get<TweetThread>(apiUrl.toString(), {
+  const response = await axios.get<Tweets>(apiUrl.toString(), {
     headers: {
       Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
       redirect: 'follow',
@@ -119,6 +116,24 @@ const getTweetById = async (id: string): Promise<Tweet> => {
   return response.data
 }
 
+const getTweetsByIds = async (ids: string[]): Promise<Tweets> => {
+  const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets?ids='
+  const apiUrl = new URL(BASE_ENDPOINT + ids.join() + getTweetFields())
+
+  if (!TWITTER_BEARER_TOKEN) {
+    throw new Error('No Twitter bearer token found')
+  }
+
+  const response = await axios.get<Tweets>(apiUrl.toString(), {
+    headers: {
+      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
+      redirect: 'follow',
+    },
+  })
+
+  return response.data
+}
+
 const titleForAuthor = (author: { name: string }) => {
   return `${author.name} on Twitter`
 }
@@ -134,6 +149,134 @@ const formatTimestamp = (timestamp: string) => {
   )
 }
 
+const getTweetsFromResponse = (response: Tweets): Tweet[] => {
+  const tweets = []
+  for (const t of response.data) {
+    const media = response.includes.media?.filter((m) =>
+      t.attachments?.media_keys?.includes(m.media_key)
+    )
+    const tweet: Tweet = {
+      data: t,
+      includes: {
+        users: response.includes.users,
+        media,
+      },
+    }
+    tweets.push(tweet)
+  }
+  return tweets
+}
+
+const getOldTweets = async (conversationId: string): Promise<Tweet[]> => {
+  const tweetIds = await getTweetIds(conversationId)
+  const response = await getTweetsByIds(tweetIds)
+  return getTweetsFromResponse(response)
+}
+
+const getRecentTweets = async (conversationId: string): Promise<Tweet[]> => {
+  const thread = await getTweetThread(conversationId)
+  if (thread.meta.result_count === 0) {
+    return []
+  }
+  // tweets are in reverse chronological order in the thread
+  return getTweetsFromResponse(thread).reverse()
+}
+
+/**
+ * Wait for `ms` amount of milliseconds
+ * @param {number} ms
+ */
+const waitFor = (ms: number) =>
+  new Promise((resolve) => setTimeout(resolve, ms))
+
+/**
+ * Get tweets(even older than 7 days) using puppeteer
+ * @param {string} tweetId
+ */
+const getTweetIds = async (tweetId: string): Promise<string[]> => {
+  const pageURL = `https://twitter.com/anyone/status/${tweetId}`
+
+  // Modify this variable to control the size of viewport
+  const factor = 0.2
+  const height = Math.floor(2000 / factor)
+  const width = Math.floor(1700 / factor)
+
+  const browser = await puppeteer.launch({
+    executablePath: process.env.CHROMIUM_PATH,
+    headless: true,
+    defaultViewport: {
+      width,
+      height,
+    },
+    args: [
+      `--force-device-scale-factor=${factor}`,
+      `--window-size=${width},${height}`,
+    ],
+  })
+
+  try {
+    const page = await browser.newPage()
+
+    await page.goto(pageURL, {
+      waitUntil: 'networkidle2',
+    })
+
+    await waitFor(4000)
+
+    return (await page.evaluate(async () => {
+      const MAX_THREAD_DEPTH = 100
+      const ids: string[] = []
+
+      /**
+       * Wait for `ms` amount of milliseconds
+       * @param {number} ms
+       */
+      const waitFor = (ms: number) =>
+        new Promise((resolve) => setTimeout(resolve, ms))
+
+      // Find the first Show thread button and click it
+      const showRepliesButton = Array.from(
+        document.querySelectorAll('div[dir="auto"]')
+      )
+        .filter(
+          (node) => node.children[0] && node.children[0].tagName === 'SPAN'
+        )
+        .find((node) => node.children[0].innerHTML === 'Show replies')
+
+      if (showRepliesButton) {
+        ;(showRepliesButton as HTMLElement).click()
+
+        await waitFor(2000)
+      }
+
+      const timeNodes = Array.from(document.querySelectorAll('time'))
+
+      for (let i = 0; i < timeNodes.length && i < MAX_THREAD_DEPTH; i++) {
+        const timeContainerAnchor: HTMLAnchorElement | HTMLSpanElement | null =
+          timeNodes[i].parentElement
+        if (!timeContainerAnchor) continue
+
+        if (timeContainerAnchor.tagName === 'SPAN') continue
+
+        const href = timeContainerAnchor.getAttribute('href')
+        if (!href) continue
+
+        const id = href.split('/').reverse()[0]
+        if (!id) continue
+
+        ids.push(id)
+      }
+
+      return ids
+    })) as string[]
+  } catch (error) {
+    console.log(error)
+    return []
+  } finally {
+    await browser.close()
+  }
+}
+
 export class TwitterHandler extends ContentHandler {
   constructor() {
     super()
@@ -164,25 +307,15 @@ export class TwitterHandler extends ContentHandler {
     const authorImage = author.profile_image_url.replace('_normal', '_400x400')
     const description = _.escape(tweetData.text)
 
-    const tweets = [tweet]
-    // we want to get the full thread
-    const thread = await getTweetThread(conversationId, author.username)
-    if (thread.meta.result_count > 0) {
-      // tweets are in reverse chronological order in the thread
-      for (const t of thread.data.reverse()) {
-        // get the tweet media if it exists
-        const media = thread.includes.media?.filter((m) =>
-          t.attachments?.media_keys?.includes(m.media_key)
-        )
-        const tweet: Tweet = {
-          data: t,
-          includes: {
-            users: thread.includes.users,
-            media,
-          },
-        }
-        tweets.push(tweet)
-      }
+    let tweets: Tweet[]
+    if (
+      new Date(tweet.data.created_at).getTime() <
+      Date.now() - 7 * 24 * 60 * 60 * 1000
+    ) {
+      // tweet is older than 7 days, so we need to use puppeteer to get the older tweets
+      tweets = await getOldTweets(conversationId)
+    } else {
+      tweets = [tweet, ...(await getRecentTweets(conversationId))]
     }
 
     let tweetsContent = ''

yarn.lock

@@ -12333,6 +12333,11 @@ devtools-protocol@0.0.1019158:
   resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1019158.tgz#4b08d06108a784a2134313149626ba55f030a86f"
   integrity sha512-wvq+KscQ7/6spEV7czhnZc9RM/woz1AY+/Vpd8/h2HFMwJSdTliu7f/yr1A6vDdJfKICZsShqsYpEQbdhg8AFQ==
 
+devtools-protocol@0.0.1045489:
+  version "0.0.1045489"
+  resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1045489.tgz#f959ad560b05acd72d55644bc3fb8168a83abf28"
+  integrity sha512-D+PTmWulkuQW4D1NTiCRCFxF7pQPn0hgp4YyX4wAQ6xYXKOadSWPR3ENGDQ47MW/Ewc9v2rpC/UEEGahgBYpSQ==
+
 devtools-protocol@0.0.901419:
   version "0.0.901419"
   resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.901419.tgz#79b5459c48fe7e1c5563c02bd72f8fec3e0cebcd"
@@ -21372,6 +21377,22 @@ puppeteer-core@^16.1.0:
     unbzip2-stream "1.4.3"
     ws "8.8.1"
 
+puppeteer-core@^19.1.1:
+  version "19.1.1"
+  resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-19.1.1.tgz#6416ff925a9cc78523c490482a17a2998f7c0626"
+  integrity sha512-jV26Ke0VFel4MoXLjqm50uAW2uwksTP6Md1tvtXqWqXM5FyboKI6E9YYJ1qEQilUAqlhgGq8xLN5+SL8bPz/kw==
+  dependencies:
+    cross-fetch "3.1.5"
+    debug "4.3.4"
+    devtools-protocol "0.0.1045489"
+    extract-zip "2.0.1"
+    https-proxy-agent "5.0.1"
+    proxy-from-env "1.1.0"
+    rimraf "3.0.2"
+    tar-fs "2.1.1"
+    unbzip2-stream "1.4.3"
+    ws "8.9.0"
+
 puppeteer@^10.1.0:
   version "10.4.0"
   resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-10.4.0.tgz#a6465ff97fda0576c4ac29601406f67e6fea3dc7"
@@ -25896,6 +25917,11 @@ ws@8.8.1, ws@^8.2.3, ws@^8.3.0, ws@^8.4.2:
   resolved "https://registry.yarnpkg.com/ws/-/ws-8.8.1.tgz#5dbad0feb7ade8ecc99b830c1d77c913d4955ff0"
   integrity sha512-bGy2JzvzkPowEJV++hF07hAD6niYSr0JzBNo/J29WsB57A2r7Wlc1UFcTR9IzrPvuNVO4B8LGqF8qcpsVOhJCA==
 
+ws@8.9.0:
+  version "8.9.0"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.9.0.tgz#2a994bb67144be1b53fe2d23c53c028adeb7f45e"
+  integrity sha512-Ja7nszREasGaYUYCI2k4lCKIRTt+y7XuqVoHR44YpI49TtryyqbqvDMn5eqfW7e6HzTukDRIsXqzVHScqRcafg==
+
 "ws@^5.2.0 || ^6.0.0 || ^7.0.0", ws@^7.3.1, ws@^7.4.6:
   version "7.5.7"
   resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.7.tgz#9e0ac77ee50af70d58326ecff7e85eb3fa375e67"