From 343c8fb5cca811dd31174dd0ca44ac36e6ef78f7 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 25 Oct 2022 18:25:49 +0800
Subject: [PATCH] Get tweet reply ids from thread id

---
 .../src/websites/twitter-handler.ts           | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts
index 16138c1e1..3aa820468 100644
--- a/packages/content-handler/src/websites/twitter-handler.ts
+++ b/packages/content-handler/src/websites/twitter-handler.ts
@@ -140,6 +140,81 @@ export class TwitterHandler extends ContentHandler {
     this.name = 'Twitter'
   }
 
+  async getTweetIdsFromThreadId(tweetID: string): Promise<string[]> {
+    const pageURL = `https://twitter.com/anyone/status/${tweetID}`
+
+    // Modify this variable to control the size of viewport
+    const factor = 0.2
+    const height = Math.floor(2000 / factor)
+    const width = Math.floor(1700 / factor)
+
+    const browser = await puppeteer.launch({
+      headless: true,
+      defaultViewport: {
+        width,
+        height,
+      },
+      args: [
+        `--force-device-scale-factor=${factor}`,
+        `--window-size=${width},${height}`,
+      ],
+    })
+
+    const page = await browser.newPage()
+
+    await page.goto(pageURL, {
+      waitUntil: 'networkidle2',
+      timeout: 5 * 60 * 1000,
+    })
+
+    await waitFor(4000)
+
+    /** @type {string[]} */
+    const tweetIDs = await page.evaluate(async () => {
+      const ids = []
+
+      /**
+       * Wait for `ms` amount of milliseconds
+       * @param {number} ms
+       */
+      const waitFor = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+      // Find the first Show thread button and click it
+      const showRepliesButton = [
+        ...document.querySelectorAll('div[dir="auto"]'),
+      ]
+        .filter(
+          (node) => node.children[0] && node.children[0].tagName === 'SPAN'
+        )
+        .find((node) => node.children[0].innerHTML === 'Show replies')
+
+      if (showRepliesButton) {
+        showRepliesButton.click()
+
+        await waitFor(2000)
+      }
+
+      const timeNodes = Array.from(document.querySelectorAll('time'))
+
+      for (const timeNode of timeNodes) {
+        /** @type {HTMLAnchorElement | HTMLSpanElement} */
+        const timeContainerAnchor = timeNode.parentElement
+
+        if (timeContainerAnchor.tagName === 'SPAN') continue
+
+        const id = timeContainerAnchor.href.split('/').reverse()[0]
+
+        ids.push(id)
+      }
+
+      return ids
+    })
+
+    await browser.close()
+
+    return [tweetID, ...tweetIDs]
+  }
+
   shouldPreHandle(url: string): boolean {
     return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
   }