From fb2c70a6fd2ab4e52ad33a373f0e8dc0c3275b8b Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 15 Jun 2023 18:33:26 +0800
Subject: [PATCH 1/5] change the page viewport when scraping twitter thread

---
 .../src/websites/twitter-handler.ts           | 94 +++++++++----------
 1 file changed, 43 insertions(+), 51 deletions(-)

diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts
index c1878e498..d80eaacf7 100644
--- a/packages/content-handler/src/websites/twitter-handler.ts
+++ b/packages/content-handler/src/websites/twitter-handler.ts
@@ -217,6 +217,12 @@ const getTweetIds = async (
     context = await browser.createIncognitoBrowserContext()
     const page = await context.newPage()
 
+    // Modify this variable to control the size of viewport
+    const deviceScaleFactor = 0.2
+    const height = Math.floor(2000 / deviceScaleFactor)
+    const width = Math.floor(1700 / deviceScaleFactor)
+    await page.setViewport({ width, height, deviceScaleFactor })
+
     await page.goto(pageURL, {
       waitUntil: 'networkidle0',
       timeout: 60000, // 60 seconds
@@ -230,62 +236,48 @@ const getTweetIds = async (
       const waitFor = (ms: number) =>
         new Promise((resolve) => setTimeout(resolve, ms))
 
-      const ids: Set<string> = new Set()
+      const ids = []
 
-      const distance = 1080
-      let scrollHeight = document.body.scrollHeight
-      let currentHeight = 0
-      // keep scrolling until there are no more elements
-      while (currentHeight < scrollHeight) {
-        const timeNodes = Array.from(document.querySelectorAll('time'))
+      // Find the first Show thread button and click it
+      const showRepliesButton = Array.from(
+        document.querySelectorAll('div[dir]')
+      )
+        .filter(
+          (node) => node.children[0] && node.children[0].tagName === 'SPAN'
+        )
+        .find((node) => node.children[0].innerHTML === 'Show replies')
 
-        for (let i = 0; i < timeNodes.length; i++) {
-          const timeContainerAnchor:
-            | HTMLAnchorElement
-            | HTMLSpanElement
-            | null = timeNodes[i].parentElement
-          if (!timeContainerAnchor) continue
+      if (showRepliesButton) {
+        ;(showRepliesButton as HTMLElement).click()
 
-          if (timeContainerAnchor.tagName === 'SPAN') continue
-
-          const href = timeContainerAnchor.getAttribute('href')
-          if (!href) continue
-
-          // Get the tweet id and username from the href: https://twitter.com/username/status/1234567890
-          const match = href.match(/\/([^/]+)\/status\/(\d+)/)
-          if (!match) continue
-
-          const id = match[2]
-          const username = match[1]
-
-          // stop at non-author replies
-          if (username !== author) return Array.from(ids)
-          ids.add(id)
-        }
-
-        window.scrollBy(0, distance)
-        await waitFor(500)
-        currentHeight += distance
-
-        // Find the show replies button and click it
-        if (currentHeight >= scrollHeight) {
-          const showRepliesButton = Array.from(
-            document.querySelectorAll('div[dir]')
-          )
-            .filter(
-              (node) => node.children[0] && node.children[0].tagName === 'SPAN'
-            )
-            .find((node) => node.children[0].innerHTML === 'Show replies')
-
-          if (showRepliesButton) {
-            ;(showRepliesButton as HTMLElement).click()
-            await waitFor(1000)
-            scrollHeight = document.body.scrollHeight
-          }
-        }
+        await waitFor(2000)
       }
 
-      return Array.from(ids)
+      const timeNodes = Array.from(document.querySelectorAll('time'))
+
+      for (const timeNode of timeNodes) {
+        /** @type {HTMLAnchorElement | HTMLSpanElement} */
+        const timeContainerAnchor: HTMLAnchorElement | HTMLSpanElement | null =
+          timeNode.parentElement
+        if (!timeContainerAnchor) continue
+
+        if (timeContainerAnchor.tagName === 'SPAN') continue
+
+        const href = timeContainerAnchor.getAttribute('href')
+        if (!href) continue
+
+        // Get the tweet id and username from the href: https://twitter.com/username/status/1234567890
+        const match = href.match(/\/([^/]+)\/status\/(\d+)/)
+        if (!match) continue
+
+        const id = match[2]
+        const username = match[1]
+
+        // skip non-author replies
+        username === author && ids.push(id)
+      }
+
+      return ids
     }, author)) as string[]
   } catch (error) {
     console.error('Error getting tweets', error)

From 65758f34693d8c7f295499487ea02620c007e814 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 20 Jun 2023 14:56:54 +0800
Subject: [PATCH 2/5] feat: add nitter handler to scrape tweets from
 twitter.com and nitter.net

---
 packages/content-handler/src/index.ts         |   2 +
 .../src/websites/nitter-handler.ts            | 306 ++++++++++++++++++
 2 files changed, 308 insertions(+)
 create mode 100644 packages/content-handler/src/websites/nitter-handler.ts

diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts
index c8f5d7baf..c1299fce9 100644
--- a/packages/content-handler/src/index.ts
+++ b/packages/content-handler/src/index.ts
@@ -27,6 +27,7 @@ import { DerstandardHandler } from './websites/derstandard-handler'
 import { GitHubHandler } from './websites/github-handler'
 import { ImageHandler } from './websites/image-handler'
 import { MediumHandler } from './websites/medium-handler'
+import { NitterHandler } from './websites/nitter-handler'
 import { PdfHandler } from './websites/pdf-handler'
 import { PipedVideoHandler } from './websites/piped-video-handler'
 import { ScrapingBeeHandler } from './websites/scrapingBee-handler'
@@ -77,6 +78,7 @@ const contentHandlers: ContentHandler[] = [
   new EnergyWorldHandler(),
   new PipedVideoHandler(),
   new WeixinQqHandler(),
+  new NitterHandler(),
 ]
 
 const newsletterHandlers: ContentHandler[] = [
diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts
new file mode 100644
index 000000000..5162485a5
--- /dev/null
+++ b/packages/content-handler/src/websites/nitter-handler.ts
@@ -0,0 +1,306 @@
+import _, { truncate } from 'lodash'
+import { DateTime } from 'luxon'
+import { Browser, BrowserContext } from 'puppeteer-core'
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+interface Tweet {
+  id: string
+  url: string
+  author: {
+    username: string
+    name: string
+    profileImageUrl: string
+  }
+  text: string
+  entities: {
+    urls: {
+      url: string
+      display_url: string
+    }[]
+    photos: string[]
+    videos: string[]
+  }
+  createdAt: string
+}
+
+export class NitterHandler extends ContentHandler {
+  // matches twitter.com and nitter.net urls
+  URL_MATCH =
+    /((twitter\.com)|(nitter\.net))\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
+  ADDRESS = 'https://nitter.net'
+
+  constructor() {
+    super()
+    this.name = 'Nitter'
+  }
+
+  async getTweets(browser: Browser, username: string, tweetId: string) {
+    const url = `${this.ADDRESS}/${username}/status/${tweetId}`
+
+    async function genTweets(): Promise<Tweet[]> {
+      const waitFor = (ms: number) =>
+        new Promise((resolve) => setTimeout(resolve, ms))
+
+      let context: BrowserContext | undefined
+      try {
+        context = await browser.createIncognitoBrowserContext()
+        const page = await context.newPage()
+
+        // // Modify this variable to control the size of viewport
+        // const deviceScaleFactor = 0.2
+        // const height = Math.floor(2000 / deviceScaleFactor)
+        // const width = Math.floor(1700 / deviceScaleFactor)
+        // await page.setViewport({ width, height, deviceScaleFactor })
+
+        await page.goto(url, {
+          waitUntil: 'networkidle2',
+          timeout: 30000, // 30 seconds
+        })
+
+        const tweets = (await page.evaluate(
+          async (username, id, url) => {
+            function authorParser(header: Element) {
+              const avatar = header
+                .querySelector('.tweet-avatar img')
+                ?.getAttribute('src')
+              if (!avatar) {
+                return null
+              }
+              const name = header
+                .querySelector('.fullname')
+                ?.getAttribute('title')
+              if (!name) {
+                return null
+              }
+              return {
+                avatar,
+                name,
+              }
+            }
+
+            function attachmentParser(attachments: Element | null) {
+              if (!attachments) return { photos: [], videos: [] }
+
+              const photos = Array.from(
+                attachments.querySelectorAll('img')
+              ).map((i) => i.getAttribute('src') ?? '')
+              const videos = Array.from(
+                attachments.querySelectorAll('source')
+              ).map((i) => i.getAttribute('src') ?? '')
+              return {
+                photos,
+                videos,
+              }
+            }
+
+            function parseTweet(tweet: Element): Tweet | null {
+              const header = tweet.querySelector('.tweet-header')
+              if (!header) {
+                return null
+              }
+              const author = authorParser(header)
+              if (!author) {
+                return null
+              }
+
+              const body = tweet.querySelector('.tweet-body')
+              if (!body) {
+                return null
+              }
+
+              const createdAt = body
+                .querySelector('.tweet-date a')
+                ?.getAttribute('title')
+              if (!createdAt) {
+                return null
+              }
+
+              const content = body.querySelector('.tweet-content')
+              if (!content) {
+                return null
+              }
+              const text = content.textContent
+              if (!text) {
+                return null
+              }
+              const urls = Array.from(content.querySelectorAll('a')).map(
+                (a) => ({
+                  url: a.getAttribute('href') ?? '',
+                  display_url: a.textContent ?? '',
+                })
+              )
+
+              const attachments = body.querySelector('.attachments')
+              const { photos, videos } = attachmentParser(attachments)
+
+              return {
+                id,
+                author: {
+                  username,
+                  name: author.name,
+                  profileImageUrl: author.avatar,
+                },
+                createdAt,
+                text,
+                url,
+                entities: {
+                  urls,
+                  photos,
+                  videos,
+                },
+              }
+            }
+
+            const tweets: Tweet[] = []
+            // get the main thread including tweets and threads
+            const mainThread = document.querySelector('.main-thread')
+            if (!mainThread) {
+              return []
+            }
+            const timelineItems = Array.from(
+              mainThread.querySelectorAll('.timeline-item')
+            )
+            for (let i = 0; i < timelineItems.length; i++) {
+              const item = timelineItems[i]
+              if (item.classList.contains('show-more')) {
+                // click the show more button
+                ;(item as HTMLAnchorElement).click()
+                await waitFor(2000)
+
+                // get the new timeline items and add them to the list
+                const newTimelineItems = Array.from(
+                  mainThread.querySelectorAll('.timeline-item')
+                )
+                timelineItems.push(...newTimelineItems)
+                continue
+              }
+
+              const tweet = parseTweet(item)
+              tweet && tweets.push(tweet)
+            }
+
+            return tweets
+          },
+          username,
+          tweetId,
+          url
+        )) as Tweet[]
+
+        console.log('tweets', tweets)
+        return tweets
+      } catch (error) {
+        console.error('Error getting tweets', error)
+
+        return []
+      } finally {
+        if (context) {
+          await context.close()
+        }
+      }
+    }
+
+    return genTweets()
+  }
+
+  tweetIdAndUsernameFromUrl = (url: string) => {
+    const match = url.toString().match(this.URL_MATCH)
+    return {
+      tweetId: match?.[5],
+      username: match?.[4],
+    }
+  }
+
+  titleForTweet = (author: { name: string }, text: string) => {
+    return `${author.name} on Twitter: ${truncate(text.replace(/http\S+/, ''), {
+      length: 100,
+    })}`
+  }
+
+  formatTimestamp = (timestamp: string) => {
+    return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(
+      DateTime.DATETIME_FULL
+    )
+  }
+
+  shouldPreHandle(url: string): boolean {
+    return this.URL_MATCH.test(url.toString())
+  }
+
+  async preHandle(url: string, browser: Browser): Promise<PreHandleResult> {
+    const { tweetId, username } = this.tweetIdAndUsernameFromUrl(url)
+    if (!tweetId || !username) {
+      throw new Error('could not find tweet id or username in url')
+    }
+    const tweets = await this.getTweets(browser, username, tweetId)
+
+    const tweet = tweets[0]
+    const author = tweet.author
+    // escape html entities in title
+    const title = this.titleForTweet(author, tweet.text)
+    const escapedTitle = _.escape(title)
+    const authorImage = `${this.ADDRESS}/${author.profileImageUrl.replace(
+      '_normal',
+      '_400x400'
+    )}`
+    const description = _.escape(tweet.text)
+
+    let tweetsContent = ''
+    for (const tweet of tweets) {
+      let text = tweet.text
+      if (tweet.entities && tweet.entities.urls) {
+        for (const urlObj of tweet.entities.urls) {
+          text = text.replace(
+            urlObj.url,
+            `<a href="${urlObj.url}">${urlObj.display_url}</a>`
+          )
+        }
+      }
+
+      const includesHtml =
+        tweet.entities.photos
+          ?.map(
+            (url) =>
+              `<a class="media-link" href=${this.ADDRESS}/${url}>
+          <picture>
+            <img class="tweet-img" src=${this.ADDRESS}/${url} />
+          </picture>
+          </a>`
+          )
+          .join('\n') ?? ''
+
+      tweetsContent += `
+      <p>${text}</p>
+      ${includesHtml}
+    `
+    }
+
+    const tweetUrl = `
+       — <a href="https://twitter.com/${author.username}">${
+      author.username
+    }</a> <span itemscope itemtype="https://schema.org/Person" itemprop="author">${
+      author.name
+    }</span> <a href="${url}">${this.formatTimestamp(tweet.createdAt)}</a>
+    `
+
+    const content = `
+<html>
+    <head>
+      <meta property="og:image" content="${authorImage}" />
+      <meta property="og:image:secure_url" content="${authorImage}" />
+      <meta property="og:title" content="${escapedTitle}" />
+      <meta property="og:description" content="${description}" />
+      <meta property="article:published_time" content="${tweet.createdAt}" />
+      <meta property="og:site_name" content="Twitter" />
+      <meta property="og:type" content="tweet" />
+    </head>
+    <body>
+      <div>
+        ${tweetsContent}
+        ${tweetUrl}
+      </div>
+    </body>
+</html>`
+
+    return { content, url, title }
+  }
+}

From ff9fcef08db95760cc6f8bfebecb9562f4768c84 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 20 Jun 2023 14:59:14 +0800
Subject: [PATCH 3/5] disable twitter-handler

---
 packages/content-handler/src/index.ts                  |  2 --
 .../content-handler/src/websites/nitter-handler.ts     | 10 +++++-----
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts
index c1299fce9..57d113a0d 100644
--- a/packages/content-handler/src/index.ts
+++ b/packages/content-handler/src/index.ts
@@ -33,7 +33,6 @@ import { PipedVideoHandler } from './websites/piped-video-handler'
 import { ScrapingBeeHandler } from './websites/scrapingBee-handler'
 import { StackOverflowHandler } from './websites/stack-overflow-handler'
 import { TDotCoHandler } from './websites/t-dot-co-handler'
-import { TwitterHandler } from './websites/twitter-handler'
 import { WeixinQqHandler } from './websites/weixin-qq-handler'
 import { WikipediaHandler } from './websites/wikipedia-handler'
 import { YoutubeHandler } from './websites/youtube-handler'
@@ -65,7 +64,6 @@ const contentHandlers: ContentHandler[] = [
   new PdfHandler(),
   new ScrapingBeeHandler(),
   new TDotCoHandler(),
-  new TwitterHandler(),
   new YoutubeHandler(),
   new WikipediaHandler(),
   new GitHubHandler(),
diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts
index 5162485a5..c88202221 100644
--- a/packages/content-handler/src/websites/nitter-handler.ts
+++ b/packages/content-handler/src/websites/nitter-handler.ts
@@ -46,11 +46,11 @@ export class NitterHandler extends ContentHandler {
         context = await browser.createIncognitoBrowserContext()
         const page = await context.newPage()
 
-        // // Modify this variable to control the size of viewport
-        // const deviceScaleFactor = 0.2
-        // const height = Math.floor(2000 / deviceScaleFactor)
-        // const width = Math.floor(1700 / deviceScaleFactor)
-        // await page.setViewport({ width, height, deviceScaleFactor })
+        // Modify this variable to control the size of viewport
+        const deviceScaleFactor = 0.2
+        const height = Math.floor(2000 / deviceScaleFactor)
+        const width = Math.floor(1700 / deviceScaleFactor)
+        await page.setViewport({ width, height, deviceScaleFactor })
 
         await page.goto(url, {
           waitUntil: 'networkidle2',

From 89f89875d634b086fa6375187f51f3b5eb744301 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 20 Jun 2023 17:35:59 +0800
Subject: [PATCH 4/5] fix date parsing

---
 .../src/websites/nitter-handler.ts            | 82 ++++++++++---------
 1 file changed, 43 insertions(+), 39 deletions(-)

diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts
index c88202221..fbd0e1e7b 100644
--- a/packages/content-handler/src/websites/nitter-handler.ts
+++ b/packages/content-handler/src/websites/nitter-handler.ts
@@ -38,9 +38,6 @@ export class NitterHandler extends ContentHandler {
     const url = `${this.ADDRESS}/${username}/status/${tweetId}`
 
     async function genTweets(): Promise<Tweet[]> {
-      const waitFor = (ms: number) =>
-        new Promise((resolve) => setTimeout(resolve, ms))
-
       let context: BrowserContext | undefined
       try {
         context = await browser.createIncognitoBrowserContext()
@@ -59,6 +56,10 @@ export class NitterHandler extends ContentHandler {
 
         const tweets = (await page.evaluate(
           async (username, id, url) => {
+            async function waitFor(ms: number) {
+              return new Promise((resolve) => setTimeout(resolve, ms))
+            }
+
             function authorParser(header: Element) {
               const avatar = header
                 .querySelector('.tweet-avatar img')
@@ -78,6 +79,12 @@ export class NitterHandler extends ContentHandler {
               }
             }
 
+            function dateParser(tweetDate: string) {
+              const validDateTime = tweetDate.replace(' · ', ' ')
+
+              return new Date(validDateTime).toISOString()
+            }
+
             function attachmentParser(attachments: Element | null) {
               if (!attachments) return { photos: [], videos: [] }
 
@@ -108,21 +115,19 @@ export class NitterHandler extends ContentHandler {
                 return null
               }
 
-              const createdAt = body
+              const tweetDate = body
                 .querySelector('.tweet-date a')
                 ?.getAttribute('title')
-              if (!createdAt) {
+              if (!tweetDate) {
                 return null
               }
+              const createdAt = dateParser(tweetDate)
 
               const content = body.querySelector('.tweet-content')
               if (!content) {
                 return null
               }
-              const text = content.textContent
-              if (!text) {
-                return null
-              }
+              const text = content.textContent ?? ''
               const urls = Array.from(content.querySelectorAll('a')).map(
                 (a) => ({
                   url: a.getAttribute('href') ?? '',
@@ -186,7 +191,6 @@ export class NitterHandler extends ContentHandler {
           url
         )) as Tweet[]
 
-        console.log('tweets', tweets)
         return tweets
       } catch (error) {
         console.error('Error getting tweets', error)
@@ -202,11 +206,12 @@ export class NitterHandler extends ContentHandler {
     return genTweets()
   }
 
-  tweetIdAndUsernameFromUrl = (url: string) => {
-    const match = url.toString().match(this.URL_MATCH)
+  parseTweetUrl = (url: string) => {
+    const match = url.match(this.URL_MATCH)
     return {
-      tweetId: match?.[5],
+      domain: match?.[1],
       username: match?.[4],
+      tweetId: match?.[5],
     }
   }
 
@@ -227,9 +232,9 @@ export class NitterHandler extends ContentHandler {
   }
 
   async preHandle(url: string, browser: Browser): Promise<PreHandleResult> {
-    const { tweetId, username } = this.tweetIdAndUsernameFromUrl(url)
-    if (!tweetId || !username) {
-      throw new Error('could not find tweet id or username in url')
+    const { tweetId, username, domain } = this.parseTweetUrl(url)
+    if (!tweetId || !username || !domain) {
+      throw new Error('could not parse tweet url')
     }
     const tweets = await this.getTweets(browser, username, tweetId)
 
@@ -238,7 +243,7 @@ export class NitterHandler extends ContentHandler {
     // escape html entities in title
     const title = this.titleForTweet(author, tweet.text)
     const escapedTitle = _.escape(title)
-    const authorImage = `${this.ADDRESS}/${author.profileImageUrl.replace(
+    const authorImage = `${this.ADDRESS}${author.profileImageUrl.replace(
       '_normal',
       '_400x400'
     )}`
@@ -260,9 +265,9 @@ export class NitterHandler extends ContentHandler {
         tweet.entities.photos
           ?.map(
             (url) =>
-              `<a class="media-link" href=${this.ADDRESS}/${url}>
+              `<a class="media-link" href=${this.ADDRESS}${url}>
           <picture>
-            <img class="tweet-img" src=${this.ADDRESS}/${url} />
+            <img class="tweet-img" src=${this.ADDRESS}${url} />
           </picture>
           </a>`
           )
@@ -275,31 +280,30 @@ export class NitterHandler extends ContentHandler {
     }
 
     const tweetUrl = `
-       — <a href="https://twitter.com/${author.username}">${
+       — <a href="https://${domain}/${author.username}">${
       author.username
     }</a> <span itemscope itemtype="https://schema.org/Person" itemprop="author">${
       author.name
-    }</span> <a href="${url}">${this.formatTimestamp(tweet.createdAt)}</a>
-    `
+    }</span> <a href="${url}">${this.formatTimestamp(tweet.createdAt)}</a>`
 
     const content = `
-<html>
-    <head>
-      <meta property="og:image" content="${authorImage}" />
-      <meta property="og:image:secure_url" content="${authorImage}" />
-      <meta property="og:title" content="${escapedTitle}" />
-      <meta property="og:description" content="${description}" />
-      <meta property="article:published_time" content="${tweet.createdAt}" />
-      <meta property="og:site_name" content="Twitter" />
-      <meta property="og:type" content="tweet" />
-    </head>
-    <body>
-      <div>
-        ${tweetsContent}
-        ${tweetUrl}
-      </div>
-    </body>
-</html>`
+      <html>
+          <head>
+            <meta property="og:image" content="${authorImage}" />
+            <meta property="og:image:secure_url" content="${authorImage}" />
+            <meta property="og:title" content="${escapedTitle}" />
+            <meta property="og:description" content="${description}" />
+            <meta property="article:published_time" content="${tweet.createdAt}" />
+            <meta property="og:site_name" content="Twitter" />
+            <meta property="og:type" content="tweet" />
+          </head>
+          <body>
+            <div>
+              ${tweetsContent}
+              ${tweetUrl}
+            </div>
+          </body>
+      </html>`
 
     return { content, url, title }
   }

From a109d0383fd01fcbf61e9e71ea31fc54a3d0ece5 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 20 Jun 2023 19:17:34 +0800
Subject: [PATCH 5/5] go to the next thread page to scrape long thread

---
 .../src/websites/nitter-handler.ts            | 275 +++++++++---------
 1 file changed, 135 insertions(+), 140 deletions(-)

diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts
index fbd0e1e7b..f5c5a8eab 100644
--- a/packages/content-handler/src/websites/nitter-handler.ts
+++ b/packages/content-handler/src/websites/nitter-handler.ts
@@ -1,10 +1,10 @@
+import { parseHTML } from 'linkedom'
 import _, { truncate } from 'lodash'
 import { DateTime } from 'luxon'
 import { Browser, BrowserContext } from 'puppeteer-core'
 import { ContentHandler, PreHandleResult } from '../content-handler'
 
 interface Tweet {
-  id: string
   url: string
   author: {
     username: string
@@ -37,159 +37,154 @@ export class NitterHandler extends ContentHandler {
   async getTweets(browser: Browser, username: string, tweetId: string) {
     const url = `${this.ADDRESS}/${username}/status/${tweetId}`
 
-    async function genTweets(): Promise<Tweet[]> {
+    async function genTweets(address: string): Promise<Tweet[]> {
+      function authorParser(header: Element) {
+        const avatar = header
+          .querySelector('.tweet-avatar img')
+          ?.getAttribute('src')
+        if (!avatar) {
+          return null
+        }
+        const name = header.querySelector('.fullname')?.getAttribute('title')
+        if (!name) {
+          return null
+        }
+        return {
+          avatar,
+          name,
+        }
+      }
+
+      function dateParser(tweetDate: string) {
+        const validDateTime = tweetDate.replace(' · ', ' ')
+
+        return new Date(validDateTime).toISOString()
+      }
+
+      function attachmentParser(attachments: Element | null) {
+        if (!attachments) return { photos: [], videos: [] }
+
+        const photos = Array.from(attachments.querySelectorAll('img')).map(
+          (i) => i.getAttribute('src') ?? ''
+        )
+        const videos = Array.from(attachments.querySelectorAll('source')).map(
+          (i) => i.getAttribute('src') ?? ''
+        )
+        return {
+          photos,
+          videos,
+        }
+      }
+
+      function parseTweet(tweet: Element): Tweet | null {
+        const header = tweet.querySelector('.tweet-header')
+        if (!header) {
+          return null
+        }
+        const author = authorParser(header)
+        if (!author) {
+          return null
+        }
+
+        const body = tweet.querySelector('.tweet-body')
+        if (!body) {
+          return null
+        }
+
+        const tweetDate = body
+          .querySelector('.tweet-date a')
+          ?.getAttribute('title')
+        if (!tweetDate) {
+          return null
+        }
+        const createdAt = dateParser(tweetDate)
+
+        const content = body.querySelector('.tweet-content')
+        if (!content) {
+          return null
+        }
+        const text = content.textContent ?? ''
+        const urls = Array.from(content.querySelectorAll('a')).map((a) => ({
+          url: a.getAttribute('href') ?? '',
+          display_url: a.textContent ?? '',
+        }))
+
+        const attachments = body.querySelector('.attachments')
+        const { photos, videos } = attachmentParser(attachments)
+
+        return {
+          author: {
+            username,
+            name: author.name,
+            profileImageUrl: author.avatar,
+          },
+          createdAt,
+          text,
+          url,
+          entities: {
+            urls,
+            photos,
+            videos,
+          },
+        }
+      }
+
       let context: BrowserContext | undefined
       try {
+        const tweets: Tweet[] = []
+
         context = await browser.createIncognitoBrowserContext()
         const page = await context.newPage()
-
-        // Modify this variable to control the size of viewport
-        const deviceScaleFactor = 0.2
-        const height = Math.floor(2000 / deviceScaleFactor)
-        const width = Math.floor(1700 / deviceScaleFactor)
-        await page.setViewport({ width, height, deviceScaleFactor })
-
         await page.goto(url, {
           waitUntil: 'networkidle2',
           timeout: 30000, // 30 seconds
         })
 
-        const tweets = (await page.evaluate(
-          async (username, id, url) => {
-            async function waitFor(ms: number) {
-              return new Promise((resolve) => setTimeout(resolve, ms))
+        const html = await page.content()
+        const document = parseHTML(html).document
+
+        // get the main thread including tweets and threads
+        const mainThread = document.querySelector('.main-thread')
+        if (!mainThread) {
+          return []
+        }
+        const timelineItems = Array.from(
+          mainThread.querySelectorAll('.timeline-item')
+        )
+        for (let i = 0; i < timelineItems.length; i++) {
+          const item = timelineItems[i]
+          if (item.classList.contains('more-replies')) {
+            const newUrl = item.querySelector('a')?.getAttribute('href')
+            if (!newUrl) {
+              break
             }
 
-            function authorParser(header: Element) {
-              const avatar = header
-                .querySelector('.tweet-avatar img')
-                ?.getAttribute('src')
-              if (!avatar) {
-                return null
-              }
-              const name = header
-                .querySelector('.fullname')
-                ?.getAttribute('title')
-              if (!name) {
-                return null
-              }
-              return {
-                avatar,
-                name,
-              }
-            }
+            // go to new url and wait for it to load
+            await page.goto(`${address}${newUrl}`, {
+              waitUntil: 'networkidle2',
+              timeout: 30000, // 30 seconds
+            })
 
-            function dateParser(tweetDate: string) {
-              const validDateTime = tweetDate.replace(' · ', ' ')
-
-              return new Date(validDateTime).toISOString()
-            }
-
-            function attachmentParser(attachments: Element | null) {
-              if (!attachments) return { photos: [], videos: [] }
-
-              const photos = Array.from(
-                attachments.querySelectorAll('img')
-              ).map((i) => i.getAttribute('src') ?? '')
-              const videos = Array.from(
-                attachments.querySelectorAll('source')
-              ).map((i) => i.getAttribute('src') ?? '')
-              return {
-                photos,
-                videos,
-              }
-            }
-
-            function parseTweet(tweet: Element): Tweet | null {
-              const header = tweet.querySelector('.tweet-header')
-              if (!header) {
-                return null
-              }
-              const author = authorParser(header)
-              if (!author) {
-                return null
-              }
-
-              const body = tweet.querySelector('.tweet-body')
-              if (!body) {
-                return null
-              }
-
-              const tweetDate = body
-                .querySelector('.tweet-date a')
-                ?.getAttribute('title')
-              if (!tweetDate) {
-                return null
-              }
-              const createdAt = dateParser(tweetDate)
-
-              const content = body.querySelector('.tweet-content')
-              if (!content) {
-                return null
-              }
-              const text = content.textContent ?? ''
-              const urls = Array.from(content.querySelectorAll('a')).map(
-                (a) => ({
-                  url: a.getAttribute('href') ?? '',
-                  display_url: a.textContent ?? '',
-                })
-              )
-
-              const attachments = body.querySelector('.attachments')
-              const { photos, videos } = attachmentParser(attachments)
-
-              return {
-                id,
-                author: {
-                  username,
-                  name: author.name,
-                  profileImageUrl: author.avatar,
-                },
-                createdAt,
-                text,
-                url,
-                entities: {
-                  urls,
-                  photos,
-                  videos,
-                },
-              }
-            }
-
-            const tweets: Tweet[] = []
-            // get the main thread including tweets and threads
-            const mainThread = document.querySelector('.main-thread')
-            if (!mainThread) {
-              return []
-            }
-            const timelineItems = Array.from(
-              mainThread.querySelectorAll('.timeline-item')
+            const document = parseHTML(await page.content()).document
+            const nextThread = document.querySelector(
+              '.main-thread .after-tweet'
             )
-            for (let i = 0; i < timelineItems.length; i++) {
-              const item = timelineItems[i]
-              if (item.classList.contains('show-more')) {
-                // click the show more button
-                ;(item as HTMLAnchorElement).click()
-                await waitFor(2000)
-
-                // get the new timeline items and add them to the list
-                const newTimelineItems = Array.from(
-                  mainThread.querySelectorAll('.timeline-item')
-                )
-                timelineItems.push(...newTimelineItems)
-                continue
-              }
-
-              const tweet = parseTweet(item)
-              tweet && tweets.push(tweet)
+            if (!nextThread) {
+              break
             }
 
-            return tweets
-          },
-          username,
-          tweetId,
-          url
-        )) as Tweet[]
+            // get the new timeline items and add them to the list
+            const newTimelineItems = Array.from(
+              nextThread.querySelectorAll('.timeline-item')
+            )
+
+            timelineItems.push(...newTimelineItems)
+            continue
+          }
+
+          const tweet = parseTweet(item)
+          tweet && tweets.push(tweet)
+        }
 
         return tweets
       } catch (error) {
@@ -203,7 +198,7 @@ export class NitterHandler extends ContentHandler {
       }
     }
 
-    return genTweets()
+    return genTweets(this.ADDRESS)
   }
 
   parseTweetUrl = (url: string) => {