Merge pull request #2420 from omnivore-app/fix/nitter-scraper

Fix format problems brought by other instances

packages/content-handler/src/websites/nitter-handler.ts

@@ -133,17 +133,20 @@ export class NitterHandler extends ContentHandler {
     function parseTweet(tweet: Element): Tweet | null {
       const header = tweet.querySelector('.tweet-header')
       if (!header) {
+        console.error('no header found', tweet)
         return null
       }
       const author = authorParser(header)
 
       const body = tweet.querySelector('.tweet-body')
       if (!body) {
+        console.error('no body found', tweet)
         return null
       }
 
       const tweetDateElement = body.querySelector('.tweet-date a')
       if (!tweetDateElement) {
+        console.error('no tweet date found', tweet)
         return null
       }
       const createdAt = dateParser(tweetDateElement)
@@ -151,6 +154,7 @@ export class NitterHandler extends ContentHandler {
 
       const content = body.querySelector('.tweet-content')
       if (!content) {
+        console.error('no content found', tweet)
         return null
       }
       const text = content.textContent ?? ''
@@ -183,7 +187,7 @@ export class NitterHandler extends ContentHandler {
       const option = {
         timeout: 20000, // 20 seconds
       }
-      let html: any
+      let html = ''
       // get instances from redis
       const instances = await this.getInstances(redisClient)
       for (const instance of instances) {
@@ -194,8 +198,7 @@ export class NitterHandler extends ContentHandler {
           const latency = Math.floor(Date.now() - startTime)
           console.debug('latency', latency)
 
-          // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-          html = response.data
+          html = response.data as string
           this.instance = instance
 
           await this.incrementInstanceScore(redisClient, instance, latency)
@@ -215,6 +218,7 @@ export class NitterHandler extends ContentHandler {
         }
       }
       if (!this.instance || !html) {
+        console.error('no instance or html found')
         return []
       }
 
@@ -223,25 +227,53 @@ export class NitterHandler extends ContentHandler {
       // get the main thread including tweets and threads
       const mainThread = document.querySelector('.main-thread')
       if (!mainThread) {
+        console.error('no main thread found')
         return []
       }
       const timelineItems = Array.from(
         mainThread.querySelectorAll('.timeline-item')
       )
+      if (timelineItems.length === 0) {
+        console.error('no timeline items found')
+        return []
+      }
       for (let i = 0; i < timelineItems.length; i++) {
         const item = timelineItems[i]
-        if (item.classList.contains('more-replies')) {
+        const classList = item.classList
+        // skip unavailable tweets and earlier replies
+        if (
+          classList.contains('unavailable') ||
+          classList.contains('earlier-replies')
+        ) {
+          console.info('skip unavailable tweets and earlier replies')
+          continue
+        }
+        // if there are more replies, get them
+        if (classList.contains('more-replies')) {
           const newUrl = item.querySelector('a')?.getAttribute('href')
           if (!newUrl) {
+            console.error('no new url', newUrl)
             break
           }
 
-          // go to new url and wait for it to load
-          const response = await axios.get(`${this.instance}${newUrl}`, option)
+          let html = ''
+          try {
+            // go to new url and wait for it to load
+            const response = await axios.get(
+              `${this.instance}${newUrl}`,
+              option
+            )
 
-          const document = parseHTML(response.data).document
+            html = response.data as string
+          } catch (error) {
+            console.error('Error getting tweets', error)
+            break
+          }
+
+          const document = parseHTML(html).document
           const nextThread = document.querySelector('.main-thread .after-tweet')
           if (!nextThread) {
+            console.error('no next thread found')
             break
           }
 
@@ -256,7 +288,7 @@ export class NitterHandler extends ContentHandler {
 
         const tweet = parseTweet(item)
         // filter out replies
-        if (tweet && tweet.author.username === username) {
+        if (tweet && tweet.author.username.toLowerCase() === username) {
           tweets.push(tweet)
         }
       }
@@ -317,7 +349,7 @@ export class NitterHandler extends ContentHandler {
     )}`
     const description = _.escape(tweet.text)
     const imageDomain =
-      domain === 'twitter.com'
+      domain.toLowerCase() === 'twitter.com'
         ? 'https://pbs.twimg.com'
         : 'https://nitter.net/pic'