From 41f0c5b3a33445d59886e8491fafbe27e0e64374 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Jun 2023 16:26:07 +0800 Subject: [PATCH 1/3] skip saving unavailable and earlier replies --- .../src/websites/nitter-handler.ts | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index b69b2e275..7ac6b83fa 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -183,7 +183,7 @@ export class NitterHandler extends ContentHandler { const option = { timeout: 20000, // 20 seconds } - let html: any + let html = '' // get instances from redis const instances = await this.getInstances(redisClient) for (const instance of instances) { @@ -194,8 +194,8 @@ export class NitterHandler extends ContentHandler { const latency = Math.floor(Date.now() - startTime) console.debug('latency', latency) - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - html = response.data + html = response.data as string + console.debug('html', html) this.instance = instance await this.incrementInstanceScore(redisClient, instance, latency) @@ -230,16 +230,37 @@ export class NitterHandler extends ContentHandler { ) for (let i = 0; i < timelineItems.length; i++) { const item = timelineItems[i] - if (item.classList.contains('more-replies')) { + const classList = item.classList + // skip unavailable tweets and earlier replies + if ( + classList.contains('unavailable') || + classList.contains('earlier-replies') + ) { + console.debug('skip unavailable tweets and earlier replies', item) + continue + } + // if there are more replies, get them + if (classList.contains('more-replies')) { const newUrl = item.querySelector('a')?.getAttribute('href') if (!newUrl) { break } - // go to new url and wait for it to load - const response = await axios.get(`${this.instance}${newUrl}`, option) + let html = '' + try { + // go to new url and wait for it to load + const response = await axios.get( + `${this.instance}${newUrl}`, + option + ) - const document = parseHTML(response.data).document + html = response.data as string + } catch (error) { + console.info('Error parsing html', error) + break + } + + const document = parseHTML(html).document const nextThread = document.querySelector('.main-thread .after-tweet') if (!nextThread) { break From a5ddc9acedf8cb82e3a6ab5a60576285e07c1c7e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Jun 2023 16:56:03 +0800 Subject: [PATCH 2/3] lowercase username before comparing --- .../src/websites/nitter-handler.ts | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index 7ac6b83fa..6f0e1dda5 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -133,17 +133,20 @@ export class NitterHandler extends ContentHandler { function parseTweet(tweet: Element): Tweet | null { const header = tweet.querySelector('.tweet-header') if (!header) { + console.error('no header found', tweet) return null } const author = authorParser(header) const body = tweet.querySelector('.tweet-body') if (!body) { + console.error('no body found', tweet) return null } const tweetDateElement = body.querySelector('.tweet-date a') if (!tweetDateElement) { + console.error('no tweet date found', tweet) return null } const createdAt = dateParser(tweetDateElement) @@ -151,6 +154,7 @@ export class NitterHandler extends ContentHandler { const content = body.querySelector('.tweet-content') if (!content) { + console.error('no content found', tweet) return null } const text = content.textContent ?? '' @@ -195,7 +199,6 @@ export class NitterHandler extends ContentHandler { console.debug('latency', latency) html = response.data as string - console.debug('html', html) this.instance = instance await this.incrementInstanceScore(redisClient, instance, latency) @@ -215,6 +218,7 @@ export class NitterHandler extends ContentHandler { } } if (!this.instance || !html) { + console.error('no instance or html found') return [] } @@ -223,11 +227,16 @@ export class NitterHandler extends ContentHandler { // get the main thread including tweets and threads const mainThread = document.querySelector('.main-thread') if (!mainThread) { + console.error('no main thread found') return [] } const timelineItems = Array.from( mainThread.querySelectorAll('.timeline-item') ) + if (timelineItems.length === 0) { + console.error('no timeline items found') + return [] + } for (let i = 0; i < timelineItems.length; i++) { const item = timelineItems[i] const classList = item.classList @@ -236,13 +245,14 @@ export class NitterHandler extends ContentHandler { classList.contains('unavailable') || classList.contains('earlier-replies') ) { - console.debug('skip unavailable tweets and earlier replies', item) + console.info('skip unavailable tweets and earlier replies') continue } // if there are more replies, get them if (classList.contains('more-replies')) { const newUrl = item.querySelector('a')?.getAttribute('href') if (!newUrl) { + console.error('new url', newUrl) break } @@ -256,13 +266,14 @@ export class NitterHandler extends ContentHandler { html = response.data as string } catch (error) { - console.info('Error parsing html', error) + console.error('Error getting tweets', error) break } const document = parseHTML(html).document const nextThread = document.querySelector('.main-thread .after-tweet') if (!nextThread) { + console.error('no next thread found') break } @@ -277,7 +288,7 @@ export class NitterHandler extends ContentHandler { const tweet = parseTweet(item) // filter out replies - if (tweet && tweet.author.username === username) { + if (tweet && tweet.author.username.toLowerCase() === username) { tweets.push(tweet) } } @@ -338,7 +349,7 @@ export class NitterHandler extends ContentHandler { )}` const description = _.escape(tweet.text) const imageDomain = - domain === 'twitter.com' + domain.toLowerCase() === 'twitter.com' ? 'https://pbs.twimg.com' : 'https://nitter.net/pic' From 978a21d160fa744e30583800ad663493e7a5f3ac Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Jun 2023 16:59:34 +0800 Subject: [PATCH 3/3] fix typo --- packages/content-handler/src/websites/nitter-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index 6f0e1dda5..c938c9e96 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -252,7 +252,7 @@ export class NitterHandler extends ContentHandler { if (classList.contains('more-replies')) { const newUrl = item.querySelector('a')?.getAttribute('href') if (!newUrl) { - console.error('new url', newUrl) + console.error('no new url', newUrl) break }