diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index b69b2e275..c938c9e96 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -133,17 +133,20 @@ export class NitterHandler extends ContentHandler { function parseTweet(tweet: Element): Tweet | null { const header = tweet.querySelector('.tweet-header') if (!header) { + console.error('no header found', tweet) return null } const author = authorParser(header) const body = tweet.querySelector('.tweet-body') if (!body) { + console.error('no body found', tweet) return null } const tweetDateElement = body.querySelector('.tweet-date a') if (!tweetDateElement) { + console.error('no tweet date found', tweet) return null } const createdAt = dateParser(tweetDateElement) @@ -151,6 +154,7 @@ export class NitterHandler extends ContentHandler { const content = body.querySelector('.tweet-content') if (!content) { + console.error('no content found', tweet) return null } const text = content.textContent ?? '' @@ -183,7 +187,7 @@ export class NitterHandler extends ContentHandler { const option = { timeout: 20000, // 20 seconds } - let html: any + let html = '' // get instances from redis const instances = await this.getInstances(redisClient) for (const instance of instances) { @@ -194,8 +198,7 @@ export class NitterHandler extends ContentHandler { const latency = Math.floor(Date.now() - startTime) console.debug('latency', latency) - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - html = response.data + html = response.data as string this.instance = instance await this.incrementInstanceScore(redisClient, instance, latency) @@ -215,6 +218,7 @@ export class NitterHandler extends ContentHandler { } } if (!this.instance || !html) { + console.error('no instance or html found') return [] } @@ -223,25 +227,53 @@ export class NitterHandler extends ContentHandler { // get the main thread including tweets and threads const mainThread = document.querySelector('.main-thread') if (!mainThread) { + console.error('no main thread found') return [] } const timelineItems = Array.from( mainThread.querySelectorAll('.timeline-item') ) + if (timelineItems.length === 0) { + console.error('no timeline items found') + return [] + } for (let i = 0; i < timelineItems.length; i++) { const item = timelineItems[i] - if (item.classList.contains('more-replies')) { + const classList = item.classList + // skip unavailable tweets and earlier replies + if ( + classList.contains('unavailable') || + classList.contains('earlier-replies') + ) { + console.info('skip unavailable tweets and earlier replies') + continue + } + // if there are more replies, get them + if (classList.contains('more-replies')) { const newUrl = item.querySelector('a')?.getAttribute('href') if (!newUrl) { + console.error('no new url', newUrl) break } - // go to new url and wait for it to load - const response = await axios.get(`${this.instance}${newUrl}`, option) + let html = '' + try { + // go to new url and wait for it to load + const response = await axios.get( + `${this.instance}${newUrl}`, + option + ) - const document = parseHTML(response.data).document + html = response.data as string + } catch (error) { + console.error('Error getting tweets', error) + break + } + + const document = parseHTML(html).document const nextThread = document.querySelector('.main-thread .after-tweet') if (!nextThread) { + console.error('no next thread found') break } @@ -256,7 +288,7 @@ export class NitterHandler extends ContentHandler { const tweet = parseTweet(item) // filter out replies - if (tweet && tweet.author.username === username) { + if (tweet && tweet.author.username.toLowerCase() === username) { tweets.push(tweet) } } @@ -317,7 +349,7 @@ export class NitterHandler extends ContentHandler { )}` const description = _.escape(tweet.text) const imageDomain = - domain === 'twitter.com' + domain.toLowerCase() === 'twitter.com' ? 'https://pbs.twimg.com' : 'https://nitter.net/pic'