prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

skip saving unavailable and earlier replies

Browse source
This commit is contained in:
Hongbo Wu 2023-06-23 16:26:07 +08:00
parent f94c6317ea
commit 41f0c5b3a3
1 changed files with 28 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
packages/content-handler/src/websites/nitter-handler.ts
View file

@ -183,7 +183,7 @@ export class NitterHandler extends ContentHandler {
const option = {
timeout: 20000, // 20 seconds
}
let html: any
let html = ''
// get instances from redis
const instances = await this.getInstances(redisClient)
for (const instance of instances) {
@ -194,8 +194,8 @@ export class NitterHandler extends ContentHandler {
const latency = Math.floor(Date.now() - startTime)
console.debug('latency', latency)
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
html = response.data
html = response.data as string
console.debug('html', html)
this.instance = instance
await this.incrementInstanceScore(redisClient, instance, latency)
@ -230,16 +230,37 @@ export class NitterHandler extends ContentHandler {
)
for (let i = 0; i < timelineItems.length; i++) {
const item = timelineItems[i]
if (item.classList.contains('more-replies')) {
const classList = item.classList
// skip unavailable tweets and earlier replies
if (
classList.contains('unavailable') ||
classList.contains('earlier-replies')
) {
console.debug('skip unavailable tweets and earlier replies', item)
continue
}
// if there are more replies, get them
if (classList.contains('more-replies')) {
const newUrl = item.querySelector('a')?.getAttribute('href')
if (!newUrl) {
break
}
// go to new url and wait for it to load
const response = await axios.get(`${this.instance}${newUrl}`, option)
let html = ''
try {
// go to new url and wait for it to load
const response = await axios.get(
`${this.instance}${newUrl}`,
option
)
const document = parseHTML(response.data).document
html = response.data as string
} catch (error) {
console.info('Error parsing html', error)
break
}
const document = parseHTML(html).document
const nextThread = document.querySelector('.main-thread .after-tweet')
if (!nextThread) {
break