From 4607754f58007a7049d846bb784963ee5d22161b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Jun 2023 11:36:30 +0800 Subject: [PATCH 1/7] reduce timeout to 20s --- packages/content-handler/src/websites/nitter-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index 9cf53abcb..423cbc65a 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -142,7 +142,7 @@ export class NitterHandler extends ContentHandler { try { const tweets: Tweet[] = [] const option = { - timeout: 60000, // 60 seconds + timeout: 20000, // 20 seconds } let html: any // use the first instance that works From ae504a970beeced41880aaccdfe57ad22a25021a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Jun 2023 12:17:41 +0800 Subject: [PATCH 2/7] save instances in redis as a sorted set --- packages/content-handler/package.json | 1 + packages/content-handler/src/redis.ts | 26 +++++++ .../src/websites/nitter-handler.ts | 72 ++++++++++++++++--- yarn.lock | 51 +++++++++++++ 4 files changed, 141 insertions(+), 9 deletions(-) create mode 100644 packages/content-handler/src/redis.ts diff --git a/packages/content-handler/package.json b/packages/content-handler/package.json index 239e31376..7de5e338b 100644 --- a/packages/content-handler/package.json +++ b/packages/content-handler/package.json @@ -35,6 +35,7 @@ "lodash": "^4.17.21", "luxon": "^3.0.4", "puppeteer-core": "^19.1.1", + "redis": "^4.6.7", "underscore": "^1.13.6", "uuid": "^9.0.0" } diff --git a/packages/content-handler/src/redis.ts b/packages/content-handler/src/redis.ts new file mode 100644 index 000000000..350cb09f6 --- /dev/null +++ b/packages/content-handler/src/redis.ts @@ -0,0 +1,26 @@ +import { createClient } from 'redis' + +export const createRedisClient = async (url?: string, cert?: string) => { + const redisClient = createClient({ + url, + socket: { + tls: url?.startsWith('rediss://'), // rediss:// is the protocol for TLS + cert: cert?.replace(/\\n/g, '\n'), // replace \n with new line + rejectUnauthorized: false, // for self-signed certs + connectTimeout: 10000, // 10 seconds + reconnectStrategy(retries: number): number | Error { + if (retries > 10) { + return new Error('Retries exhausted') + } + return 1000 + }, + }, + }) + + redisClient.on('error', (err) => console.error('Redis Client Error', err)) + + await redisClient.connect() + console.log('Redis Client Connected:', url) + + return redisClient +} diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index 423cbc65a..6d1c68d8b 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -2,7 +2,12 @@ import axios from 'axios' import { parseHTML } from 'linkedom' import _, { truncate } from 'lodash' import { DateTime } from 'luxon' +import { createClient } from 'redis' import { ContentHandler, PreHandleResult } from '../content-handler' +import { createRedisClient } from '../redis' + +// explicitly create the return type of RedisClient +type RedisClient = ReturnType interface Tweet { url: string @@ -31,14 +36,15 @@ export class NitterHandler extends ContentHandler { URL_MATCH = /((twitter\.com)|(nitter\.net))\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ INSTANCES = [ - 'https://nitter.net', - 'https://nitter.lacontrevoie.fr', - 'https://nitter.1d4.us', - 'https://nitter.kavin.rocks', - 'https://nitter.it', - 'https://twitter.owacon.moe', - 'https://singapore.unofficialbird.com', + { value: 'https://nitter.net', score: 0 }, + { value: 'https://nitter.lacontrevoie.fr', score: 0 }, + { value: 'https://nitter.1d4.us', score: 0 }, + { value: 'https://nitter.kavin.rocks', score: 0 }, + { value: 'https://nitter.it', score: 0 }, + { value: 'https://twitter.owacon.moe', score: 0 }, + { value: 'https://singapore.unofficialbird.com', score: 0 }, ] + REDIS_KEY = 'nitter-instances' private instance: string @@ -48,6 +54,44 @@ export class NitterHandler extends ContentHandler { this.instance = '' } + async getInstance(redisClient: RedisClient) { + const instances = await redisClient.zRangeByScore( + this.REDIS_KEY, + '-inf', + '+inf', + { + LIMIT: { + count: 1, + offset: 0, + }, + } + ) + + // if no instance is found, save the default instances + if (instances.length === 0) { + await redisClient.zAdd(this.REDIS_KEY, this.INSTANCES) + return this.INSTANCES[0].value + } + + return instances[0] + } + + async incrementInstanceScore( + redisClient: RedisClient, + instance: string, + score = 1 + ) { + await redisClient.zIncrBy(this.REDIS_KEY, score, instance) + } + + async decrementInstanceScore( + redisClient: RedisClient, + instance: string, + score = 1 + ) { + await redisClient.zIncrBy(this.REDIS_KEY, score, instance) + } + async getTweets(username: string, tweetId: string) { function authorParser(header: Element) { const profileImageUrl = @@ -139,22 +183,30 @@ export class NitterHandler extends ContentHandler { } } + const redisClient = await createRedisClient() + try { const tweets: Tweet[] = [] const option = { timeout: 20000, // 20 seconds } let html: any - // use the first instance that works - for (const instance of this.INSTANCES) { + // get instance from redis + for (let i = 0; i < this.INSTANCES.length; i++) { + const instance = await this.getInstance(redisClient) + try { const url = `${instance}/${username}/status/${tweetId}` const response = await axios.get(url, option) // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment html = response.data this.instance = instance + + await this.incrementInstanceScore(redisClient, instance) break } catch (error) { + await this.decrementInstanceScore(redisClient, instance) + if (axios.isAxiosError(error)) { console.info(`Error getting tweets from ${instance}`, error.message) } else { @@ -214,6 +266,8 @@ export class NitterHandler extends ContentHandler { console.error('Error getting tweets', error) return [] + } finally { + await redisClient?.quit() } } diff --git a/yarn.lock b/yarn.lock index 025cd3b97..178efa557 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5436,6 +5436,11 @@ resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.0.2.tgz#42b82ec399a92db05e29fffcdfd9235a5fc15cdf" integrity sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw== +"@redis/bloom@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" + integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== + "@redis/client@1.3.0": version "1.3.0" resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.3.0.tgz#c62ccd707f16370a2dc2f9e158a28b7da049fa77" @@ -5445,11 +5450,25 @@ generic-pool "3.8.2" yallist "4.0.0" +"@redis/client@1.5.8": + version "1.5.8" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.8.tgz#a375ba7861825bd0d2dc512282b8bff7b98dbcb1" + integrity sha512-xzElwHIO6rBAqzPeVnCzgvrnBEcFL1P0w8P65VNLRkdVW8rOE58f52hdj0BDgmsdOm4f1EoXPZtH4Fh7M/qUpw== + dependencies: + cluster-key-slot "1.1.2" + generic-pool "3.9.0" + yallist "4.0.0" + "@redis/graph@1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.0.1.tgz#eabc58ba99cd70d0c907169c02b55497e4ec8a99" integrity sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ== +"@redis/graph@1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519" + integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg== + "@redis/json@1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1" @@ -5460,11 +5479,21 @@ resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.0.tgz#7abb18d431f27ceafe6bcb4dd83a3fa67e9ab4df" integrity sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ== +"@redis/search@1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.3.tgz#b5a6837522ce9028267fe6f50762a8bcfd2e998b" + integrity sha512-4Dg1JjvCevdiCBTZqjhKkGoC5/BcB7k9j99kdMnaXFXg8x4eyOIVg9487CMv7/BUVkFLZCaIh8ead9mU15DNng== + "@redis/time-series@1.0.3": version "1.0.3" resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4" integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA== +"@redis/time-series@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717" + integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng== + "@remusao/guess-url-type@^1.1.2": version "1.2.1" resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57" @@ -11860,6 +11889,11 @@ cluster-key-slot@1.1.0: resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== +cluster-key-slot@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + cmd-shim@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd" @@ -15355,6 +15389,11 @@ generic-pool@3.8.2: resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9" integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg== +generic-pool@3.9.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" + integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== + gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -24194,6 +24233,18 @@ redis@^4.3.1: "@redis/search" "1.1.0" "@redis/time-series" "1.0.3" +redis@^4.6.7: + version "4.6.7" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.7.tgz#c73123ad0b572776223f172ec78185adb72a6b57" + integrity sha512-KrkuNJNpCwRm5vFJh0tteMxW8SaUzkm5fBH7eL5hd/D0fAkzvapxbfGPP/r+4JAXdQuX7nebsBkBqA2RHB7Usw== + dependencies: + "@redis/bloom" "1.2.0" + "@redis/client" "1.5.8" + "@redis/graph" "1.1.0" + "@redis/json" "1.0.4" + "@redis/search" "1.1.3" + "@redis/time-series" "1.0.4" + reflect-metadata@^0.1.13: version "0.1.13" resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08" From 9ef40b3ed95064b4b9b86a04f60e2227c4e514d9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Jun 2023 12:27:34 +0800 Subject: [PATCH 3/7] deduct latency from the score of the member in sorted set --- .../src/websites/nitter-handler.ts | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index 6d1c68d8b..d2fa1fb10 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -54,26 +54,20 @@ export class NitterHandler extends ContentHandler { this.instance = '' } - async getInstance(redisClient: RedisClient) { + async getInstances(redisClient: RedisClient) { const instances = await redisClient.zRangeByScore( this.REDIS_KEY, '-inf', - '+inf', - { - LIMIT: { - count: 1, - offset: 0, - }, - } + '+inf' ) // if no instance is found, save the default instances if (instances.length === 0) { await redisClient.zAdd(this.REDIS_KEY, this.INSTANCES) - return this.INSTANCES[0].value + return this.INSTANCES.map((i) => i.value) } - return instances[0] + return instances } async incrementInstanceScore( @@ -84,14 +78,6 @@ export class NitterHandler extends ContentHandler { await redisClient.zIncrBy(this.REDIS_KEY, score, instance) } - async decrementInstanceScore( - redisClient: RedisClient, - instance: string, - score = 1 - ) { - await redisClient.zIncrBy(this.REDIS_KEY, score, instance) - } - async getTweets(username: string, tweetId: string) { function authorParser(header: Element) { const profileImageUrl = @@ -191,21 +177,27 @@ export class NitterHandler extends ContentHandler { timeout: 20000, // 20 seconds } let html: any - // get instance from redis - for (let i = 0; i < this.INSTANCES.length; i++) { - const instance = await this.getInstance(redisClient) - + // get instances from redis + const instances = await this.getInstances(redisClient) + for (const instance of instances) { try { const url = `${instance}/${username}/status/${tweetId}` + const startTime = Date.now() const response = await axios.get(url, option) + const latency = Date.now() - startTime + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment html = response.data this.instance = instance - await this.incrementInstanceScore(redisClient, instance) + await this.incrementInstanceScore( + redisClient, + instance, + -latency / 1000 + ) break } catch (error) { - await this.decrementInstanceScore(redisClient, instance) + await this.incrementInstanceScore(redisClient, instance, -20000) if (axios.isAxiosError(error)) { console.info(`Error getting tweets from ${instance}`, error.message) From b746046727dfb820ff4a3cb96f08ca2b7dc38e9e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Jun 2023 12:34:31 +0800 Subject: [PATCH 4/7] update default score --- .../content-handler/src/websites/nitter-handler.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index d2fa1fb10..e0953c90d 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -36,13 +36,13 @@ export class NitterHandler extends ContentHandler { URL_MATCH = /((twitter\.com)|(nitter\.net))\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ INSTANCES = [ - { value: 'https://nitter.net', score: 0 }, - { value: 'https://nitter.lacontrevoie.fr', score: 0 }, - { value: 'https://nitter.1d4.us', score: 0 }, - { value: 'https://nitter.kavin.rocks', score: 0 }, - { value: 'https://nitter.it', score: 0 }, - { value: 'https://twitter.owacon.moe', score: 0 }, - { value: 'https://singapore.unofficialbird.com', score: 0 }, + { value: 'https://nitter.1d4.us', score: 7 }, + { value: 'https://nitter.net', score: 6 }, // the official instance + { value: 'https://nitter.lacontrevoie.fr', score: 5 }, + { value: 'https://nitter.kavin.rocks', score: 4 }, + { value: 'https://nitter.it', score: 3 }, + { value: 'https://singapore.unofficialbird.com', score: 2 }, + { value: 'nitter.fly.dev', score: 1 }, ] REDIS_KEY = 'nitter-instances' From fc0257b480790e517726536aa394ab5261e04ed1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Jun 2023 12:52:27 +0800 Subject: [PATCH 5/7] sort by descending order --- .../src/websites/nitter-handler.ts | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index e0953c90d..dea868489 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -36,13 +36,13 @@ export class NitterHandler extends ContentHandler { URL_MATCH = /((twitter\.com)|(nitter\.net))\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ INSTANCES = [ - { value: 'https://nitter.1d4.us', score: 7 }, - { value: 'https://nitter.net', score: 6 }, // the official instance - { value: 'https://nitter.lacontrevoie.fr', score: 5 }, - { value: 'https://nitter.kavin.rocks', score: 4 }, - { value: 'https://nitter.it', score: 3 }, - { value: 'https://singapore.unofficialbird.com', score: 2 }, - { value: 'nitter.fly.dev', score: 1 }, + { value: 'https://nitter.1d4.us', score: 0 }, + { value: 'https://nitter.net', score: 1 }, // the official instance + { value: 'https://nitter.lacontrevoie.fr', score: 2 }, + { value: 'https://nitter.kavin.rocks', score: 3 }, + { value: 'https://nitter.it', score: 4 }, + { value: 'https://singapore.unofficialbird.com', score: 5 }, + { value: 'nitter.fly.dev', score: 6 }, ] REDIS_KEY = 'nitter-instances' @@ -55,15 +55,19 @@ export class NitterHandler extends ContentHandler { } async getInstances(redisClient: RedisClient) { - const instances = await redisClient.zRangeByScore( - this.REDIS_KEY, - '-inf', - '+inf' - ) + const instances = await redisClient.zRange(this.REDIS_KEY, '-inf', '+inf', { + REV: true, + BY: 'SCORE', + }) + console.debug('instances', instances) // if no instance is found, save the default instances if (instances.length === 0) { - await redisClient.zAdd(this.REDIS_KEY, this.INSTANCES) + const result = await redisClient.zAdd(this.REDIS_KEY, this.INSTANCES, { + NX: true, // only add if the key does not exist + }) + console.debug('zAdd result', result) + return this.INSTANCES.map((i) => i.value) } @@ -184,20 +188,16 @@ export class NitterHandler extends ContentHandler { const url = `${instance}/${username}/status/${tweetId}` const startTime = Date.now() const response = await axios.get(url, option) - const latency = Date.now() - startTime + const latency = (Date.now() - startTime) / 1000000 // convert to seconds // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment html = response.data this.instance = instance - await this.incrementInstanceScore( - redisClient, - instance, - -latency / 1000 - ) + await this.incrementInstanceScore(redisClient, instance, latency) break } catch (error) { - await this.incrementInstanceScore(redisClient, instance, -20000) + await this.incrementInstanceScore(redisClient, instance, 20) if (axios.isAxiosError(error)) { console.info(`Error getting tweets from ${instance}`, error.message) From 716fdf9a619adc871ab11a4b13d54ad6e534f7a0 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Jun 2023 13:19:18 +0800 Subject: [PATCH 6/7] expire key after 1 day --- .../src/websites/nitter-handler.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index dea868489..534f93ae9 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -55,8 +55,8 @@ export class NitterHandler extends ContentHandler { } async getInstances(redisClient: RedisClient) { + // get instances by score in ascending order const instances = await redisClient.zRange(this.REDIS_KEY, '-inf', '+inf', { - REV: true, BY: 'SCORE', }) console.debug('instances', instances) @@ -66,7 +66,11 @@ export class NitterHandler extends ContentHandler { const result = await redisClient.zAdd(this.REDIS_KEY, this.INSTANCES, { NX: true, // only add if the key does not exist }) - console.debug('zAdd result', result) + console.debug('add instances', result) + + // expire the key after 1 day + const exp = await redisClient.expire(this.REDIS_KEY, 60 * 60 * 24) + console.debug('instances expire in 1 day', exp) return this.INSTANCES.map((i) => i.value) } @@ -188,7 +192,8 @@ export class NitterHandler extends ContentHandler { const url = `${instance}/${username}/status/${tweetId}` const startTime = Date.now() const response = await axios.get(url, option) - const latency = (Date.now() - startTime) / 1000000 // convert to seconds + const latency = Math.floor(Date.now() - startTime) + console.debug('latency', latency) // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment html = response.data @@ -197,7 +202,11 @@ export class NitterHandler extends ContentHandler { await this.incrementInstanceScore(redisClient, instance, latency) break } catch (error) { - await this.incrementInstanceScore(redisClient, instance, 20) + await this.incrementInstanceScore( + redisClient, + instance, + option.timeout + ) if (axios.isAxiosError(error)) { console.info(`Error getting tweets from ${instance}`, error.message) From e2b66be75dabc15b4f9b20c28503810265086a46 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Jun 2023 13:33:38 +0800 Subject: [PATCH 7/7] fix dependecies --- packages/content-handler/package.json | 2 +- packages/content-handler/src/redis.ts | 8 ++- .../src/websites/nitter-handler.ts | 6 +-- yarn.lock | 51 ------------------- 4 files changed, 9 insertions(+), 58 deletions(-) diff --git a/packages/content-handler/package.json b/packages/content-handler/package.json index 7de5e338b..adda03962 100644 --- a/packages/content-handler/package.json +++ b/packages/content-handler/package.json @@ -35,7 +35,7 @@ "lodash": "^4.17.21", "luxon": "^3.0.4", "puppeteer-core": "^19.1.1", - "redis": "^4.6.7", + "redis": "^4.3.1", "underscore": "^1.13.6", "uuid": "^9.0.0" } diff --git a/packages/content-handler/src/redis.ts b/packages/content-handler/src/redis.ts index 350cb09f6..2ecf068c6 100644 --- a/packages/content-handler/src/redis.ts +++ b/packages/content-handler/src/redis.ts @@ -1,6 +1,12 @@ import { createClient } from 'redis' -export const createRedisClient = async (url?: string, cert?: string) => { +// explicitly create the return type of RedisClient +export type RedisClient = ReturnType + +export const createRedisClient = async ( + url?: string, + cert?: string +): Promise => { const redisClient = createClient({ url, socket: { diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index 534f93ae9..8fb4732c8 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -2,12 +2,8 @@ import axios from 'axios' import { parseHTML } from 'linkedom' import _, { truncate } from 'lodash' import { DateTime } from 'luxon' -import { createClient } from 'redis' import { ContentHandler, PreHandleResult } from '../content-handler' -import { createRedisClient } from '../redis' - -// explicitly create the return type of RedisClient -type RedisClient = ReturnType +import { createRedisClient, RedisClient } from '../redis' interface Tweet { url: string diff --git a/yarn.lock b/yarn.lock index 178efa557..025cd3b97 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5436,11 +5436,6 @@ resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.0.2.tgz#42b82ec399a92db05e29fffcdfd9235a5fc15cdf" integrity sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw== -"@redis/bloom@1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" - integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== - "@redis/client@1.3.0": version "1.3.0" resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.3.0.tgz#c62ccd707f16370a2dc2f9e158a28b7da049fa77" @@ -5450,25 +5445,11 @@ generic-pool "3.8.2" yallist "4.0.0" -"@redis/client@1.5.8": - version "1.5.8" - resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.8.tgz#a375ba7861825bd0d2dc512282b8bff7b98dbcb1" - integrity sha512-xzElwHIO6rBAqzPeVnCzgvrnBEcFL1P0w8P65VNLRkdVW8rOE58f52hdj0BDgmsdOm4f1EoXPZtH4Fh7M/qUpw== - dependencies: - cluster-key-slot "1.1.2" - generic-pool "3.9.0" - yallist "4.0.0" - "@redis/graph@1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.0.1.tgz#eabc58ba99cd70d0c907169c02b55497e4ec8a99" integrity sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ== -"@redis/graph@1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519" - integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg== - "@redis/json@1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1" @@ -5479,21 +5460,11 @@ resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.0.tgz#7abb18d431f27ceafe6bcb4dd83a3fa67e9ab4df" integrity sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ== -"@redis/search@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.3.tgz#b5a6837522ce9028267fe6f50762a8bcfd2e998b" - integrity sha512-4Dg1JjvCevdiCBTZqjhKkGoC5/BcB7k9j99kdMnaXFXg8x4eyOIVg9487CMv7/BUVkFLZCaIh8ead9mU15DNng== - "@redis/time-series@1.0.3": version "1.0.3" resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4" integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA== -"@redis/time-series@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717" - integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng== - "@remusao/guess-url-type@^1.1.2": version "1.2.1" resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57" @@ -11889,11 +11860,6 @@ cluster-key-slot@1.1.0: resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== -cluster-key-slot@1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" - integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== - cmd-shim@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd" @@ -15389,11 +15355,6 @@ generic-pool@3.8.2: resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9" integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg== -generic-pool@3.9.0: - version "3.9.0" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" - integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== - gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -24233,18 +24194,6 @@ redis@^4.3.1: "@redis/search" "1.1.0" "@redis/time-series" "1.0.3" -redis@^4.6.7: - version "4.6.7" - resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.7.tgz#c73123ad0b572776223f172ec78185adb72a6b57" - integrity sha512-KrkuNJNpCwRm5vFJh0tteMxW8SaUzkm5fBH7eL5hd/D0fAkzvapxbfGPP/r+4JAXdQuX7nebsBkBqA2RHB7Usw== - dependencies: - "@redis/bloom" "1.2.0" - "@redis/client" "1.5.8" - "@redis/graph" "1.1.0" - "@redis/json" "1.0.4" - "@redis/search" "1.1.3" - "@redis/time-series" "1.0.4" - reflect-metadata@^0.1.13: version "0.1.13" resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08"