From fc25d6d6153dc7d472b3ad26b919340924c78aef Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 11 Jan 2024 21:03:59 +0800 Subject: [PATCH 01/76] Enqueue rss-refresh-feed jobs in bull --- packages/api/package.json | 1 + packages/api/src/jobqueue.ts | 29 ++++++++++++++++++++++++++++ packages/api/src/utils/createTask.ts | 8 ++++++++ 3 files changed, 38 insertions(+) create mode 100644 packages/api/src/jobqueue.ts diff --git a/packages/api/package.json b/packages/api/package.json index 432de1b5d..c41153433 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -47,6 +47,7 @@ "apollo-server-express": "^3.6.3", "axios": "^0.27.2", "bcryptjs": "^2.4.3", + "bullmq": "^5.1.1", "cookie": "^0.5.0", "cookie-parser": "^1.4.5", "cors": "^2.8.5", diff --git a/packages/api/src/jobqueue.ts b/packages/api/src/jobqueue.ts new file mode 100644 index 000000000..92cd224f7 --- /dev/null +++ b/packages/api/src/jobqueue.ts @@ -0,0 +1,29 @@ +import { env } from './env' + +import { Queue } from 'bullmq' +import Redis from 'ioredis' + +const redisOptions = () => { + if (env.redis.url?.startsWith('rediss://') && env.redis.cert) { + return { + tls: { + cert: env.redis.cert?.replace(/\\n/g, '\n'), + }, + } + } + return {} +} + +const connection = new Redis(env.redis.url ?? '', redisOptions()) + +export const rssRefreshFeedJobQueue = new Queue('rssRefreshFeed', { + connection, +}) + +export const addRefreshFeedJob = async (jobid: string, payload: any) => { + return rssRefreshFeedJobQueue.add('rssRefreshFeed', payload, { + jobId: jobid, + removeOnComplete: true, + removeOnFail: true, + }) +} diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 7e332c967..021ec8291 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -19,6 +19,8 @@ import { generateVerificationToken, OmnivoreAuthorizationHeader } from './auth' import { CreateTaskError } from './errors' import { logger } from './logger' import View = google.cloud.tasks.v2.Task.View +import { addRefreshFeedJob } from '../jobqueue' +import { stringToHash } from './helpers' // Instantiates a client. const client = new CloudTasksClient() @@ -649,6 +651,12 @@ export const enqueueRssFeedFetch = async ( folders: subscriptionGroup.folders, } + let jobid = `refresh-feed_${stringToHash( + subscriptionGroup.url + )}_${stringToHash(JSON.stringify(subscriptionGroup.userIds.sort()))}` + + await addRefreshFeedJob(jobid, payload) + // If there is no Google Cloud Project Id exposed, it means that we are in local environment if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { if (env.queue.rssFeedTaskHandlerUrl) { From d29c26d93ba1a255e844d0f149a803a6ffb07599 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 11 Jan 2024 22:59:33 +0800 Subject: [PATCH 02/76] Some redis setup --- packages/api/src/jobqueue.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/api/src/jobqueue.ts b/packages/api/src/jobqueue.ts index 92cd224f7..5a644f7af 100644 --- a/packages/api/src/jobqueue.ts +++ b/packages/api/src/jobqueue.ts @@ -8,10 +8,14 @@ const redisOptions = () => { return { tls: { cert: env.redis.cert?.replace(/\\n/g, '\n'), + rejectUnauthorized: false, }, + maxRetriesPerRequest: null, } } - return {} + return { + maxRetriesPerRequest: null, + } } const connection = new Redis(env.redis.url ?? '', redisOptions()) From b4ce07efd3493f63763fffda3368d1436dd70056 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 11:00:05 +0800 Subject: [PATCH 03/76] Better handling of cases when redis isnt configured --- packages/api/src/jobqueue.ts | 41 +++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/packages/api/src/jobqueue.ts b/packages/api/src/jobqueue.ts index 5a644f7af..d9f38e176 100644 --- a/packages/api/src/jobqueue.ts +++ b/packages/api/src/jobqueue.ts @@ -1,31 +1,38 @@ import { env } from './env' -import { Queue } from 'bullmq' +import { Queue, RedisOptions } from 'bullmq' import Redis from 'ioredis' -const redisOptions = () => { - if (env.redis.url?.startsWith('rediss://') && env.redis.cert) { +const createRSSRefreshFeedQueue = (): Queue | undefined => { + if (!env.redis.url) { + return undefined + } + const redisOptions = (): RedisOptions => { + if (env.redis.url?.startsWith('rediss://') && env.redis.cert) { + return { + tls: { + cert: env.redis.cert?.replace(/\\n/g, '\n'), + rejectUnauthorized: false, + }, + maxRetriesPerRequest: null, + } + } return { - tls: { - cert: env.redis.cert?.replace(/\\n/g, '\n'), - rejectUnauthorized: false, - }, maxRetriesPerRequest: null, } } - return { - maxRetriesPerRequest: null, - } + + const connection = new Redis(env.redis.url ?? '', redisOptions()) + return new Queue('rssRefreshFeed', { connection }) } -const connection = new Redis(env.redis.url ?? '', redisOptions()) - -export const rssRefreshFeedJobQueue = new Queue('rssRefreshFeed', { - connection, -}) - export const addRefreshFeedJob = async (jobid: string, payload: any) => { - return rssRefreshFeedJobQueue.add('rssRefreshFeed', payload, { + const rssRefreshFeedJobQueue = createRSSRefreshFeedQueue() + + if (!rssRefreshFeedJobQueue) { + return false + } + return rssRefreshFeedJobQueue?.add('rssRefreshFeed', payload, { jobId: jobid, removeOnComplete: true, removeOnFail: true, From 1af4bf69287ecf7c6642a7084988315d7061df97 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 11:28:15 +0800 Subject: [PATCH 04/76] Add bull admin panel --- pkg/bull-queue-admin/Dockerfile | 17 + pkg/bull-queue-admin/index.js | 120 +++ pkg/bull-queue-admin/package.json | 21 + pkg/bull-queue-admin/views/login.ejs | 86 +++ pkg/bull-queue-admin/yarn.lock | 1047 ++++++++++++++++++++++++++ 5 files changed, 1291 insertions(+) create mode 100644 pkg/bull-queue-admin/Dockerfile create mode 100644 pkg/bull-queue-admin/index.js create mode 100644 pkg/bull-queue-admin/package.json create mode 100644 pkg/bull-queue-admin/views/login.ejs create mode 100644 pkg/bull-queue-admin/yarn.lock diff --git a/pkg/bull-queue-admin/Dockerfile b/pkg/bull-queue-admin/Dockerfile new file mode 100644 index 000000000..8e8876ccf --- /dev/null +++ b/pkg/bull-queue-admin/Dockerfile @@ -0,0 +1,17 @@ +FROM node:14.18-alpine + +WORKDIR /app + +COPY package.json . +COPY yarn.lock . + +RUN yarn install --frozen-lockfile + +COPY . . + +ENV PORT=8080 +EXPOSE 8080 + +ENV NODE_ENV production + +CMD yarn start diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js new file mode 100644 index 000000000..2b598ddf3 --- /dev/null +++ b/pkg/bull-queue-admin/index.js @@ -0,0 +1,120 @@ +const { createBullBoard } = require('@bull-board/api') +const { BullMQAdapter } = require('@bull-board/api/bullMQAdapter') +const { ExpressAdapter } = require('@bull-board/express') +const { Queue: QueueMQ, Worker } = require('bullmq') +const session = require('express-session') +const bodyParser = require('body-parser') +const passport = require('passport') +const LocalStrategy = require('passport-local').Strategy +const { ensureLoggedIn } = require('connect-ensure-login') +const express = require('express') + +const readYamlFile = require('read-yaml-file') + +passport.use( + new LocalStrategy(function (username, password, cb) { + readYamlFile(process.env.SECRETS_FILE).then((secrets) => { + if ( + secrets.ADMIN_USER_PASSWORD && + username === secrets.ADMIN_USER_EMAIL && + password === secrets.ADMIN_USER_PASSWORD + ) { + return cb(null, { user: 'bull-board' }) + } + return cb(null, false) + }) + }) +) + +// Configure Passport authenticated session persistence. +// +// In order to restore authentication state across HTTP requests, Passport needs +// to serialize users into and deserialize users out of the session. The +// typical implementation of this is as simple as supplying the user ID when +// serializing, and querying the user record by ID from the database when +// deserializing. +passport.serializeUser((user, cb) => { + cb(null, user) +}) + +passport.deserializeUser((user, cb) => { + cb(null, user) +}) + +const sleep = (t) => new Promise((resolve) => setTimeout(resolve, t * 1000)) + +const redisOptions = { + port: 6379, + host: 'localhost', + password: '', + tls: false, +} + +const createQueueMQ = (name) => new QueueMQ(name, { connection: redisOptions }) + +const run = async () => { + const rssRefreshFeed = createQueueMQ('rssRefreshFeed') + + const serverAdapter = new ExpressAdapter() + serverAdapter.setBasePath('/ui') + + createBullBoard({ + queues: [new BullMQAdapter(rssRefreshFeed)], + serverAdapter, + }) + + const app = express() + // Configure view engine to render EJS templates. + app.set('views', __dirname + '/views') + app.set('view engine', 'ejs') + + app.use( + session({ secret: 'keyboard cat', saveUninitialized: true, resave: true }) + ) + app.use(bodyParser.urlencoded({ extended: false })) + + // Initialize Passport and restore authentication state, if any, from the session. + app.use(passport.initialize({})) + app.use(passport.session({})) + + app.get('/ui/login', (req, res) => { + res.render('login', { invalid: req.query.invalid === 'true' }) + }) + + app.post( + '/ui/login', + passport.authenticate('local', { + failureRedirect: '/ui/login?invalid=true', + }), + (req, res) => { + res.redirect('/ui') + } + ) + + app.use('/add', (req, res) => { + const opts = req.query.opts || {} + + if (opts.delay) { + opts.delay = +opts.delay * 1000 // delay must be a number + } + + rssRefreshFeed.add('Add', { title: req.query.title }, opts) + + res.json({ + ok: true, + }) + }) + + app.use( + '/ui', + ensureLoggedIn({ redirectTo: '/ui/login' }), + serverAdapter.getRouter() + ) + + app.listen(3000, () => { + console.log('Running on 3000...') + }) +} + +// eslint-disable-next-line no-console +run().catch((e) => console.error(e)) diff --git a/pkg/bull-queue-admin/package.json b/pkg/bull-queue-admin/package.json new file mode 100644 index 000000000..b1526249a --- /dev/null +++ b/pkg/bull-queue-admin/package.json @@ -0,0 +1,21 @@ +{ + "name": "queue-admin", + "version": "1.0.0", + "description": "Custom version of bull-board for adm", + "main": "index.js", + "scripts": { + "start": "node index.js", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "dependencies": { + "@bull-board/express": "^3.10.4", + "body-parser": "^1.20.0", + "bullmq": "^4.6.0", + "connect-ensure-login": "^0.1.1", + "express": "^4.17.3", + "express-session": "^1.17.2", + "passport": "^0.6.0", + "passport-local": "^1.0.0", + "read-yaml-file": "^2.1.0" + } +} \ No newline at end of file diff --git a/pkg/bull-queue-admin/views/login.ejs b/pkg/bull-queue-admin/views/login.ejs new file mode 100644 index 000000000..32ee8f161 --- /dev/null +++ b/pkg/bull-queue-admin/views/login.ejs @@ -0,0 +1,86 @@ + + + + \ No newline at end of file diff --git a/pkg/bull-queue-admin/yarn.lock b/pkg/bull-queue-admin/yarn.lock new file mode 100644 index 000000000..6fc96d716 --- /dev/null +++ b/pkg/bull-queue-admin/yarn.lock @@ -0,0 +1,1047 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@bull-board/api@3.11.1": + version "3.11.1" + resolved "https://registry.npmjs.org/@bull-board/api/-/api-3.11.1.tgz" + integrity sha512-ElwX7sM+Ng4ZL9KUsbDubRE+r2hu/gss85OsROeE9bmyfkW14jOJkgr5MKUyjTTgPEeMs1Mw55TgQs2vxoWBiA== + dependencies: + redis-info "^3.0.8" + +"@bull-board/express@^3.10.4": + version "3.11.1" + resolved "https://registry.npmjs.org/@bull-board/express/-/express-3.11.1.tgz" + integrity sha512-+a5IQilu/CxvnGC0z6oJPANn7eF/Q61VU86JR9vRtmaY96mKKny0NuAyulJ2/Y9JznMCBOLUsXWL6fm9zD9kJw== + dependencies: + "@bull-board/api" "3.11.1" + "@bull-board/ui" "3.11.1" + ejs "3.1.7" + express "4.17.3" + +"@bull-board/ui@3.11.1": + version "3.11.1" + resolved "https://registry.npmjs.org/@bull-board/ui/-/ui-3.11.1.tgz" + integrity sha512-SRrfvxHF/WaBICiAFuWAoAlTvoBYUBmX94oRbSKzVILRFZMe3gs0hN071BFohrn4yOTFHAkWPN7cjMbaqHwCag== + dependencies: + "@bull-board/api" "3.11.1" + +"@ioredis/commands@^1.1.1": + version "1.2.0" + resolved "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz" + integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg== + +"@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2": + version "3.0.2" + resolved "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz" + integrity sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ== + +accepts@~1.3.8: + version "1.3.8" + resolved "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz" + integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== + dependencies: + mime-types "~2.1.34" + negotiator "0.6.3" + +ansi-styles@^4.1.0: + version "4.3.0" + resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + +argparse@^2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz" + integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== + +array-flatten@1.1.1: + version "1.1.1" + resolved "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" + integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== + +async@^3.2.3: + version "3.2.5" + resolved "https://registry.npmjs.org/async/-/async-3.2.5.tgz" + integrity sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg== + +balanced-match@^1.0.0: + version "1.0.2" + resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== + +body-parser@^1.20.0: + version "1.20.2" + resolved "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz" + integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== + dependencies: + bytes "3.1.2" + content-type "~1.0.5" + debug "2.6.9" + depd "2.0.0" + destroy "1.2.0" + http-errors "2.0.0" + iconv-lite "0.4.24" + on-finished "2.4.1" + qs "6.11.0" + raw-body "2.5.2" + type-is "~1.6.18" + unpipe "1.0.0" + +body-parser@1.19.2: + version "1.19.2" + resolved "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz" + integrity sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw== + dependencies: + bytes "3.1.2" + content-type "~1.0.4" + debug "2.6.9" + depd "~1.1.2" + http-errors "1.8.1" + iconv-lite "0.4.24" + on-finished "~2.3.0" + qs "6.9.7" + raw-body "2.4.3" + type-is "~1.6.18" + +body-parser@1.20.1: + version "1.20.1" + resolved "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz" + integrity sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw== + dependencies: + bytes "3.1.2" + content-type "~1.0.4" + debug "2.6.9" + depd "2.0.0" + destroy "1.2.0" + http-errors "2.0.0" + iconv-lite "0.4.24" + on-finished "2.4.1" + qs "6.11.0" + raw-body "2.5.1" + type-is "~1.6.18" + unpipe "1.0.0" + +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + +bullmq@^4.6.0: + version "4.17.0" + resolved "https://registry.npmjs.org/bullmq/-/bullmq-4.17.0.tgz" + integrity sha512-URnHgB01rlCP8RTpmW3kFnvv3vdd2aI1OcBMYQwnqODxGiJUlz9MibDVXE83mq7ee1eS1IvD9lMQqGszX6E5Pw== + dependencies: + cron-parser "^4.6.0" + glob "^8.0.3" + ioredis "^5.3.2" + lodash "^4.17.21" + msgpackr "^1.6.2" + node-abort-controller "^3.1.1" + semver "^7.5.4" + tslib "^2.0.0" + uuid "^9.0.0" + +bytes@3.1.2: + version "3.1.2" + resolved "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz" + integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== + +call-bind@^1.0.0: + version "1.0.5" + resolved "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz" + integrity sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ== + dependencies: + function-bind "^1.1.2" + get-intrinsic "^1.2.1" + set-function-length "^1.1.1" + +chalk@^4.0.2: + version "4.1.2" + resolved "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +cluster-key-slot@^1.1.0: + version "1.1.2" + resolved "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" + integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== + +connect-ensure-login@^0.1.1: + version "0.1.1" + resolved "https://registry.npmjs.org/connect-ensure-login/-/connect-ensure-login-0.1.1.tgz" + integrity sha512-u0LGY/YXgkqdD4uVz8Gkri0adby5SBPYIFQvIa4FjBIaZUN0yuZPVHsjAA2bUiprGyjh8NlqxPUcIzzfTiXhQQ== + +content-disposition@0.5.4: + version "0.5.4" + resolved "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz" + integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ== + dependencies: + safe-buffer "5.2.1" + +content-type@~1.0.4, content-type@~1.0.5: + version "1.0.5" + resolved "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz" + integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA== + +cookie-signature@1.0.6: + version "1.0.6" + resolved "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz" + integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== + +cookie@0.4.2: + version "0.4.2" + resolved "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz" + integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA== + +cookie@0.5.0: + version "0.5.0" + resolved "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz" + integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== + +cron-parser@^4.6.0: + version "4.9.0" + resolved "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz" + integrity sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q== + dependencies: + luxon "^3.2.1" + +debug@^4.3.4: + version "4.3.4" + resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== + dependencies: + ms "2.1.2" + +debug@2.6.9: + version "2.6.9" + resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + +define-data-property@^1.1.1: + version "1.1.1" + resolved "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz" + integrity sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ== + dependencies: + get-intrinsic "^1.2.1" + gopd "^1.0.1" + has-property-descriptors "^1.0.0" + +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + +depd@~1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz" + integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ== + +depd@~2.0.0, depd@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz" + integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== + +destroy@~1.0.4: + version "1.0.4" + resolved "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz" + integrity sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg== + +destroy@1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz" + integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== + +ee-first@1.1.1: + version "1.1.1" + resolved "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" + integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== + +ejs@3.1.7: + version "3.1.7" + resolved "https://registry.npmjs.org/ejs/-/ejs-3.1.7.tgz" + integrity sha512-BIar7R6abbUxDA3bfXrO4DSgwo8I+fB5/1zgujl3HLLjwd6+9iOnrT+t3grn2qbk9vOgBubXOFwX2m9axoFaGw== + dependencies: + jake "^10.8.5" + +encodeurl@~1.0.2: + version "1.0.2" + resolved "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz" + integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== + +escape-html@~1.0.3: + version "1.0.3" + resolved "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz" + integrity sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow== + +etag@~1.8.1: + version "1.8.1" + resolved "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz" + integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== + +express-session@^1.17.2: + version "1.17.3" + resolved "https://registry.npmjs.org/express-session/-/express-session-1.17.3.tgz" + integrity sha512-4+otWXlShYlG1Ma+2Jnn+xgKUZTMJ5QD3YvfilX3AcocOAbIkVylSWEklzALe/+Pu4qV6TYBj5GwOBFfdKqLBw== + dependencies: + cookie "0.4.2" + cookie-signature "1.0.6" + debug "2.6.9" + depd "~2.0.0" + on-headers "~1.0.2" + parseurl "~1.3.3" + safe-buffer "5.2.1" + uid-safe "~2.1.5" + +express@^4.17.3: + version "4.18.2" + resolved "https://registry.npmjs.org/express/-/express-4.18.2.tgz" + integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ== + dependencies: + accepts "~1.3.8" + array-flatten "1.1.1" + body-parser "1.20.1" + content-disposition "0.5.4" + content-type "~1.0.4" + cookie "0.5.0" + cookie-signature "1.0.6" + debug "2.6.9" + depd "2.0.0" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + finalhandler "1.2.0" + fresh "0.5.2" + http-errors "2.0.0" + merge-descriptors "1.0.1" + methods "~1.1.2" + on-finished "2.4.1" + parseurl "~1.3.3" + path-to-regexp "0.1.7" + proxy-addr "~2.0.7" + qs "6.11.0" + range-parser "~1.2.1" + safe-buffer "5.2.1" + send "0.18.0" + serve-static "1.15.0" + setprototypeof "1.2.0" + statuses "2.0.1" + type-is "~1.6.18" + utils-merge "1.0.1" + vary "~1.1.2" + +express@4.17.3: + version "4.17.3" + resolved "https://registry.npmjs.org/express/-/express-4.17.3.tgz" + integrity sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg== + dependencies: + accepts "~1.3.8" + array-flatten "1.1.1" + body-parser "1.19.2" + content-disposition "0.5.4" + content-type "~1.0.4" + cookie "0.4.2" + cookie-signature "1.0.6" + debug "2.6.9" + depd "~1.1.2" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + finalhandler "~1.1.2" + fresh "0.5.2" + merge-descriptors "1.0.1" + methods "~1.1.2" + on-finished "~2.3.0" + parseurl "~1.3.3" + path-to-regexp "0.1.7" + proxy-addr "~2.0.7" + qs "6.9.7" + range-parser "~1.2.1" + safe-buffer "5.2.1" + send "0.17.2" + serve-static "1.14.2" + setprototypeof "1.2.0" + statuses "~1.5.0" + type-is "~1.6.18" + utils-merge "1.0.1" + vary "~1.1.2" + +filelist@^1.0.4: + version "1.0.4" + resolved "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz" + integrity sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q== + dependencies: + minimatch "^5.0.1" + +finalhandler@~1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz" + integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== + dependencies: + debug "2.6.9" + encodeurl "~1.0.2" + escape-html "~1.0.3" + on-finished "~2.3.0" + parseurl "~1.3.3" + statuses "~1.5.0" + unpipe "~1.0.0" + +finalhandler@1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz" + integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== + dependencies: + debug "2.6.9" + encodeurl "~1.0.2" + escape-html "~1.0.3" + on-finished "2.4.1" + parseurl "~1.3.3" + statuses "2.0.1" + unpipe "~1.0.0" + +forwarded@0.2.0: + version "0.2.0" + resolved "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz" + integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== + +fresh@0.5.2: + version "0.5.2" + resolved "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz" + integrity sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q== + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" + integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== + +function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== + +get-intrinsic@^1.0.2, get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2: + version "1.2.2" + resolved "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz" + integrity sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA== + dependencies: + function-bind "^1.1.2" + has-proto "^1.0.1" + has-symbols "^1.0.3" + hasown "^2.0.0" + +glob@^8.0.3: + version "8.1.0" + resolved "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz" + integrity sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^5.0.1" + once "^1.3.0" + +gopd@^1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz" + integrity sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA== + dependencies: + get-intrinsic "^1.1.3" + +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== + +has-property-descriptors@^1.0.0: + version "1.0.1" + resolved "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz" + integrity sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg== + dependencies: + get-intrinsic "^1.2.2" + +has-proto@^1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz" + integrity sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg== + +has-symbols@^1.0.3: + version "1.0.3" + resolved "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz" + integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== + +hasown@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz" + integrity sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA== + dependencies: + function-bind "^1.1.2" + +http-errors@1.8.1: + version "1.8.1" + resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz" + integrity sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g== + dependencies: + depd "~1.1.2" + inherits "2.0.4" + setprototypeof "1.2.0" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.1" + +http-errors@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz" + integrity sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ== + dependencies: + depd "2.0.0" + inherits "2.0.4" + setprototypeof "1.2.0" + statuses "2.0.1" + toidentifier "1.0.1" + +iconv-lite@0.4.24: + version "0.4.24" + resolved "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz" + integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== + dependencies: + safer-buffer ">= 2.1.2 < 3" + +inflight@^1.0.4: + version "1.0.6" + resolved "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz" + integrity sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA== + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2, inherits@2.0.4: + version "2.0.4" + resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +ioredis@^5.3.2: + version "5.3.2" + resolved "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz" + integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + +ipaddr.js@1.9.1: + version "1.9.1" + resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz" + integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== + +jake@^10.8.5: + version "10.8.7" + resolved "https://registry.npmjs.org/jake/-/jake-10.8.7.tgz" + integrity sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w== + dependencies: + async "^3.2.3" + chalk "^4.0.2" + filelist "^1.0.4" + minimatch "^3.1.2" + +js-yaml@^4.0.0: + version "4.1.0" + resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== + dependencies: + argparse "^2.0.1" + +lodash.defaults@^4.2.0: + version "4.2.0" + resolved "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz" + integrity sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ== + +lodash.isarguments@^3.1.0: + version "3.1.0" + resolved "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz" + integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== + +lodash@^4.17.11, lodash@^4.17.21: + version "4.17.21" + resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== + +lru-cache@^6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz" + integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== + dependencies: + yallist "^4.0.0" + +luxon@^3.2.1: + version "3.4.4" + resolved "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz" + integrity sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA== + +media-typer@0.3.0: + version "0.3.0" + resolved "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" + integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== + +merge-descriptors@1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz" + integrity sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w== + +methods@~1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" + integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== + +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@~2.1.24, mime-types@~2.1.34: + version "2.1.35" + resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + +mime@1.6.0: + version "1.6.0" + resolved "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== + +minimatch@^3.1.2: + version "3.1.2" + resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== + dependencies: + brace-expansion "^1.1.7" + +minimatch@^5.0.1: + version "5.1.6" + resolved "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz" + integrity sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g== + dependencies: + brace-expansion "^2.0.1" + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz" + integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A== + +ms@2.1.2: + version "2.1.2" + resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== + +ms@2.1.3: + version "2.1.3" + resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +msgpackr-extract@^3.0.2: + version "3.0.2" + resolved "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz" + integrity sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A== + dependencies: + node-gyp-build-optional-packages "5.0.7" + optionalDependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64" "3.0.2" + "@msgpackr-extract/msgpackr-extract-darwin-x64" "3.0.2" + "@msgpackr-extract/msgpackr-extract-linux-arm" "3.0.2" + "@msgpackr-extract/msgpackr-extract-linux-arm64" "3.0.2" + "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" + "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" + +msgpackr@^1.6.2: + version "1.10.1" + resolved "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.1.tgz" + integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== + optionalDependencies: + msgpackr-extract "^3.0.2" + +negotiator@0.6.3: + version "0.6.3" + resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz" + integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== + +node-abort-controller@^3.1.1: + version "3.1.1" + resolved "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz" + integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== + +node-gyp-build-optional-packages@5.0.7: + version "5.0.7" + resolved "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz" + integrity sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w== + +object-inspect@^1.9.0: + version "1.13.1" + resolved "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz" + integrity sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ== + +on-finished@~2.3.0: + version "2.3.0" + resolved "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz" + integrity sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww== + dependencies: + ee-first "1.1.1" + +on-finished@2.4.1: + version "2.4.1" + resolved "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz" + integrity sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg== + dependencies: + ee-first "1.1.1" + +on-headers@~1.0.2: + version "1.0.2" + resolved "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz" + integrity sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA== + +once@^1.3.0: + version "1.4.0" + resolved "https://registry.npmjs.org/once/-/once-1.4.0.tgz" + integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== + dependencies: + wrappy "1" + +parseurl@~1.3.3: + version "1.3.3" + resolved "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz" + integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== + +passport-local@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/passport-local/-/passport-local-1.0.0.tgz" + integrity sha512-9wCE6qKznvf9mQYYbgJ3sVOHmCWoUNMVFoZzNoznmISbhnNNPhN9xfY3sLmScHMetEJeoY7CXwfhCe7argfQow== + dependencies: + passport-strategy "1.x.x" + +passport-strategy@1.x.x: + version "1.0.0" + resolved "https://registry.npmjs.org/passport-strategy/-/passport-strategy-1.0.0.tgz" + integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== + +passport@^0.6.0: + version "0.6.0" + resolved "https://registry.npmjs.org/passport/-/passport-0.6.0.tgz" + integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== + dependencies: + passport-strategy "1.x.x" + pause "0.0.1" + utils-merge "^1.0.1" + +path-to-regexp@0.1.7: + version "0.1.7" + resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz" + integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== + +pause@0.0.1: + version "0.0.1" + resolved "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz" + integrity sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg== + +proxy-addr@~2.0.7: + version "2.0.7" + resolved "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz" + integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== + dependencies: + forwarded "0.2.0" + ipaddr.js "1.9.1" + +qs@6.11.0: + version "6.11.0" + resolved "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz" + integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== + dependencies: + side-channel "^1.0.4" + +qs@6.9.7: + version "6.9.7" + resolved "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz" + integrity sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw== + +random-bytes@~1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz" + integrity sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ== + +range-parser@~1.2.1: + version "1.2.1" + resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz" + integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== + +raw-body@2.4.3: + version "2.4.3" + resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz" + integrity sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g== + dependencies: + bytes "3.1.2" + http-errors "1.8.1" + iconv-lite "0.4.24" + unpipe "1.0.0" + +raw-body@2.5.1: + version "2.5.1" + resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz" + integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== + dependencies: + bytes "3.1.2" + http-errors "2.0.0" + iconv-lite "0.4.24" + unpipe "1.0.0" + +raw-body@2.5.2: + version "2.5.2" + resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz" + integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA== + dependencies: + bytes "3.1.2" + http-errors "2.0.0" + iconv-lite "0.4.24" + unpipe "1.0.0" + +read-yaml-file@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/read-yaml-file/-/read-yaml-file-2.1.0.tgz" + integrity sha512-UkRNRIwnhG+y7hpqnycCL/xbTk7+ia9VuVTC0S+zVbwd65DI9eUpRMfsWIGrCWxTU/mi+JW8cHQCrv+zfCbEPQ== + dependencies: + js-yaml "^4.0.0" + strip-bom "^4.0.0" + +redis-errors@^1.0.0, redis-errors@^1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz" + integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w== + +redis-info@^3.0.8: + version "3.1.0" + resolved "https://registry.npmjs.org/redis-info/-/redis-info-3.1.0.tgz" + integrity sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg== + dependencies: + lodash "^4.17.11" + +redis-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz" + integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A== + dependencies: + redis-errors "^1.0.0" + +safe-buffer@5.2.1: + version "5.2.1" + resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + +"safer-buffer@>= 2.1.2 < 3": + version "2.1.2" + resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== + +semver@^7.5.4: + version "7.5.4" + resolved "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== + dependencies: + lru-cache "^6.0.0" + +send@0.17.2: + version "0.17.2" + resolved "https://registry.npmjs.org/send/-/send-0.17.2.tgz" + integrity sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww== + dependencies: + debug "2.6.9" + depd "~1.1.2" + destroy "~1.0.4" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + fresh "0.5.2" + http-errors "1.8.1" + mime "1.6.0" + ms "2.1.3" + on-finished "~2.3.0" + range-parser "~1.2.1" + statuses "~1.5.0" + +send@0.18.0: + version "0.18.0" + resolved "https://registry.npmjs.org/send/-/send-0.18.0.tgz" + integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== + dependencies: + debug "2.6.9" + depd "2.0.0" + destroy "1.2.0" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + fresh "0.5.2" + http-errors "2.0.0" + mime "1.6.0" + ms "2.1.3" + on-finished "2.4.1" + range-parser "~1.2.1" + statuses "2.0.1" + +serve-static@1.14.2: + version "1.14.2" + resolved "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz" + integrity sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ== + dependencies: + encodeurl "~1.0.2" + escape-html "~1.0.3" + parseurl "~1.3.3" + send "0.17.2" + +serve-static@1.15.0: + version "1.15.0" + resolved "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz" + integrity sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g== + dependencies: + encodeurl "~1.0.2" + escape-html "~1.0.3" + parseurl "~1.3.3" + send "0.18.0" + +set-function-length@^1.1.1: + version "1.1.1" + resolved "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz" + integrity sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ== + dependencies: + define-data-property "^1.1.1" + get-intrinsic "^1.2.1" + gopd "^1.0.1" + has-property-descriptors "^1.0.0" + +setprototypeof@1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz" + integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== + +side-channel@^1.0.4: + version "1.0.4" + resolved "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz" + integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== + dependencies: + call-bind "^1.0.0" + get-intrinsic "^1.0.2" + object-inspect "^1.9.0" + +standard-as-callback@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz" + integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== + +"statuses@>= 1.5.0 < 2", statuses@~1.5.0: + version "1.5.0" + resolved "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz" + integrity sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA== + +statuses@2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz" + integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== + +strip-bom@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz" + integrity sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w== + +supports-color@^7.1.0: + version "7.2.0" + resolved "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== + dependencies: + has-flag "^4.0.0" + +toidentifier@1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz" + integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== + +tslib@^2.0.0: + version "2.6.2" + resolved "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + +type-is@~1.6.18: + version "1.6.18" + resolved "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz" + integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== + dependencies: + media-typer "0.3.0" + mime-types "~2.1.24" + +uid-safe@~2.1.5: + version "2.1.5" + resolved "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz" + integrity sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA== + dependencies: + random-bytes "~1.0.0" + +unpipe@~1.0.0, unpipe@1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" + integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== + +utils-merge@^1.0.1, utils-merge@1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" + integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== + +uuid@^9.0.0: + version "9.0.1" + resolved "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz" + integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== + +vary@~1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" + integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== + +wrappy@1: + version "1.0.2" + resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" + integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== + +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== From d98c2c2cf5660d22e79ca36faac71b6f544fef55 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 11:51:55 +0800 Subject: [PATCH 05/76] Redis config --- pkg/bull-queue-admin/index.js | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index 2b598ddf3..016bc26c3 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -41,18 +41,27 @@ passport.deserializeUser((user, cb) => { cb(null, user) }) -const sleep = (t) => new Promise((resolve) => setTimeout(resolve, t * 1000)) - -const redisOptions = { - port: 6379, - host: 'localhost', - password: '', - tls: false, -} - -const createQueueMQ = (name) => new QueueMQ(name, { connection: redisOptions }) - const run = async () => { + const secrets = readYamlFile(process.env.SECRETS_FILE) + const redisOptions = () => { + if (secrets.REDIS_URL?.startsWith('rediss://') && secrets.REDIS_CERT) { + return { + tls: { + cert: secrets.REDIS_CERT?.replace(/\\n/g, '\n'), + rejectUnauthorized: false, + }, + maxRetriesPerRequest: null, + } + } + return { + maxRetriesPerRequest: null, + } + } + const createQueueMQ = (name) => + new QueueMQ(secrets.REDIS_URL, { + connection: redisOptions, + }) + const rssRefreshFeed = createQueueMQ('rssRefreshFeed') const serverAdapter = new ExpressAdapter() From 273e593a930227678918eaeec7233db4ccc9c048 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 13:18:04 +0800 Subject: [PATCH 06/76] Fix init of queue workers --- pkg/bull-queue-admin/index.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index 016bc26c3..ca32a2bc4 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -57,12 +57,10 @@ const run = async () => { maxRetriesPerRequest: null, } } - const createQueueMQ = (name) => - new QueueMQ(secrets.REDIS_URL, { - connection: redisOptions, - }) - const rssRefreshFeed = createQueueMQ('rssRefreshFeed') + const rssRefreshFeed = createQueueMQ('rssRefreshFeed', { + connection: redisOptions, + }) const serverAdapter = new ExpressAdapter() serverAdapter.setBasePath('/ui') From 2f3e2d98d8a28c80a16a0d488a4d6011fc07fc09 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 13:26:27 +0800 Subject: [PATCH 07/76] Fix instantiation --- pkg/admin/src/index.ts | 1 - pkg/bull-queue-admin/index.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/admin/src/index.ts b/pkg/admin/src/index.ts index 894e4b733..20875919c 100644 --- a/pkg/admin/src/index.ts +++ b/pkg/admin/src/index.ts @@ -56,7 +56,6 @@ const ADMIN_USER_EMAIL = const router = AdminJsExpress.buildAuthenticatedRouter(adminBro, { authenticate: async (email, password) => { const user = await AdminUser.findOne({ email }) - console.log('looked up user: ', user) if (user) { const matched = await compare(password, user.password) console.log(' -- failed match') diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index ca32a2bc4..e01ca9fd7 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -58,7 +58,7 @@ const run = async () => { } } - const rssRefreshFeed = createQueueMQ('rssRefreshFeed', { + const rssRefreshFeed = new QueueMQ('rssRefreshFeed', { connection: redisOptions, }) From 91442640209762a4af374b924416498ad4c941cb Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 13:35:39 +0800 Subject: [PATCH 08/76] Use redis connection --- pkg/bull-queue-admin/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index e01ca9fd7..b257e439d 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -58,8 +58,9 @@ const run = async () => { } } + const connection = new Redis(secrets.REDIS_URL, redisOptions()) const rssRefreshFeed = new QueueMQ('rssRefreshFeed', { - connection: redisOptions, + connection: connection, }) const serverAdapter = new ExpressAdapter() From b3616af783ced467343caf6eab49595dd3903ec4 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 13:41:44 +0800 Subject: [PATCH 09/76] Add ioredis --- pkg/bull-queue-admin/index.js | 1 + pkg/bull-queue-admin/yarn.lock | 219 ++++++++++++++++++--------------- 2 files changed, 123 insertions(+), 97 deletions(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index b257e439d..d53974099 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -2,6 +2,7 @@ const { createBullBoard } = require('@bull-board/api') const { BullMQAdapter } = require('@bull-board/api/bullMQAdapter') const { ExpressAdapter } = require('@bull-board/express') const { Queue: QueueMQ, Worker } = require('bullmq') +const { Redis } = require('ioredis') const session = require('express-session') const bodyParser = require('body-parser') const passport = require('passport') diff --git a/pkg/bull-queue-admin/yarn.lock b/pkg/bull-queue-admin/yarn.lock index 6fc96d716..3762bfe5e 100644 --- a/pkg/bull-queue-admin/yarn.lock +++ b/pkg/bull-queue-admin/yarn.lock @@ -36,6 +36,31 @@ resolved "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz" integrity sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ== +"@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.2": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz#f954f34355712212a8e06c465bc06c40852c6bb3" + integrity sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw== + +"@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.2": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz#45c63037f045c2b15c44f80f0393fa24f9655367" + integrity sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg== + +"@msgpackr-extract/msgpackr-extract-linux-arm@3.0.2": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz#35707efeafe6d22b3f373caf9e8775e8920d1399" + integrity sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA== + +"@msgpackr-extract/msgpackr-extract-linux-x64@3.0.2": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz#091b1218b66c341f532611477ef89e83f25fae4f" + integrity sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA== + +"@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz#0f164b726869f71da3c594171df5ebc1c4b0a407" + integrity sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ== + accepts@~1.3.8: version "1.3.8" resolved "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz" @@ -71,24 +96,6 @@ balanced-match@^1.0.0: resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== -body-parser@^1.20.0: - version "1.20.2" - resolved "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz" - integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== - dependencies: - bytes "3.1.2" - content-type "~1.0.5" - debug "2.6.9" - depd "2.0.0" - destroy "1.2.0" - http-errors "2.0.0" - iconv-lite "0.4.24" - on-finished "2.4.1" - qs "6.11.0" - raw-body "2.5.2" - type-is "~1.6.18" - unpipe "1.0.0" - body-parser@1.19.2: version "1.19.2" resolved "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz" @@ -123,6 +130,24 @@ body-parser@1.20.1: type-is "~1.6.18" unpipe "1.0.0" +body-parser@^1.20.0: + version "1.20.2" + resolved "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz" + integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== + dependencies: + bytes "3.1.2" + content-type "~1.0.5" + debug "2.6.9" + depd "2.0.0" + destroy "1.2.0" + http-errors "2.0.0" + iconv-lite "0.4.24" + on-finished "2.4.1" + qs "6.11.0" + raw-body "2.5.2" + type-is "~1.6.18" + unpipe "1.0.0" + brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz" @@ -236,13 +261,6 @@ cron-parser@^4.6.0: dependencies: luxon "^3.2.1" -debug@^4.3.4: - version "4.3.4" - resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== - dependencies: - ms "2.1.2" - debug@2.6.9: version "2.6.9" resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" @@ -250,6 +268,13 @@ debug@2.6.9: dependencies: ms "2.0.0" +debug@^4.3.4: + version "4.3.4" + resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== + dependencies: + ms "2.1.2" + define-data-property@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz" @@ -264,26 +289,26 @@ denque@^2.1.0: resolved "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz" integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== +depd@2.0.0, depd@~2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz" + integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== + depd@~1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz" integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ== -depd@~2.0.0, depd@2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz" - integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== +destroy@1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz" + integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== destroy@~1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz" integrity sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg== -destroy@1.2.0: - version "1.2.0" - resolved "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz" - integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== - ee-first@1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" @@ -325,6 +350,42 @@ express-session@^1.17.2: safe-buffer "5.2.1" uid-safe "~2.1.5" +express@4.17.3: + version "4.17.3" + resolved "https://registry.npmjs.org/express/-/express-4.17.3.tgz" + integrity sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg== + dependencies: + accepts "~1.3.8" + array-flatten "1.1.1" + body-parser "1.19.2" + content-disposition "0.5.4" + content-type "~1.0.4" + cookie "0.4.2" + cookie-signature "1.0.6" + debug "2.6.9" + depd "~1.1.2" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + finalhandler "~1.1.2" + fresh "0.5.2" + merge-descriptors "1.0.1" + methods "~1.1.2" + on-finished "~2.3.0" + parseurl "~1.3.3" + path-to-regexp "0.1.7" + proxy-addr "~2.0.7" + qs "6.9.7" + range-parser "~1.2.1" + safe-buffer "5.2.1" + send "0.17.2" + serve-static "1.14.2" + setprototypeof "1.2.0" + statuses "~1.5.0" + type-is "~1.6.18" + utils-merge "1.0.1" + vary "~1.1.2" + express@^4.17.3: version "4.18.2" resolved "https://registry.npmjs.org/express/-/express-4.18.2.tgz" @@ -362,42 +423,6 @@ express@^4.17.3: utils-merge "1.0.1" vary "~1.1.2" -express@4.17.3: - version "4.17.3" - resolved "https://registry.npmjs.org/express/-/express-4.17.3.tgz" - integrity sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg== - dependencies: - accepts "~1.3.8" - array-flatten "1.1.1" - body-parser "1.19.2" - content-disposition "0.5.4" - content-type "~1.0.4" - cookie "0.4.2" - cookie-signature "1.0.6" - debug "2.6.9" - depd "~1.1.2" - encodeurl "~1.0.2" - escape-html "~1.0.3" - etag "~1.8.1" - finalhandler "~1.1.2" - fresh "0.5.2" - merge-descriptors "1.0.1" - methods "~1.1.2" - on-finished "~2.3.0" - parseurl "~1.3.3" - path-to-regexp "0.1.7" - proxy-addr "~2.0.7" - qs "6.9.7" - range-parser "~1.2.1" - safe-buffer "5.2.1" - send "0.17.2" - serve-static "1.14.2" - setprototypeof "1.2.0" - statuses "~1.5.0" - type-is "~1.6.18" - utils-merge "1.0.1" - vary "~1.1.2" - filelist@^1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz" @@ -405,19 +430,6 @@ filelist@^1.0.4: dependencies: minimatch "^5.0.1" -finalhandler@~1.1.2: - version "1.1.2" - resolved "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz" - integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== - dependencies: - debug "2.6.9" - encodeurl "~1.0.2" - escape-html "~1.0.3" - on-finished "~2.3.0" - parseurl "~1.3.3" - statuses "~1.5.0" - unpipe "~1.0.0" - finalhandler@1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz" @@ -431,6 +443,19 @@ finalhandler@1.2.0: statuses "2.0.1" unpipe "~1.0.0" +finalhandler@~1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz" + integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== + dependencies: + debug "2.6.9" + encodeurl "~1.0.2" + escape-html "~1.0.3" + on-finished "~2.3.0" + parseurl "~1.3.3" + statuses "~1.5.0" + unpipe "~1.0.0" + forwarded@0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz" @@ -716,13 +741,6 @@ object-inspect@^1.9.0: resolved "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz" integrity sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ== -on-finished@~2.3.0: - version "2.3.0" - resolved "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz" - integrity sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww== - dependencies: - ee-first "1.1.1" - on-finished@2.4.1: version "2.4.1" resolved "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz" @@ -730,6 +748,13 @@ on-finished@2.4.1: dependencies: ee-first "1.1.1" +on-finished@~2.3.0: + version "2.3.0" + resolved "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz" + integrity sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww== + dependencies: + ee-first "1.1.1" + on-headers@~1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz" @@ -969,16 +994,16 @@ standard-as-callback@^2.1.0: resolved "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz" integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== -"statuses@>= 1.5.0 < 2", statuses@~1.5.0: - version "1.5.0" - resolved "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz" - integrity sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA== - statuses@2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz" integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== +"statuses@>= 1.5.0 < 2", statuses@~1.5.0: + version "1.5.0" + resolved "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz" + integrity sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA== + strip-bom@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz" @@ -1016,12 +1041,12 @@ uid-safe@~2.1.5: dependencies: random-bytes "~1.0.0" -unpipe@~1.0.0, unpipe@1.0.0: +unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== -utils-merge@^1.0.1, utils-merge@1.0.1: +utils-merge@1.0.1, utils-merge@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== From 0af8c47cfc6436800935151d8977bb7270ec80f9 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 14:21:07 +0800 Subject: [PATCH 10/76] Add ioredis package explicitly --- pkg/bull-queue-admin/package.json | 3 ++- pkg/bull-queue-admin/yarn.lock | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/bull-queue-admin/package.json b/pkg/bull-queue-admin/package.json index b1526249a..ba8c27b95 100644 --- a/pkg/bull-queue-admin/package.json +++ b/pkg/bull-queue-admin/package.json @@ -14,8 +14,9 @@ "connect-ensure-login": "^0.1.1", "express": "^4.17.3", "express-session": "^1.17.2", + "ioredis": "^5.3.2", "passport": "^0.6.0", "passport-local": "^1.0.0", "read-yaml-file": "^2.1.0" } -} \ No newline at end of file +} diff --git a/pkg/bull-queue-admin/yarn.lock b/pkg/bull-queue-admin/yarn.lock index 3762bfe5e..9f4c0606a 100644 --- a/pkg/bull-queue-admin/yarn.lock +++ b/pkg/bull-queue-admin/yarn.lock @@ -577,7 +577,7 @@ inherits@2, inherits@2.0.4: ioredis@^5.3.2: version "5.3.2" - resolved "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7" integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA== dependencies: "@ioredis/commands" "^1.1.1" From 78c1590ed76d99132361979379158d8384267723 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 14:50:17 +0800 Subject: [PATCH 11/76] Temp logging --- pkg/bull-queue-admin/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index d53974099..bb042aa0a 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -59,6 +59,7 @@ const run = async () => { } } + console.log('secrets: ', secrets) const connection = new Redis(secrets.REDIS_URL, redisOptions()) const rssRefreshFeed = new QueueMQ('rssRefreshFeed', { connection: connection, From 1559514c61b119b6b4ba655f35512024fc046bf3 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 14:54:15 +0800 Subject: [PATCH 12/76] Await on secrets loading --- pkg/bull-queue-admin/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index bb042aa0a..48b7d1964 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -43,7 +43,7 @@ passport.deserializeUser((user, cb) => { }) const run = async () => { - const secrets = readYamlFile(process.env.SECRETS_FILE) + const secrets = await readYamlFile(process.env.SECRETS_FILE) const redisOptions = () => { if (secrets.REDIS_URL?.startsWith('rediss://') && secrets.REDIS_CERT) { return { @@ -59,7 +59,6 @@ const run = async () => { } } - console.log('secrets: ', secrets) const connection = new Redis(secrets.REDIS_URL, redisOptions()) const rssRefreshFeed = new QueueMQ('rssRefreshFeed', { connection: connection, From 77e09954cd9fbdb93b854a326dbf0c81ac42c89d Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 15:04:39 +0800 Subject: [PATCH 13/76] More init debugging --- pkg/bull-queue-admin/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index 48b7d1964..dc64cda02 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -44,7 +44,8 @@ passport.deserializeUser((user, cb) => { const run = async () => { const secrets = await readYamlFile(process.env.SECRETS_FILE) - const redisOptions = () => { + const redisOptions = (secrets) => { + console.log('secrets: ', secrets) if (secrets.REDIS_URL?.startsWith('rediss://') && secrets.REDIS_CERT) { return { tls: { @@ -59,7 +60,7 @@ const run = async () => { } } - const connection = new Redis(secrets.REDIS_URL, redisOptions()) + const connection = new Redis(secrets.REDIS_URL, redisOptions(secrets)) const rssRefreshFeed = new QueueMQ('rssRefreshFeed', { connection: connection, }) From 154ccfe78730b749b57573838c3c1da4afaea32c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 15:11:54 +0800 Subject: [PATCH 14/76] More init debugging --- pkg/bull-queue-admin/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index dc64cda02..6da8a719c 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -60,6 +60,7 @@ const run = async () => { } } + console.log('redis options:', redisOptions(secrets)) const connection = new Redis(secrets.REDIS_URL, redisOptions(secrets)) const rssRefreshFeed = new QueueMQ('rssRefreshFeed', { connection: connection, From a36a516f047688079c35a1e11bfe1bd83f170386 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 20:25:35 +0800 Subject: [PATCH 15/76] Add some more debugging --- pkg/bull-queue-admin/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index 6da8a719c..eff2cbdf2 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -62,6 +62,8 @@ const run = async () => { console.log('redis options:', redisOptions(secrets)) const connection = new Redis(secrets.REDIS_URL, redisOptions(secrets)) + console.log('set connection: ', connection) + const rssRefreshFeed = new QueueMQ('rssRefreshFeed', { connection: connection, }) From 4ecb47f80f74576a6bc74ac53ab5b32ae0784f19 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 20:41:10 +0800 Subject: [PATCH 16/76] Use yaml escaping in the cert file --- pkg/bull-queue-admin/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index eff2cbdf2..21569142b 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -49,7 +49,7 @@ const run = async () => { if (secrets.REDIS_URL?.startsWith('rediss://') && secrets.REDIS_CERT) { return { tls: { - cert: secrets.REDIS_CERT?.replace(/\\n/g, '\n'), + cert: secrets.REDIS_CERT, rejectUnauthorized: false, }, maxRetriesPerRequest: null, From b136d5c4bc6d67625a1d14bbfbb6f006ffef750f Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 21:07:01 +0800 Subject: [PATCH 17/76] Refactor --- pkg/bull-queue-admin/index.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index 21569142b..c1152b674 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -45,7 +45,6 @@ passport.deserializeUser((user, cb) => { const run = async () => { const secrets = await readYamlFile(process.env.SECRETS_FILE) const redisOptions = (secrets) => { - console.log('secrets: ', secrets) if (secrets.REDIS_URL?.startsWith('rediss://') && secrets.REDIS_CERT) { return { tls: { @@ -60,8 +59,13 @@ const run = async () => { } } - console.log('redis options:', redisOptions(secrets)) - const connection = new Redis(secrets.REDIS_URL, redisOptions(secrets)) + const connection = new Redis(secrets.REDIS_URL, { + tls: { + cert: secrets.REDIS_CERT, + rejectUnauthorized: false, + }, + maxRetriesPerRequest: null, + }) console.log('set connection: ', connection) const rssRefreshFeed = new QueueMQ('rssRefreshFeed', { From 362f87bb09c3e6ba4b22d7e91b4ac945b095ab5e Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 21:36:33 +0800 Subject: [PATCH 18/76] Clean up --- pkg/bull-queue-admin/index.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index c1152b674..2157b040c 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -1,7 +1,7 @@ const { createBullBoard } = require('@bull-board/api') const { BullMQAdapter } = require('@bull-board/api/bullMQAdapter') const { ExpressAdapter } = require('@bull-board/express') -const { Queue: QueueMQ, Worker } = require('bullmq') +const { Queue } = require('bullmq') const { Redis } = require('ioredis') const session = require('express-session') const bodyParser = require('body-parser') @@ -59,7 +59,9 @@ const run = async () => { } } - const connection = new Redis(secrets.REDIS_URL, { + const connection = new Redis({ + host: secrets.REDIS_HOST, + port: secrets.REDIS_PORT, tls: { cert: secrets.REDIS_CERT, rejectUnauthorized: false, @@ -68,7 +70,7 @@ const run = async () => { }) console.log('set connection: ', connection) - const rssRefreshFeed = new QueueMQ('rssRefreshFeed', { + const rssRefreshFeed = new Queue('rssRefreshFeed', { connection: connection, }) From 2f2206707ad1ea40673409accb497a58da0bb470 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 21:53:42 +0800 Subject: [PATCH 19/76] redis config --- pkg/bull-queue-admin/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index 2157b040c..1c0c4b462 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -60,9 +60,9 @@ const run = async () => { } const connection = new Redis({ - host: secrets.REDIS_HOST, - port: secrets.REDIS_PORT, tls: { + host: secrets.REDIS_HOST, + port: secrets.REDIS_PORT, cert: secrets.REDIS_CERT, rejectUnauthorized: false, }, From 181c0253cb1200fb3ce77ad7b2b8edb3e1cf05f5 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 12 Jan 2024 22:04:53 +0800 Subject: [PATCH 20/76] Update docker base image --- pkg/bull-queue-admin/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bull-queue-admin/Dockerfile b/pkg/bull-queue-admin/Dockerfile index 8e8876ccf..0cb4230f2 100644 --- a/pkg/bull-queue-admin/Dockerfile +++ b/pkg/bull-queue-admin/Dockerfile @@ -1,4 +1,4 @@ -FROM node:14.18-alpine +FROM node:18.16 WORKDIR /app From bca68372c9a38fb399a4ad5ae385d4be30c59afe Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Sat, 13 Jan 2024 08:59:41 +0800 Subject: [PATCH 21/76] Fix cert import --- pkg/bull-queue-admin/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index 1c0c4b462..1e5832b92 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -45,10 +45,10 @@ passport.deserializeUser((user, cb) => { const run = async () => { const secrets = await readYamlFile(process.env.SECRETS_FILE) const redisOptions = (secrets) => { - if (secrets.REDIS_URL?.startsWith('rediss://') && secrets.REDIS_CERT) { + if (secrets.REDIS_URL?.startsWith('rediss://') && process.env.REDIS_CERT) { return { tls: { - cert: secrets.REDIS_CERT, + cert: process.env.REDIS_CERT?.replace(/\\n/g, '\n'), rejectUnauthorized: false, }, maxRetriesPerRequest: null, @@ -63,7 +63,7 @@ const run = async () => { tls: { host: secrets.REDIS_HOST, port: secrets.REDIS_PORT, - cert: secrets.REDIS_CERT, + cert: process.env.REDIS_CERT?.replace(/\\n/g, '\n'), rejectUnauthorized: false, }, maxRetriesPerRequest: null, From 45fcde8ebc66105c71a4fc3b40aa0cb29c088f8a Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Sat, 13 Jan 2024 15:41:29 +0800 Subject: [PATCH 22/76] Update redis config --- packages/api/src/jobqueue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/jobqueue.ts b/packages/api/src/jobqueue.ts index d9f38e176..9897f2281 100644 --- a/packages/api/src/jobqueue.ts +++ b/packages/api/src/jobqueue.ts @@ -11,7 +11,7 @@ const createRSSRefreshFeedQueue = (): Queue | undefined => { if (env.redis.url?.startsWith('rediss://') && env.redis.cert) { return { tls: { - cert: env.redis.cert?.replace(/\\n/g, '\n'), + ca: env.redis.cert?.replace(/\\n/g, '\n'), rejectUnauthorized: false, }, maxRetriesPerRequest: null, From 20df5c621205ef4da12e9b31a15e8c36c149a3f8 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Sat, 13 Jan 2024 18:45:24 +0800 Subject: [PATCH 23/76] Parse redis cert on load --- packages/api/src/jobqueue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/jobqueue.ts b/packages/api/src/jobqueue.ts index 9897f2281..2cb8367c9 100644 --- a/packages/api/src/jobqueue.ts +++ b/packages/api/src/jobqueue.ts @@ -11,7 +11,7 @@ const createRSSRefreshFeedQueue = (): Queue | undefined => { if (env.redis.url?.startsWith('rediss://') && env.redis.cert) { return { tls: { - ca: env.redis.cert?.replace(/\\n/g, '\n'), + ca: env.redis.cert, rejectUnauthorized: false, }, maxRetriesPerRequest: null, From 447bba23ba25730b49d8b8d552695da99689cef7 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Sat, 13 Jan 2024 20:41:49 +0800 Subject: [PATCH 24/76] Fix init --- packages/api/src/jobqueue.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/jobqueue.ts b/packages/api/src/jobqueue.ts index 2cb8367c9..4ca026bff 100644 --- a/packages/api/src/jobqueue.ts +++ b/packages/api/src/jobqueue.ts @@ -26,9 +26,9 @@ const createRSSRefreshFeedQueue = (): Queue | undefined => { return new Queue('rssRefreshFeed', { connection }) } -export const addRefreshFeedJob = async (jobid: string, payload: any) => { - const rssRefreshFeedJobQueue = createRSSRefreshFeedQueue() +const rssRefreshFeedJobQueue = createRSSRefreshFeedQueue() +export const addRefreshFeedJob = async (jobid: string, payload: any) => { if (!rssRefreshFeedJobQueue) { return false } From b4914f534603a55612d46cf220bca09e72c78a91 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Sun, 14 Jan 2024 12:30:32 +0800 Subject: [PATCH 25/76] Fix creation of redis clients --- packages/api/src/jobqueue.ts | 32 ++++++-------------------------- packages/api/src/redis.ts | 2 ++ packages/api/src/server.ts | 9 ++++++++- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/packages/api/src/jobqueue.ts b/packages/api/src/jobqueue.ts index 4ca026bff..b7539c2b4 100644 --- a/packages/api/src/jobqueue.ts +++ b/packages/api/src/jobqueue.ts @@ -1,34 +1,14 @@ -import { env } from './env' - -import { Queue, RedisOptions } from 'bullmq' -import Redis from 'ioredis' +import { Queue } from 'bullmq' +import { mqRedisClient } from './redis' const createRSSRefreshFeedQueue = (): Queue | undefined => { - if (!env.redis.url) { - return undefined - } - const redisOptions = (): RedisOptions => { - if (env.redis.url?.startsWith('rediss://') && env.redis.cert) { - return { - tls: { - ca: env.redis.cert, - rejectUnauthorized: false, - }, - maxRetriesPerRequest: null, - } - } - return { - maxRetriesPerRequest: null, - } - } - - const connection = new Redis(env.redis.url ?? '', redisOptions()) - return new Queue('rssRefreshFeed', { connection }) + return new Queue('rssRefreshFeed', { + connection: mqRedisClient, + }) } -const rssRefreshFeedJobQueue = createRSSRefreshFeedQueue() - export const addRefreshFeedJob = async (jobid: string, payload: any) => { + const rssRefreshFeedJobQueue = createRSSRefreshFeedQueue() if (!rssRefreshFeedJobQueue) { return false } diff --git a/packages/api/src/redis.ts b/packages/api/src/redis.ts index ebc2b75d6..27c241a06 100644 --- a/packages/api/src/redis.ts +++ b/packages/api/src/redis.ts @@ -1,5 +1,7 @@ import { Redis } from 'ioredis' import { env } from './env' +import { Redis } from 'ioredis' +import { RedisOptions } from 'bullmq' const url = env.redis.url const cert = env.redis.cert diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 4f1fe52bb..d93a66a16 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -15,7 +15,7 @@ import { config, loggers } from 'winston' import { makeApolloServer } from './apollo' import { appDataSource } from './data_source' import { env } from './env' -import { redisClient } from './redis' +import { redisClient, mqRedisClient } from './redis' import { articleRouter } from './routers/article_router' import { authRouter } from './routers/auth/auth_router' import { mobileAuthRouter } from './routers/auth/mobile/mobile_auth_router' @@ -163,6 +163,13 @@ const main = async (): Promise => { console.log('Redis Client Connected:', env.redis.url) } + // redis for message queue + if (env.redis.url) { + mqRedisClient?.on('error', (err) => { + console.error('Redis Client Error', err) + }) + } + const { app, apollo, httpServer } = createApp() await apollo.start() From 819052f51fe4d83cb8feb5ec49f87459a665aeb2 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 15 Jan 2024 09:50:32 +0800 Subject: [PATCH 26/76] Add queue-processor endpoint to the api --- packages/api/package.json | 2 + packages/api/src/queue-processor.ts | 91 ++++++++++++++++++++++++ packages/api/src/routers/svc/rss_feed.ts | 78 ++++++++++---------- yarn.lock | 89 ++++++++++++++++++++--- 4 files changed, 214 insertions(+), 46 deletions(-) create mode 100644 packages/api/src/queue-processor.ts diff --git a/packages/api/package.json b/packages/api/package.json index c41153433..7d5420686 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -5,7 +5,9 @@ "scripts": { "build": "tsc && yarn copy-files", "dev": "ts-node-dev --files src/server.ts", + "dev_qp": "ts-node-dev --files src/queue-processor.ts", "start": "node dist/server.js", + "start_queue_processor": "node dist/queue-processor.js", "lint": "eslint src --ext ts,js,tsx,jsx", "lint:fix": "eslint src --fix --ext ts,js,tsx,jsx", "test:typecheck": "tsc --noEmit", diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts new file mode 100644 index 000000000..b8afa9a25 --- /dev/null +++ b/packages/api/src/queue-processor.ts @@ -0,0 +1,91 @@ +/* eslint-disable @typescript-eslint/no-floating-promises */ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +/* eslint-disable @typescript-eslint/require-await */ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import express, { Express } from 'express' +import { appDataSource } from './data_source' +import { env } from './env' +import { redisClient, mqRedisClient } from './redis' + +import { Worker, Job, QueueEvents } from 'bullmq' +import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds' +import { refreshFeed } from './jobs/rss/refreshFeed' + +export const QUEUE_NAME = 'omnivore-backend-queue' + +const main = async () => { + console.log('calling queue-processor start') + const app: Express = express() + const port = process.env.PORT || 3002 + + await appDataSource.initialize() + + // respond healthy to auto-scaler. + app.get('/_ah/health', (req, res) => res.sendStatus(200)) + + // redis is optional + if (env.redis.url) { + redisClient.on('error', (err) => { + console.error('Redis Client Error', err) + }) + + await redisClient.connect() + console.log('Redis Client Connected:', env.redis.url) + } + + // redis for message queue + if (env.redis.url) { + mqRedisClient?.on('error', (err) => { + console.error('Redis Client Error', err) + }) + } + + const worker = new Worker( + QUEUE_NAME, + async (job: Job) => { + switch (job.name) { + case 'refresh-all-feeds': { + return await refreshAllFeeds(appDataSource, mqRedisClient) + } + case 'refresh-feed': { + return await refreshFeed(redisClient, job.data) + } + } + return true + }, + { + connection: mqRedisClient, + } + ) + + const queueEvents = new QueueEvents(QUEUE_NAME, { + connection: mqRedisClient, + }) + + queueEvents.on('added', async (job) => { + console.log('added job: ', job.jobId) + }) + + queueEvents.on('removed', async (job) => { + console.log('removed job: ', job.jobId) + }) + + queueEvents.on('completed', async (job) => { + console.log('completed job: ', job.jobId) + }) + + process.on('SIGINT', async () => { + console.log('[queue-processor]: Received SIGTERM. Shutting down.') + await redisClient.disconnect() + mqRedisClient.disconnect() + }) + + app.listen(port, () => { + console.log(`[queue-processor]: started`) + }) +} + +// only call main if the file was called from the CLI and wasn't required from another module +if (require.main === module) { + main().catch((e) => console.error(e)) +} diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 671fbebd5..2c5be9bd7 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -12,6 +12,8 @@ import { RssSubscriptionGroup, } from '../../utils/createTask' import { logger } from '../../utils/logger' +import { queueRSSRefreshAllFeedsJob } from '../../jobs/rss/refreshAllFeeds' +import { mqRedisClient } from '../../redis' export function rssFeedRouter() { const router = express.Router() @@ -28,44 +30,46 @@ export function rssFeedRouter() { return res.status(200).send('Expired') } - // get active rss feed subscriptions scheduled for fetch and group by feed url - const subscriptionGroups = (await getRepository(Subscription).query( - ` - SELECT - url, - ARRAY_AGG(id) AS "subscriptionIds", - ARRAY_AGG(user_id) AS "userIds", - ARRAY_AGG(last_fetched_at) AS "fetchedDates", - ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", - ARRAY_AGG(last_fetched_checksum) AS checksums, - ARRAY_AGG(fetch_content) AS "fetchContents", - ARRAY_AGG(coalesce(folder, $3)) AS folders - FROM - omnivore.subscriptions - WHERE - type = $1 - AND status = $2 - AND (scheduled_at <= NOW() OR scheduled_at IS NULL) - GROUP BY - url - `, - [ - SubscriptionType.Rss, - SubscriptionStatus.Active, - DEFAULT_SUBSCRIPTION_FOLDER, - ] - )) as RssSubscriptionGroup[] + await queueRSSRefreshAllFeedsJob(mqRedisClient) - // create a cloud taks to fetch rss feed item for each subscription - await Promise.all( - subscriptionGroups.map((subscriptionGroup) => { - try { - return enqueueRssFeedFetch(subscriptionGroup) - } catch (error) { - logger.info('error creating rss feed fetch task', error) - } - }) - ) + // // get active rss feed subscriptions scheduled for fetch and group by feed url + // const subscriptionGroups = (await getRepository(Subscription).query( + // ` + // SELECT + // url, + // ARRAY_AGG(id) AS "subscriptionIds", + // ARRAY_AGG(user_id) AS "userIds", + // ARRAY_AGG(last_fetched_at) AS "fetchedDates", + // ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", + // ARRAY_AGG(last_fetched_checksum) AS checksums, + // ARRAY_AGG(fetch_content) AS "fetchContents", + // ARRAY_AGG(coalesce(folder, $3)) AS folders + // FROM + // omnivore.subscriptions + // WHERE + // type = $1 + // AND status = $2 + // AND (scheduled_at <= NOW() OR scheduled_at IS NULL) + // GROUP BY + // url + // `, + // [ + // SubscriptionType.Rss, + // SubscriptionStatus.Active, + // DEFAULT_SUBSCRIPTION_FOLDER, + // ] + // )) as RssSubscriptionGroup[] + + // // create a cloud taks to fetch rss feed item for each subscription + // await Promise.all( + // subscriptionGroups.map((subscriptionGroup) => { + // try { + // return enqueueRssFeedFetch(subscriptionGroup) + // } catch (error) { + // logger.info('error creating rss feed fetch task', error) + // } + // }) + // ) } catch (error) { logger.info('error fetching rss feeds', error) return res.status(500).send('Internal Server Error') diff --git a/yarn.lock b/yarn.lock index 4512cf5f5..2a5ec5f3d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5980,6 +5980,11 @@ resolved "https://registry.yarnpkg.com/@sqltools/formatter/-/formatter-1.2.3.tgz#1185726610acc37317ddab11c3c7f9066966bd20" integrity sha512-O3uyB/JbkAEMZaP3YqyHH7TMnex7tWyCbCI4EfJdOCoN6HIhqdJBWTM6aCCiWQ/5f5wxjgU735QAIpJbjDvmzg== +"@sqltools/formatter@^1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@sqltools/formatter/-/formatter-1.2.5.tgz#3abc203c79b8c3e90fd6c156a0c62d5403520e12" + integrity sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw== + "@stitches/react@^1.2.5": version "1.2.8" resolved "https://registry.yarnpkg.com/@stitches/react/-/react-1.2.8.tgz#954f8008be8d9c65c4e58efa0937f32388ce3a38" @@ -7390,6 +7395,16 @@ "@types/qs" "*" "@types/serve-static" "*" +"@types/express@^4.17.21": + version "4.17.21" + resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.21.tgz#c26d4a151e60efe0084b23dc3369ebc631ed192d" + integrity sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ== + dependencies: + "@types/body-parser" "*" + "@types/express-serve-static-core" "^4.17.33" + "@types/qs" "*" + "@types/serve-static" "*" + "@types/filesystem@*": version "0.0.32" resolved "https://registry.yarnpkg.com/@types/filesystem/-/filesystem-0.0.32.tgz#307df7cc084a2293c3c1a31151b178063e0a8edf" @@ -7791,6 +7806,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.68.tgz#3155f64a961b3d8d10246c80657f9a7292e3421a" integrity sha512-sG3hPIQwJLoewrN7cr0dwEy+yF5nD4D/4FxtQpFciRD/xwUzgD+G05uxZHv5mhfXo4F9Jkp13jjn0CC2q325sg== +"@types/node@^20.11.0": + version "20.11.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.0.tgz#8e0b99e70c0c1ade1a86c4a282f7b7ef87c9552f" + integrity sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ== + dependencies: + undici-types "~5.26.4" + "@types/nodemailer@^6.4.4": version "6.4.4" resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.4.tgz#c265f7e7a51df587597b3a49a023acaf0c741f4b" @@ -9315,6 +9337,11 @@ app-root-path@^3.0.0: resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-3.0.0.tgz#210b6f43873227e18a4b810a032283311555d5ad" integrity sha512-qMcx+Gy2UZynHjOHOIXPNvpf+9cjvk3cWrBBK7zg4gH9+clobJRb9NGzcT7mQTcV/6Gm/1WelUtqxVXnNlrwcw== +app-root-path@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-3.1.0.tgz#5971a2fc12ba170369a7a1ef018c71e6e47c2e86" + integrity sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA== + apparatus@^0.0.10: version "0.0.10" resolved "https://registry.yarnpkg.com/apparatus/-/apparatus-0.0.10.tgz#81ea756772ada77863db54ceee8202c109bdca3e" @@ -12724,7 +12751,7 @@ dateformat@^3.0.0, dateformat@^3.0.3: resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae" integrity sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q== -dayjs@1.x, dayjs@^1.10.4, dayjs@^1.11.7: +dayjs@1.x, dayjs@^1.10.4, dayjs@^1.11.7, dayjs@^1.11.9: version "1.11.10" resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.10.tgz#68acea85317a6e164457d6d6947564029a6a16a0" integrity sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ== @@ -13452,16 +13479,16 @@ dotenv@^16.0.1: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d" integrity sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ== +dotenv@^16.0.3, dotenv@~16.3.1: + version "16.3.1" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.3.1.tgz#369034de7d7e5b120972693352a3bf112172cc3e" + integrity sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ== + dotenv@^8.0.0, dotenv@^8.2.0: version "8.6.0" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b" integrity sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g== -dotenv@~16.3.1: - version "16.3.1" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.3.1.tgz#369034de7d7e5b120972693352a3bf112172cc3e" - integrity sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ== - downshift@^6.0.15: version "6.1.7" resolved "https://registry.yarnpkg.com/downshift/-/downshift-6.1.7.tgz#fdb4c4e4f1d11587985cd76e21e8b4b3fa72e44c" @@ -14732,7 +14759,7 @@ express-rate-limit@^6.3.0: resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-6.11.1.tgz#52e05c5d379cd5d06ae29665862436eb712e414a" integrity sha512-8+UpWtQY25lJaa4+3WxDBGDcAu4atcTruSs3QSL5VPEplYy6kmk84wutG9rUkkK5LmMQQ7TFHWLZYITwVNbbEg== -express@^4.16.4, express@^4.17.1: +express@^4.16.4, express@^4.17.1, express@^4.18.2: version "4.18.2" resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59" integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ== @@ -16083,7 +16110,7 @@ glob@7.2.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glo once "^1.3.0" path-is-absolute "^1.0.0" -glob@^10.2.2: +glob@^10.2.2, glob@^10.3.10: version "10.3.10" resolved "https://registry.yarnpkg.com/glob/-/glob-10.3.10.tgz#0351ebb809fd187fe421ab96af83d3a70715df4b" integrity sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g== @@ -21777,6 +21804,11 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +mkdirp@^2.1.3: + version "2.1.6" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-2.1.6.tgz#964fbcb12b2d8c5d6fbc62a963ac95a273e2cc19" + integrity sha512-+hEnITedc8LAtIP9u3HJDFIdcLV2vXP33sqLLIzkv1Db1zO/1OxbvYf0Y1OC/S/Qo5dxHXepofhmxL02PsKe+A== + mkdirp@~0.3.5: version "0.3.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.5.tgz#de3e5f8961c88c787ee1368df849ac4413eca8d7" @@ -25964,6 +25996,14 @@ read-pkg@^7.1.0: parse-json "^5.2.0" type-fest "^2.0.0" +read-yaml-file@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/read-yaml-file/-/read-yaml-file-2.1.0.tgz#c5866712db9ef5343b4d02c2413bada53c41c4a9" + integrity sha512-UkRNRIwnhG+y7hpqnycCL/xbTk7+ia9VuVTC0S+zVbwd65DI9eUpRMfsWIGrCWxTU/mi+JW8cHQCrv+zfCbEPQ== + dependencies: + js-yaml "^4.0.0" + strip-bom "^4.0.0" + read@1, read@^1.0.7, read@~1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/read/-/read-1.0.7.tgz#b3da19bd052431a97671d44a42634adf710b40c4" @@ -29124,7 +29164,7 @@ tslib@^1.0.0, tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== -tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.3.1, tslib@^2.4.0: +tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.3.1, tslib@^2.4.0, tslib@^2.5.0: version "2.6.2" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== @@ -29340,6 +29380,27 @@ typeorm-naming-strategies@^4.1.0: resolved "https://registry.yarnpkg.com/typeorm-naming-strategies/-/typeorm-naming-strategies-4.1.0.tgz#1ec6eb296c8d7b69bb06764d5b9083ff80e814a9" integrity sha512-vPekJXzZOTZrdDvTl1YoM+w+sUIfQHG4kZTpbFYoTsufyv9NIBRe4Q+PdzhEAFA2std3D9LZHEb1EjE9zhRpiQ== +typeorm@^0.3.19: + version "0.3.19" + resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.3.19.tgz#a985ce8ae36d266018e44fed5e27a4a5da34ad2a" + integrity sha512-OGelrY5qEoAU80mR1iyvmUHiKCPUydL6xp6bebXzS7jyv/X70Gp/jBWRAfF4qGOfy2A7orMiGRfwsBUNbEL65g== + dependencies: + "@sqltools/formatter" "^1.2.5" + app-root-path "^3.1.0" + buffer "^6.0.3" + chalk "^4.1.2" + cli-highlight "^2.1.11" + dayjs "^1.11.9" + debug "^4.3.4" + dotenv "^16.0.3" + glob "^10.3.10" + mkdirp "^2.1.3" + reflect-metadata "^0.1.13" + sha.js "^2.4.11" + tslib "^2.5.0" + uuid "^9.0.0" + yargs "^17.6.2" + typeorm@^0.3.4: version "0.3.7" resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.3.7.tgz#5776ed5058f0acb75d64723b39ff458d21de64c1" @@ -29378,6 +29439,11 @@ typescript@^4.4.4: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== +typescript@^5.3.3: + version "5.3.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.3.3.tgz#b3ce6ba258e72e6305ba66f5c9b452aaee3ffe37" + integrity sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw== + ua-parser-js@^0.7.30: version "0.7.33" resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.33.tgz#1d04acb4ccef9293df6f70f2c3d22f3030d8b532" @@ -29456,6 +29522,11 @@ underscore@^1.13.4, underscore@^1.9.1: resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.4.tgz#7886b46bbdf07f768e0052f1828e1dcab40c0dee" integrity sha512-BQFnUDuAQ4Yf/cYY5LNrK9NCJFKriaRbD9uR1fTeXnBeoa97W0i41qkZfGO9pSo8I5KzjAcSY2XYtdf0oKd7KQ== +undici-types@~5.26.4: + version "5.26.5" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" + integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== + undici@^4.9.3: version "4.14.1" resolved "https://registry.yarnpkg.com/undici/-/undici-4.14.1.tgz#7633b143a8a10d6d63335e00511d071e8d52a1d9" From 7aa76b70fecb3aa4d8faafb4e5f42e676666aa63 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 15 Jan 2024 10:52:00 +0800 Subject: [PATCH 27/76] Add missing jobs --- packages/api/src/jobs/rss/refreshAllFeeds.ts | 88 +++ packages/api/src/jobs/rss/refreshFeed.ts | 674 +++++++++++++++++++ packages/api/src/jobs/rss/task.ts | 48 ++ 3 files changed, 810 insertions(+) create mode 100644 packages/api/src/jobs/rss/refreshAllFeeds.ts create mode 100644 packages/api/src/jobs/rss/refreshFeed.ts create mode 100644 packages/api/src/jobs/rss/task.ts diff --git a/packages/api/src/jobs/rss/refreshAllFeeds.ts b/packages/api/src/jobs/rss/refreshAllFeeds.ts new file mode 100644 index 000000000..6f1824c2d --- /dev/null +++ b/packages/api/src/jobs/rss/refreshAllFeeds.ts @@ -0,0 +1,88 @@ +import Redis from 'ioredis' +import { DataSource } from 'typeorm' +import { stringToHash } from '../../utils/helpers' +import { RssSubscriptionGroup } from '../../utils/createTask' +import { Queue } from 'bullmq' +import { QUEUE_NAME } from '../../queue-processor' + +export const refreshAllFeeds = async ( + db: DataSource, + redis: Redis +): Promise => { + const subscriptionGroups = (await db.createEntityManager().query( + ` + SELECT + url, + ARRAY_AGG(id) AS "subscriptionIds", + ARRAY_AGG(user_id) AS "userIds", + ARRAY_AGG(last_fetched_at) AS "fetchedDates", + ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", + ARRAY_AGG(last_fetched_checksum) AS checksums, + ARRAY_AGG(fetch_content) AS "fetchContents", + ARRAY_AGG(coalesce(folder, $3)) AS folders + FROM + omnivore.subscriptions + WHERE + type = $1 + AND status = $2 + AND (scheduled_at <= NOW() OR scheduled_at IS NULL) + GROUP BY + url + `, + ['RSS', 'ACTIVE', 'following'] + )) as RssSubscriptionGroup[] + + for (let group of subscriptionGroups) { + let jobid = `refresh-feed_${stringToHash(group.url)}_${stringToHash( + JSON.stringify(group.userIds.sort()) + )}` + const payload = { + subscriptionIds: group.subscriptionIds, + feedUrl: group.url, + lastFetchedTimestamps: group.fetchedDates.map( + (timestamp) => timestamp?.getTime() || 0 + ), // unix timestamp in milliseconds + lastFetchedChecksums: group.checksums, + scheduledTimestamps: group.scheduledDates.map((timestamp) => + timestamp.getTime() + ), // unix timestamp in milliseconds + userIds: group.userIds, + fetchContents: group.fetchContents, + folders: group.folders, + } + + await queueRSSRefreshFeedJob(redis, jobid, payload) + } + + return true +} + +const createBackendQueue = (redis: Redis): Queue | undefined => { + return new Queue(QUEUE_NAME, { + connection: redis, + }) +} + +export const queueRSSRefreshAllFeedsJob = async (redis: Redis) => { + const queue = createBackendQueue(redis) + if (!queue) { + return false + } + return queue.add('refresh-all-feeds', {}) +} + +export const queueRSSRefreshFeedJob = async ( + redis: Redis, + jobid: string, + payload: any +) => { + const queue = createBackendQueue(redis) + if (!queue) { + return false + } + return queue.add('refresh-feed', payload, { + jobId: jobid, + removeOnComplete: true, + removeOnFail: true, + }) +} diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts new file mode 100644 index 000000000..95bfdae35 --- /dev/null +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -0,0 +1,674 @@ +import axios from 'axios' +import crypto from 'crypto' +import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import +import * as jwt from 'jsonwebtoken' +import { parseHTML } from 'linkedom' +import { createClient } from 'redis' +import Parser, { Item } from 'rss-parser' +import { promisify } from 'util' +import { CONTENT_FETCH_URL, createCloudTask } from './task' + +type FolderType = 'following' | 'inbox' +// explicitly create the return type of RedisClient +type RedisClient = ReturnType + +interface RefreshFeedRequest { + subscriptionIds: string[] + feedUrl: string + lastFetchedTimestamps: number[] // unix timestamp in milliseconds + scheduledTimestamps: number[] // unix timestamp in milliseconds + lastFetchedChecksums: string[] + userIds: string[] + fetchContents: boolean[] + folders: FolderType[] +} + +export const isRefreshFeedRequest = (data: any): data is RefreshFeedRequest => { + console.log('body: ', data) + return ( + 'subscriptionIds' in data && + 'feedUrl' in data && + 'lastFetchedTimestamps' in data && + 'scheduledTimestamps' in data && + 'userIds' in data && + 'lastFetchedChecksums' in data && + 'fetchContents' in data && + 'folders' in data + ) +} + +// link can be a string or an object +type RssFeedItemLink = string | { $: { rel?: string; href: string } } +type RssFeed = Parser.Output<{ + published?: string + updated?: string + created?: string + link?: RssFeedItemLink + links?: RssFeedItemLink[] +}> & { + lastBuildDate?: string + 'syn:updatePeriod'?: string + 'syn:updateFrequency'?: string + 'sy:updatePeriod'?: string + 'sy:updateFrequency'?: string +} +type RssFeedItemMedia = { + $: { url: string; width?: string; height?: string; medium?: string } +} +export type RssFeedItem = Item & { + 'media:thumbnail'?: RssFeedItemMedia + 'media:content'?: RssFeedItemMedia[] + link: string +} + +export const isOldItem = (item: RssFeedItem, lastFetchedAt: number) => { + // existing items and items that were published before 24h + const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() + return ( + publishedAt <= new Date(lastFetchedAt) || + publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) + ) +} + +const feedFetchFailedRedisKey = (feedUrl: string) => + `feed-fetch-failure:${feedUrl}` + +const isFeedBlocked = async (feedUrl: string, redisClient: RedisClient) => { + const key = feedFetchFailedRedisKey(feedUrl) + try { + const result = await redisClient.get(key) + // if the feed has failed to fetch more than certain times, block it + const maxFailures = parseInt(process.env.MAX_FEED_FETCH_FAILURES ?? '10') + if (result && parseInt(result) > maxFailures) { + console.log('feed is blocked: ', feedUrl) + return true + } + } catch (error) { + console.error('Failed to check feed block status', feedUrl, error) + } + + return false +} + +const incrementFeedFailure = async ( + feedUrl: string, + redisClient: RedisClient +) => { + const key = feedFetchFailedRedisKey(feedUrl) + try { + const result = await redisClient.incr(key) + // expire the key in 1 day + await redisClient.expire(key, 24 * 60 * 60) + + return result + } catch (error) { + console.error('Failed to block feed', feedUrl, error) + return null + } +} + +export const isContentFetchBlocked = (feedUrl: string) => { + if (feedUrl.startsWith('https://arxiv.org/')) { + return true + } + return false +} + +const getThumbnail = (item: RssFeedItem) => { + if (item['media:thumbnail']) { + return item['media:thumbnail'].$.url + } + + return item['media:content']?.find((media) => media.$.medium === 'image')?.$ + .url +} + +export const fetchAndChecksum = async (url: string) => { + try { + const response = await axios.get(url, { + responseType: 'arraybuffer', + timeout: 60_000, + maxRedirects: 10, + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', + Accept: + 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml, text/html;q=0.4', + }, + }) + + const hash = crypto.createHash('sha256') + hash.update(response.data as Buffer) + + const dataStr = (response.data as Buffer).toString() + + return { url, content: dataStr, checksum: hash.digest('hex') } + } catch (error) { + console.log(`Failed to fetch or hash content from ${url}.`, error) + return null + } +} + +const parseFeed = async (url: string, content: string) => { + try { + // check if url is a telegram channel + const telegramRegex = /https:\/\/t\.me\/([a-zA-Z0-9_]+)/ + const telegramMatch = url.match(telegramRegex) + if (telegramMatch) { + const dom = parseHTML(content).document + const title = dom.querySelector('meta[property="og:title"]') + // post has attribute data-post + const posts = dom.querySelectorAll('[data-post]') + const items = Array.from(posts) + .map((post) => { + const id = post.getAttribute('data-post') + if (!id) { + return null + } + + const url = `https://t.me/${telegramMatch[1]}/${id}` + // find the