From 72d89308c537c6a70e3d149f316043522aa4d5be Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 21 Aug 2024 12:24:12 +0800
Subject: [PATCH] process up to 10 jobs concurrently

---
 packages/content-fetch/src/worker.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/content-fetch/src/worker.ts b/packages/content-fetch/src/worker.ts
index 731178c94..33cb44668 100644
--- a/packages/content-fetch/src/worker.ts
+++ b/packages/content-fetch/src/worker.ts
@@ -40,6 +40,10 @@ export const createWorker = (
     {
       connection: redisDataSource.queueRedisClient,
       autorun: true, // start processing jobs immediately
+      limiter: {
+        max: 10, // process up to 10 jobs concurrently
+        duration: 1000, // 1 second
+      },
     }
   )