From c9ea622461cadd7be4588e5bfe44823814caa5fd Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 25 Oct 2022 12:42:32 +0800
Subject: [PATCH 1/3] Use auth token to identify client request in rate limiter

---
 packages/api/src/server.ts | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts
index 2efb4e765..e5c4dfc56 100755
--- a/packages/api/src/server.ts
+++ b/packages/api/src/server.ts
@@ -94,12 +94,20 @@ export const createApp = (): {
   app.use(json({ limit: '100mb' }))
   app.use(urlencoded({ limit: '100mb', extended: true }))
 
-  if (!env.dev.isLocal) {
+  if (env.dev.isLocal) {
     const apiLimiter = rateLimit({
       windowMs: 60 * 1000, // 1 minute
       max: 50, // Limit each IP to 10 requests per `window` (here, per minute)
       standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
       legacyHeaders: false, // Disable the `X-RateLimit-*` headers
+      keyGenerator: (req) => {
+        return (
+          req.header('authorization') ||
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+          (req.cookies['auth'] as string) ||
+          req.ip
+        )
+      },
     })
     // Apply the rate limiting middleware to API calls only
     app.use('/api/', apiLimiter)

From 57ba9afbfa603421fbc82bb3ce7acf051091964a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 25 Oct 2022 13:17:57 +0800
Subject: [PATCH 2/3] Remove debugging

---
 packages/api/src/server.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts
index e5c4dfc56..0438c6921 100755
--- a/packages/api/src/server.ts
+++ b/packages/api/src/server.ts
@@ -94,7 +94,7 @@ export const createApp = (): {
   app.use(json({ limit: '100mb' }))
   app.use(urlencoded({ limit: '100mb', extended: true }))
 
-  if (env.dev.isLocal) {
+  if (!env.dev.isLocal) {
     const apiLimiter = rateLimit({
       windowMs: 60 * 1000, // 1 minute
       max: 50, // Limit each IP to 10 requests per `window` (here, per minute)

From 2bad70fda91abf4ea5e9ce10b99e04a09462376a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 25 Oct 2022 16:59:48 +0800
Subject: [PATCH 3/3] Catch puppeteer navigation timeout error

---
 packages/puppeteer-parse/index.js | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js
index d3fadf998..b602ff1ef 100644
--- a/packages/puppeteer-parse/index.js
+++ b/packages/puppeteer-parse/index.js
@@ -304,19 +304,19 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
     console.log('error with handler: ', e);
   }
 
-  var context, page, finalUrl;
-  if ((!content || !title) && contentType !== 'application/pdf') {
-    const result = await retrievePage(url)
-    if (result && result.context) { context = result.context }
-    if (result && result.page) { page = result.page }
-    if (result && result.finalUrl) { finalUrl = result.finalUrl }
-    if (result && result.contentType) { contentType = result.contentType }
-    console.log('context, page, finalUrl, contentType', context, page, finalUrl, contentType);
-  } else {
-    finalUrl = url
-  }
-
+  let context, page, finalUrl;
   try {
+    if ((!content || !title) && contentType !== 'application/pdf') {
+      const result = await retrievePage(url)
+      if (result && result.context) { context = result.context }
+      if (result && result.page) { page = result.page }
+      if (result && result.finalUrl) { finalUrl = result.finalUrl }
+      if (result && result.contentType) { contentType = result.contentType }
+      console.log('context, page, finalUrl, contentType', context, page, finalUrl, contentType);
+    } else {
+      finalUrl = url
+    }
+
     if (contentType === 'application/pdf') {
       const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId);
       const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId);
@@ -641,7 +641,7 @@ async function retrievePage(url) {
   });
 
   try {
-    const response = await page.goto(url, { waitUntil: ['networkidle2'] });
+    const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] });
     const finalUrl = response.url();
     const contentType = response.headers()['content-type'];