Merge pull request #3199 from omnivore-app/fix/integration-handler-rate-limit

fix/integration handler rate limit

packages/api/src/resolvers/integrations/index.ts

@@ -111,8 +111,10 @@ export const setIntegrationResolver = authorized<
       integration.taskName = taskName
     } else if (integrationToSave.taskName) {
       // delete the task if disable integration and task exists
-      await deleteTask(integrationToSave.taskName)
-      log.info('task deleted', integrationToSave.taskName)
+      const result = await deleteTask(integrationToSave.taskName)
+      if (result) {
+        log.info('task deleted', integrationToSave.taskName)
+      }
 
       // update task name in integration
       await updateIntegration(

packages/api/src/resolvers/types.ts

@@ -13,6 +13,7 @@ export interface Claims {
   scope?: string // scope is used for api key like page:search
   exp?: number
   email?: string
+  system?: boolean
 }
 
 export type ClaimsToSet = {

packages/api/src/server.ts

@@ -37,7 +37,11 @@ import { webhooksServiceRouter } from './routers/svc/webhooks'
 import { textToSpeechRouter } from './routers/text_to_speech'
 import { userRouter } from './routers/user_router'
 import { sentryConfig } from './sentry'
-import { getClaimsByToken, getTokenByRequest } from './utils/auth'
+import {
+  getClaimsByToken,
+  getTokenByRequest,
+  isSystemRequest,
+} from './utils/auth'
 import { corsConfig } from './utils/corsConfig'
 import { buildLogger, buildLoggerTransport } from './utils/logger'
 
@@ -79,8 +83,9 @@ export const createApp = (): {
     keyGenerator: (req) => {
       return getTokenByRequest(req) || req.ip
     },
-    // skip preflight requests and test requests
-    skip: (req) => req.method === 'OPTIONS' || env.dev.isLocal,
+    // skip preflight requests and test requests and system requests
+    skip: (req) =>
+      req.method === 'OPTIONS' || env.dev.isLocal || isSystemRequest(req),
   })
 
   // Apply the rate limiting middleware to API calls only

packages/api/src/utils/auth.ts

@@ -69,17 +69,12 @@ export const claimsFromApiKey = async (key: string): Promise<Claims> => {
 export const getClaimsByToken = async (
   token: string | undefined
 ): Promise<Claims | undefined> => {
-  let claims: Claims | undefined
-
   if (!token) {
     return undefined
   }
 
   try {
-    jwt.verify(token, env.server.jwtSecret) &&
-      (claims = jwt.decode(token) as Claims)
-
-    return claims
+    return jwt.verify(token, env.server.jwtSecret) as Claims
   } catch (e) {
     if (
       e instanceof jwt.JsonWebTokenError &&
@@ -133,3 +128,17 @@ export const getTokenByRequest = (req: express.Request): string | undefined => {
     (req.cookies?.auth as string)
   )
 }
+
+export const isSystemRequest = (req: express.Request): boolean => {
+  const token = getTokenByRequest(req)
+  if (!token) {
+    return false
+  }
+
+  try {
+    const claims = jwt.verify(token, env.server.jwtSecret) as Claims
+    return !claims.system
+  } catch (e) {
+    return false
+  }
+}

packages/api/src/utils/createTask.ts

@@ -196,7 +196,7 @@ export const getTask = async (
 
 export const deleteTask = async (
   taskName: string
-): Promise<google.protobuf.IEmpty> => {
+): Promise<google.protobuf.IEmpty | null> => {
   // If we are in local environment
   if (env.dev.isLocal) {
     return taskName
@@ -206,9 +206,13 @@ export const deleteTask = async (
     name: taskName,
   }
 
-  const [response] = await client.deleteTask(request)
-
-  return response
+  try {
+    const [response] = await client.deleteTask(request)
+    return response
+  } catch (error) {
+    logError(error)
+    return null
+  }
 }
 
 /**

packages/integration-handler/src/index.ts

@@ -4,6 +4,7 @@ import { stringify } from 'csv-stringify'
 import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
 import * as jwt from 'jsonwebtoken'
 import { DateTime } from 'luxon'
+import { promisify } from 'util'
 import { v4 as uuidv4 } from 'uuid'
 import { getIntegrationClient, updateIntegration } from './integrations'
 import { State } from './integrations/integration'
@@ -29,6 +30,7 @@ Sentry.GCPFunction.init({
 })
 
 const storage = new Storage()
+const signToken = promisify(jwt.sign)
 
 export const wait = (ms: number): Promise<void> => {
   return new Promise((resolve) => {
@@ -46,6 +48,19 @@ const createGCSFile = (bucket: string, filename: string): File => {
   return storage.bucket(bucket).file(filename)
 }
 
+const createSystemToken = async (
+  claims: Claims,
+  secret: string
+): Promise<string> => {
+  return signToken(
+    {
+      ...claims,
+      system: true,
+    },
+    secret
+  ) as Promise<string>
+}
+
 export const exporter = Sentry.GCPFunction.wrapHttpFunction(
   async (req, res) => {
     console.log('start to export to integration')
@@ -76,20 +91,23 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction(
         return res.status(200).send('Bad Request')
       }
 
+      const systemToken = await createSystemToken(claims, JWT_SECRET)
+
       const { integrationId, syncAt, integrationName } = req.body
       const client = getIntegrationClient(integrationName)
 
       // get paginated items from the backend
+      const first = '50'
       let hasMore = true
       let after = '0'
       while (hasMore) {
         console.log('searching for items...')
         const response = await search(
           REST_BACKEND_ENDPOINT,
-          token,
+          systemToken,
           client.highlightOnly,
           new Date(syncAt),
-          '50',
+          first,
           after
         )
 
@@ -124,7 +142,7 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction(
           items[items.length - 1].updatedAt,
           integrationName,
           claims.token,
-          token,
+          systemToken,
           'EXPORT'
         )
 
@@ -179,6 +197,8 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction(
       return res.status(200).send('Bad Request')
     }
 
+    const systemToken = await createSystemToken(claims, JWT_SECRET)
+
     let writeStream: NodeJS.WritableStream | undefined
     try {
       const userId = claims.uid
@@ -279,7 +299,7 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction(
         new Date(syncedAt),
         req.body.integrationName,
         claims.token,
-        token,
+        systemToken,
         'IMPORT',
         null
       )