Fix uploading endpoint returns 504 (#289)

* add timeout = 60s for uploading to private bucket

* add debug logs

* send response to client to close connection

* add tests for upload

* add PUBSUB_VERIFICATION_TOKEN in .env.test

packages/api/.env.test

@@ -26,3 +26,4 @@ TWITTER_BEARER_TOKEN=
 PREVIEW_IMAGE_WRAPPER_ID='selected_highlight_wrapper'
 SEGMENT_WRITE_KEY='test'
 REMINDER_TASK_HANDLER_URL=http://localhost:4000/svc/reminders/trigger
+PUBSUB_VERIFICATION_TOKEN='123456'

packages/api/src/routers/svc/pages.ts

@@ -37,6 +37,9 @@ export function pageServiceRouter() {
 
       const contentType = 'application/json'
       const bucketName = env.fileUpload.gcsUploadPrivateBucket
+
+      console.log('generate upload url')
+
       const uploadUrl = await generateUploadSignedUrl(
         `${req.params.folder}/${data.userId}/${DateTime.now().toFormat(
           'yyyy-LL-dd'
@@ -44,12 +47,15 @@ export function pageServiceRouter() {
         contentType,
         bucketName
       )
+
+      console.log('start uploading', uploadUrl)
+
       await uploadToSignedUrl(
         uploadUrl,
         Buffer.from(msgStr, 'utf8'),
         contentType
       )
-      res.status(200)
+      res.status(200).send('OK')
     } catch (err) {
       console.log('upload page data failed', err)
       res.status(500).send(err)

packages/api/src/utils/uploads.ts

@@ -2,7 +2,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import { env } from '../env'
 import { GetSignedUrlConfig, Storage } from '@google-cloud/storage'
-import axios, { AxiosResponse } from 'axios'
+import axios from 'axios'
 
 /* On GAE/Prod, we shall rely on default app engine service account credentials.
  * Two changes needed: 1) add default service account to our uploads GCS Bucket
@@ -102,12 +102,17 @@ export const uploadToSignedUrl = async (
   uploadUrl: string,
   data: Buffer,
   contentType: string
-): Promise<AxiosResponse> => {
-  return axios.put(uploadUrl, data, {
+): Promise<void> => {
+  if (env.dev.isLocal) {
+    return
+  }
+
+  await axios.put(uploadUrl, data, {
     headers: {
       'Content-Type': contentType,
     },
     maxBodyLength: 1000000000,
     maxContentLength: 100000000,
+    timeout: 30000,
   })
 }

packages/api/test/routers/pages.test.ts

@@ -0,0 +1,23 @@
+import { request } from '../util'
+import 'mocha'
+
+describe('Pages Router', () => {
+  const token = process.env.PUBSUB_VERIFICATION_TOKEN || ''
+
+  describe('upload', () => {
+    it('upload data to GCS', async () => {
+      const data = {
+        message: {
+          data: Buffer.from(JSON.stringify({ userId: 'userId' })).toString(
+            'base64'
+          ),
+          publishTime: new Date().toISOString(),
+        },
+      }
+      await request
+        .post(`/svc/pubsub/pages/upload/createdPage?token=${token}`)
+        .send(data)
+        .expect(200)
+    })
+  })
+})