From 0419472c2e7aca79501d8d803ae875254c70fc5d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 11 Aug 2022 19:14:22 +0800 Subject: [PATCH] Upload audio file with public access right --- packages/api/src/routers/svc/upload.ts | 48 +++++++++++--------------- packages/api/src/utils/textToSpeech.ts | 24 ++++++------- packages/api/src/utils/uploads.ts | 24 +++++-------- 3 files changed, 38 insertions(+), 58 deletions(-) diff --git a/packages/api/src/routers/svc/upload.ts b/packages/api/src/routers/svc/upload.ts index ebc935f86..c2378225d 100644 --- a/packages/api/src/routers/svc/upload.ts +++ b/packages/api/src/routers/svc/upload.ts @@ -3,60 +3,52 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' import { readPushSubscription } from '../../datalayer/pubsub' -import { generateUploadSignedUrl, uploadToSignedUrl } from '../../utils/uploads' +import { uploadToBucket } from '../../utils/uploads' import { v4 as uuidv4 } from 'uuid' import { env } from '../../env' import { DateTime } from 'luxon' +import { buildLogger } from '../../utils/logger' + +const logger = buildLogger('app.dispatch') export function uploadServiceRouter() { const router = express.Router() router.post('/:folder', async (req, res) => { - console.log('upload data to folder', req.params.folder) + logger.info('upload data to folder', req.params.folder) const { message: msgStr, expired } = readPushSubscription(req) if (!msgStr) { - res.status(400).send('Bad Request') - return + return res.status(400).send('Bad Request') } if (expired) { - console.log('discarding expired message') - res.status(200).send('Expired') - return + logger.info('discarding expired message') + return res.status(200).send('Expired') } try { const data: { userId: string; type: string } = JSON.parse(msgStr) if (!data.userId || !data.type) { - console.log('No userId or type found in message') - res.status(400).send('Bad Request') - return + logger.info('No userId or type found in message') + return res.status(400).send('Bad Request') } - const contentType = 'application/json' - const bucketName = env.fileUpload.gcsUploadPrivateBucket + const filePath = `${req.params.folder}/${data.type}/${ + data.userId + }/${DateTime.now().toFormat('yyyy-LL-dd')}/${uuidv4()}.json` - console.log('generate upload url') - - const uploadUrl = await generateUploadSignedUrl( - `${req.params.folder}/${data.type}/${ - data.userId - }/${DateTime.now().toFormat('yyyy-LL-dd')}/${uuidv4()}.json`, - contentType, - bucketName - ) - - console.log('start uploading', uploadUrl) - - await uploadToSignedUrl( - uploadUrl, + logger.info('uploading data to', filePath) + await uploadToBucket( + filePath, Buffer.from(msgStr, 'utf8'), - contentType + { contentType: 'application/json' }, + env.fileUpload.gcsUploadPrivateBucket ) + res.status(200).send('OK') } catch (err) { - console.log('upload page data failed', err) + logger.error('upload page data failed', err) res.status(500).send(err) } }) diff --git a/packages/api/src/utils/textToSpeech.ts b/packages/api/src/utils/textToSpeech.ts index d9cf0af7e..34e2e860b 100644 --- a/packages/api/src/utils/textToSpeech.ts +++ b/packages/api/src/utils/textToSpeech.ts @@ -1,16 +1,10 @@ import * as AWS from 'aws-sdk' import { buildLogger } from './logger' import { SynthesizeSpeechInput } from 'aws-sdk/clients/polly' -import { - generateUploadFilePathName, - generateUploadSignedUrl, - getFilePublicUrl, - uploadToSignedUrl, -} from './uploads' +import { getFilePublicUrl, uploadToBucket } from './uploads' export interface TextToSpeechInput { id: string - title: string text: string voice?: string textType?: 'text' | 'ssml' @@ -74,16 +68,18 @@ export const createAudioWithSpeechMarks = async ( try { const audio = await createAudio(input) // upload audio to google cloud storage - logger.info('generating upload url...') - const filePathName = generateUploadFilePathName(input.id, input.title) - const contentType = 'audio/mpeg' - const uploadUrl = await generateUploadSignedUrl(filePathName, contentType) + const filePath = `speech/${input.id}.mp3` - logger.info('start uploading...', { uploadUrl }) - await uploadToSignedUrl(uploadUrl, audio, contentType) + logger.info('start uploading...', { filePath }) + await uploadToBucket(filePath, audio, { + contentType: 'audio/mpeg', + public: true, + }) // get public url for audio file - const publicUrl = getFilePublicUrl(filePathName) + const publicUrl = getFilePublicUrl(filePath) + logger.info('upload complete', { publicUrl }) + const speechMarks = await createSpeechMarks(input) return { audioUrl: publicUrl, diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 6c4492479..351799a34 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -2,7 +2,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { env } from '../env' import { GetSignedUrlConfig, Storage } from '@google-cloud/storage' -import axios from 'axios' /* On GAE/Prod, we shall rely on default app engine service account credentials. * Two changes needed: 1) add default service account to our uploads GCS Bucket @@ -102,21 +101,14 @@ export const generateUploadFilePathName = ( return `u/${id}/${fileName}` } -export const uploadToSignedUrl = async ( - uploadUrl: string, +export const uploadToBucket = async ( + filePath: string, data: Buffer, - contentType: string + options?: { contentType?: string; public?: boolean }, + selectedBucket?: string ): Promise => { - // if (env.dev.isLocal) { - // return - // } - - await axios.put(uploadUrl, data, { - headers: { - 'Content-Type': contentType, - }, - maxBodyLength: 1000000000, - maxContentLength: 100000000, - timeout: 30000, - }) + await storage + .bucket(selectedBucket || bucketName) + .file(filePath) + .save(data, options) }