From 5d95e6c7340764c063374722166b141b09ebb008 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 15 Feb 2022 22:22:13 +0800 Subject: [PATCH 1/5] add pdf attachment upload --- .../api/src/routers/svc/pdf_attachments.ts | 120 ++++++++++++++++++ packages/api/src/server.ts | 2 + packages/inbound-email-handler/src/index.ts | 16 ++- packages/inbound-email-handler/src/pdf.ts | 91 +++++++++++++ 4 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 packages/api/src/routers/svc/pdf_attachments.ts create mode 100644 packages/inbound-email-handler/src/pdf.ts diff --git a/packages/api/src/routers/svc/pdf_attachments.ts b/packages/api/src/routers/svc/pdf_attachments.ts new file mode 100644 index 000000000..192f781ee --- /dev/null +++ b/packages/api/src/routers/svc/pdf_attachments.ts @@ -0,0 +1,120 @@ +import express from 'express' +import { env } from '../../env' +import * as jwt from 'jsonwebtoken' +import { PageType, UploadFileStatus } from '../../generated/graphql' +import { + generateUploadFilePathName, + generateUploadSignedUrl, + getStorageFileDetails, +} from '../../utils/uploads' +import { initModels } from '../../server' +import { kx } from '../../datalayer/knex_config' +import { analytics } from '../../utils/analytics' + +export function pdfAttachmentsRouter() { + const router = express.Router() + + // eslint-disable-next-line @typescript-eslint/no-misused-promises + router.post('/upload', async (req, res) => { + console.log('pdf-attachments/upload') + + const { email, fileName } = req.body as { + email: string + fileName: string + } + + const token = req?.headers?.authorization + if (!token || !jwt.verify(token, env.server.jwtSecret)) { + return res.status(401).send('UNAUTHORIZED') + } + + const models = initModels(kx, false) + const user = await models.user.getWhere({ email }) + + if (!user) { + return res.status(401).send('UNAUTHORIZED') + } + + analytics.track({ + userId: user.id, + event: 'pdf-attachment-upload', + properties: { + env: env.server.apiEnv, + }, + }) + + const contentType = 'application/pdf' + const uploadFileData = await models.uploadFile.create({ + url: '', + userId: user.id, + fileName: fileName, + status: UploadFileStatus.Initialized, + contentType: contentType, + }) + + if (uploadFileData.id) { + const uploadFilePathName = generateUploadFilePathName( + uploadFileData.id, + fileName + ) + const uploadSignedUrl = await generateUploadSignedUrl( + uploadFilePathName, + contentType + ) + res.send({ + id: uploadFileData.id, + url: uploadSignedUrl, + }) + } else { + res.status(400).send('BAD REQUEST') + } + }) + + // eslint-disable-next-line @typescript-eslint/no-misused-promises + router.post('create-article', async (req, res) => { + console.log('pdf-attachments/create-article') + + const { email, uploadFileId } = req.body as { + email: string + uploadFileId: string + } + + const token = req?.headers?.authorization + if (!token || !jwt.verify(token, env.server.jwtSecret)) { + return res.status(401).send('UNAUTHORIZED') + } + + const models = initModels(kx, false) + const user = await models.user.getWhere({ email }) + + if (!user) { + return res.status(401).send('UNAUTHORIZED') + } + + analytics.track({ + userId: user.id, + event: 'create-article', + properties: { + env: env.server.apiEnv, + }, + }) + + const uploadFile = await models.uploadFile.getWhere({ + id: uploadFileId, + userId: user.id, + }) + if (!uploadFile) { + return res.status(400).send('BAD REQUEST') + } + const uploadFileDetails = await getStorageFileDetails( + uploadFileId, + uploadFile.fileName + ) + const uploadFileHash = uploadFileDetails.md5Hash + const userArticleUrl = uploadFileDetails.fileUrl + const canonicalUrl = uploadFile.url + const pageType = PageType.File + }) + + return router +} diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index c72e0f853..b0352da67 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -37,6 +37,7 @@ import { emailsServiceRouter } from './routers/svc/emails' import ReminderModel from './datalayer/reminders' import { remindersServiceRouter } from './routers/svc/reminders' import { ApolloServer } from 'apollo-server-express' +import { pdfAttachmentsRouter } from './routers/svc/pdf_attachments' const PORT = process.env.PORT || 4000 @@ -97,6 +98,7 @@ export const createApp = (): { app.use('/svc/pubsub/newsletters', newsletterServiceRouter()) app.use('/svc/pubsub/emails', emailsServiceRouter()) app.use('/svc/reminders', remindersServiceRouter()) + app.use('/svc/pdf-attachments', pdfAttachmentsRouter()) if (env.dev.isLocal) { app.use('/local/debug', localDebugRouter()) diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index 121c1194a..57c4a73ac 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -13,6 +13,7 @@ import { isNewsletter, } from './newsletter' import { PubSub } from '@google-cloud/pubsub' +import { handlePdfAttachment } from './pdf' const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived' const pubsub = new PubSub() @@ -23,9 +24,14 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( const parsed: Record = {} for (const part of parts) { - const { name, data } = part + const { name, data, type, filename } = part if (name && data) { parsed[name] = data.toString() + } else if (type === 'application/pdf' && data) { + parsed['pdf-attachment-data'] = data.toString() + parsed['pdf-attachment-filename'] = filename + ? filename + : 'attachment.pdf' } else { console.log('no data or name for ', part) } @@ -76,7 +82,15 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( if (isConfirmationEmail(from)) { console.log('handleConfirmation', from, recipientAddress) await handleConfirmation(recipientAddress, subject) + } else if (parsed['pdf-attachment']) { + console.log('handle PDF attachment', from, recipientAddress) + await handlePdfAttachment( + recipientAddress, + parsed['pdf-attachment-filename'], + parsed['pdf-attachment-data'] + ) } + // queue non-newsletter emails await pubsub.topic(NON_NEWSLETTER_EMAIL_TOPIC).publishMessage({ json: { diff --git a/packages/inbound-email-handler/src/pdf.ts b/packages/inbound-email-handler/src/pdf.ts new file mode 100644 index 000000000..6cefe4c55 --- /dev/null +++ b/packages/inbound-email-handler/src/pdf.ts @@ -0,0 +1,91 @@ +import axios, { AxiosResponse } from 'axios' +import { promisify } from 'util' +import * as jwt from 'jsonwebtoken' + +const signToken = promisify(jwt.sign) + +type UploadResponse = { + id: string + url: string +} + +export const handlePdfAttachment = async ( + email: string, + fileName: string, + data: string +): Promise => { + const uploadResult = await getUploadIdAndSignedUrl(email, fileName) + await uploadToSignedUrl(uploadResult.url, data) + await createArticle(email, uploadResult.id) +} + +const getUploadIdAndSignedUrl = async ( + email: string, + fileName: string +): Promise => { + if (process.env.JWT_SECRET === undefined) { + throw new Error('JWT_SECRET is not defined') + } + const auth = await signToken(email, process.env.JWT_SECRET) + const data = { + fileName, + email, + } + + if (process.env.REST_BACKEND_ENDPOINT === undefined) { + throw new Error('REST_BACKEND_ENDPOINT is not defined') + } + const response = await axios.post( + `${process.env.REST_BACKEND_ENDPOINT}/svc/pdf-attachments/upload`, + data, + { + headers: { + Authorization: `${auth as string}`, + 'Content-Type': 'application/json', + }, + } + ) + return response.data as UploadResponse +} + +const uploadToSignedUrl = ( + uploadUrl: string, + data: string +): Promise => { + return axios.put(uploadUrl, data, { + headers: { + 'Content-Type': 'application/pdf', + }, + maxBodyLength: 1000000000, + maxContentLength: 100000000, + }) +} + +const createArticle = async ( + email: string, + uploadFileId: string +): Promise => { + const data = { + email, + uploadFileId, + } + + if (process.env.JWT_SECRET === undefined) { + throw new Error('JWT_SECRET is not defined') + } + const auth = await signToken(email, process.env.JWT_SECRET) + + if (process.env.REST_BACKEND_ENDPOINT === undefined) { + throw new Error('REST_BACKEND_ENDPOINT is not defined') + } + return axios.post( + `${process.env.REST_BACKEND_ENDPOINT}/svc/pdf-attachments/create-article`, + data, + { + headers: { + Authorization: `${auth as string}`, + 'Content-Type': 'application/json', + }, + } + ) +} From b826a101c5f55d7083f275ba78c0d1af0c0b0db9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Feb 2022 09:24:16 +0800 Subject: [PATCH 2/5] add pdf attachment create-article --- .../api/src/routers/svc/pdf_attachments.ts | 48 +++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/packages/api/src/routers/svc/pdf_attachments.ts b/packages/api/src/routers/svc/pdf_attachments.ts index 192f781ee..01e6c5903 100644 --- a/packages/api/src/routers/svc/pdf_attachments.ts +++ b/packages/api/src/routers/svc/pdf_attachments.ts @@ -6,10 +6,12 @@ import { generateUploadFilePathName, generateUploadSignedUrl, getStorageFileDetails, + makeStorageFilePublic, } from '../../utils/uploads' import { initModels } from '../../server' import { kx } from '../../datalayer/knex_config' import { analytics } from '../../utils/analytics' +import normalizeUrl from 'normalize-url' export function pdfAttachmentsRouter() { const router = express.Router() @@ -71,7 +73,7 @@ export function pdfAttachmentsRouter() { }) // eslint-disable-next-line @typescript-eslint/no-misused-promises - router.post('create-article', async (req, res) => { + router.post('/create-article', async (req, res) => { console.log('pdf-attachments/create-article') const { email, uploadFileId } = req.body as { @@ -93,7 +95,7 @@ export function pdfAttachmentsRouter() { analytics.track({ userId: user.id, - event: 'create-article', + event: 'pdf-attachment-create-article', properties: { env: env.server.apiEnv, }, @@ -111,9 +113,49 @@ export function pdfAttachmentsRouter() { uploadFile.fileName ) const uploadFileHash = uploadFileDetails.md5Hash - const userArticleUrl = uploadFileDetails.fileUrl const canonicalUrl = uploadFile.url const pageType = PageType.File + + const saveTime = new Date() + const articleToSave = { + url: normalizeUrl(canonicalUrl, { + stripHash: true, + stripWWW: false, + }), + pageType: pageType, + hash: uploadFileHash, + uploadFileId: uploadFileId, + title: uploadFile.fileName, + content: '', + } + + const uploadFileData = await kx.transaction(async (tx) => { + return models.uploadFile.setFileUploadComplete(uploadFileId, tx) + }) + if (!uploadFileData || !uploadFileData.id || !uploadFileData.fileName) { + return res.status(400).send('BAD REQUEST') + } + const uploadFileUrlOverride = await makeStorageFilePublic( + uploadFileData.id, + uploadFileData.fileName + ) + + await kx.transaction(async (tx) => { + const articleRecord = await models.article.create(articleToSave, tx) + await models.userArticle.create( + { + userId: user.id, + slug: '', + savedAt: saveTime, + articleId: articleRecord.id, + articleUrl: uploadFileUrlOverride, + articleHash: articleRecord.hash, + }, + tx + ) + }) + + res.send('OK') }) return router From 94601971322a5dd912505814e5e0ad72419ba3f0 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Feb 2022 11:34:01 +0800 Subject: [PATCH 3/5] add more logs for debugging --- .../api/src/routers/svc/pdf_attachments.ts | 7 ++++--- packages/inbound-email-handler/src/pdf.ts | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/api/src/routers/svc/pdf_attachments.ts b/packages/api/src/routers/svc/pdf_attachments.ts index 01e6c5903..5abbeb927 100644 --- a/packages/api/src/routers/svc/pdf_attachments.ts +++ b/packages/api/src/routers/svc/pdf_attachments.ts @@ -129,12 +129,13 @@ export function pdfAttachmentsRouter() { content: '', } - const uploadFileData = await kx.transaction(async (tx) => { - return models.uploadFile.setFileUploadComplete(uploadFileId, tx) - }) + const uploadFileData = await models.uploadFile.setFileUploadComplete( + uploadFileId + ) if (!uploadFileData || !uploadFileData.id || !uploadFileData.fileName) { return res.status(400).send('BAD REQUEST') } + const uploadFileUrlOverride = await makeStorageFilePublic( uploadFileData.id, uploadFileData.fileName diff --git a/packages/inbound-email-handler/src/pdf.ts b/packages/inbound-email-handler/src/pdf.ts index 6cefe4c55..45bd1bed3 100644 --- a/packages/inbound-email-handler/src/pdf.ts +++ b/packages/inbound-email-handler/src/pdf.ts @@ -14,9 +14,19 @@ export const handlePdfAttachment = async ( fileName: string, data: string ): Promise => { - const uploadResult = await getUploadIdAndSignedUrl(email, fileName) - await uploadToSignedUrl(uploadResult.url, data) - await createArticle(email, uploadResult.id) + console.log('handlePdfAttachment', email, fileName) + + try { + const uploadResult = await getUploadIdAndSignedUrl(email, fileName) + if (!uploadResult.url || !uploadResult.id) { + console.log('failed to create upload request', uploadResult) + return + } + await uploadToSignedUrl(uploadResult.url, data) + await createArticle(email, uploadResult.id) + } catch (error) { + console.error('handlePdfAttachment error', error) + } } const getUploadIdAndSignedUrl = async ( @@ -48,7 +58,7 @@ const getUploadIdAndSignedUrl = async ( return response.data as UploadResponse } -const uploadToSignedUrl = ( +const uploadToSignedUrl = async ( uploadUrl: string, data: string ): Promise => { From 77ba7531f3a0922272174e43aacdf9a47cc17889 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Feb 2022 15:26:11 +0800 Subject: [PATCH 4/5] add tests for upload --- .../api/src/routers/svc/pdf_attachments.ts | 27 +++++----- .../api/test/routers/pdf_attachments.test.ts | 49 +++++++++++++++++++ ....do.add_delete_cascade_on_upload_files.sql | 14 ++++++ ...ndo.add_delete_cascade_on_upload_files.sql | 13 +++++ packages/inbound-email-handler/package.json | 5 +- packages/inbound-email-handler/src/index.ts | 2 +- yarn.lock | 17 ------- 7 files changed, 94 insertions(+), 33 deletions(-) create mode 100644 packages/api/test/routers/pdf_attachments.test.ts create mode 100755 packages/db/migrations/0068.do.add_delete_cascade_on_upload_files.sql create mode 100755 packages/db/migrations/0068.undo.add_delete_cascade_on_upload_files.sql diff --git a/packages/api/src/routers/svc/pdf_attachments.ts b/packages/api/src/routers/svc/pdf_attachments.ts index 5abbeb927..356589267 100644 --- a/packages/api/src/routers/svc/pdf_attachments.ts +++ b/packages/api/src/routers/svc/pdf_attachments.ts @@ -12,6 +12,7 @@ import { initModels } from '../../server' import { kx } from '../../datalayer/knex_config' import { analytics } from '../../utils/analytics' import normalizeUrl from 'normalize-url' +import { getNewsletterEmail } from '../../services/newsletters' export function pdfAttachmentsRouter() { const router = express.Router() @@ -30,13 +31,13 @@ export function pdfAttachmentsRouter() { return res.status(401).send('UNAUTHORIZED') } - const models = initModels(kx, false) - const user = await models.user.getWhere({ email }) - - if (!user) { + const newsletterEmail = await getNewsletterEmail(email) + if (!newsletterEmail || !newsletterEmail.user) { return res.status(401).send('UNAUTHORIZED') } + const user = newsletterEmail.user + analytics.track({ userId: user.id, event: 'pdf-attachment-upload', @@ -46,6 +47,7 @@ export function pdfAttachmentsRouter() { }) const contentType = 'application/pdf' + const models = initModels(kx, false) const uploadFileData = await models.uploadFile.create({ url: '', userId: user.id, @@ -59,10 +61,10 @@ export function pdfAttachmentsRouter() { uploadFileData.id, fileName ) - const uploadSignedUrl = await generateUploadSignedUrl( - uploadFilePathName, - contentType - ) + const uploadSignedUrl = + env.server.apiEnv === 'prod' + ? await generateUploadSignedUrl(uploadFilePathName, contentType) + : 'http://localhost:3000/uploads/' + uploadFilePathName res.send({ id: uploadFileData.id, url: uploadSignedUrl, @@ -86,13 +88,13 @@ export function pdfAttachmentsRouter() { return res.status(401).send('UNAUTHORIZED') } - const models = initModels(kx, false) - const user = await models.user.getWhere({ email }) - - if (!user) { + const newsletterEmail = await getNewsletterEmail(email) + if (!newsletterEmail || !newsletterEmail.user) { return res.status(401).send('UNAUTHORIZED') } + const user = newsletterEmail.user + analytics.track({ userId: user.id, event: 'pdf-attachment-create-article', @@ -101,6 +103,7 @@ export function pdfAttachmentsRouter() { }, }) + const models = initModels(kx, false) const uploadFile = await models.uploadFile.getWhere({ id: uploadFileId, userId: user.id, diff --git a/packages/api/test/routers/pdf_attachments.test.ts b/packages/api/test/routers/pdf_attachments.test.ts new file mode 100644 index 000000000..ee1ee0152 --- /dev/null +++ b/packages/api/test/routers/pdf_attachments.test.ts @@ -0,0 +1,49 @@ +import { + createTestNewsletterEmail, + createTestUser, + deleteTestUser, +} from '../db' +import { request } from '../util' +import { User } from '../../src/entity/user' +import 'mocha' +import * as jwt from 'jsonwebtoken' +import { expect } from 'chai' + +describe('PDF attachments Router', () => { + const username = 'fakeUser' + const newsletterEmail = 'fakeEmail' + + let user: User + let authToken: string + + before(async () => { + // create test user and login + user = await createTestUser(username) + + await createTestNewsletterEmail(user, newsletterEmail) + authToken = jwt.sign(newsletterEmail, process.env.JWT_SECRET || '') + }) + + after(async () => { + // clean up + await deleteTestUser(username) + }) + + describe('upload', () => { + it('create upload file request and return id and url', async () => { + const testFile = 'testFile.pdf' + + const res = await request + .post('/svc/pdf-attachments/upload') + .send({ + email: newsletterEmail, + fileName: testFile, + }) + .set('Authorization', `${authToken}`) + .expect(200) + + expect(res.body.id).to.be.a('string') + expect(res.body.url).to.be.a('string') + }) + }) +}) diff --git a/packages/db/migrations/0068.do.add_delete_cascade_on_upload_files.sql b/packages/db/migrations/0068.do.add_delete_cascade_on_upload_files.sql new file mode 100755 index 000000000..5cbdc49cd --- /dev/null +++ b/packages/db/migrations/0068.do.add_delete_cascade_on_upload_files.sql @@ -0,0 +1,14 @@ +-- Type: DO +-- Name: add_delete_cascade_on_upload_files +-- Description: Add delete cascade on user_id field on upload_files table + +BEGIN; + +ALTER TABLE omnivore.upload_files + DROP CONSTRAINT upload_files_user_id_fkey, + ADD CONSTRAINT upload_files_user_id_fkey + FOREIGN KEY (user_id) + REFERENCES omnivore.user(id) + ON DELETE CASCADE; + +COMMIT; diff --git a/packages/db/migrations/0068.undo.add_delete_cascade_on_upload_files.sql b/packages/db/migrations/0068.undo.add_delete_cascade_on_upload_files.sql new file mode 100755 index 000000000..a4c0727a7 --- /dev/null +++ b/packages/db/migrations/0068.undo.add_delete_cascade_on_upload_files.sql @@ -0,0 +1,13 @@ +-- Type: UNDO +-- Name: add_delete_cascade_on_upload_files +-- Description: Add delete cascade on user_id field on upload_files table + +BEGIN; + +ALTER TABLE omnivore.upload_files + DROP CONSTRAINT upload_files_user_id_fkey, + ADD CONSTRAINT upload_files_user_id_fkey + FOREIGN KEY (user_id) + REFERENCES omnivore.user(id); + +COMMIT; diff --git a/packages/inbound-email-handler/package.json b/packages/inbound-email-handler/package.json index 704439c60..a3b9231db 100644 --- a/packages/inbound-email-handler/package.json +++ b/packages/inbound-email-handler/package.json @@ -22,7 +22,6 @@ "devDependencies": { "@types/json-bigint": "^1.0.1", "@types/node": "^14.11.2", - "@types/quoted-printable": "^1.0.0", "eslint-plugin-prettier": "^4.0.0" }, "dependencies": { @@ -31,8 +30,8 @@ "@sendgrid/client": "^7.6.0", "@sentry/serverless": "^6.16.1", "axios": "^0.26.0", + "jsonwebtoken": "^8.5.1", "parse-headers": "^2.0.4", - "parse-multipart-data": "^1.2.1", - "quoted-printable": "^1.0.1" + "parse-multipart-data": "^1.2.1" } } diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index 57c4a73ac..5b79b2cfc 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -82,7 +82,7 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( if (isConfirmationEmail(from)) { console.log('handleConfirmation', from, recipientAddress) await handleConfirmation(recipientAddress, subject) - } else if (parsed['pdf-attachment']) { + } else if (parsed['pdf-attachment-filename']) { console.log('handle PDF attachment', from, recipientAddress) await handlePdfAttachment( recipientAddress, diff --git a/yarn.lock b/yarn.lock index 9ce1db3aa..d020d8059 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4906,11 +4906,6 @@ resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw== -"@types/quoted-printable@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@types/quoted-printable/-/quoted-printable-1.0.0.tgz#903f81f5d81698d361c176b5b52970cd2bc9868e" - integrity sha512-hgFjmHmgT5M8SvDVe+tMhiUb3xViwqkEAM/sTpWCpO0B2Z7RGAgwiQaxPcLVk4KLiZmqj7BMXZvaQQdX6uPM6A== - "@types/range-parser@*": version "1.2.4" resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" @@ -14538,13 +14533,6 @@ quick-lru@^4.0.1: resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g== -quoted-printable@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/quoted-printable/-/quoted-printable-1.0.1.tgz#9eebf5eb3d11eef022b264fd2d2b6b2bb3b84cc3" - integrity sha1-nuv16z0R7vAismT9LStrK7O4TMM= - dependencies: - utf8 "^2.1.0" - randombytes@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" @@ -16991,11 +16979,6 @@ user-home@^1.1.1: resolved "https://registry.yarnpkg.com/user-home/-/user-home-1.1.1.tgz#2b5be23a32b63a7c9deb8d0f28d485724a3df190" integrity sha1-K1viOjK2Onyd640PKNSFcko98ZA= -utf8@^2.1.0: - version "2.1.2" - resolved "https://registry.yarnpkg.com/utf8/-/utf8-2.1.2.tgz#1fa0d9270e9be850d9b05027f63519bf46457d96" - integrity sha1-H6DZJw6b6FDZsFAn9jUZv0ZFfZY= - util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" From d53e94aeb21cba30761c8528a1e661c4d0adfb4f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Feb 2022 16:14:14 +0800 Subject: [PATCH 5/5] add tests for create article --- .../api/src/routers/svc/pdf_attachments.ts | 165 ++++++++++-------- packages/api/test/db.ts | 4 + .../api/test/routers/pdf_attachments.test.ts | 33 +++- ...ete_cascade_on_upload_file_id_to_pages.sql | 14 ++ ...ete_cascade_on_upload_file_id_to_pages.sql | 13 ++ 5 files changed, 151 insertions(+), 78 deletions(-) create mode 100755 packages/db/migrations/0069.do.add_delete_cascade_on_upload_file_id_to_pages.sql create mode 100755 packages/db/migrations/0069.undo.add_delete_cascade_on_upload_file_id_to_pages.sql diff --git a/packages/api/src/routers/svc/pdf_attachments.ts b/packages/api/src/routers/svc/pdf_attachments.ts index 356589267..6ec01d68a 100644 --- a/packages/api/src/routers/svc/pdf_attachments.ts +++ b/packages/api/src/routers/svc/pdf_attachments.ts @@ -11,7 +11,6 @@ import { import { initModels } from '../../server' import { kx } from '../../datalayer/knex_config' import { analytics } from '../../utils/analytics' -import normalizeUrl from 'normalize-url' import { getNewsletterEmail } from '../../services/newsletters' export function pdfAttachmentsRouter() { @@ -46,31 +45,36 @@ export function pdfAttachmentsRouter() { }, }) - const contentType = 'application/pdf' - const models = initModels(kx, false) - const uploadFileData = await models.uploadFile.create({ - url: '', - userId: user.id, - fileName: fileName, - status: UploadFileStatus.Initialized, - contentType: contentType, - }) - - if (uploadFileData.id) { - const uploadFilePathName = generateUploadFilePathName( - uploadFileData.id, - fileName - ) - const uploadSignedUrl = - env.server.apiEnv === 'prod' - ? await generateUploadSignedUrl(uploadFilePathName, contentType) - : 'http://localhost:3000/uploads/' + uploadFilePathName - res.send({ - id: uploadFileData.id, - url: uploadSignedUrl, + try { + const contentType = 'application/pdf' + const models = initModels(kx, false) + const uploadFileData = await models.uploadFile.create({ + url: '', + userId: user.id, + fileName: fileName, + status: UploadFileStatus.Initialized, + contentType: contentType, }) - } else { - res.status(400).send('BAD REQUEST') + + if (uploadFileData.id) { + const uploadFilePathName = generateUploadFilePathName( + uploadFileData.id, + fileName + ) + const uploadSignedUrl = + env.server.apiEnv === 'prod' + ? await generateUploadSignedUrl(uploadFilePathName, contentType) + : 'http://localhost:3000/uploads/' + uploadFilePathName + res.send({ + id: uploadFileData.id, + url: uploadSignedUrl, + }) + } else { + res.status(400).send('BAD REQUEST') + } + } catch (err) { + console.error(err) + return res.status(500).send('INTERNAL_SERVER_ERROR') } }) @@ -103,63 +107,70 @@ export function pdfAttachmentsRouter() { }, }) - const models = initModels(kx, false) - const uploadFile = await models.uploadFile.getWhere({ - id: uploadFileId, - userId: user.id, - }) - if (!uploadFile) { - return res.status(400).send('BAD REQUEST') - } - const uploadFileDetails = await getStorageFileDetails( - uploadFileId, - uploadFile.fileName - ) - const uploadFileHash = uploadFileDetails.md5Hash - const canonicalUrl = uploadFile.url - const pageType = PageType.File + try { + const models = initModels(kx, false) + const uploadFile = await models.uploadFile.getWhere({ + id: uploadFileId, + userId: user.id, + }) + if (!uploadFile) { + return res.status(400).send('BAD REQUEST') + } - const saveTime = new Date() - const articleToSave = { - url: normalizeUrl(canonicalUrl, { - stripHash: true, - stripWWW: false, - }), - pageType: pageType, - hash: uploadFileHash, - uploadFileId: uploadFileId, - title: uploadFile.fileName, - content: '', - } + const uploadFileDetails = + env.server.apiEnv === 'prod' + ? await getStorageFileDetails(uploadFileId, uploadFile.fileName) + : { md5Hash: '', size: 0 } + const uploadFileHash = uploadFileDetails.md5Hash + const pageType = PageType.File - const uploadFileData = await models.uploadFile.setFileUploadComplete( - uploadFileId - ) - if (!uploadFileData || !uploadFileData.id || !uploadFileData.fileName) { - return res.status(400).send('BAD REQUEST') - } + const saveTime = new Date() + const articleToSave = { + url: '', + pageType: pageType, + hash: uploadFileHash, + uploadFileId: uploadFileId, + title: uploadFile.fileName, + content: '', + } - const uploadFileUrlOverride = await makeStorageFilePublic( - uploadFileData.id, - uploadFileData.fileName - ) - - await kx.transaction(async (tx) => { - const articleRecord = await models.article.create(articleToSave, tx) - await models.userArticle.create( - { - userId: user.id, - slug: '', - savedAt: saveTime, - articleId: articleRecord.id, - articleUrl: uploadFileUrlOverride, - articleHash: articleRecord.hash, - }, - tx + const uploadFileData = await models.uploadFile.setFileUploadComplete( + uploadFileId ) - }) + if (!uploadFileData || !uploadFileData.id || !uploadFileData.fileName) { + return res.status(400).send('BAD REQUEST') + } - res.send('OK') + const uploadFileUrlOverride = + env.server.apiEnv === 'prod' + ? await makeStorageFilePublic( + uploadFileData.id, + uploadFileData.fileName + ) + : 'http://localhost:3000/uploads/' + + uploadFileData.id + + '/' + + uploadFileData.fileName + + const link = await kx.transaction(async (tx) => { + const articleRecord = await models.article.create(articleToSave, tx) + return models.userArticle.create( + { + userId: user.id, + slug: '', + savedAt: saveTime, + articleId: articleRecord.id, + articleUrl: uploadFileUrlOverride, + articleHash: articleRecord.hash, + }, + tx + ) + }) + res.send({ id: link.id }) + } catch (err) { + console.log(err) + res.status(500).send(err) + } }) return router diff --git a/packages/api/test/db.ts b/packages/api/test/db.ts index d405472aa..68596e748 100644 --- a/packages/api/test/db.ts +++ b/packages/api/test/db.ts @@ -196,3 +196,7 @@ export const getDeviceToken = async ( export const getUser = async (id: string): Promise => { return getRepository(User).findOne(id) } + +export const getLink = async (id: string): Promise => { + return getRepository(Link).findOne(id) +} diff --git a/packages/api/test/routers/pdf_attachments.test.ts b/packages/api/test/routers/pdf_attachments.test.ts index ee1ee0152..6e9756f41 100644 --- a/packages/api/test/routers/pdf_attachments.test.ts +++ b/packages/api/test/routers/pdf_attachments.test.ts @@ -2,6 +2,7 @@ import { createTestNewsletterEmail, createTestUser, deleteTestUser, + getLink, } from '../db' import { request } from '../util' import { User } from '../../src/entity/user' @@ -35,15 +36,45 @@ describe('PDF attachments Router', () => { const res = await request .post('/svc/pdf-attachments/upload') + .set('Authorization', `${authToken}`) .send({ email: newsletterEmail, fileName: testFile, }) - .set('Authorization', `${authToken}`) .expect(200) expect(res.body.id).to.be.a('string') expect(res.body.url).to.be.a('string') }) }) + + describe('create article', () => { + it('create article with uploaded file id and url', async () => { + // upload file first + const testFile = 'testFile.pdf' + const res = await request + .post('/svc/pdf-attachments/upload') + .set('Authorization', `${authToken}`) + .send({ + email: newsletterEmail, + fileName: testFile, + }) + const uploadFileId = res.body.id + + // create article + const res2 = await request + .post('/svc/pdf-attachments/create-article') + .send({ + email: newsletterEmail, + uploadFileId: uploadFileId, + }) + .set('Authorization', `${authToken}`) + .expect(200) + + expect(res2.body.id).to.be.a('string') + const link = await getLink(res2.body.id) + + expect(link).to.exist + }) + }) }) diff --git a/packages/db/migrations/0069.do.add_delete_cascade_on_upload_file_id_to_pages.sql b/packages/db/migrations/0069.do.add_delete_cascade_on_upload_file_id_to_pages.sql new file mode 100755 index 000000000..0249065f3 --- /dev/null +++ b/packages/db/migrations/0069.do.add_delete_cascade_on_upload_file_id_to_pages.sql @@ -0,0 +1,14 @@ +-- Type: DO +-- Name: add_delete_cascade_on_upload_file_id_to_pages +-- Description: Add delete cascade on upload_file_id field on pages table + +BEGIN; + +ALTER TABLE omnivore.pages + DROP CONSTRAINT article_upload_file_id_fkey, + ADD CONSTRAINT pages_upload_file_id_fkey + FOREIGN KEY (upload_file_id) + REFERENCES omnivore.upload_files (id) + ON DELETE CASCADE; + +COMMIT; diff --git a/packages/db/migrations/0069.undo.add_delete_cascade_on_upload_file_id_to_pages.sql b/packages/db/migrations/0069.undo.add_delete_cascade_on_upload_file_id_to_pages.sql new file mode 100755 index 000000000..d9006dcf0 --- /dev/null +++ b/packages/db/migrations/0069.undo.add_delete_cascade_on_upload_file_id_to_pages.sql @@ -0,0 +1,13 @@ +-- Type: UNDO +-- Name: add_delete_cascade_on_upload_file_id_to_pages +-- Description: Add delete cascade on upload_file_id field on pages table + +BEGIN; + +ALTER TABLE omnivore.pages + DROP CONSTRAINT pages_upload_file_id_fkey, + ADD CONSTRAINT article_upload_file_id_fkey + FOREIGN KEY (upload_file_id) + REFERENCES omnivore.upload_files (id); + +COMMIT;