diff --git a/packages/api/src/routers/svc/pdf_attachments.ts b/packages/api/src/routers/svc/email_attachment.ts similarity index 91% rename from packages/api/src/routers/svc/pdf_attachments.ts rename to packages/api/src/routers/svc/email_attachment.ts index 6a858ad23..3f2f2ebac 100644 --- a/packages/api/src/routers/svc/pdf_attachments.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -1,34 +1,35 @@ import express from 'express' +import { setClaims } from '../../datalayer/helpers' +import { kx } from '../../datalayer/knex_config' +import { createPubSubClient } from '../../datalayer/pubsub' +import { createPage } from '../../elastic/pages' +import { ArticleSavingRequestStatus, Page } from '../../elastic/types' import { env } from '../../env' import { PageType, UploadFileStatus } from '../../generated/graphql' +import { initModels } from '../../server' +import { getNewsletterEmail } from '../../services/newsletters' +import { updateReceivedEmail } from '../../services/received_emails' +import { analytics } from '../../utils/analytics' +import { getClaimsByToken } from '../../utils/auth' +import { generateSlug } from '../../utils/helpers' import { generateUploadFilePathName, generateUploadSignedUrl, getStorageFileDetails, makeStorageFilePublic, } from '../../utils/uploads' -import { initModels } from '../../server' -import { kx } from '../../datalayer/knex_config' -import { analytics } from '../../utils/analytics' -import { getNewsletterEmail } from '../../services/newsletters' -import { setClaims } from '../../datalayer/helpers' -import { generateSlug } from '../../utils/helpers' -import { createPubSubClient } from '../../datalayer/pubsub' -import { ArticleSavingRequestStatus, Page } from '../../elastic/types' -import { createPage } from '../../elastic/pages' -import { getClaimsByToken } from '../../utils/auth' -import { updateReceivedEmail } from '../../services/received_emails' -export function pdfAttachmentsRouter() { +export function emailAttachmentRouter() { const router = express.Router() // eslint-disable-next-line @typescript-eslint/no-misused-promises router.post('/upload', async (req, res) => { - console.log('pdf-attachments/upload') + console.log('email-attachment/upload') - const { email, fileName } = req.body as { + const { email, fileName, contentType } = req.body as { email: string fileName: string + contentType: string } const token = req?.headers?.authorization @@ -45,14 +46,13 @@ export function pdfAttachmentsRouter() { analytics.track({ userId: user.id, - event: 'pdf_attachment_upload', + event: 'email_attachment_upload', properties: { env: env.server.apiEnv, }, }) try { - const contentType = 'application/pdf' const models = initModels(kx, false) const uploadFileData = await models.uploadFile.create({ url: '', @@ -86,7 +86,7 @@ export function pdfAttachmentsRouter() { // eslint-disable-next-line @typescript-eslint/no-misused-promises router.post('/create-article', async (req, res) => { - console.log('pdf-attachments/create-article') + console.log('email-attachment/create-article') const { email, uploadFileId, subject, receivedEmailId } = req.body as { email: string @@ -109,7 +109,7 @@ export function pdfAttachmentsRouter() { analytics.track({ userId: user.id, - event: 'pdf_attachment_create_article', + event: 'email_attachment_create_article', properties: { env: env.server.apiEnv, }, @@ -144,18 +144,21 @@ export function pdfAttachmentsRouter() { ) const uploadFileHash = uploadFileDetails.md5Hash - const pageType = PageType.File + const pageType = + uploadFile.contentType === 'application/pdf' + ? PageType.File + : PageType.Book const title = subject || uploadFileData.fileName const articleToSave: Page = { + id: '', url: uploadFileUrlOverride, - pageType: pageType, + pageType, hash: uploadFileHash, - uploadFileId: uploadFileId, + uploadFileId, title, content: '', userId: user.id, slug: generateSlug(title), - id: '', createdAt: new Date(), savedAt: new Date(), readingProgressPercent: 0, diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 58044982a..43a3eed3e 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -38,10 +38,10 @@ import { notificationRouter } from './routers/notification_router' import { pageRouter } from './routers/page_router' import { contentServiceRouter } from './routers/svc/content' import { emailsServiceRouter } from './routers/svc/emails' +import { emailAttachmentRouter } from './routers/svc/email_attachment' import { integrationsServiceRouter } from './routers/svc/integrations' import { linkServiceRouter } from './routers/svc/links' import { newsletterServiceRouter } from './routers/svc/newsletters' -import { pdfAttachmentsRouter } from './routers/svc/pdf_attachments' import { remindersServiceRouter } from './routers/svc/reminders' import { uploadServiceRouter } from './routers/svc/upload' import { webhooksServiceRouter } from './routers/svc/webhooks' @@ -145,7 +145,7 @@ export const createApp = (): { app.use('/svc/pubsub/webhooks', webhooksServiceRouter()) app.use('/svc/pubsub/integrations', integrationsServiceRouter()) app.use('/svc/reminders', remindersServiceRouter()) - app.use('/svc/pdf-attachments', pdfAttachmentsRouter()) + app.use('/svc/email-attachment', emailAttachmentRouter()) if (env.dev.isLocal) { app.use('/local/debug', localDebugRouter()) diff --git a/packages/inbound-email-handler/src/pdf.ts b/packages/inbound-email-handler/src/attachment.ts similarity index 58% rename from packages/inbound-email-handler/src/pdf.ts rename to packages/inbound-email-handler/src/attachment.ts index 6f60c5d62..273555bae 100644 --- a/packages/inbound-email-handler/src/pdf.ts +++ b/packages/inbound-email-handler/src/attachment.ts @@ -1,41 +1,63 @@ import axios, { AxiosResponse } from 'axios' -import { promisify } from 'util' import * as jwt from 'jsonwebtoken' +import { promisify } from 'util' const signToken = promisify(jwt.sign) +export interface Attachment { + contentType: string + data: Buffer + filename: string | undefined +} + type UploadResponse = { id: string url: string } -export const handlePdfAttachment = async ( +export const isAttachment = (contentType: string, data: Buffer): boolean => { + return ( + (contentType === 'application/pdf' || + contentType === 'application/epub+zip') && + data.length > 0 + ) +} + +export const handleAttachments = async ( email: string, - fileName: string | undefined, - data: Buffer, subject: string, + attachments: Attachment[], receivedEmailId: string ): Promise => { - console.log('handlePdfAttachment', email, fileName) + for await (const attachment of attachments) { + const { contentType, data } = attachment + const filename = + attachment.filename || contentType === 'application/pdf' + ? 'attachment.pdf' + : 'attachment.epub' - fileName = fileName || 'attachment.pdf' - - try { - const uploadResult = await getUploadIdAndSignedUrl(email, fileName) - if (!uploadResult.url || !uploadResult.id) { - console.log('failed to create upload request', uploadResult) - return + try { + const uploadResult = await getUploadIdAndSignedUrl( + email, + filename, + contentType + ) + if (!uploadResult.url || !uploadResult.id) { + console.log('failed to create upload request', uploadResult) + return + } + await uploadToSignedUrl(uploadResult.url, data, contentType) + await createArticle(email, uploadResult.id, subject, receivedEmailId) + } catch (error) { + console.error('handleAttachments error', error) } - await uploadToSignedUrl(uploadResult.url, data) - await createArticle(email, uploadResult.id, subject, receivedEmailId) - } catch (error) { - console.error('handlePdfAttachment error', error) } } const getUploadIdAndSignedUrl = async ( email: string, - fileName: string + fileName: string, + contentType: string ): Promise => { if (process.env.JWT_SECRET === undefined) { throw new Error('JWT_SECRET is not defined') @@ -44,13 +66,14 @@ const getUploadIdAndSignedUrl = async ( const data = { fileName, email, + contentType, } if (process.env.INTERNAL_SVC_ENDPOINT === undefined) { throw new Error('REST_BACKEND_ENDPOINT is not defined') } const response = await axios.post( - `${process.env.INTERNAL_SVC_ENDPOINT}svc/pdf-attachments/upload`, + `${process.env.INTERNAL_SVC_ENDPOINT}svc/email-attachment/upload`, data, { headers: { @@ -64,11 +87,12 @@ const getUploadIdAndSignedUrl = async ( const uploadToSignedUrl = async ( uploadUrl: string, - data: Buffer + data: Buffer, + contentType: string ): Promise => { return axios.put(uploadUrl, data, { headers: { - 'Content-Type': 'application/pdf', + 'Content-Type': contentType, }, maxBodyLength: 1000000000, maxContentLength: 100000000, @@ -97,7 +121,7 @@ const createArticle = async ( throw new Error('REST_BACKEND_ENDPOINT is not defined') } return axios.post( - `${process.env.INTERNAL_SVC_ENDPOINT}svc/pdf-attachments/create-article`, + `${process.env.INTERNAL_SVC_ENDPOINT}svc/email-attachment/create-article`, data, { headers: { diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index 806e1a455..6f88f5ff7 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -12,6 +12,7 @@ import * as jwt from 'jsonwebtoken' import parseHeaders from 'parse-headers' import * as multipart from 'parse-multipart-data' import { promisify } from 'util' +import { Attachment, handleAttachments, isAttachment } from './attachment' import { handleGoogleConfirmationEmail, isGoogleConfirmationEmail, @@ -19,7 +20,6 @@ import { parseAuthor, parseUnsubscribe, } from './newsletter' -import { handlePdfAttachment } from './pdf' interface SaveReceivedEmailResponse { id: string @@ -91,17 +91,14 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( try { const parts = multipart.parse(req.body, 'xYzZY') const parsed: Record = {} - - let pdfAttachment: Buffer | undefined - let pdfAttachmentName: string | undefined + const attachments: Attachment[] = [] for (const part of parts) { const { name, data, type, filename } = part if (name && data) { parsed[name] = data.toString() - } else if (type === 'application/pdf' && data) { - pdfAttachment = data - pdfAttachmentName = filename + } else if (isAttachment(type, data)) { + attachments.push({ data, contentType: type, filename }) } else { console.log('no data or name for ', part) } @@ -157,16 +154,10 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( }) return res.send('ok') } - if (pdfAttachment) { - console.log('handle PDF attachment', from, to) - // save the pdf attachment as an article - await handlePdfAttachment( - to, - pdfAttachmentName, - pdfAttachment, - subject, - receivedEmailId - ) + if (attachments.length > 0) { + console.debug('handle attachments', from, to, subject) + // save the attachments as articles + await handleAttachments(to, subject, attachments, receivedEmailId) return res.send('ok') } // all other emails are considered newsletters