Merge pull request #2083 from omnivore-app/feature/epud-in-email-attachment

feature/epud in email attachment

packages/api/src/routers/svc/email_attachment.ts

@@ -1,34 +1,35 @@
 import express from 'express'
+import { setClaims } from '../../datalayer/helpers'
+import { kx } from '../../datalayer/knex_config'
+import { createPubSubClient } from '../../datalayer/pubsub'
+import { createPage } from '../../elastic/pages'
+import { ArticleSavingRequestStatus, Page } from '../../elastic/types'
 import { env } from '../../env'
 import { PageType, UploadFileStatus } from '../../generated/graphql'
+import { initModels } from '../../server'
+import { getNewsletterEmail } from '../../services/newsletters'
+import { updateReceivedEmail } from '../../services/received_emails'
+import { analytics } from '../../utils/analytics'
+import { getClaimsByToken } from '../../utils/auth'
+import { generateSlug } from '../../utils/helpers'
 import {
   generateUploadFilePathName,
   generateUploadSignedUrl,
   getStorageFileDetails,
   makeStorageFilePublic,
 } from '../../utils/uploads'
-import { initModels } from '../../server'
-import { kx } from '../../datalayer/knex_config'
-import { analytics } from '../../utils/analytics'
-import { getNewsletterEmail } from '../../services/newsletters'
-import { setClaims } from '../../datalayer/helpers'
-import { generateSlug } from '../../utils/helpers'
-import { createPubSubClient } from '../../datalayer/pubsub'
-import { ArticleSavingRequestStatus, Page } from '../../elastic/types'
-import { createPage } from '../../elastic/pages'
-import { getClaimsByToken } from '../../utils/auth'
-import { updateReceivedEmail } from '../../services/received_emails'
 
-export function pdfAttachmentsRouter() {
+export function emailAttachmentRouter() {
   const router = express.Router()
 
   // eslint-disable-next-line @typescript-eslint/no-misused-promises
   router.post('/upload', async (req, res) => {
-    console.log('pdf-attachments/upload')
+    console.log('email-attachment/upload')
 
-    const { email, fileName } = req.body as {
+    const { email, fileName, contentType } = req.body as {
       email: string
       fileName: string
+      contentType: string
     }
 
     const token = req?.headers?.authorization
@@ -45,14 +46,13 @@ export function pdfAttachmentsRouter() {
 
     analytics.track({
       userId: user.id,
-      event: 'pdf_attachment_upload',
+      event: 'email_attachment_upload',
       properties: {
         env: env.server.apiEnv,
       },
     })
 
     try {
-      const contentType = 'application/pdf'
       const models = initModels(kx, false)
       const uploadFileData = await models.uploadFile.create({
         url: '',
@@ -86,7 +86,7 @@ export function pdfAttachmentsRouter() {
 
   // eslint-disable-next-line @typescript-eslint/no-misused-promises
   router.post('/create-article', async (req, res) => {
-    console.log('pdf-attachments/create-article')
+    console.log('email-attachment/create-article')
 
     const { email, uploadFileId, subject, receivedEmailId } = req.body as {
       email: string
@@ -109,7 +109,7 @@ export function pdfAttachmentsRouter() {
 
     analytics.track({
       userId: user.id,
-      event: 'pdf_attachment_create_article',
+      event: 'email_attachment_create_article',
       properties: {
         env: env.server.apiEnv,
       },
@@ -144,18 +144,21 @@ export function pdfAttachmentsRouter() {
       )
 
       const uploadFileHash = uploadFileDetails.md5Hash
-      const pageType = PageType.File
+      const pageType =
+        uploadFile.contentType === 'application/pdf'
+          ? PageType.File
+          : PageType.Book
       const title = subject || uploadFileData.fileName
       const articleToSave: Page = {
+        id: '',
         url: uploadFileUrlOverride,
-        pageType: pageType,
+        pageType,
         hash: uploadFileHash,
-        uploadFileId: uploadFileId,
+        uploadFileId,
         title,
         content: '',
         userId: user.id,
         slug: generateSlug(title),
-        id: '',
         createdAt: new Date(),
         savedAt: new Date(),
         readingProgressPercent: 0,

packages/api/src/server.ts

@@ -38,10 +38,10 @@ import { notificationRouter } from './routers/notification_router'
 import { pageRouter } from './routers/page_router'
 import { contentServiceRouter } from './routers/svc/content'
 import { emailsServiceRouter } from './routers/svc/emails'
+import { emailAttachmentRouter } from './routers/svc/email_attachment'
 import { integrationsServiceRouter } from './routers/svc/integrations'
 import { linkServiceRouter } from './routers/svc/links'
 import { newsletterServiceRouter } from './routers/svc/newsletters'
-import { pdfAttachmentsRouter } from './routers/svc/pdf_attachments'
 import { remindersServiceRouter } from './routers/svc/reminders'
 import { uploadServiceRouter } from './routers/svc/upload'
 import { webhooksServiceRouter } from './routers/svc/webhooks'
@@ -145,7 +145,7 @@ export const createApp = (): {
   app.use('/svc/pubsub/webhooks', webhooksServiceRouter())
   app.use('/svc/pubsub/integrations', integrationsServiceRouter())
   app.use('/svc/reminders', remindersServiceRouter())
-  app.use('/svc/pdf-attachments', pdfAttachmentsRouter())
+  app.use('/svc/email-attachment', emailAttachmentRouter())
 
   if (env.dev.isLocal) {
     app.use('/local/debug', localDebugRouter())

packages/inbound-email-handler/src/attachment.ts

@@ -1,41 +1,63 @@
 import axios, { AxiosResponse } from 'axios'
-import { promisify } from 'util'
 import * as jwt from 'jsonwebtoken'
+import { promisify } from 'util'
 
 const signToken = promisify(jwt.sign)
 
+export interface Attachment {
+  contentType: string
+  data: Buffer
+  filename: string | undefined
+}
+
 type UploadResponse = {
   id: string
   url: string
 }
 
-export const handlePdfAttachment = async (
+export const isAttachment = (contentType: string, data: Buffer): boolean => {
+  return (
+    (contentType === 'application/pdf' ||
+      contentType === 'application/epub+zip') &&
+    data.length > 0
+  )
+}
+
+export const handleAttachments = async (
   email: string,
-  fileName: string | undefined,
-  data: Buffer,
   subject: string,
+  attachments: Attachment[],
   receivedEmailId: string
 ): Promise<void> => {
-  console.log('handlePdfAttachment', email, fileName)
+  for await (const attachment of attachments) {
+    const { contentType, data } = attachment
+    const filename =
+      attachment.filename || contentType === 'application/pdf'
+        ? 'attachment.pdf'
+        : 'attachment.epub'
 
-  fileName = fileName || 'attachment.pdf'
-
-  try {
-    const uploadResult = await getUploadIdAndSignedUrl(email, fileName)
-    if (!uploadResult.url || !uploadResult.id) {
-      console.log('failed to create upload request', uploadResult)
-      return
+    try {
+      const uploadResult = await getUploadIdAndSignedUrl(
+        email,
+        filename,
+        contentType
+      )
+      if (!uploadResult.url || !uploadResult.id) {
+        console.log('failed to create upload request', uploadResult)
+        return
+      }
+      await uploadToSignedUrl(uploadResult.url, data, contentType)
+      await createArticle(email, uploadResult.id, subject, receivedEmailId)
+    } catch (error) {
+      console.error('handleAttachments error', error)
     }
-    await uploadToSignedUrl(uploadResult.url, data)
-    await createArticle(email, uploadResult.id, subject, receivedEmailId)
-  } catch (error) {
-    console.error('handlePdfAttachment error', error)
   }
 }
 
 const getUploadIdAndSignedUrl = async (
   email: string,
-  fileName: string
+  fileName: string,
+  contentType: string
 ): Promise<UploadResponse> => {
   if (process.env.JWT_SECRET === undefined) {
     throw new Error('JWT_SECRET is not defined')
@@ -44,13 +66,14 @@ const getUploadIdAndSignedUrl = async (
   const data = {
     fileName,
     email,
+    contentType,
   }
 
   if (process.env.INTERNAL_SVC_ENDPOINT === undefined) {
     throw new Error('REST_BACKEND_ENDPOINT is not defined')
   }
   const response = await axios.post(
-    `${process.env.INTERNAL_SVC_ENDPOINT}svc/pdf-attachments/upload`,
+    `${process.env.INTERNAL_SVC_ENDPOINT}svc/email-attachment/upload`,
     data,
     {
       headers: {
@@ -64,11 +87,12 @@ const getUploadIdAndSignedUrl = async (
 
 const uploadToSignedUrl = async (
   uploadUrl: string,
-  data: Buffer
+  data: Buffer,
+  contentType: string
 ): Promise<AxiosResponse> => {
   return axios.put(uploadUrl, data, {
     headers: {
-      'Content-Type': 'application/pdf',
+      'Content-Type': contentType,
     },
     maxBodyLength: 1000000000,
     maxContentLength: 100000000,
@@ -97,7 +121,7 @@ const createArticle = async (
     throw new Error('REST_BACKEND_ENDPOINT is not defined')
   }
   return axios.post(
-    `${process.env.INTERNAL_SVC_ENDPOINT}svc/pdf-attachments/create-article`,
+    `${process.env.INTERNAL_SVC_ENDPOINT}svc/email-attachment/create-article`,
     data,
     {
       headers: {

packages/inbound-email-handler/src/index.ts

@@ -12,6 +12,7 @@ import * as jwt from 'jsonwebtoken'
 import parseHeaders from 'parse-headers'
 import * as multipart from 'parse-multipart-data'
 import { promisify } from 'util'
+import { Attachment, handleAttachments, isAttachment } from './attachment'
 import {
   handleGoogleConfirmationEmail,
   isGoogleConfirmationEmail,
@@ -19,7 +20,6 @@ import {
   parseAuthor,
   parseUnsubscribe,
 } from './newsletter'
-import { handlePdfAttachment } from './pdf'
 
 interface SaveReceivedEmailResponse {
   id: string
@@ -91,17 +91,14 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
     try {
       const parts = multipart.parse(req.body, 'xYzZY')
       const parsed: Record<string, string> = {}
-
-      let pdfAttachment: Buffer | undefined
-      let pdfAttachmentName: string | undefined
+      const attachments: Attachment[] = []
 
       for (const part of parts) {
         const { name, data, type, filename } = part
         if (name && data) {
           parsed[name] = data.toString()
-        } else if (type === 'application/pdf' && data) {
-          pdfAttachment = data
-          pdfAttachmentName = filename
+        } else if (isAttachment(type, data)) {
+          attachments.push({ data, contentType: type, filename })
         } else {
           console.log('no data or name for ', part)
         }
@@ -157,16 +154,10 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
           })
           return res.send('ok')
         }
-        if (pdfAttachment) {
-          console.log('handle PDF attachment', from, to)
-          // save the pdf attachment as an article
-          await handlePdfAttachment(
-            to,
-            pdfAttachmentName,
-            pdfAttachment,
-            subject,
-            receivedEmailId
-          )
+        if (attachments.length > 0) {
+          console.debug('handle attachments', from, to, subject)
+          // save the attachments as articles
+          await handleAttachments(to, subject, attachments, receivedEmailId)
           return res.send('ok')
         }
         // all other emails are considered newsletters