diff --git a/packages/api/src/services/update_pdf_content.ts b/packages/api/src/services/update_pdf_content.ts index 75d3e8438..c96b67dce 100644 --- a/packages/api/src/services/update_pdf_content.ts +++ b/packages/api/src/services/update_pdf_content.ts @@ -8,16 +8,17 @@ import { findUploadFileById, setFileUploadComplete } from './upload_file' export interface UpdateContentMessage { fileId: string - content: string + content?: string title?: string author?: string description?: string + state?: LibraryItemState } export const isUpdateContentMessage = ( data: any ): data is UpdateContentMessage => { - return 'fileId' in data && 'content' in data + return 'fileId' in data } export const updateContentForFileItem = async (msg: UpdateContentMessage) => { @@ -51,16 +52,16 @@ export const updateContentForFileItem = async (msg: UpdateContentMessage) => { } const itemToUpdate: QueryDeepPartialEntity = { - originalContent: msg.content, + title: msg.title, + description: msg.description, + author: msg.author, + // content may not be present if we failed to parse the file + readableContent: msg.content, + // This event is fired after the file is fully uploaded, + // so along with updating content, we mark it as + // succeeded or failed based on the message state + state: msg.state || LibraryItemState.Succeeded, } - if (msg.title) itemToUpdate.title = msg.title - if (msg.author) itemToUpdate.author = msg.author - if (msg.description) itemToUpdate.description = msg.description - - // This event is fired after the file is fully uploaded, - // so along with updating content, we mark it as - // succeeded. - itemToUpdate.state = LibraryItemState.Succeeded try { const uploadFileData = await setFileUploadComplete( @@ -80,7 +81,8 @@ export const updateContentForFileItem = async (msg: UpdateContentMessage) => { logger.info('Updating library item text', { id: libraryItem.id, result, - content: msg.content.substring(0, 20), + content: msg.content?.substring(0, 20), + state: msg.state, }) return true diff --git a/packages/api/src/services/upload_file.ts b/packages/api/src/services/upload_file.ts index 1d284916f..148614352 100644 --- a/packages/api/src/services/upload_file.ts +++ b/packages/api/src/services/upload_file.ts @@ -18,6 +18,7 @@ import { } from '../utils/uploads' import { validateUrl } from './create_page_save_request' import { createOrUpdateLibraryItem } from './library_item' +import { v4 as uuid } from 'uuid' const isFileUrl = (url: string): boolean => { const parsedUrl = new URL(url) @@ -90,8 +91,18 @@ export const uploadFile = async ( } } + let url = input.url + + const uploadFileId = uuid() + const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName) + // If this is a file URL, we swap in a special URL + if (isFileUrl(url)) { + url = `https://omnivore.app/attachments/${uploadFilePathName}` + } + const uploadFileData = await authTrx((t) => t.getRepository(UploadFile).save({ + id: uploadFileId, url: input.url, user: { id: uid }, fileName, @@ -99,24 +110,11 @@ export const uploadFile = async ( contentType: input.contentType, }) ) - const uploadFileId = uploadFileData.id - const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName) const uploadSignedUrl = await generateUploadSignedUrl( uploadFilePathName, input.contentType ) - // If this is a file URL, we swap in a special URL - const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` - if (isFileUrl(input.url)) { - await authTrx(async (tx) => { - await tx.getRepository(UploadFile).update(uploadFileId, { - url: attachmentUrl, - status: UploadFileStatus.Initialized, - }) - }) - } - const itemType = itemTypeForContentType(input.contentType) if (input.createPageEntry) { // If we have a file:// URL, don't try to match it @@ -125,7 +123,7 @@ export const uploadFile = async ( const item = await createOrUpdateLibraryItem( { id: input.clientRequestId || undefined, - originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, + originalUrl: url, user: { id: uid }, title, readableContent: '', diff --git a/packages/pdf-handler/src/index.ts b/packages/pdf-handler/src/index.ts index e1623372b..cf3f3359c 100644 --- a/packages/pdf-handler/src/index.ts +++ b/packages/pdf-handler/src/index.ts @@ -1,7 +1,7 @@ import { GetSignedUrlConfig, Storage } from '@google-cloud/storage' import * as Sentry from '@sentry/serverless' import { parsePdf } from './pdf' -import { queueUpdatePageJob } from './job' +import { queueUpdatePageJob, State } from './job' Sentry.GCPFunction.init({ dsn: process.env.SENTRY_DSN, @@ -50,10 +50,11 @@ const getDocumentUrl = async ( export const updatePageContent = async ( fileId: string, - content: string, + content?: string, title?: string, author?: string, - description?: string + description?: string, + state?: State ): Promise => { const job = await queueUpdatePageJob({ fileId, @@ -61,6 +62,7 @@ export const updatePageContent = async ( title, author, description, + state, }) return job.id } @@ -86,45 +88,68 @@ export const pdfHandler = Sentry.GCPFunction.wrapHttpFunction( if ('message' in req.body && 'data' in req.body.message) { const pubSubMessage = req.body.message.data as string const data = getStorageEventData(pubSubMessage) - if (data) { - try { - if (shouldHandle(data)) { - console.log('handling pdf data', data) - - const url = await getDocumentUrl(data) - console.log('PDF url: ', url) - if (!url) { - console.log('Could not fetch PDF', data.bucket, data.name) - return res.status(404).send('Could not fetch PDF') - } - - const parsed = await parsePdf(url) - const result = await updatePageContent( - data.name, - parsed.content, - parsed.title, - parsed.author, - parsed.description - ) - console.log( - 'publish result', - result, - 'title', - parsed.title, - 'author', - parsed.author - ) - } else { - console.log('not handling pdf data', data) - } - } catch (err) { - console.log('error handling event', { err, data }) - return res.status(500).send('Error handling event') - } + if (!data) { + console.log('no data found in pubsub message') + return res.send('ok') + } + + if (!shouldHandle(data)) { + console.log('not handling pdf data', data) + return res.send('ok') + } + + console.log('handling pdf data', data) + + let content, + title, + author, + description, + state: State = 'SUCCEEDED' // Default to succeeded even if we fail to parse + + try { + const url = await getDocumentUrl(data) + console.log('PDF url: ', url) + if (!url) { + console.log('Could not fetch PDF', data.bucket, data.name) + // If we can't fetch the PDF, mark it as failed + state = 'FAILED' + + return res.status(404).send('Could not fetch PDF') + } + + // Parse the PDF to update the content and metadata + const parsed = await parsePdf(url) + content = parsed.content + title = parsed.title + author = parsed.author + description = parsed.description + } catch (err) { + console.log('error parsing pdf', { err, data }) + + return res.status(500).send('Error parsing pdf') + } finally { + // Always update the state, even if we fail to parse + const result = await updatePageContent( + data.name, + content, + title, + author, + description, + state + ) + console.log( + 'publish result', + result, + 'title', + title, + 'author', + author, + 'state', + state + ) } - } else { - console.log('no pubsub message') } + res.send('ok') } ) diff --git a/packages/pdf-handler/src/job.ts b/packages/pdf-handler/src/job.ts index d244079d3..ca404aa0f 100644 --- a/packages/pdf-handler/src/job.ts +++ b/packages/pdf-handler/src/job.ts @@ -8,12 +8,15 @@ const queue = new Queue(QUEUE_NAME, { connection: redisDataSource.queueRedisClient, }) +export type State = 'SUCCEEDED' | 'FAILED' + type UpdatePageJobData = { fileId: string - content: string + content?: string title?: string author?: string description?: string + state?: State } export const queueUpdatePageJob = async (data: UpdatePageJobData) => { diff --git a/packages/pdf-handler/src/redis_data_source.ts b/packages/pdf-handler/src/redis_data_source.ts index 1a95f9d6e..1c02c2a47 100644 --- a/packages/pdf-handler/src/redis_data_source.ts +++ b/packages/pdf-handler/src/redis_data_source.ts @@ -1,4 +1,5 @@ import Redis, { RedisOptions } from 'ioredis' +import 'dotenv/config' export type RedisDataSourceOptions = { REDIS_URL?: string diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json index f8e669fc9..bd97a0260 100644 --- a/packages/text-to-speech/package.json +++ b/packages/text-to-speech/package.json @@ -23,7 +23,6 @@ "deploy": "yarn build && yarn gcloud-deploy" }, "devDependencies": { - "@types/fluent-ffmpeg": "^2.1.20", "@types/html-to-text": "^8.1.1", "@types/natural": "^5.1.1", "@types/node": "^14.11.2", @@ -33,13 +32,11 @@ "mocha": "^10.0.0" }, "dependencies": { - "@ffmpeg-installer/ffmpeg": "^1.1.0", "@google-cloud/functions-framework": "3.1.2", "@google-cloud/storage": "^7.0.1", "@sentry/serverless": "^7.77.0", "axios": "^0.27.2", "dotenv": "^16.0.1", - "fluent-ffmpeg": "^2.1.2", "html-to-text": "^8.2.1", "ioredis": "^5.3.2", "jsonwebtoken": "^8.5.1", diff --git a/yarn.lock b/yarn.lock index 074fd4c76..71a8f77bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2633,60 +2633,6 @@ dependencies: text-decoding "^1.0.0" -"@ffmpeg-installer/darwin-arm64@4.1.5": - version "4.1.5" - resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz#b7b5c262dd96d1aea4807514e1cdcf6e11f82743" - integrity sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA== - -"@ffmpeg-installer/darwin-x64@4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz#48e1706c690e628148482bfb64acb67472089aaa" - integrity sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw== - -"@ffmpeg-installer/ffmpeg@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.1.0.tgz#87fdb9e7d180e8d78f7903f9441e36f978938a90" - integrity sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg== - optionalDependencies: - "@ffmpeg-installer/darwin-arm64" "4.1.5" - "@ffmpeg-installer/darwin-x64" "4.1.0" - "@ffmpeg-installer/linux-arm" "4.1.3" - "@ffmpeg-installer/linux-arm64" "4.1.4" - "@ffmpeg-installer/linux-ia32" "4.1.0" - "@ffmpeg-installer/linux-x64" "4.1.0" - "@ffmpeg-installer/win32-ia32" "4.1.0" - "@ffmpeg-installer/win32-x64" "4.1.0" - -"@ffmpeg-installer/linux-arm64@4.1.4": - version "4.1.4" - resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz#7219f3f901bb67f7926cb060b56b6974a6cad29f" - integrity sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg== - -"@ffmpeg-installer/linux-arm@4.1.3": - version "4.1.3" - resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz#c554f105ed5f10475ec25d7bec94926ce18db4c1" - integrity sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg== - -"@ffmpeg-installer/linux-ia32@4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz#adad70b0d0d9d8d813983d6e683c5a338a75e442" - integrity sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ== - -"@ffmpeg-installer/linux-x64@4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz#b4a5d89c4e12e6d9306dbcdc573df716ec1c4323" - integrity sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A== - -"@ffmpeg-installer/win32-ia32@4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz#6eac4fb691b64c02e7a116c1e2d167f3e9b40638" - integrity sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw== - -"@ffmpeg-installer/win32-x64@4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz#17e8699b5798d4c60e36e2d6326a8ebe5e95a2c5" - integrity sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg== - "@firebase/app-types@0.9.0": version "0.9.0" resolved "https://registry.yarnpkg.com/@firebase/app-types/-/app-types-0.9.0.tgz#35b5c568341e9e263b29b3d2ba0e9cfc9ec7f01e" @@ -7986,13 +7932,6 @@ resolved "https://registry.yarnpkg.com/@types/firefox-webext-browser/-/firefox-webext-browser-94.0.1.tgz#52afb975253dc0fd350d5d58c7fe9fd1a01f64a1" integrity sha512-I6iHRQJSTZ+gYt2IxdH2RRAMvcUyK8v5Ig7fHQR0IwUNYP7hz9+cziBVIKxLCO6XI7fiyRsNOWObfl3/4Js2Lg== -"@types/fluent-ffmpeg@^2.1.20": - version "2.1.20" - resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz#3b5f42fc8263761d58284fa46ee6759a64ce54ac" - integrity sha512-B+OvhCdJ3LgEq2PhvWNOiB/EfwnXLElfMCgc4Z1K5zXgSfo9I6uGKwR/lqmNPFQuebNnes7re3gqkV77SyypLg== - dependencies: - "@types/node" "*" - "@types/fs-extra@^11.0.1": version "11.0.1" resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-11.0.1.tgz#f542ec47810532a8a252127e6e105f487e0a6ea5" @@ -10404,11 +10343,6 @@ async-validator@^4.1.0: resolved "https://registry.yarnpkg.com/async-validator/-/async-validator-4.2.5.tgz#c96ea3332a521699d0afaaceed510a54656c6339" integrity sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg== -async@>=0.2.9: - version "3.2.4" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" - integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== - async@^2.6.2: version "2.6.4" resolved "https://registry.yarnpkg.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221" @@ -11679,6 +11613,7 @@ caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001251, caniuse-lite@^1.0.300012 version "1.0.30001600" resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001600.tgz" integrity sha512-+2S9/2JFhYmYaDpZvo0lKkfvuKIglrx68MwOBqMGHhQsNkLjB5xtc/TGoEPs+MxjSyN/72qer2g97nzR641mOQ== + capital-case@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669" @@ -16064,14 +15999,6 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.2.tgz#64bfed5cb68fe3ca78b3eb214ad97b63bedce561" integrity sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA== -fluent-ffmpeg@^2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz#c952de2240f812ebda0aa8006d7776ee2acf7d74" - integrity sha512-IZTB4kq5GK0DPp7sGQ0q/BWurGHffRtQQwVkiqDgeO6wYJLLV5ZhgNOQ65loZxxuPMKZKZcICCUnaGtlxBiR0Q== - dependencies: - async ">=0.2.9" - which "^1.1.1" - flush-write-stream@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8" @@ -26575,6 +26502,11 @@ rc@^1.2.7, rc@^1.2.8: minimist "^1.2.0" strip-json-comments "~2.0.1" +re-resizable@^6.9.11: + version "6.9.11" + resolved "https://registry.yarnpkg.com/re-resizable/-/re-resizable-6.9.11.tgz#f356e27877f12d926d076ab9ad9ff0b95912b475" + integrity sha512-a3hiLWck/NkmyLvGWUuvkAmN1VhwAz4yOhS6FdMTaxCUVN9joIWkT11wsO68coG/iEYuwn+p/7qAmfQzRhiPLQ== + react-color@^2.19.3: version "2.19.3" resolved "https://registry.yarnpkg.com/react-color/-/react-color-2.19.3.tgz#ec6c6b4568312a3c6a18420ab0472e146aa5683d" @@ -31764,7 +31696,7 @@ which@2.0.2, which@^2.0.1, which@^2.0.2: dependencies: isexe "^2.0.0" -which@^1.1.1, which@^1.2.14, which@^1.2.9: +which@^1.2.14, which@^1.2.9: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==