Merge pull request #3811 from omnivore-app/feature/pdf-state-job

feat: always update the state of the pdf item to succeeded/failed, even if we fail to parse the content
This commit is contained in:
Hongbo Wu 2024-04-15 13:54:28 +08:00 committed by GitHub
commit cc599fba10
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 103 additions and 145 deletions

View file

@ -8,16 +8,17 @@ import { findUploadFileById, setFileUploadComplete } from './upload_file'
export interface UpdateContentMessage {
fileId: string
content: string
content?: string
title?: string
author?: string
description?: string
state?: LibraryItemState
}
export const isUpdateContentMessage = (
data: any
): data is UpdateContentMessage => {
return 'fileId' in data && 'content' in data
return 'fileId' in data
}
export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
@ -51,16 +52,16 @@ export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
}
const itemToUpdate: QueryDeepPartialEntity<LibraryItem> = {
originalContent: msg.content,
title: msg.title,
description: msg.description,
author: msg.author,
// content may not be present if we failed to parse the file
readableContent: msg.content,
// This event is fired after the file is fully uploaded,
// so along with updating content, we mark it as
// succeeded or failed based on the message state
state: msg.state || LibraryItemState.Succeeded,
}
if (msg.title) itemToUpdate.title = msg.title
if (msg.author) itemToUpdate.author = msg.author
if (msg.description) itemToUpdate.description = msg.description
// This event is fired after the file is fully uploaded,
// so along with updating content, we mark it as
// succeeded.
itemToUpdate.state = LibraryItemState.Succeeded
try {
const uploadFileData = await setFileUploadComplete(
@ -80,7 +81,8 @@ export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
logger.info('Updating library item text', {
id: libraryItem.id,
result,
content: msg.content.substring(0, 20),
content: msg.content?.substring(0, 20),
state: msg.state,
})
return true

View file

@ -18,6 +18,7 @@ import {
} from '../utils/uploads'
import { validateUrl } from './create_page_save_request'
import { createOrUpdateLibraryItem } from './library_item'
import { v4 as uuid } from 'uuid'
const isFileUrl = (url: string): boolean => {
const parsedUrl = new URL(url)
@ -90,8 +91,18 @@ export const uploadFile = async (
}
}
let url = input.url
const uploadFileId = uuid()
const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName)
// If this is a file URL, we swap in a special URL
if (isFileUrl(url)) {
url = `https://omnivore.app/attachments/${uploadFilePathName}`
}
const uploadFileData = await authTrx((t) =>
t.getRepository(UploadFile).save({
id: uploadFileId,
url: input.url,
user: { id: uid },
fileName,
@ -99,24 +110,11 @@ export const uploadFile = async (
contentType: input.contentType,
})
)
const uploadFileId = uploadFileData.id
const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName)
const uploadSignedUrl = await generateUploadSignedUrl(
uploadFilePathName,
input.contentType
)
// If this is a file URL, we swap in a special URL
const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}`
if (isFileUrl(input.url)) {
await authTrx(async (tx) => {
await tx.getRepository(UploadFile).update(uploadFileId, {
url: attachmentUrl,
status: UploadFileStatus.Initialized,
})
})
}
const itemType = itemTypeForContentType(input.contentType)
if (input.createPageEntry) {
// If we have a file:// URL, don't try to match it
@ -125,7 +123,7 @@ export const uploadFile = async (
const item = await createOrUpdateLibraryItem(
{
id: input.clientRequestId || undefined,
originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url,
originalUrl: url,
user: { id: uid },
title,
readableContent: '',

View file

@ -1,7 +1,7 @@
import { GetSignedUrlConfig, Storage } from '@google-cloud/storage'
import * as Sentry from '@sentry/serverless'
import { parsePdf } from './pdf'
import { queueUpdatePageJob } from './job'
import { queueUpdatePageJob, State } from './job'
Sentry.GCPFunction.init({
dsn: process.env.SENTRY_DSN,
@ -50,10 +50,11 @@ const getDocumentUrl = async (
export const updatePageContent = async (
fileId: string,
content: string,
content?: string,
title?: string,
author?: string,
description?: string
description?: string,
state?: State
): Promise<string | undefined> => {
const job = await queueUpdatePageJob({
fileId,
@ -61,6 +62,7 @@ export const updatePageContent = async (
title,
author,
description,
state,
})
return job.id
}
@ -86,45 +88,68 @@ export const pdfHandler = Sentry.GCPFunction.wrapHttpFunction(
if ('message' in req.body && 'data' in req.body.message) {
const pubSubMessage = req.body.message.data as string
const data = getStorageEventData(pubSubMessage)
if (data) {
try {
if (shouldHandle(data)) {
console.log('handling pdf data', data)
const url = await getDocumentUrl(data)
console.log('PDF url: ', url)
if (!url) {
console.log('Could not fetch PDF', data.bucket, data.name)
return res.status(404).send('Could not fetch PDF')
}
const parsed = await parsePdf(url)
const result = await updatePageContent(
data.name,
parsed.content,
parsed.title,
parsed.author,
parsed.description
)
console.log(
'publish result',
result,
'title',
parsed.title,
'author',
parsed.author
)
} else {
console.log('not handling pdf data', data)
}
} catch (err) {
console.log('error handling event', { err, data })
return res.status(500).send('Error handling event')
}
if (!data) {
console.log('no data found in pubsub message')
return res.send('ok')
}
if (!shouldHandle(data)) {
console.log('not handling pdf data', data)
return res.send('ok')
}
console.log('handling pdf data', data)
let content,
title,
author,
description,
state: State = 'SUCCEEDED' // Default to succeeded even if we fail to parse
try {
const url = await getDocumentUrl(data)
console.log('PDF url: ', url)
if (!url) {
console.log('Could not fetch PDF', data.bucket, data.name)
// If we can't fetch the PDF, mark it as failed
state = 'FAILED'
return res.status(404).send('Could not fetch PDF')
}
// Parse the PDF to update the content and metadata
const parsed = await parsePdf(url)
content = parsed.content
title = parsed.title
author = parsed.author
description = parsed.description
} catch (err) {
console.log('error parsing pdf', { err, data })
return res.status(500).send('Error parsing pdf')
} finally {
// Always update the state, even if we fail to parse
const result = await updatePageContent(
data.name,
content,
title,
author,
description,
state
)
console.log(
'publish result',
result,
'title',
title,
'author',
author,
'state',
state
)
}
} else {
console.log('no pubsub message')
}
res.send('ok')
}
)

View file

@ -8,12 +8,15 @@ const queue = new Queue(QUEUE_NAME, {
connection: redisDataSource.queueRedisClient,
})
export type State = 'SUCCEEDED' | 'FAILED'
type UpdatePageJobData = {
fileId: string
content: string
content?: string
title?: string
author?: string
description?: string
state?: State
}
export const queueUpdatePageJob = async (data: UpdatePageJobData) => {

View file

@ -1,4 +1,5 @@
import Redis, { RedisOptions } from 'ioredis'
import 'dotenv/config'
export type RedisDataSourceOptions = {
REDIS_URL?: string

View file

@ -23,7 +23,6 @@
"deploy": "yarn build && yarn gcloud-deploy"
},
"devDependencies": {
"@types/fluent-ffmpeg": "^2.1.20",
"@types/html-to-text": "^8.1.1",
"@types/natural": "^5.1.1",
"@types/node": "^14.11.2",
@ -33,13 +32,11 @@
"mocha": "^10.0.0"
},
"dependencies": {
"@ffmpeg-installer/ffmpeg": "^1.1.0",
"@google-cloud/functions-framework": "3.1.2",
"@google-cloud/storage": "^7.0.1",
"@sentry/serverless": "^7.77.0",
"axios": "^0.27.2",
"dotenv": "^16.0.1",
"fluent-ffmpeg": "^2.1.2",
"html-to-text": "^8.2.1",
"ioredis": "^5.3.2",
"jsonwebtoken": "^8.5.1",

View file

@ -2633,60 +2633,6 @@
dependencies:
text-decoding "^1.0.0"
"@ffmpeg-installer/darwin-arm64@4.1.5":
version "4.1.5"
resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz#b7b5c262dd96d1aea4807514e1cdcf6e11f82743"
integrity sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA==
"@ffmpeg-installer/darwin-x64@4.1.0":
version "4.1.0"
resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz#48e1706c690e628148482bfb64acb67472089aaa"
integrity sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==
"@ffmpeg-installer/ffmpeg@^1.1.0":
version "1.1.0"
resolved "https://registry.yarnpkg.com/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.1.0.tgz#87fdb9e7d180e8d78f7903f9441e36f978938a90"
integrity sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg==
optionalDependencies:
"@ffmpeg-installer/darwin-arm64" "4.1.5"
"@ffmpeg-installer/darwin-x64" "4.1.0"
"@ffmpeg-installer/linux-arm" "4.1.3"
"@ffmpeg-installer/linux-arm64" "4.1.4"
"@ffmpeg-installer/linux-ia32" "4.1.0"
"@ffmpeg-installer/linux-x64" "4.1.0"
"@ffmpeg-installer/win32-ia32" "4.1.0"
"@ffmpeg-installer/win32-x64" "4.1.0"
"@ffmpeg-installer/linux-arm64@4.1.4":
version "4.1.4"
resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz#7219f3f901bb67f7926cb060b56b6974a6cad29f"
integrity sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==
"@ffmpeg-installer/linux-arm@4.1.3":
version "4.1.3"
resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz#c554f105ed5f10475ec25d7bec94926ce18db4c1"
integrity sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==
"@ffmpeg-installer/linux-ia32@4.1.0":
version "4.1.0"
resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz#adad70b0d0d9d8d813983d6e683c5a338a75e442"
integrity sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==
"@ffmpeg-installer/linux-x64@4.1.0":
version "4.1.0"
resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz#b4a5d89c4e12e6d9306dbcdc573df716ec1c4323"
integrity sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==
"@ffmpeg-installer/win32-ia32@4.1.0":
version "4.1.0"
resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz#6eac4fb691b64c02e7a116c1e2d167f3e9b40638"
integrity sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==
"@ffmpeg-installer/win32-x64@4.1.0":
version "4.1.0"
resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz#17e8699b5798d4c60e36e2d6326a8ebe5e95a2c5"
integrity sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==
"@firebase/app-types@0.9.0":
version "0.9.0"
resolved "https://registry.yarnpkg.com/@firebase/app-types/-/app-types-0.9.0.tgz#35b5c568341e9e263b29b3d2ba0e9cfc9ec7f01e"
@ -7986,13 +7932,6 @@
resolved "https://registry.yarnpkg.com/@types/firefox-webext-browser/-/firefox-webext-browser-94.0.1.tgz#52afb975253dc0fd350d5d58c7fe9fd1a01f64a1"
integrity sha512-I6iHRQJSTZ+gYt2IxdH2RRAMvcUyK8v5Ig7fHQR0IwUNYP7hz9+cziBVIKxLCO6XI7fiyRsNOWObfl3/4Js2Lg==
"@types/fluent-ffmpeg@^2.1.20":
version "2.1.20"
resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz#3b5f42fc8263761d58284fa46ee6759a64ce54ac"
integrity sha512-B+OvhCdJ3LgEq2PhvWNOiB/EfwnXLElfMCgc4Z1K5zXgSfo9I6uGKwR/lqmNPFQuebNnes7re3gqkV77SyypLg==
dependencies:
"@types/node" "*"
"@types/fs-extra@^11.0.1":
version "11.0.1"
resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-11.0.1.tgz#f542ec47810532a8a252127e6e105f487e0a6ea5"
@ -10404,11 +10343,6 @@ async-validator@^4.1.0:
resolved "https://registry.yarnpkg.com/async-validator/-/async-validator-4.2.5.tgz#c96ea3332a521699d0afaaceed510a54656c6339"
integrity sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg==
async@>=0.2.9:
version "3.2.4"
resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c"
integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==
async@^2.6.2:
version "2.6.4"
resolved "https://registry.yarnpkg.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221"
@ -11679,6 +11613,7 @@ caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001251, caniuse-lite@^1.0.300012
version "1.0.30001600"
resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001600.tgz"
integrity sha512-+2S9/2JFhYmYaDpZvo0lKkfvuKIglrx68MwOBqMGHhQsNkLjB5xtc/TGoEPs+MxjSyN/72qer2g97nzR641mOQ==
capital-case@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669"
@ -16064,14 +15999,6 @@ flatted@^3.1.0:
resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.2.tgz#64bfed5cb68fe3ca78b3eb214ad97b63bedce561"
integrity sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==
fluent-ffmpeg@^2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz#c952de2240f812ebda0aa8006d7776ee2acf7d74"
integrity sha512-IZTB4kq5GK0DPp7sGQ0q/BWurGHffRtQQwVkiqDgeO6wYJLLV5ZhgNOQ65loZxxuPMKZKZcICCUnaGtlxBiR0Q==
dependencies:
async ">=0.2.9"
which "^1.1.1"
flush-write-stream@^1.0.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"
@ -26575,6 +26502,11 @@ rc@^1.2.7, rc@^1.2.8:
minimist "^1.2.0"
strip-json-comments "~2.0.1"
re-resizable@^6.9.11:
version "6.9.11"
resolved "https://registry.yarnpkg.com/re-resizable/-/re-resizable-6.9.11.tgz#f356e27877f12d926d076ab9ad9ff0b95912b475"
integrity sha512-a3hiLWck/NkmyLvGWUuvkAmN1VhwAz4yOhS6FdMTaxCUVN9joIWkT11wsO68coG/iEYuwn+p/7qAmfQzRhiPLQ==
react-color@^2.19.3:
version "2.19.3"
resolved "https://registry.yarnpkg.com/react-color/-/react-color-2.19.3.tgz#ec6c6b4568312a3c6a18420ab0472e146aa5683d"
@ -31764,7 +31696,7 @@ which@2.0.2, which@^2.0.1, which@^2.0.2:
dependencies:
isexe "^2.0.0"
which@^1.1.1, which@^1.2.14, which@^1.2.9:
which@^1.2.14, which@^1.2.9:
version "1.3.1"
resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==