From eddf9206d08420b345fb6124c17375e846aaaaee Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 22 Apr 2024 19:10:43 +0800 Subject: [PATCH] do not store original content in db --- .../api/src/jobs/process-youtube-video.ts | 16 ++--- packages/api/src/jobs/rss/refreshFeed.ts | 2 +- packages/api/src/jobs/save_page.ts | 61 ++++++----------- packages/api/src/resolvers/article/index.ts | 1 - packages/api/src/services/library_item.ts | 35 +++++++++- packages/api/src/services/recommendation.ts | 1 - packages/api/src/services/reports.ts | 1 - packages/api/src/services/save_email.ts | 1 - packages/api/src/services/save_page.ts | 4 -- packages/api/src/utils/uploads.ts | 66 ++++++++++++++----- 10 files changed, 108 insertions(+), 80 deletions(-) diff --git a/packages/api/src/jobs/process-youtube-video.ts b/packages/api/src/jobs/process-youtube-video.ts index 7369aedb6..adbfdee71 100644 --- a/packages/api/src/jobs/process-youtube-video.ts +++ b/packages/api/src/jobs/process-youtube-video.ts @@ -321,11 +321,7 @@ export const processYouTubeVideo = async ( undefined, jobData.userId ) - if ( - !libraryItem || - libraryItem.state !== LibraryItemState.Succeeded || - !libraryItem.originalContent - ) { + if (!libraryItem || libraryItem.state !== LibraryItemState.Succeeded) { logger.info( `Not ready to get YouTube metadata job state: ${ libraryItem?.state ?? 'null' @@ -382,7 +378,7 @@ export const processYouTubeVideo = async ( // enqueue a job to process the full transcript const updatedContent = await addTranscriptPlaceholdReadableContent( libraryItem.originalUrl, - libraryItem.originalContent + libraryItem.readableContent ) if (updatedContent) { @@ -438,11 +434,7 @@ export const processYouTubeTranscript = async ( undefined, jobData.userId ) - if ( - !libraryItem || - libraryItem.state !== LibraryItemState.Succeeded || - !libraryItem.originalContent - ) { + if (!libraryItem || libraryItem.state !== LibraryItemState.Succeeded) { logger.info( `Not ready to get YouTube metadata job state: ${ libraryItem?.state ?? 'null' @@ -481,7 +473,7 @@ export const processYouTubeTranscript = async ( ) const updatedContent = await addTranscriptToReadableContent( libraryItem.originalUrl, - libraryItem.originalContent, + libraryItem.readableContent, transcriptHTML ) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 06faefde2..4b7995d5c 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -381,7 +381,7 @@ const createItemWithFeedContent = async ( rssFeedUrl: feedUrl, savedAt: item.isoDate, publishedAt: item.isoDate, - originalContent: feedContent || '', + originalContent: '', source: 'rss-feeder', state: ArticleSavingRequestStatus.ContentNotFetched, clientRequestId: '', diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 3f2839bbe..07869306d 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -7,12 +7,12 @@ import { CreateLabelInput, } from '../generated/graphql' import { userRepository } from '../repository/user' +import { downloadOriginalContent } from '../services/library_item' import { saveFile } from '../services/save_file' import { savePage } from '../services/save_page' import { uploadFile } from '../services/upload_file' import { logError, logger } from '../utils/logger' import { downloadFromUrl, uploadToSignedUrl } from '../utils/uploads' -import { downloadStringFromBucket } from '../utils/uploads' const signToken = promisify(jwt.sign) @@ -128,29 +128,27 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { finalUrl, title, contentType, - urlHash, + state, } = data - let isImported, - isSaved, - state = data.state + let isImported, isSaved - try { - logger.info('savePageJob', { + logger.info('savePageJob', { + userId, + url, + finalUrl, + }) + + const user = await userRepository.findById(userId) + if (!user) { + logger.error('Unable to save job, user can not be found.', { userId, url, - finalUrl, }) + // if the user is not found, we do not retry + return false + } - const user = await userRepository.findById(userId) - if (!user) { - logger.error('Unable to save job, user can not be found.', { - userId, - url, - }) - // if the user is not found, we do not retry - return false - } - + try { // for pdf content, we need to upload the pdf if (contentType === 'application/pdf') { const uploadResult = await uploadPdf( @@ -163,7 +161,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { { url: finalUrl, uploadFileId: uploadResult.uploadFileId, - state: state ? (state as ArticleSavingRequestStatus) : undefined, + state: (state as ArticleSavingRequestStatus) || undefined, labels, source, folder, @@ -183,25 +181,8 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { return true } - let originalContent - if (!urlHash) { - logger.info(`content is not uploaded: ${finalUrl}`) - // set the state to failed if we don't have content - originalContent = 'Failed to fetch content' - state = ArticleSavingRequestStatus.Failed - } else { - // download content from the bucket - const downloaded = await downloadStringFromBucket( - `originalContent/${urlHash}` - ) - if (!downloaded) { - logger.error('error while downloading content from bucket') - originalContent = 'Failed to fetch content' - state = ArticleSavingRequestStatus.Failed - } else { - originalContent = downloaded - } - } + // download content from the bucket + const originalContent = (await downloadOriginalContent(finalUrl)).toString() // for non-pdf content, we need to save the page const result = await savePage( @@ -210,8 +191,8 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { clientRequestId: articleSavingRequestId, title, originalContent, - state: state ? (state as ArticleSavingRequestStatus) : undefined, - labels: labels, + state: (state as ArticleSavingRequestStatus) || undefined, + labels, rssFeedUrl, savedAt: savedAt ? new Date(savedAt) : new Date(), publishedAt: publishedAt ? new Date(publishedAt) : null, diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 026cb7988..f1da74eef 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -302,7 +302,6 @@ export const createArticleResolver = authorized< userId: uid, slug, croppedPathname, - originalHtml: domContent, itemType, preparedDocument, uploadFileHash, diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index a7376da30..0ee80ecee 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -21,9 +21,14 @@ import { redisDataSource } from '../redis_data_source' import { authTrx, getColumns, queryBuilderToRawSql } from '../repository' import { libraryItemRepository } from '../repository/library_item' import { Merge, PickTuple } from '../util' -import { deepDelete, setRecentlySavedItemInRedis } from '../utils/helpers' +import { + deepDelete, + setRecentlySavedItemInRedis, + stringToHash, +} from '../utils/helpers' import { logger } from '../utils/logger' import { parseSearchQuery } from '../utils/search' +import { downloadFileFromBucket, uploadToBucket } from '../utils/uploads' import { HighlightEvent } from './highlights' import { addLabelsToLibraryItem, LabelEvent } from './labels' @@ -1016,6 +1021,14 @@ export const createOrUpdateLibraryItem = async ( pubsub = createPubSubClient(), skipPubSub = false ): Promise => { + // if (libraryItem.originalContent && !urlHash) { + // // upload original content to GCS + // await uploadContent(libraryItem.originalUrl, libraryItem.originalContent) + + // // remove original content + // delete libraryItem.originalContent + // } + const newLibraryItem = await authTrx( async (tx) => { const repo = tx.withRepository(libraryItemRepository) @@ -1663,3 +1676,23 @@ export const filterItemEvents = ( throw new Error('Unexpected state.') } + +const originalContentFilename = (originalUrl: string) => + `originalContent/${stringToHash(originalUrl)}` + +export const uploadOriginalContent = async ( + originalUrl: string, + originalContent: string +) => { + await uploadToBucket( + originalContentFilename(originalUrl), + Buffer.from(originalContent), + { + public: false, + } + ) +} + +export const downloadOriginalContent = async (originalUrl: string) => { + return downloadFileFromBucket(originalContentFilename(originalUrl)) +} diff --git a/packages/api/src/services/recommendation.ts b/packages/api/src/services/recommendation.ts index 9e58224f4..07d189383 100644 --- a/packages/api/src/services/recommendation.ts +++ b/packages/api/src/services/recommendation.ts @@ -47,7 +47,6 @@ export const addRecommendation = async ( author: item.author, description: item.description, originalUrl: item.originalUrl, - originalContent: item.originalContent, contentReader: item.contentReader, directionality: item.directionality, itemLanguage: item.itemLanguage, diff --git a/packages/api/src/services/reports.ts b/packages/api/src/services/reports.ts index 33928c4f3..ee71f6807 100644 --- a/packages/api/src/services/reports.ts +++ b/packages/api/src/services/reports.ts @@ -25,7 +25,6 @@ export const saveContentDisplayReport = async ( const report = await getRepository(ContentDisplayReport).save({ user: { id: uid }, content: item.readableContent, - originalHtml: item.originalContent || undefined, originalUrl: item.originalUrl, reportComment: input.reportComment, libraryItemId: item.id, diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 699db6919..020237662 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -91,7 +91,6 @@ export const saveEmail = async ( user: { id: input.userId }, slug, readableContent: content, - originalContent: input.originalContent, description: metadata?.description || parseResult.parsedContent?.excerpt, title: input.title, author: input.author, diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index c5ba13989..5dab1fe1b 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -124,7 +124,6 @@ export const savePage = async ( croppedPathname, parsedContent: parseResult.parsedContent, itemType: parseResult.pageType, - originalHtml: parseResult.domContent, canonicalUrl: parseResult.canonicalUrl, savedAt: input.savedAt ? new Date(input.savedAt) : new Date(), publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, @@ -197,7 +196,6 @@ export const savePage = async ( export const parsedContentToLibraryItem = ({ url, userId, - originalHtml, itemId, parsedContent, slug, @@ -224,7 +222,6 @@ export const parsedContentToLibraryItem = ({ croppedPathname: string itemType: string parsedContent: Readability.ParseResult | null - originalHtml?: string | null itemId?: string | null title?: string | null preparedDocument?: PreparedDocumentInput | null @@ -246,7 +243,6 @@ export const parsedContentToLibraryItem = ({ id: itemId || undefined, slug, user: { id: userId }, - originalContent: originalHtml, readableContent: parsedContent?.content || '', description: parsedContent?.excerpt, title: diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index e53fe079d..ed4951600 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -154,23 +154,53 @@ export const isFileExists = async (filePath: string): Promise => { return exists } -export const downloadStringFromBucket = async ( - filePath: string -): Promise => { - try { - const file = storage.bucket(bucketName).file(filePath) +export const downloadFromUrl = async ( + contentObjUrl: string, + timeout?: number +) => { + // download the content as stream and max 10MB + const response = await axios.get(contentObjUrl, { + responseType: 'stream', + maxContentLength, + timeout, + }) - const [exists] = await file.exists() - if (!exists) { - logger.error(`File not found: ${filePath}`) - return null - } - - // Download the file contents as a string - const [data] = await file.download() - return data.toString() - } catch (error) { - logger.info('Error downloading file:', error) - return null - } + return response.data +} + +export const uploadToSignedUrl = async ( + uploadSignedUrl: string, + data: Buffer, + contentType: string, + timeout?: number +) => { + // upload the stream to the signed url + await axios.put(uploadSignedUrl, data, { + headers: { + 'Content-Type': contentType, + }, + maxBodyLength: maxContentLength, + timeout, + }) +} + +export const isFileExists = async (filePath: string): Promise => { + const [exists] = await storage.bucket(bucketName).file(filePath).exists() + return exists +} + +export const downloadFileFromBucket = async ( + filePath: string +): Promise => { + const file = storage.bucket(bucketName).file(filePath) + + const [exists] = await file.exists() + if (!exists) { + logger.error(`File not found: ${filePath}`) + throw new Error('File not found') + } + + // Download the file contents as a string + const [data] = await file.download() + return data }