do not store original content in db

packages/api/src/jobs/process-youtube-video.ts

@@ -321,11 +321,7 @@ export const processYouTubeVideo = async (
       undefined,
       jobData.userId
     )
-    if (
-      !libraryItem ||
-      libraryItem.state !== LibraryItemState.Succeeded ||
-      !libraryItem.originalContent
-    ) {
+    if (!libraryItem || libraryItem.state !== LibraryItemState.Succeeded) {
       logger.info(
         `Not ready to get YouTube metadata job state: ${
           libraryItem?.state ?? 'null'
@@ -382,7 +378,7 @@ export const processYouTubeVideo = async (
         // enqueue a job to process the full transcript
         const updatedContent = await addTranscriptPlaceholdReadableContent(
           libraryItem.originalUrl,
-          libraryItem.originalContent
+          libraryItem.readableContent
         )
 
         if (updatedContent) {
@@ -438,11 +434,7 @@ export const processYouTubeTranscript = async (
       undefined,
       jobData.userId
     )
-    if (
-      !libraryItem ||
-      libraryItem.state !== LibraryItemState.Succeeded ||
-      !libraryItem.originalContent
-    ) {
+    if (!libraryItem || libraryItem.state !== LibraryItemState.Succeeded) {
       logger.info(
         `Not ready to get YouTube metadata job state: ${
           libraryItem?.state ?? 'null'
@@ -481,7 +473,7 @@ export const processYouTubeTranscript = async (
       )
       const updatedContent = await addTranscriptToReadableContent(
         libraryItem.originalUrl,
-        libraryItem.originalContent,
+        libraryItem.readableContent,
         transcriptHTML
       )
 

packages/api/src/jobs/rss/refreshFeed.ts

@@ -381,7 +381,7 @@ const createItemWithFeedContent = async (
         rssFeedUrl: feedUrl,
         savedAt: item.isoDate,
         publishedAt: item.isoDate,
-        originalContent: feedContent || '',
+        originalContent: '',
         source: 'rss-feeder',
         state: ArticleSavingRequestStatus.ContentNotFetched,
         clientRequestId: '',

packages/api/src/jobs/save_page.ts

@@ -7,12 +7,12 @@ import {
   CreateLabelInput,
 } from '../generated/graphql'
 import { userRepository } from '../repository/user'
+import { downloadOriginalContent } from '../services/library_item'
 import { saveFile } from '../services/save_file'
 import { savePage } from '../services/save_page'
 import { uploadFile } from '../services/upload_file'
 import { logError, logger } from '../utils/logger'
 import { downloadFromUrl, uploadToSignedUrl } from '../utils/uploads'
-import { downloadStringFromBucket } from '../utils/uploads'
 
 const signToken = promisify(jwt.sign)
 
@@ -128,29 +128,27 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
     finalUrl,
     title,
     contentType,
-    urlHash,
+    state,
   } = data
-  let isImported,
-    isSaved,
-    state = data.state
+  let isImported, isSaved
 
-  try {
-    logger.info('savePageJob', {
+  logger.info('savePageJob', {
+    userId,
+    url,
+    finalUrl,
+  })
+
+  const user = await userRepository.findById(userId)
+  if (!user) {
+    logger.error('Unable to save job, user can not be found.', {
       userId,
       url,
-      finalUrl,
     })
+    // if the user is not found, we do not retry
+    return false
+  }
 
-    const user = await userRepository.findById(userId)
-    if (!user) {
-      logger.error('Unable to save job, user can not be found.', {
-        userId,
-        url,
-      })
-      // if the user is not found, we do not retry
-      return false
-    }
-
+  try {
     // for pdf content, we need to upload the pdf
     if (contentType === 'application/pdf') {
       const uploadResult = await uploadPdf(
@@ -163,7 +161,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
         {
           url: finalUrl,
           uploadFileId: uploadResult.uploadFileId,
-          state: state ? (state as ArticleSavingRequestStatus) : undefined,
+          state: (state as ArticleSavingRequestStatus) || undefined,
           labels,
           source,
           folder,
@@ -183,25 +181,8 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
       return true
     }
 
-    let originalContent
-    if (!urlHash) {
-      logger.info(`content is not uploaded: ${finalUrl}`)
-      // set the state to failed if we don't have content
-      originalContent = 'Failed to fetch content'
-      state = ArticleSavingRequestStatus.Failed
-    } else {
-      // download content from the bucket
-      const downloaded = await downloadStringFromBucket(
-        `originalContent/${urlHash}`
-      )
-      if (!downloaded) {
-        logger.error('error while downloading content from bucket')
-        originalContent = 'Failed to fetch content'
-        state = ArticleSavingRequestStatus.Failed
-      } else {
-        originalContent = downloaded
-      }
-    }
+    // download content from the bucket
+    const originalContent = (await downloadOriginalContent(finalUrl)).toString()
 
     // for non-pdf content, we need to save the page
     const result = await savePage(
@@ -210,8 +191,8 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
         clientRequestId: articleSavingRequestId,
         title,
         originalContent,
-        state: state ? (state as ArticleSavingRequestStatus) : undefined,
-        labels: labels,
+        state: (state as ArticleSavingRequestStatus) || undefined,
+        labels,
         rssFeedUrl,
         savedAt: savedAt ? new Date(savedAt) : new Date(),
         publishedAt: publishedAt ? new Date(publishedAt) : null,

packages/api/src/resolvers/article/index.ts

@@ -302,7 +302,6 @@ export const createArticleResolver = authorized<
         userId: uid,
         slug,
         croppedPathname,
-        originalHtml: domContent,
         itemType,
         preparedDocument,
         uploadFileHash,

packages/api/src/services/library_item.ts

@@ -21,9 +21,14 @@ import { redisDataSource } from '../redis_data_source'
 import { authTrx, getColumns, queryBuilderToRawSql } from '../repository'
 import { libraryItemRepository } from '../repository/library_item'
 import { Merge, PickTuple } from '../util'
-import { deepDelete, setRecentlySavedItemInRedis } from '../utils/helpers'
+import {
+  deepDelete,
+  setRecentlySavedItemInRedis,
+  stringToHash,
+} from '../utils/helpers'
 import { logger } from '../utils/logger'
 import { parseSearchQuery } from '../utils/search'
+import { downloadFileFromBucket, uploadToBucket } from '../utils/uploads'
 import { HighlightEvent } from './highlights'
 import { addLabelsToLibraryItem, LabelEvent } from './labels'
 
@@ -1016,6 +1021,14 @@ export const createOrUpdateLibraryItem = async (
   pubsub = createPubSubClient(),
   skipPubSub = false
 ): Promise<LibraryItem> => {
+  // if (libraryItem.originalContent && !urlHash) {
+  //   // upload original content to GCS
+  //   await uploadContent(libraryItem.originalUrl, libraryItem.originalContent)
+
+  //   // remove original content
+  //   delete libraryItem.originalContent
+  // }
+
   const newLibraryItem = await authTrx(
     async (tx) => {
       const repo = tx.withRepository(libraryItemRepository)
@@ -1663,3 +1676,23 @@ export const filterItemEvents = (
 
   throw new Error('Unexpected state.')
 }
+
+const originalContentFilename = (originalUrl: string) =>
+  `originalContent/${stringToHash(originalUrl)}`
+
+export const uploadOriginalContent = async (
+  originalUrl: string,
+  originalContent: string
+) => {
+  await uploadToBucket(
+    originalContentFilename(originalUrl),
+    Buffer.from(originalContent),
+    {
+      public: false,
+    }
+  )
+}
+
+export const downloadOriginalContent = async (originalUrl: string) => {
+  return downloadFileFromBucket(originalContentFilename(originalUrl))
+}

packages/api/src/services/recommendation.ts

@@ -47,7 +47,6 @@ export const addRecommendation = async (
         author: item.author,
         description: item.description,
         originalUrl: item.originalUrl,
-        originalContent: item.originalContent,
         contentReader: item.contentReader,
         directionality: item.directionality,
         itemLanguage: item.itemLanguage,

packages/api/src/services/reports.ts

@@ -25,7 +25,6 @@ export const saveContentDisplayReport = async (
   const report = await getRepository(ContentDisplayReport).save({
     user: { id: uid },
     content: item.readableContent,
-    originalHtml: item.originalContent || undefined,
     originalUrl: item.originalUrl,
     reportComment: input.reportComment,
     libraryItemId: item.id,

packages/api/src/services/save_email.ts

@@ -91,7 +91,6 @@ export const saveEmail = async (
       user: { id: input.userId },
       slug,
       readableContent: content,
-      originalContent: input.originalContent,
       description: metadata?.description || parseResult.parsedContent?.excerpt,
       title: input.title,
       author: input.author,

packages/api/src/services/save_page.ts

@@ -124,7 +124,6 @@ export const savePage = async (
     croppedPathname,
     parsedContent: parseResult.parsedContent,
     itemType: parseResult.pageType,
-    originalHtml: parseResult.domContent,
     canonicalUrl: parseResult.canonicalUrl,
     savedAt: input.savedAt ? new Date(input.savedAt) : new Date(),
     publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined,
@@ -197,7 +196,6 @@ export const savePage = async (
 export const parsedContentToLibraryItem = ({
   url,
   userId,
-  originalHtml,
   itemId,
   parsedContent,
   slug,
@@ -224,7 +222,6 @@ export const parsedContentToLibraryItem = ({
   croppedPathname: string
   itemType: string
   parsedContent: Readability.ParseResult | null
-  originalHtml?: string | null
   itemId?: string | null
   title?: string | null
   preparedDocument?: PreparedDocumentInput | null
@@ -246,7 +243,6 @@ export const parsedContentToLibraryItem = ({
     id: itemId || undefined,
     slug,
     user: { id: userId },
-    originalContent: originalHtml,
     readableContent: parsedContent?.content || '',
     description: parsedContent?.excerpt,
     title:

packages/api/src/utils/uploads.ts

@@ -154,23 +154,53 @@ export const isFileExists = async (filePath: string): Promise<boolean> => {
   return exists
 }
 
-export const downloadStringFromBucket = async (
-  filePath: string
-): Promise<string | null> => {
-  try {
-    const file = storage.bucket(bucketName).file(filePath)
+export const downloadFromUrl = async (
+  contentObjUrl: string,
+  timeout?: number
+) => {
+  // download the content as stream and max 10MB
+  const response = await axios.get<Buffer>(contentObjUrl, {
+    responseType: 'stream',
+    maxContentLength,
+    timeout,
+  })
 
-    const [exists] = await file.exists()
-    if (!exists) {
-      logger.error(`File not found: ${filePath}`)
-      return null
-    }
-
-    // Download the file contents as a string
-    const [data] = await file.download()
-    return data.toString()
-  } catch (error) {
-    logger.info('Error downloading file:', error)
-    return null
-  }
+  return response.data
+}
+
+export const uploadToSignedUrl = async (
+  uploadSignedUrl: string,
+  data: Buffer,
+  contentType: string,
+  timeout?: number
+) => {
+  // upload the stream to the signed url
+  await axios.put(uploadSignedUrl, data, {
+    headers: {
+      'Content-Type': contentType,
+    },
+    maxBodyLength: maxContentLength,
+    timeout,
+  })
+}
+
+export const isFileExists = async (filePath: string): Promise<boolean> => {
+  const [exists] = await storage.bucket(bucketName).file(filePath).exists()
+  return exists
+}
+
+export const downloadFileFromBucket = async (
+  filePath: string
+): Promise<Buffer> => {
+  const file = storage.bucket(bucketName).file(filePath)
+
+  const [exists] = await file.exists()
+  if (!exists) {
+    logger.error(`File not found: ${filePath}`)
+    throw new Error('File not found')
+  }
+
+  // Download the file contents as a string
+  const [data] = await file.download()
+  return data
 }