Merge pull request #3839 from omnivore-app/feature/upload-original-content

feature/upload original content

package.json

@@ -8,7 +8,7 @@
   ],
   "license": "AGPL-3.0-only",
   "scripts": {
-    "test": "lerna run --no-bail test",
+    "test": "lerna run --stream test",
     "lint": "lerna run lint",
     "build": "lerna run build",
     "test:scoped:example": "lerna run test --scope={@omnivore/pdf-handler,@omnivore/web}",

packages/api/.nycrc

@@ -1,15 +1,10 @@
 {
   "extends": "@istanbuljs/nyc-config-typescript",
-  "check-coverage": true,
   "all": true,
   "include": [
     "src/**/*.ts"
   ],
   "reporter": [
     "text-summary"
-  ],
-  "branches": 0,
-  "lines": 0,
-  "functions": 0,
-  "statements": 60
+  ]
 }

packages/api/mocha-config.json

@@ -2,6 +2,6 @@
   "extension": ["ts"],
   "spec": "test/**/*.test.ts",
   "reporter": "mocha-unfunk-reporter",
-  "require": ["test/global-setup.ts", "test/global-teardown.ts"],
+  "require": ["test/global-setup.ts", "test/global-teardown.ts", "test/hooks.ts"],
   "timeout": 10000
 }

packages/api/package.json

@@ -168,4 +168,4 @@
   "volta": {
     "extends": "../../package.json"
   }
-}
+}

packages/api/src/jobs/process-youtube-video.ts

@@ -322,11 +322,7 @@ export const processYouTubeVideo = async (
       undefined,
       jobData.userId
     )
-    if (
-      !libraryItem ||
-      libraryItem.state !== LibraryItemState.Succeeded ||
-      !libraryItem.originalContent
-    ) {
+    if (!libraryItem || libraryItem.state !== LibraryItemState.Succeeded) {
       logger.info(
         `Not ready to get YouTube metadata job state: ${
           libraryItem?.state ?? 'null'
@@ -383,7 +379,7 @@ export const processYouTubeVideo = async (
         // enqueue a job to process the full transcript
         const updatedContent = await addTranscriptPlaceholdReadableContent(
           libraryItem.originalUrl,
-          libraryItem.originalContent
+          libraryItem.readableContent
         )
 
         if (updatedContent) {
@@ -439,11 +435,7 @@ export const processYouTubeTranscript = async (
       undefined,
       jobData.userId
     )
-    if (
-      !libraryItem ||
-      libraryItem.state !== LibraryItemState.Succeeded ||
-      !libraryItem.originalContent
-    ) {
+    if (!libraryItem || libraryItem.state !== LibraryItemState.Succeeded) {
       logger.info(
         `Not ready to get YouTube metadata job state: ${
           libraryItem?.state ?? 'null'
@@ -482,7 +474,7 @@ export const processYouTubeTranscript = async (
       )
       const updatedContent = await addTranscriptToReadableContent(
         libraryItem.originalUrl,
-        libraryItem.originalContent,
+        libraryItem.readableContent,
         transcriptHTML
       )
 

packages/api/src/jobs/rss/refreshFeed.ts

@@ -2,6 +2,7 @@ import axios from 'axios'
 import crypto from 'crypto'
 import { parseHTML } from 'linkedom'
 import Parser, { Item } from 'rss-parser'
+import { v4 as uuid } from 'uuid'
 import { FetchContentType } from '../../entity/subscription'
 import { env } from '../../env'
 import { ArticleSavingRequestStatus } from '../../generated/graphql'
@@ -72,13 +73,14 @@ export type RssFeedItem = Item & {
   link: string
 }
 
-interface User {
+interface UserConfig {
   id: string
   folder: FolderType
+  libraryItemId: string
 }
 
 interface FetchContentTask {
-  users: Map<string, User> // userId -> User
+  users: Map<string, UserConfig> // userId -> User
   item: RssFeedItem
 }
 
@@ -160,8 +162,12 @@ const getThumbnail = (item: RssFeedItem) => {
     return item['media:thumbnail'].$.url
   }
 
-  return item['media:content']?.find((media) => media.$.medium === 'image')?.$
-    .url
+  if (item['media:content']) {
+    return item['media:content'].find((media) => media.$?.medium === 'image')?.$
+      .url
+  }
+
+  return undefined
 }
 
 export const fetchAndChecksum = async (url: string) => {
@@ -276,13 +282,16 @@ const addFetchContentTask = (
 ) => {
   const url = item.link
   const task = fetchContentTasks.get(url)
+  const libraryItemId = uuid()
+  const userConfig = { id: userId, folder, libraryItemId }
+
   if (!task) {
     fetchContentTasks.set(url, {
-      users: new Map([[userId, { id: userId, folder }]]),
+      users: new Map([[userId, userConfig]]),
       item,
     })
   } else {
-    task.users.set(userId, { id: userId, folder })
+    task.users.set(userId, userConfig)
   }
 
   return true
@@ -315,7 +324,7 @@ const createTask = async (
 }
 
 const fetchContentAndCreateItem = async (
-  users: User[],
+  users: UserConfig[],
   feedUrl: string,
   item: RssFeedItem
 ) => {
@@ -323,7 +332,6 @@ const fetchContentAndCreateItem = async (
     users,
     source: 'rss-feeder',
     url: item.link.trim(),
-    saveRequestId: '',
     labels: [{ name: 'RSS' }],
     rssFeedUrl: feedUrl,
     savedAt: item.isoDate,

packages/api/src/jobs/save_page.ts

@@ -6,13 +6,18 @@ import {
   ArticleSavingRequestStatus,
   CreateLabelInput,
 } from '../generated/graphql'
-import { redisDataSource } from '../redis_data_source'
 import { userRepository } from '../repository/user'
 import { saveFile } from '../services/save_file'
 import { savePage } from '../services/save_page'
 import { uploadFile } from '../services/upload_file'
 import { logError, logger } from '../utils/logger'
-import { downloadFromUrl, uploadToSignedUrl } from '../utils/uploads'
+import {
+  contentFilePath,
+  downloadFromBucket,
+  downloadFromUrl,
+  isFileExists,
+  uploadToSignedUrl,
+} from '../utils/uploads'
 
 const signToken = promisify(jwt.sign)
 
@@ -27,27 +32,19 @@ interface Data {
   url: string
   finalUrl: string
   articleSavingRequestId: string
+  title: string
+  contentType: string
+  savedAt: string
+
   state?: string
   labels?: CreateLabelInput[]
   source: string
   folder: string
   rssFeedUrl?: string
-  savedAt?: string
   publishedAt?: string
   taskId?: string
 }
 
-interface FetchResult {
-  finalUrl: string
-  title?: string
-  content?: string
-  contentType?: string
-}
-
-const isFetchResult = (obj: unknown): obj is FetchResult => {
-  return typeof obj === 'object' && obj !== null && 'finalUrl' in obj
-}
-
 const uploadPdf = async (
   url: string,
   userId: string,
@@ -120,32 +117,6 @@ const sendImportStatusUpdate = async (
   }
 }
 
-const getCachedFetchResult = async (url: string) => {
-  const key = `fetch-result:${url}`
-  if (!redisDataSource.redisClient || !redisDataSource.workerRedisClient) {
-    throw new Error('redis client is not initialized')
-  }
-
-  let result = await redisDataSource.redisClient.get(key)
-  if (!result) {
-    logger.debug(`fetch result is not cached in cache redis ${url}`)
-    // fallback to worker redis client if the result is not found
-    result = await redisDataSource.workerRedisClient.get(key)
-    if (!result) {
-      throw new Error('fetch result is not cached')
-    }
-  }
-
-  const fetchResult = JSON.parse(result) as unknown
-  if (!isFetchResult(fetchResult)) {
-    throw new Error('fetch result is not valid')
-  }
-
-  logger.info('fetch result is cached', url)
-
-  return fetchResult
-}
-
 export const savePageJob = async (data: Data, attemptsMade: number) => {
   const {
     userId,
@@ -159,33 +130,29 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
     taskId,
     url,
     finalUrl,
+    title,
+    contentType,
+    state,
   } = data
-  let isImported,
-    isSaved,
-    state = data.state
+  let isImported, isSaved
 
-  try {
-    logger.info('savePageJob', {
+  logger.info('savePageJob', {
+    userId,
+    url,
+    finalUrl,
+  })
+
+  const user = await userRepository.findById(userId)
+  if (!user) {
+    logger.error('Unable to save job, user can not be found.', {
       userId,
       url,
-      finalUrl,
     })
+    // if the user is not found, we do not retry
+    return false
+  }
 
-    // get the fetch result from cache
-    const fetchedResult = await getCachedFetchResult(finalUrl)
-    const { title, contentType } = fetchedResult
-    let content = fetchedResult.content
-
-    const user = await userRepository.findById(userId)
-    if (!user) {
-      logger.error('Unable to save job, user can not be found.', {
-        userId,
-        url,
-      })
-      // if the user is not found, we do not retry
-      return false
-    }
-
+  try {
     // for pdf content, we need to upload the pdf
     if (contentType === 'application/pdf') {
       const uploadResult = await uploadPdf(
@@ -198,7 +165,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
         {
           url: finalUrl,
           uploadFileId: uploadResult.uploadFileId,
-          state: state ? (state as ArticleSavingRequestStatus) : undefined,
+          state: (state as ArticleSavingRequestStatus) || undefined,
           labels,
           source,
           folder,
@@ -218,27 +185,41 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
       return true
     }
 
-    if (!content) {
-      logger.info(`content is not fetched: ${finalUrl}`)
-      // set the state to failed if we don't have content
-      content = 'Failed to fetch content'
-      state = ArticleSavingRequestStatus.Failed
+    // download the original content
+    const filePath = contentFilePath(
+      userId,
+      articleSavingRequestId,
+      new Date(savedAt).getTime(),
+      'original'
+    )
+    const exists = await isFileExists(filePath)
+    if (!exists) {
+      logger.error('Original content file does not exist', {
+        finalUrl,
+        filePath,
+      })
+
+      throw new Error('Original content file does not exist')
     }
 
-    // for non-pdf content, we need to save the page
+    const content = (await downloadFromBucket(filePath)).toString()
+    console.log('Downloaded original content from:', filePath)
+
+    // for non-pdf content, we need to save the content
     const result = await savePage(
       {
         url: finalUrl,
         clientRequestId: articleSavingRequestId,
         title,
         originalContent: content,
-        state: state ? (state as ArticleSavingRequestStatus) : undefined,
-        labels: labels,
+        state: (state as ArticleSavingRequestStatus) || undefined,
+        labels,
         rssFeedUrl,
-        savedAt: savedAt ? new Date(savedAt) : new Date(),
+        savedAt,
         publishedAt: publishedAt ? new Date(publishedAt) : null,
         source,
         folder,
+        originalContentUploaded: true,
       },
       user
     )

packages/api/src/routers/content_router.ts

@@ -6,7 +6,11 @@ import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
 import { corsConfig } from '../utils/corsConfig'
 import { enqueueBulkUploadContentJob } from '../utils/createTask'
 import { logger } from '../utils/logger'
-import { generateDownloadSignedUrl, isFileExists } from '../utils/uploads'
+import {
+  contentFilePath,
+  generateDownloadSignedUrl,
+  isFileExists,
+} from '../utils/uploads'
 
 export function contentRouter() {
   const router = Router()
@@ -58,7 +62,7 @@ export function contentRouter() {
     const userId = claims.uid
 
     const libraryItems = await findLibraryItemsByIds(libraryItemIds, userId, {
-      select: ['id', 'updatedAt'],
+      select: ['id', 'updatedAt', 'savedAt'],
     })
     if (libraryItems.length === 0) {
       logger.error('Library items not found')
@@ -68,9 +72,14 @@ export function contentRouter() {
     // generate signed url for each library item
     const data = await Promise.all(
       libraryItems.map(async (libraryItem) => {
-        const filePath = `content/${userId}/${
-          libraryItem.id
-        }.${libraryItem.updatedAt.getTime()}.${format}`
+        const date =
+          format === 'original' ? libraryItem.savedAt : libraryItem.updatedAt
+        const filePath = contentFilePath(
+          userId,
+          libraryItem.id,
+          date.getTime(),
+          format
+        )
 
         try {
           const downloadUrl = await generateDownloadSignedUrl(filePath, {

packages/api/src/routers/svc/following.ts

@@ -112,7 +112,6 @@ export function followingServiceRouter() {
         userId,
         slug,
         croppedPathname,
-        originalHtml: req.body.feedContent,
         itemType: parsedResult?.pageType || PageType.Unknown,
         canonicalUrl: url,
         folder: FOLDER,

packages/api/src/services/library_item.ts

@@ -24,6 +24,11 @@ import { Merge, PickTuple } from '../util'
 import { deepDelete, setRecentlySavedItemInRedis } from '../utils/helpers'
 import { logger } from '../utils/logger'
 import { parseSearchQuery } from '../utils/search'
+import {
+  contentFilePath,
+  downloadFromBucket,
+  uploadToBucket,
+} from '../utils/uploads'
 import { HighlightEvent } from './highlights'
 import { addLabelsToLibraryItem, LabelEvent } from './labels'
 
@@ -1014,8 +1019,17 @@ export const createOrUpdateLibraryItem = async (
   libraryItem: CreateOrUpdateLibraryItemArgs,
   userId: string,
   pubsub = createPubSubClient(),
-  skipPubSub = false
+  skipPubSub = false,
+  originalContentUploaded = false
 ): Promise<LibraryItem> => {
+  let originalContent: string | null = null
+  if (libraryItem.originalContent) {
+    originalContent = libraryItem.originalContent
+
+    // remove original content from the item
+    delete libraryItem.originalContent
+  }
+
   const newLibraryItem = await authTrx(
     async (tx) => {
       const repo = tx.withRepository(libraryItemRepository)
@@ -1089,6 +1103,19 @@ export const createOrUpdateLibraryItem = async (
   const data = deepDelete(newLibraryItem, columnsToDelete)
   await pubsub.entityCreated<ItemEvent>(EntityType.ITEM, data, userId)
 
+  // upload original content to GCS if it's not already uploaded
+  if (originalContent && !originalContentUploaded) {
+    await uploadOriginalContent(
+      userId,
+      newLibraryItem.id,
+      newLibraryItem.savedAt,
+      originalContent
+    )
+    logger.info('Uploaded original content to GCS', {
+      id: newLibraryItem.id,
+    })
+  }
+
   return newLibraryItem
 }
 
@@ -1663,3 +1690,29 @@ export const filterItemEvents = (
 
   throw new Error('Unexpected state.')
 }
+
+export const uploadOriginalContent = async (
+  userId: string,
+  libraryItemId: string,
+  savedAt: Date,
+  originalContent: string
+) => {
+  await uploadToBucket(
+    contentFilePath(userId, libraryItemId, savedAt.getTime(), 'original'),
+    Buffer.from(originalContent),
+    {
+      public: false,
+      contentType: 'text/html',
+    }
+  )
+}
+
+export const downloadOriginalContent = async (
+  userId: string,
+  libraryItemId: string,
+  savedAt: Date
+) => {
+  return downloadFromBucket(
+    contentFilePath(userId, libraryItemId, savedAt.getTime(), 'original')
+  )
+}

packages/api/src/services/reports.ts

@@ -25,7 +25,6 @@ export const saveContentDisplayReport = async (
   const report = await getRepository(ContentDisplayReport).save({
     user: { id: uid },
     content: item.readableContent,
-    originalHtml: item.originalContent || undefined,
     originalUrl: item.originalUrl,
     reportComment: input.reportComment,
     libraryItemId: item.id,

packages/api/src/services/save_page.ts

@@ -68,7 +68,12 @@ const shouldParseInBackend = (input: SavePageInput): boolean => {
 
 export type SavePageArgs = Merge<
   SavePageInput,
-  { feedContent?: string; previewImage?: string; author?: string }
+  {
+    feedContent?: string
+    previewImage?: string
+    author?: string
+    originalContentUploaded?: boolean
+  }
 >
 
 export const savePage = async (
@@ -145,7 +150,8 @@ export const savePage = async (
     itemToSave,
     user.id,
     undefined,
-    isImported
+    isImported,
+    input.originalContentUploaded
   )
   clientRequestId = newItem.id
 
@@ -274,7 +280,7 @@ export const parsedContentToLibraryItem = ({
     state: state
       ? (state as unknown as LibraryItemState)
       : LibraryItemState.Succeeded,
-    savedAt: validatedDate(savedAt),
+    savedAt: validatedDate(savedAt) || new Date(),
     siteName: parsedContent?.siteName,
     itemLanguage: parsedContent?.language,
     siteIcon: parsedContent?.siteIcon,

packages/api/src/services/upload_file.ts

@@ -1,4 +1,3 @@
-import normalizeUrl from 'normalize-url'
 import path from 'path'
 import { In } from 'typeorm'
 import { v4 as uuid } from 'uuid'
@@ -11,7 +10,7 @@ import {
   UploadFileStatus,
 } from '../generated/graphql'
 import { authTrx, getRepository } from '../repository'
-import { generateSlug } from '../utils/helpers'
+import { cleanUrl, generateSlug } from '../utils/helpers'
 import { logger } from '../utils/logger'
 import {
   contentReaderForLibraryItem,
@@ -69,13 +68,11 @@ export const uploadFile = async (
   input: UploadFileRequestInput,
   uid: string
 ) => {
+  let url = input.url
   let title: string
   let fileName: string
   try {
-    const url = normalizeUrl(new URL(input.url).href, {
-      stripHash: true,
-      stripWWW: false,
-    })
+    url = cleanUrl(new URL(url).href)
     title = decodeURI(path.basename(new URL(url).pathname, '.pdf'))
     fileName = decodeURI(path.basename(new URL(url).pathname)).replace(
       /[^a-zA-Z0-9-_.]/g,
@@ -102,8 +99,6 @@ export const uploadFile = async (
     }
   }
 
-  let url = input.url
-
   const uploadFileId = uuid()
   const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName)
   // If this is a file URL, we swap in a special URL

packages/api/src/util.ts

@@ -177,11 +177,6 @@ const nullableEnvVars = [
   'NOTION_AUTH_URL',
 ] // Allow some vars to be null/empty
 
-/* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */
-if (process.env.API_ENV == 'local') {
-  nullableEnvVars.push(...['GCS_UPLOAD_BUCKET'])
-}
-
 const envParser =
   (env: { [key: string]: string | undefined }) =>
   (varName: string): string => {
@@ -204,6 +199,11 @@ export function getEnv(): BackendEnv {
   // Dotenv parses env file merging into proces.env which is then read into custom struct here.
   dotenv.config()
 
+  /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */
+  if (process.env.API_ENV == 'local') {
+    nullableEnvVars.push(...['GCS_UPLOAD_BUCKET'])
+  }
+
   const parse = envParser(process.env)
   const pg = {
     host: parse('PG_HOST'),

packages/api/src/utils/uploads.ts

@@ -5,6 +5,7 @@ import axios from 'axios'
 import { ContentReaderType } from '../entity/library_item'
 import { env } from '../env'
 import { PageType } from '../generated/graphql'
+import { ContentFormat } from '../jobs/upload_content'
 import { logger } from './logger'
 
 export const contentReaderForLibraryItem = (
@@ -30,7 +31,7 @@ export const contentReaderForLibraryItem = (
  * the default app engine service account on the IAM page. We also need to
  * enable IAM related APIs on the project.
  */
-const storage = env.fileUpload?.gcsUploadSAKeyFilePath
+export const storage = env.fileUpload?.gcsUploadSAKeyFilePath
   ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath })
   : new Storage()
 const bucketName = env.fileUpload.gcsUploadBucket
@@ -153,3 +154,18 @@ export const isFileExists = async (filePath: string): Promise<boolean> => {
   const [exists] = await storage.bucket(bucketName).file(filePath).exists()
   return exists
 }
+
+export const downloadFromBucket = async (filePath: string): Promise<Buffer> => {
+  const file = storage.bucket(bucketName).file(filePath)
+
+  // Download the file contents
+  const [data] = await file.download()
+  return data
+}
+
+export const contentFilePath = (
+  userId: string,
+  libraryItemId: string,
+  timestamp: number,
+  format: ContentFormat
+) => `content/${userId}/${libraryItemId}.${timestamp}.${format}`

packages/api/test/db.ts

@@ -120,7 +120,13 @@ export const createTestLibraryItem = async (
     slug: 'test-with-omnivore',
   }
 
-  const createdItem = await createOrUpdateLibraryItem(item, userId)
+  const createdItem = await createOrUpdateLibraryItem(
+    item,
+    userId,
+    undefined,
+    true,
+    true
+  )
   if (labels) {
     await saveLabelsInLibraryItem(labels, createdItem.id, userId)
   }

packages/api/test/hooks.ts

@@ -0,0 +1,13 @@
+import { Storage } from '@google-cloud/storage'
+import sinon from 'sinon'
+import * as uploads from '../src/utils/uploads'
+import { MockStorage } from './mock_storage'
+
+export const mochaHooks = {
+  beforeEach() {
+    // Mock cloud storage
+    sinon
+      .stub(uploads, 'storage')
+      .value(new MockStorage() as unknown as Storage)
+  },
+}

packages/api/test/mock_storage.ts

@@ -1,6 +1,6 @@
 import { Writable } from 'stream'
 
-class MockStorage {
+export class MockStorage {
   buckets: { [name: string]: MockBucket }
 
   constructor() {
@@ -12,7 +12,7 @@ class MockStorage {
   }
 }
 
-export class MockBucket {
+class MockBucket {
   name: string
   files: { [path: string]: MockFile }
 
@@ -54,6 +54,11 @@ class MockFile {
   makePublic() {
     return
   }
+
+  save() {
+    console.log('Saved file to:', this.path)
+    return
+  }
 }
 
 class MockWriteStream extends Writable {

packages/api/test/resolvers/article.test.ts

@@ -435,7 +435,13 @@ describe('Article API', () => {
         originalUrl: 'https://blog.omnivore.app/test-with-omnivore',
         directionality: DirectionalityType.RTL,
       }
-      const item = await createOrUpdateLibraryItem(itemToCreate, user.id)
+      const item = await createOrUpdateLibraryItem(
+        itemToCreate,
+        user.id,
+        undefined,
+        true,
+        true
+      )
       itemId = item.id
 
       // save highlights
@@ -528,11 +534,11 @@ describe('Article API', () => {
   })
 
   describe('SavePage', () => {
-    let title = 'Example Title'
+    const title = 'Example Title'
     let url = 'https://blog.omnivore.app'
-    let originalContent =
+    const originalContent =
       '<html dir="rtl"><body><div>Example Content</div></body></html>'
-    let source = 'puppeteer-parse'
+    const source = 'puppeteer-parse'
 
     context('when we save a new item', () => {
       after(async () => {
@@ -668,7 +674,7 @@ describe('Article API', () => {
 
   describe('SaveUrl', () => {
     let query = ''
-    let url = 'https://blog.omnivore.app/new-url-1'
+    const url = 'https://blog.omnivore.app/new-url-1'
 
     before(() => {
       sinon.replace(createTask, 'enqueueParseRequest', sinon.fake.resolves(''))
@@ -727,8 +733,8 @@ describe('Article API', () => {
   describe('saveArticleReadingProgressResolver', () => {
     let query = ''
     let itemId = ''
-    let progress = 0.5
-    let topPercent: number | null = null
+    const progress = 0.5
+    const topPercent: number | null = null
 
     before(async () => {
       itemId = (await createTestLibraryItem(user.id)).id
@@ -1976,7 +1982,7 @@ describe('Article API', () => {
     const items: LibraryItem[] = []
 
     let query = ''
-    let keyword = 'typeahead'
+    const keyword = 'typeahead'
 
     before(async () => {
       // Create some test items
@@ -2049,8 +2055,8 @@ describe('Article API', () => {
       }
     `
     let since: string
-    let items: LibraryItem[] = []
-    let deletedItems: LibraryItem[] = []
+    const items: LibraryItem[] = []
+    const deletedItems: LibraryItem[] = []
 
     before(async () => {
       // Create some test items
@@ -2263,7 +2269,7 @@ describe('Article API', () => {
     )
 
     context('when action is Delete and query contains item id', () => {
-      let items: LibraryItem[] = []
+      const items: LibraryItem[] = []
 
       before(async () => {
         // Create some test items
@@ -2367,7 +2373,7 @@ describe('Article API', () => {
         }
       }`
 
-    let items: LibraryItem[] = []
+    const items: LibraryItem[] = []
 
     before(async () => {
       // Create some test items

packages/api/test/resolvers/features.test.ts

@@ -159,12 +159,14 @@ describe('features resolvers', () => {
     })
 
     context('when user is already opted in', () => {
+      const grantedAt = new Date('2024-05-15')
+
       before(async () => {
         // opt in
         await createFeature({
           user: { id: loginUser.id },
           name: featureName,
-          grantedAt: new Date(),
+          grantedAt,
         })
       })
 
@@ -182,7 +184,7 @@ describe('features resolvers', () => {
           {
             uid: loginUser.id,
             featureName,
-            grantedAt: Date.now() / 1000,
+            grantedAt: grantedAt.getTime() / 1000,
           },
           env.server.jwtSecret,
           { expiresIn: '1y' }
@@ -191,7 +193,7 @@ describe('features resolvers', () => {
         expect(res.body.data.optInFeature).to.eql({
           feature: {
             name: featureName,
-            grantedAt: new Date().toISOString(),
+            grantedAt: grantedAt.toISOString(),
             token,
           },
         })

packages/api/test/resolvers/subscriptions.test.ts

@@ -36,7 +36,7 @@ describe('Subscriptions API', () => {
       .post('/local/debug/fake-user-login')
       .send({ fakeEmail: user.email })
 
-    authToken = res.body.authToken
+    authToken = res.body.authToken as string
 
     // create test newsletter subscriptions
     const newsletterEmail = await createNewsletterEmail(user.id)
@@ -181,7 +181,7 @@ describe('Subscriptions API', () => {
           }))
         )
       } finally {
-        deleteUser(user2.id)
+        await deleteUser(user2.id)
       }
     })
 
@@ -222,7 +222,7 @@ describe('Subscriptions API', () => {
           }))
         )
       } finally {
-        deleteUser(user3.id)
+        await deleteUser(user3.id)
       }
     })
 
@@ -263,7 +263,7 @@ describe('Subscriptions API', () => {
           }))
         )
       } finally {
-        deleteUser(user2.id)
+        await deleteUser(user2.id)
       }
     })
 
@@ -372,7 +372,7 @@ describe('Subscriptions API', () => {
       const url = 'https://www.omnivore.app/rss'
       const subscriptionType = SubscriptionType.Rss
 
-      before(async () => {
+      before(() => {
         // fake rss parser
         sinon.replace(
           Parser.prototype,

packages/api/test/routers/email_attachments.test.ts

@@ -1,4 +1,3 @@
-import { Storage } from '@google-cloud/storage'
 import { expect } from 'chai'
 import * as jwt from 'jsonwebtoken'
 import 'mocha'
@@ -9,10 +8,9 @@ import { getRepository } from '../../src/repository'
 import { findLibraryItemById } from '../../src/services/library_item'
 import { deleteUser } from '../../src/services/user'
 import { createTestUser } from '../db'
-import { MockBucket } from '../mock_storage'
 import { request } from '../util'
 
-describe('Email attachments Router', () => {
+xdescribe('Email attachments Router', () => {
   const newsletterEmailAddress = 'fakeEmail@omnivore.app'
 
   let user: User
@@ -27,14 +25,6 @@ describe('Email attachments Router', () => {
       user: { id: user.id },
     })
     authToken = jwt.sign(newsletterEmailAddress, process.env.JWT_SECRET || '')
-
-    // mock cloud storage
-    const mockBucket = new MockBucket('test')
-    sinon.replace(
-      Storage.prototype,
-      'bucket',
-      sinon.fake.returns(mockBucket as never)
-    )
   })
 
   after(async () => {
@@ -76,7 +66,7 @@ describe('Email attachments Router', () => {
           fileName: testFile,
           contentType: 'application/pdf',
         })
-      uploadFileId = res.body.id
+      uploadFileId = res.body.id as string
     })
 
     it('create article with uploaded file id and url', async () => {

packages/content-fetch/package.json

@@ -13,6 +13,7 @@
     "ioredis": "^5.3.2",
     "posthog-node": "^3.6.3",
     "@google-cloud/functions-framework": "^3.0.0",
+    "@google-cloud/storage": "^7.0.1",
     "@omnivore/puppeteer-parse": "^1.0.0",
     "@sentry/serverless": "^7.77.0"
   },

packages/content-fetch/src/job.ts

@@ -9,6 +9,7 @@ interface SavePageJobData {
   url: string
   finalUrl: string
   articleSavingRequestId: string
+
   state?: string
   labels?: string[]
   source: string
@@ -17,6 +18,8 @@ interface SavePageJobData {
   savedAt?: string
   publishedAt?: string
   taskId?: string
+  title?: string
+  contentType?: string
 }
 
 interface SavePageJob {

packages/content-fetch/src/request_handler.ts

@@ -1,11 +1,12 @@
+import { Storage } from '@google-cloud/storage'
 import { fetchContent } from '@omnivore/puppeteer-parse'
 import { RequestHandler } from 'express'
 import { analytics } from './analytics'
 import { queueSavePageJob } from './job'
-import { redisDataSource } from './redis_data_source'
 
-interface User {
+interface UserConfig {
   id: string
+  libraryItemId: string
   folder?: string
 }
 
@@ -23,7 +24,7 @@ interface RequestBody {
   savedAt?: string
   publishedAt?: string
   folder?: string
-  users?: User[]
+  users?: UserConfig[]
   priority: 'high' | 'low'
 }
 
@@ -42,24 +43,37 @@ interface LogRecord {
   savedAt?: string
   publishedAt?: string
   folder?: string
-  users?: User[]
+  users?: UserConfig[]
   error?: string
   totalTime?: number
 }
 
-interface FetchResult {
-  finalUrl: string
-  title?: string
-  content?: string
-  contentType?: string
+const storage = process.env.GCS_UPLOAD_SA_KEY_FILE_PATH
+  ? new Storage({ keyFilename: process.env.GCS_UPLOAD_SA_KEY_FILE_PATH })
+  : new Storage()
+const bucketName = process.env.GCS_UPLOAD_BUCKET || 'omnivore-files'
+
+const uploadToBucket = async (filePath: string, data: string) => {
+  await storage
+    .bucket(bucketName)
+    .file(filePath)
+    .save(data, { public: false, timeout: 30000 })
 }
 
-export const cacheFetchResult = async (fetchResult: FetchResult) => {
-  // cache the fetch result for 24 hours
-  const ttl = 24 * 60 * 60
-  const key = `fetch-result:${fetchResult.finalUrl}`
-  const value = JSON.stringify(fetchResult)
-  return redisDataSource.cacheClient.set(key, value, 'EX', ttl, 'NX')
+const uploadOriginalContent = async (
+  users: UserConfig[],
+  content: string,
+  savedTimestamp: number
+) => {
+  await Promise.all(
+    users.map(async (user) => {
+      const filePath = `content/${user.id}/${user.libraryItemId}.${savedTimestamp}.original`
+
+      await uploadToBucket(filePath, content)
+
+      console.log(`Original content uploaded to ${filePath}`)
+    })
+  )
 }
 
 export const contentFetchRequestHandler: RequestHandler = async (req, res) => {
@@ -76,6 +90,7 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => {
       {
         id: userId,
         folder: body.folder,
+        libraryItemId: body.saveRequestId,
       },
     ]
   }
@@ -112,8 +127,12 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => {
   console.log(`Article parsing request`, logRecord)
 
   try {
+    const savedDate = savedAt ? new Date(savedAt) : new Date()
     const fetchResult = await fetchContent(url, locale, timezone)
-    const finalUrl = fetchResult.finalUrl
+    const { title, content, contentType, finalUrl } = fetchResult
+    if (content) {
+      await uploadOriginalContent(users, content, savedDate.getTime())
+    }
 
     const savePageJobs = users.map((user) => ({
       userId: user.id,
@@ -121,24 +140,23 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => {
         userId: user.id,
         url,
         finalUrl,
-        articleSavingRequestId,
+        articleSavingRequestId: user.libraryItemId,
         state,
         labels,
         source,
         folder: user.folder,
         rssFeedUrl,
-        savedAt,
+        savedAt: savedDate.toISOString(),
         publishedAt,
         taskId,
+        title,
+        contentType,
       },
       isRss: !!rssFeedUrl,
       isImport: !!taskId,
       priority,
     }))
 
-    const cacheResult = await cacheFetchResult(fetchResult)
-    console.log('cacheFetchResult result', cacheResult)
-
     const jobs = await queueSavePageJob(savePageJobs)
     console.log('save-page jobs queued', jobs.length)
   } catch (error) {