Merge 893d65d704 into 4cd5f2f02a

.eslintrc

@@ -11,6 +11,7 @@
   },
   "plugins": ["@typescript-eslint"],
   "rules": {
-    "semi": [2, "never"]
+    "semi": [2, "never"],
+    "@typescript-eslint/no-unnecessary-type-assertion": [0, "never"]
   }
 }

.github/workflows/run-tests.yaml

@@ -57,7 +57,7 @@ jobs:
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v2
         with:
-          node-version: 18.16
+          node-version: 22.12.0
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(source ~/.nvm/nvm.sh && yarn cache dir)"

.node-version

@@ -1 +1 @@
-18.16
+22.12.0

README.md

@@ -151,24 +151,7 @@ is done fetching your content you will see it in your library.
 
 ## How to deploy to your own server
 
-Omnivore was originally designed to be deployed on GCP and takes advantage
-of some of GCP's PaaS features. We are working to make Omnivore more portable
-so you can easily run the service on your own infrastructure. You can track
-progress here: <https://github.com/omnivore-app/omnivore/issues/25>
-
-To deploy Omnivore on your own hardware you will need to deploy three
-dockerized services and configure access to a postgres service. To handle
-PDF documents you will need to configure access to a Google Cloud Storage
-bucket.
-
-- `packages/api` - the backend API service
-- `packages/web` - the web frontend (can easily be deployed to vercel)
-- `packages/puppeteer-parse` - the content fetching service (can easily
-  be deployed as an AWS lambda or GCP Cloud Function)
-
-Additionally, you will need to run our database migrations to initialize
-your database. These are dockerized and can be run with the
-`packages/db` service.
+A guide for running a self hosted server can be found [here](./self-hosting/GUIDE.md)
 
 ## License
 

imageproxy/Dockerfile

@@ -1,4 +1,4 @@
-FROM willnorris/imageproxy:v0.10.0 as build
+FROM ghcr.io/willnorris/imageproxy:main as build
 
 # Above imageproxy image is built from scratch image and is barebones
 # Switching over to ubuntu base image to allow us to debug better.

packages/api/.eslintrc

@@ -10,7 +10,8 @@
     {
       "files": ["test/**/*.ts"],
       "rules": {
-        "@typescript-eslint/no-unsafe-member-access": 0
+        "@typescript-eslint/no-unsafe-member-access": 0,
+        "@typescript-eslint/no-unnecessary-type-assertion": 0
       }
     }
   ]

packages/api/Dockerfile

@@ -1,51 +1,36 @@
-FROM node:18.16 as builder
+FROM node:22.12 AS builder
 
 WORKDIR /app
 
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
-RUN apt-get update && apt-get install -y g++ make python3
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
 
-COPY package.json .
-COPY yarn.lock .
-COPY tsconfig.json .
-COPY .prettierrc .
-COPY .eslintrc .
+RUN apt-get update && apt-get install -y g++ make python3 && apt-get clean && rm -rf /var/lib/apt/lists/*
 
-COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json
-COPY /packages/api/package.json ./packages/api/package.json
-COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json
-COPY /packages/content-handler/package.json ./packages/content-handler/package.json
-COPY /packages/liqe/package.json ./packages/liqe/package.json
-COPY /packages/utils/package.json ./packages/utils/package.json
+COPY package.json yarn.lock tsconfig.json .prettierrc .eslintrc ./
+COPY packages ./packages
 
-RUN yarn install --pure-lockfile
+# Remove all except needed packages
+RUN find packages -mindepth 1 -type d \
+    ! -regex '^packages/\(api\|readabilityjs\|text-to-speech\|content-handler\|liqe\|utils\)\(/.*\)?' \
+    -exec rm -rf {} +
 
-ADD /packages/readabilityjs ./packages/readabilityjs
-ADD /packages/api ./packages/api
-ADD /packages/text-to-speech ./packages/text-to-speech
-ADD /packages/content-handler ./packages/content-handler
-ADD /packages/liqe ./packages/liqe
-ADD /packages/utils ./packages/utils
+RUN yarn install --pure-lockfile && \
+    yarn workspace @omnivore/utils build && \
+    yarn workspace @omnivore/text-to-speech-handler build && \
+    yarn workspace @omnivore/content-handler build && \
+    yarn workspace @omnivore/liqe build && \
+    yarn workspace @omnivore/api build && \
+    rm -rf /app/packages/api/node_modules /app/node_modules && \
+    yarn install --pure-lockfile --production
 
-RUN yarn workspace @omnivore/utils build
-RUN yarn workspace @omnivore/text-to-speech-handler build
-RUN yarn workspace @omnivore/content-handler build
-RUN yarn workspace @omnivore/liqe build
-RUN yarn workspace @omnivore/api build
-
-# After building, fetch the production dependencies
-RUN rm -rf /app/packages/api/node_modules
-RUN rm -rf /app/node_modules
-RUN yarn install --pure-lockfile --production
-
-FROM node:18.16 as runner
+FROM node:22.12-alpine AS runner
 LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
 
-RUN apt-get update && apt-get install -y netcat-openbsd
+RUN apk update && apk add netcat-openbsd && rm -rf /var/cache/apk/*
 
 WORKDIR /app
 
-ENV NODE_ENV production
+ENV NODE_ENV=production
 ENV NODE_OPTIONS=--max-old-space-size=4096
 ENV PORT=8080
 
@@ -59,6 +44,7 @@ COPY --from=builder /app/packages/text-to-speech/ /app/packages/text-to-speech/
 COPY --from=builder /app/packages/content-handler/ /app/packages/content-handler/
 COPY --from=builder /app/packages/liqe/ /app/packages/liqe/
 COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
 EXPOSE 8080
 
 CMD ["yarn", "workspace", "@omnivore/api", "start"]

packages/api/Dockerfile-test

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:22.12-alpine
 
 WORKDIR /app
 

packages/api/package.json

@@ -120,7 +120,10 @@
     "voca": "^1.4.0",
     "winston": "^3.3.3",
     "yaml": "^2.4.1",
-    "youtubei": "^1.5.4"
+    "youtubei": "^1.5.4",
+    "@aws-sdk/client-s3": "^3.679.0",
+    "@aws-sdk/s3-request-presigner": "^3.679.0",
+    "@aws-sdk/lib-storage": "^3.679.0"
   },
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.2",
@@ -171,9 +174,9 @@
     "ts-node-dev": "^1.1.8"
   },
   "engines": {
-    "node": "18.16.1"
+    "node": "22.12.0"
   },
   "volta": {
     "extends": "../../package.json"
   }
-}
+}

packages/api/queue-processor/Dockerfile

@@ -0,0 +1,61 @@
+FROM node:22.12 as builder
+
+WORKDIR /app
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+RUN apt-get update && apt-get install -y g++ make python3
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json
+COPY /packages/api/package.json ./packages/api/package.json
+COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json
+COPY /packages/content-handler/package.json ./packages/content-handler/package.json
+COPY /packages/liqe/package.json ./packages/liqe/package.json
+COPY /packages/utils/package.json ./packages/utils/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/readabilityjs ./packages/readabilityjs
+ADD /packages/api ./packages/api
+ADD /packages/text-to-speech ./packages/text-to-speech
+ADD /packages/content-handler ./packages/content-handler
+ADD /packages/liqe ./packages/liqe
+ADD /packages/utils ./packages/utils
+
+RUN yarn workspace @omnivore/utils build
+RUN yarn workspace @omnivore/text-to-speech-handler build
+RUN yarn workspace @omnivore/content-handler build
+RUN yarn workspace @omnivore/liqe build
+RUN yarn workspace @omnivore/api build
+
+# After building, fetch the production dependencies
+RUN rm -rf /app/packages/api/node_modules
+RUN rm -rf /app/node_modules
+RUN yarn install --pure-lockfile --production
+
+FROM node:22.12 as runner
+LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
+
+RUN apt-get update && apt-get install -y netcat-openbsd
+
+WORKDIR /app
+
+ENV NODE_ENV production
+
+COPY --from=builder /app/packages/api/dist /app/packages/api/dist
+COPY --from=builder /app/packages/readabilityjs/ /app/packages/readabilityjs/
+COPY --from=builder /app/packages/api/package.json /app/packages/api/package.json
+COPY --from=builder /app/packages/api/node_modules /app/packages/api/node_modules
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+COPY --from=builder /app/packages/text-to-speech/ /app/packages/text-to-speech/
+COPY --from=builder /app/packages/content-handler/ /app/packages/content-handler/
+COPY --from=builder /app/packages/liqe/ /app/packages/liqe/
+COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
+CMD ["yarn", "workspace", "@omnivore/api", "start_queue_processor"]

packages/api/src/data_source.ts

@@ -23,6 +23,7 @@ export const appDataSource = new DataSource({
     max: env.pg.pool.max,
     idleTimeoutMillis: 10000, // 10 seconds
   },
+
   replication: env.pg.replication
     ? {
         master: {
@@ -42,5 +43,15 @@ export const appDataSource = new DataSource({
           },
         ],
       }
-    : undefined,
+    : {
+        defaultMode: 'master',
+        master: {
+          host: env.pg.host,
+          port: env.pg.port,
+          username: env.pg.userName,
+          password: env.pg.password,
+          database: env.pg.dbName,
+        },
+        slaves: [],
+      },
 })

packages/api/src/jobs/export.ts

@@ -1,10 +1,6 @@
 import archiver, { Archiver } from 'archiver'
 import { v4 as uuidv4 } from 'uuid'
-import {
-  ContentReaderType,
-  LibraryItem,
-  LibraryItemState,
-} from '../entity/library_item'
+import { LibraryItem, LibraryItemState } from '../entity/library_item'
 import { TaskState } from '../generated/graphql'
 import { findExportById, saveExport } from '../services/export'
 import { findHighlightsByLibraryItemId } from '../services/highlights'
@@ -17,12 +13,11 @@ import { sendExportJobEmail } from '../services/send_emails'
 import { findActiveUser } from '../services/user'
 import { logger } from '../utils/logger'
 import { highlightToMarkdown } from '../utils/parser'
-import {
-  contentFilePath,
-  createGCSFile,
-  generateUploadFilePathName,
-} from '../utils/uploads'
-import { batch } from 'googleapis/build/src/apis/batch'
+import { env } from '../env'
+import { storage } from '../repository/storage/storage'
+import { File } from '../repository/storage/StorageClient'
+import { Readable } from 'stream'
+import { contentFilePath, generateUploadFilePathName } from '../utils/uploads'
 import { getRepository } from '../repository'
 import { UploadFile } from '../entity/upload_file'
 
@@ -31,6 +26,12 @@ export interface ExportJobData {
   exportId: string
 }
 
+const bucketName = env.fileUpload.gcsUploadBucket
+
+const createGCSFile = (filename: string): File => {
+  return storage.createFile(bucketName, filename)
+}
+
 export const EXPORT_JOB_NAME = 'export'
 
 const itemStateMappping = (state: LibraryItemState) => {
@@ -61,7 +62,7 @@ const uploadContent = async (
   const file = createGCSFile(filePath)
 
   // check if file is already uploaded
-  const [exists] = await file.exists()
+  const exists = await file.exists()
   if (!exists) {
     logger.info(`File not found: ${filePath}`)
 
@@ -81,10 +82,14 @@ const uploadContent = async (
       contentType: 'text/html',
       private: true,
     })
+    archive.append(Readable.from(item.readableContent), {
+      name: `content/${libraryItem.slug}.html`,
+    })
   }
 
   // append the existing file to the archive
-  archive.append(file.createReadStream(), {
+  const content = await file.download()
+  archive.append(Readable.from(content.toString()), {
     name: `content/${libraryItem.slug}.html`,
   })
 }
@@ -97,17 +102,19 @@ const uploadPdfContent = async (
     id: libraryItem.uploadFileId,
   })
   if (!upload || !upload.fileName) {
-    console.log(`upload does not have a filename: ${upload}`)
+    console.log(
+      `upload does not have a filename: ${upload?.fileName ?? 'empty'}`
+    )
     return
   }
 
   const filePath = generateUploadFilePathName(upload.id, upload.fileName)
   const file = createGCSFile(filePath)
-  const [exists] = await file.exists()
+  const exists = await file.exists()
   if (exists) {
     console.log(`adding PDF file: ${filePath}`)
     // append the existing file to the archive
-    archive.append(file.createReadStream(), {
+    archive.append(await file.download(), {
       name: `content/${libraryItem.slug}.pdf`,
     })
   }
@@ -238,9 +245,18 @@ export const exportJob = async (jobData: ExportJobData) => {
 
     // Create a write stream
     const writeStream = file.createWriteStream({
-      metadata: {
-        contentType: 'application/zip',
-      },
+      contentType: 'application/zip',
+    })
+
+    const finishedPromise = new Promise<void>((resolve, reject) => {
+      if (writeStream.closed) {
+        resolve()
+      }
+      writeStream.on('finish', () => {
+        logger.info('File successfully written to GCS')
+        resolve()
+      })
+      writeStream.on('error', reject)
     })
 
     // Handle any errors in the streams
@@ -248,10 +264,6 @@ export const exportJob = async (jobData: ExportJobData) => {
       logger.error('Error writing to GCS:', err)
     })
 
-    writeStream.on('finish', () => {
-      logger.info('File successfully written to GCS')
-    })
-
     // Initialize archiver for zipping files
     const archive = archiver('zip', {
       zlib: { level: 9 }, // Compression level
@@ -264,7 +276,6 @@ export const exportJob = async (jobData: ExportJobData) => {
 
     // Pipe the archiver output to the write stream
     archive.pipe(writeStream)
-
     let cursor = 0
     try {
       // fetch data from the database
@@ -305,17 +316,14 @@ export const exportJob = async (jobData: ExportJobData) => {
     }
 
     // Ensure that the writeStream has finished
-    await new Promise((resolve, reject) => {
-      writeStream.on('finish', resolve)
-      writeStream.on('error', reject)
-    })
+    await finishedPromise
 
     logger.info(`export completed, exported ${cursor} items`, {
       userId,
     })
 
     // generate a temporary signed url for the zip file
-    const [signedUrl] = await file.getSignedUrl({
+    const signedUrl = await storage.signedUrl(bucketName, fullPath, {
       action: 'read',
       expires: Date.now() + 168 * 60 * 60 * 1000, // one week
     })

packages/api/src/queue-processor.ts

@@ -56,7 +56,10 @@ import {
   PROCESS_YOUTUBE_VIDEO_JOB_NAME,
 } from './jobs/process-youtube-video'
 import { pruneTrashJob, PRUNE_TRASH_JOB } from './jobs/prune_trash'
-import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds'
+import {
+  REFRESH_ALL_FEEDS_JOB_NAME,
+  refreshAllFeeds,
+} from './jobs/rss/refreshAllFeeds'
 import { refreshFeed } from './jobs/rss/refreshFeed'
 import { savePageJob } from './jobs/save_page'
 import {
@@ -159,25 +162,25 @@ export const createWorker = (connection: ConnectionOptions) =>
     async (job: Job) => {
       const executeJob = async (job: Job) => {
         switch (job.name) {
-          // case 'refresh-all-feeds': {
-          //   const queue = await getQueue()
-          //   const counts = await queue?.getJobCounts('prioritized')
-          //   if (counts && counts.wait > 1000) {
-          //     return
-          //   }
-          //   return await refreshAllFeeds(appDataSource)
-          // }
-          // case 'refresh-feed': {
-          //   return await refreshFeed(job.data)
-          // }
+          case 'refresh-all-feeds': {
+            const queue = await getQueue()
+            const counts = await queue?.getJobCounts('prioritized')
+            if (counts && counts.wait > 1000) {
+              return
+            }
+            return await refreshAllFeeds(appDataSource)
+          }
+          case 'refresh-feed': {
+            return await refreshFeed(job.data)
+          }
           case 'save-page': {
             return savePageJob(job.data, job.attemptsMade)
           }
-          // case 'update-pdf-content': {
-          //   return updatePDFContentJob(job.data)
-          // }
-          // case THUMBNAIL_JOB:
-          //   return findThumbnail(job.data)
+          case 'update-pdf-content': {
+            return updatePDFContentJob(job.data)
+          }
+          case THUMBNAIL_JOB:
+            return findThumbnail(job.data)
           case TRIGGER_RULE_JOB_NAME:
             return triggerRule(job.data)
           case UPDATE_LABELS_JOB:
@@ -218,8 +221,8 @@ export const createWorker = (connection: ConnectionOptions) =>
           //   return updateHome(job.data)
           // case SCORE_LIBRARY_ITEM_JOB:
           //   return scoreLibraryItem(job.data)
-          // case GENERATE_PREVIEW_CONTENT_JOB:
-          //   return generatePreviewContent(job.data)
+          case GENERATE_PREVIEW_CONTENT_JOB:
+            return generatePreviewContent(job.data)
           case PRUNE_TRASH_JOB:
             return pruneTrashJob(job.data)
           case EXPIRE_FOLDERS_JOB_NAME:
@@ -260,6 +263,17 @@ const setupCronJobs = async () => {
       },
     }
   )
+
+  await queue.add(
+    REFRESH_ALL_FEEDS_JOB_NAME,
+    {},
+    {
+      priority: getJobPriority(REFRESH_ALL_FEEDS_JOB_NAME),
+      repeat: {
+        every: 14_400_000, // 4 Hours
+      },
+    }
+  )
 }
 
 const main = async () => {

packages/api/src/repository/storage/GcsStorageClient.ts

@@ -0,0 +1,88 @@
+import {
+  SignedUrlParameters,
+  StorageClient,
+  File,
+  SaveOptions,
+  SaveData,
+} from './StorageClient'
+import { Storage, File as GCSFile } from '@google-cloud/storage'
+
+export class GcsStorageClient implements StorageClient {
+  private storage: Storage
+
+  constructor(keyFilename: string | undefined) {
+    this.storage = new Storage({
+      keyFilename,
+    })
+  }
+
+  private convertFileToGeneric(gcsFile: GCSFile): File {
+    return {
+      isPublic: async () => {
+        const [isPublic] = await gcsFile.isPublic()
+        return isPublic
+      },
+      exists: async () => (await gcsFile.exists())[0],
+      download: async () => (await gcsFile.download())[0],
+      bucket: gcsFile.bucket.name,
+      publicUrl: () => gcsFile.publicUrl(),
+      getMetadataMd5: async () => {
+        const [metadata] = await gcsFile.getMetadata()
+        return metadata.md5Hash
+      },
+      createWriteStream: (saveOptions: SaveOptions) =>
+        gcsFile.createWriteStream({
+          metadata: { contentType: saveOptions.contentType },
+        }),
+      save: (saveData: SaveData, saveOptions: SaveOptions) =>
+        gcsFile.save(saveData, saveOptions),
+      key: gcsFile.name,
+    }
+  }
+
+  downloadFile(bucket: string, filePath: string): Promise<File> {
+    const file = this.storage.bucket(bucket).file(filePath)
+    return Promise.resolve(this.convertFileToGeneric(file))
+  }
+
+  createFile(bucket: string, filePath: string): File {
+    return this.convertFileToGeneric(this.storage.bucket(bucket).file(filePath))
+  }
+
+  async getFilesFromPrefix(bucket: string, prefix: string): Promise<File[]> {
+    const [filesWithPrefix] = await this.storage
+      .bucket(bucket)
+      .getFiles({ prefix })
+
+    return filesWithPrefix.map((it: GCSFile) => this.convertFileToGeneric(it))
+  }
+
+  async signedUrl(
+    bucket: string,
+    filePath: string,
+    options: SignedUrlParameters
+  ): Promise<string> {
+    const [url] = await this.storage
+      .bucket(bucket)
+      .file(filePath)
+      .getSignedUrl({ ...options, version: 'v4' })
+
+    return url
+  }
+
+  upload(
+    bucket: string,
+    filePath: string,
+    data: Buffer,
+    options: {
+      contentType?: string
+      public?: boolean
+      timeout?: number
+    }
+  ): Promise<void> {
+    return this.storage
+      .bucket(bucket)
+      .file(filePath)
+      .save(data, { timeout: 30000, ...options })
+  }
+}

packages/api/src/repository/storage/S3StorageClient.ts

@@ -0,0 +1,252 @@
+import {
+  SignedUrlParameters,
+  StorageClient,
+  File,
+  SaveOptions,
+  SaveData,
+} from './StorageClient'
+import { Upload } from '@aws-sdk/lib-storage'
+import {
+  GetObjectCommand,
+  GetObjectCommandOutput,
+  S3Client,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  HeadObjectCommand,
+  S3ServiceException,
+} from '@aws-sdk/client-s3'
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
+import { Readable } from 'stream'
+import * as stream from 'node:stream'
+
+// While this is listed as S3, for self hosting we will use MinIO, which is
+// S3 Compatible.
+export class S3StorageClient implements StorageClient {
+  BlankFile = class implements File {
+    bucket: string
+    key: string
+    s3Client: S3StorageClient
+    downloadedFile: File | undefined
+
+    constructor(s3StorageClass: S3StorageClient, bucket: string, file: string) {
+      this.bucket = bucket
+      this.key = file
+      this.s3Client = s3StorageClass
+    }
+
+    isPublic() {
+      return Promise.resolve(true)
+    }
+
+    publicUrl() {
+      return `${this.s3Client.urlOverride ?? ''}/${this.bucket}/${this.key}`
+    }
+
+    async download(): Promise<Buffer> {
+      this.downloadedFile = await this.s3Client.downloadFile(
+        this.bucket,
+        this.key
+      )
+      return this.downloadedFile.download()
+    }
+
+    async exists() {
+      try {
+        await this.s3Client.s3Client.send(
+          new HeadObjectCommand({
+            Bucket: this.bucket,
+            Key: this.key,
+          })
+        )
+
+        return true
+      } catch (e) {
+        if (
+          e instanceof S3ServiceException &&
+          e.$metadata.httpStatusCode == 404
+        ) {
+          return false
+        }
+
+        throw e
+      }
+    }
+
+    save(saveData: SaveData, saveOptions: SaveOptions): Promise<void> {
+      return this.s3Client.upload(this.bucket, this.key, saveData, saveOptions)
+    }
+
+    createWriteStream(saveOptions: SaveOptions) {
+      return this.s3Client.createS3UploadStream(
+        this.bucket,
+        this.key,
+        saveOptions
+      )
+    }
+
+    getMetadataMd5() {
+      return this.downloadedFile?.getMetadataMd5() || Promise.resolve('')
+    }
+  }
+
+  private s3Client: S3Client
+  private urlOverride: string | undefined
+
+  constructor(urlOverride: string | undefined) {
+    this.urlOverride = urlOverride
+    this.s3Client = new S3Client({
+      forcePathStyle: true,
+      endpoint: urlOverride,
+    })
+  }
+
+  private createS3UploadStream = (
+    bucket: string,
+    key: string,
+    saveOptions: SaveOptions
+  ) => {
+    const passThroughStream = new stream.PassThrough()
+
+    const upload = new Upload({
+      client: this.s3Client,
+      params: {
+        Bucket: bucket,
+        Key: key,
+        Body: passThroughStream,
+        ContentType: saveOptions.contentType,
+      },
+    })
+
+    void upload.done().then((res) => {
+      console.log(`Successfully Uploaded File ${res.Key ?? ''}`)
+    })
+
+    return passThroughStream
+  }
+
+  private convertFileToGeneric(
+    s3File: GetObjectCommandOutput,
+    bucket: string,
+    key: string
+  ): File {
+    return {
+      exists: () => {
+        return Promise.resolve(s3File.$metadata.httpStatusCode == 200)
+      },
+      save: async () => Promise.resolve(),
+      isPublic: async () => Promise.resolve(true),
+      download: async () => this.getFileFromReadable(s3File.Body as Readable),
+      getMetadataMd5: () => Promise.resolve(s3File.ETag),
+      createWriteStream: (saveOptions: SaveOptions) =>
+        this.createS3UploadStream(bucket, key, saveOptions),
+      publicUrl: () => `${this.urlOverride ?? ''}/${bucket}/${key}`,
+      bucket,
+      key,
+    }
+  }
+
+  private getFileFromReadable(stream: Readable): Promise<Buffer> {
+    return new Promise<Buffer>((resolve, reject) => {
+      const chunks: Buffer[] = []
+      stream.on('data', (chunk) => chunks.push(chunk))
+      stream.once('end', () => resolve(Buffer.concat(chunks)))
+      stream.once('error', reject)
+    })
+  }
+
+  async downloadFile(bucket: string, filePath: string): Promise<File> {
+    const s3File = await this.s3Client.send(
+      new GetObjectCommand({
+        Bucket: bucket,
+        Key: filePath, // path to the file you want to download,
+      })
+    )
+
+    return this.convertFileToGeneric(s3File, bucket, filePath)
+  }
+
+  createFile(bucket: string, filePath: string): File {
+    return new this.BlankFile(this, bucket, filePath) as unknown as File
+  }
+
+  async getFilesFromPrefix(bucket: string, prefix: string): Promise<File[]> {
+    const s3PrefixedFiles = await this.s3Client.send(
+      new ListObjectsV2Command({
+        Bucket: bucket,
+        Prefix: prefix, // path to the file you want to download,
+      })
+    )
+
+    const prefixKeys = s3PrefixedFiles.CommonPrefixes || []
+
+    return prefixKeys
+      .map(({ Prefix }) => Prefix)
+      .map((key: string | undefined) => {
+        return {
+          key: key || '',
+          exists: () => Promise.resolve(true),
+          isPublic: async () => Promise.resolve(true),
+          download: async () => {
+            const s3File = await this.s3Client.send(
+              new GetObjectCommand({
+                Bucket: bucket,
+                Key: key, // path to the file you want to download,
+              })
+            )
+
+            return this.getFileFromReadable(s3File.Body as Readable)
+          },
+          save: () => Promise.resolve(),
+          createWriteStream: (saveOptions: SaveOptions) =>
+            new stream.PassThrough(),
+          getMetadataMd5: () => Promise.resolve(key),
+          bucket: bucket,
+          publicUrl: () => `${this.urlOverride ?? ''}/${bucket}/${key ?? ''}`,
+        }
+      })
+  }
+
+  async signedUrl(
+    bucket: string,
+    filePath: string,
+    options: SignedUrlParameters
+  ): Promise<string> {
+    const command =
+      options.action == 'read'
+        ? new GetObjectCommand({
+            Bucket: bucket,
+            Key: filePath, // path to the file you want to download,
+          })
+        : new PutObjectCommand({
+            Bucket: bucket,
+            Key: filePath, // path to the file you want to download,
+          })
+
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-call
+    const url = await getSignedUrl(this.s3Client, command, {
+      expiresIn: 900,
+    })
+
+    return url
+  }
+
+  async upload(
+    bucket: string,
+    filePath: string,
+    data: SaveData,
+    options: {
+      contentType?: string
+      public?: boolean
+      timeout?: number
+    }
+  ): Promise<void> {
+    await this.s3Client.send(
+      new PutObjectCommand({
+        Bucket: bucket,
+        Key: filePath,
+        Body: data.toString(),
+        ContentType: options.contentType,
+      })
+    )
+  }
+}

packages/api/src/repository/storage/StorageClient.ts

@@ -0,0 +1,49 @@
+import { PipelineSource, Writable } from 'stream'
+
+export type SignedUrlParameters = {
+  action: 'read' | 'write' | 'delete' | 'resumable'
+  expires: number
+}
+
+export type SaveData = string | Buffer | PipelineSource<string | Buffer>
+export type SaveOptions = {
+  contentType?: string
+  gzip?: string | boolean
+  resumable?: boolean
+  timeout?: number
+  validation?: string | boolean
+  private?: boolean | undefined
+}
+
+export type File = {
+  isPublic: () => Promise<boolean>
+  publicUrl: () => string
+  download: () => Promise<Buffer>
+  exists: () => Promise<boolean>
+  save: (saveData: SaveData, saveOptions: SaveOptions) => Promise<void>
+  createWriteStream: (saveOptions: SaveOptions) => Writable
+  getMetadataMd5: () => Promise<string | undefined>
+  bucket: string
+  key: string
+}
+
+export interface StorageClient {
+  downloadFile(bucket: string, filePath: string): Promise<File>
+
+  createFile(bucket: string, filePath: string): File
+
+  getFilesFromPrefix(bucket: string, filePrefix: string): Promise<File[]>
+
+  upload(
+    bucket: string,
+    filePath: string,
+    data: Buffer,
+    options: { contentType?: string; public?: boolean; timeout?: number }
+  ): Promise<void>
+
+  signedUrl(
+    bucket: string,
+    filePath: string,
+    options: SignedUrlParameters
+  ): Promise<string>
+}

packages/api/src/repository/storage/storage.ts

@@ -0,0 +1,7 @@
+import { env } from '../../env'
+import { S3StorageClient } from './S3StorageClient'
+import { GcsStorageClient } from './GcsStorageClient'
+
+export const storage = env.fileUpload.useLocalStorage
+  ? new S3StorageClient(env.fileUpload.localMinioUrl)
+  : new GcsStorageClient(env.fileUpload?.gcsUploadSAKeyFilePath ?? undefined)

packages/api/src/resolvers/article/index.ts

@@ -510,12 +510,12 @@ export const saveArticleReadingProgressResolver = authorized<
       }
     }
 
-    if (env.redis.cache && env.redis.mq) {
-      if (force) {
-        // clear any cached values.
-        await clearCachedReadingPosition(uid, id)
-      }
+    if (env.redis.cache && force) {
+      // clear any cached values.
+      await clearCachedReadingPosition(uid, id)
+    }
 
+    if (env.redis.cache && env.redis.mq && !force) {
       // If redis caching and queueing are available we delay this write
       const updatedProgress =
         await dataSources.readingProgress.updateReadingProgress(uid, id, {

packages/api/src/routers/auth/auth_router.ts

@@ -29,6 +29,7 @@ import {
 import { analytics } from '../../utils/analytics'
 import {
   comparePassword,
+  generateVerificationToken,
   hashPassword,
   setAuthInCookie,
   verifyToken,
@@ -544,7 +545,7 @@ export function authRouter() {
       try {
         // hash password
         const hashedPassword = await hashPassword(password)
-        await createUser({
+        const [user] = await createUser({
           email: trimmedEmail,
           provider: 'EMAIL',
           sourceUserId: trimmedEmail,
@@ -553,12 +554,17 @@ export function authRouter() {
           pictureUrl,
           bio,
           password: hashedPassword,
-          pendingConfirmation: true,
+          pendingConfirmation: !env.dev.autoVerify,
         })
 
-        res.redirect(
-          `${env.client.url}/auth/verify-email?message=SIGNUP_SUCCESS`
-        )
+        if (env.dev.autoVerify) {
+          const token = await generateVerificationToken({ id: user.id })
+          res.redirect(`${env.client.url}/auth/confirm-email/${token}`)
+        } else {
+          res.redirect(
+            `${env.client.url}/auth/verify-email?message=SIGNUP_SUCCESS`
+          )
+        }
       } catch (e) {
         logger.info('email-signup exception:', e)
         if (isErrorWithCode(e)) {

packages/api/src/routers/auth/mobile/sign_up.ts

@@ -13,6 +13,7 @@ import {
 } from '../auth_types'
 import { decodeGoogleToken } from '../google_auth'
 import { createPendingUserToken, suggestedUsername } from '../jwt_helpers'
+import { env } from '../../../env'
 
 export async function createMobileSignUpResponse(
   isAndroid: boolean,
@@ -66,7 +67,7 @@ export async function createMobileEmailSignUpResponse(
       name: name.trim(),
       username: username.trim().toLowerCase(),
       password: hashedPassword,
-      pendingConfirmation: true,
+      pendingConfirmation: !env.dev.autoVerify,
     })
 
     return {

packages/api/src/services/newsletters.ts

@@ -113,7 +113,9 @@ const createRandomEmailAddress = (userName: string, length: number): string => {
   when rand is sdfsdf-: jacksonh-sdfsdf-e@inbox.omnivore.app
   when rand is abcdef: jacksonh-abcdefe@inbox.omnivore.app
    */
-  return `${userName}-${nanoid(length)}e@${inbox}.omnivore.app`
+  return `${userName}-${nanoid(length)}e@${
+    env.email.domain || `@${inbox}.omnivore.app`
+  }`
 }
 
 export const findNewsletterEmailById = async (

packages/api/src/services/upload_file.ts

@@ -137,7 +137,7 @@ export const uploadFile = async (
         itemType,
         uploadFile: { id: uploadFileData.id },
         slug: generateSlug(uploadFilePathName),
-        state: LibraryItemState.Processing,
+        state: LibraryItemState.Succeeded,
         contentReader: contentReaderForLibraryItem(itemType, uploadFileId),
       },
       uid

packages/api/src/util.ts

@@ -73,6 +73,7 @@ export interface BackendEnv {
   }
   dev: {
     isLocal: boolean
+    autoVerify: boolean
   }
   queue: {
     location: string
@@ -94,6 +95,11 @@ export interface BackendEnv {
     gcsUploadSAKeyFilePath: string
     gcsUploadPrivateBucket: string
     dailyUploadLimit: number
+    useLocalStorage: boolean
+    localMinioUrl: string
+  }
+  email: {
+    domain: string
   }
   sender: {
     message: string
@@ -197,10 +203,13 @@ const nullableEnvVars = [
   'PG_REPLICA_USER',
   'PG_REPLICA_PASSWORD',
   'PG_REPLICA_DB',
+  'AUTO_VERIFY',
   'INTERCOM_WEB_SECRET',
   'INTERCOM_IOS_SECRET',
   'INTERCOM_ANDROID_SECRET',
   'EXPORT_TASK_HANDLER_URL',
+  'LOCAL_MINIO_URL',
+  'LOCAL_EMAIL_DOMAIN',
 ] // Allow some vars to be null/empty
 
 const envParser =
@@ -240,6 +249,7 @@ export function getEnv(): BackendEnv {
     pool: {
       max: parseInt(parse('PG_POOL_MAX'), 10),
     },
+
     replication: parse('PG_REPLICATION') === 'true',
     replica: {
       host: parse('PG_REPLICA_HOST'),
@@ -249,6 +259,9 @@ export function getEnv(): BackendEnv {
       dbName: parse('PG_REPLICA_DB'),
     },
   }
+  const email = {
+    domain: parse('LOCAL_EMAIL_DOMAIN'),
+  }
   const server = {
     jwtSecret: parse('JWT_SECRET'),
     ssoJwtSecret: parse('SSO_JWT_SECRET'),
@@ -288,6 +301,7 @@ export function getEnv(): BackendEnv {
   }
   const dev = {
     isLocal: parse('API_ENV') == 'local',
+    autoVerify: parse('AUTO_VERIFY') === 'true',
   }
   const queue = {
     location: parse('PUPPETEER_QUEUE_LOCATION'),
@@ -318,6 +332,8 @@ export function getEnv(): BackendEnv {
     dailyUploadLimit: parse('GCS_UPLOAD_DAILY_LIMIT')
       ? parseInt(parse('GCS_UPLOAD_DAILY_LIMIT'), 10)
       : 5, // default to 5
+    useLocalStorage: parse('GCS_USE_LOCAL_HOST') == 'true',
+    localMinioUrl: parse('LOCAL_MINIO_URL'),
   }
   const sender = {
     message: parse('SENDER_MESSAGE'),
@@ -374,6 +390,7 @@ export function getEnv(): BackendEnv {
   return {
     pg,
     client,
+    email,
     server,
     google,
     posthog,

packages/api/src/utils/uploads.ts

@@ -1,12 +1,12 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import { File, GetSignedUrlConfig, Storage } from '@google-cloud/storage'
 import axios from 'axios'
 import { ContentReaderType } from '../entity/library_item'
 import { env } from '../env'
 import { PageType } from '../generated/graphql'
 import { ContentFormat } from '../jobs/upload_content'
 import { logger } from './logger'
+import { storage } from '../repository/storage/storage'
 
 export const contentReaderForLibraryItem = (
   itemType: string,
@@ -31,14 +31,12 @@ export const contentReaderForLibraryItem = (
  * the default app engine service account on the IAM page. We also need to
  * enable IAM related APIs on the project.
  */
-export const storage = env.fileUpload?.gcsUploadSAKeyFilePath
-  ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath })
-  : new Storage()
+
 const bucketName = env.fileUpload.gcsUploadBucket
 const maxContentLength = 10 * 1024 * 1024 // 10MB
 
 export const countOfFilesWithPrefix = async (prefix: string) => {
-  const [files] = await storage.bucket(bucketName).getFiles({ prefix })
+  const files = await storage.getFilesFromPrefix(bucketName, prefix)
   return files.length
 }
 
@@ -48,40 +46,29 @@ export const generateUploadSignedUrl = async (
   selectedBucket?: string
 ): Promise<string> => {
   // These options will allow temporary uploading of file with requested content type
-  const options: GetSignedUrlConfig = {
+  const options = {
     version: 'v4',
-    action: 'write',
+    action: 'write' as const,
     expires: Date.now() + 15 * 60 * 1000, // 15 minutes
     contentType: contentType,
   }
   logger.info('signed url for: ', options)
 
-  // Get a v4 signed URL for uploading file
-  const [url] = await storage
-    .bucket(selectedBucket || bucketName)
-    .file(filePathName)
-    .getSignedUrl(options)
-  return url
+  return storage.signedUrl(selectedBucket || bucketName, filePathName, options)
 }
 
 export const generateDownloadSignedUrl = async (
   filePathName: string,
   config?: {
-    bucketName?: string
     expires?: number
   }
 ): Promise<string> => {
-  const options: GetSignedUrlConfig = {
-    version: 'v4',
-    action: 'read',
-    expires: config?.expires ?? Date.now() + 240 * 60 * 1000, // four hours
+  const options = {
+    action: 'read' as const,
+    expires: Date.now() + 240 * 60 * 1000, // four hours
+    ...config,
   }
-  const [url] = await storage
-    .bucket(config?.bucketName || bucketName)
-    .file(filePathName)
-    .getSignedUrl(options)
-  logger.info(`generating download signed url: ${url}`)
-  return url
+  return storage.signedUrl(bucketName, filePathName, options)
 }
 
 export const getStorageFileDetails = async (
@@ -89,10 +76,10 @@ export const getStorageFileDetails = async (
   fileName: string
 ): Promise<{ md5Hash: string; fileUrl: string }> => {
   const filePathName = generateUploadFilePathName(id, fileName)
-  const file = storage.bucket(bucketName).file(filePathName)
-  const [metadata] = await file.getMetadata()
+  const file = await storage.downloadFile(bucketName, filePathName)
+  const metadataMd5 = await file.getMetadataMd5()
   // GCS returns MD5 Hash in base64 encoding, we convert it here to hex string
-  const md5Hash = Buffer.from(metadata.md5Hash || '', 'base64').toString('hex')
+  const md5Hash = Buffer.from(metadataMd5 || '', 'base64').toString('hex')
 
   return { md5Hash, fileUrl: file.publicUrl() }
 }
@@ -110,17 +97,10 @@ export const uploadToBucket = async (
   options?: { contentType?: string; public?: boolean; timeout?: number },
   selectedBucket?: string
 ): Promise<void> => {
-  await storage
-    .bucket(selectedBucket || bucketName)
-    .file(filePath)
-    .save(data, { timeout: 30000, ...options }) // default timeout 30s
-}
-
-export const createGCSFile = (
-  filename: string,
-  selectedBucket = bucketName
-): File => {
-  return storage.bucket(selectedBucket).file(filename)
+  await storage.upload(selectedBucket || bucketName, filePath, data, {
+    timeout: 30000,
+    ...options,
+  })
 }
 
 export const downloadFromUrl = async (
@@ -154,16 +134,14 @@ export const uploadToSignedUrl = async (
 }
 
 export const isFileExists = async (filePath: string): Promise<boolean> => {
-  const [exists] = await storage.bucket(bucketName).file(filePath).exists()
+  const file = await storage.downloadFile(bucketName, filePath)
+  const exists = await file.exists()
   return exists
 }
 
 export const downloadFromBucket = async (filePath: string): Promise<Buffer> => {
-  const file = storage.bucket(bucketName).file(filePath)
-
-  // Download the file contents
-  const [data] = await file.download()
-  return data
+  const file = await storage.downloadFile(bucketName, filePath)
+  return file.download()
 }
 
 export const contentFilePath = ({

packages/api/test/hooks.ts

@@ -1,13 +1,13 @@
-import { Storage } from '@google-cloud/storage'
-import sinon from 'sinon'
-import * as uploads from '../src/utils/uploads'
-import { MockStorage } from './mock_storage'
-
-export const mochaHooks = {
-  beforeEach() {
-    // Mock cloud storage
-    sinon
-      .stub(uploads, 'storage')
-      .value(new MockStorage() as unknown as Storage)
-  },
-}
+// import { Storage } from '@google-cloud/storage'
+// import sinon from 'sinon'
+// import * as uploads from '../src/utils/uploads'
+// import { MockStorage } from './mock_storage'
+//
+// export const mochaHooks = {
+//   beforeEach() {
+//     // Mock cloud storage
+//     sinon
+//       .stub(uploads, 'storage')
+//       .value(new MockStorage() as unknown as Storage)
+//   },
+// }

packages/content-fetch/.eslintrc

@@ -4,6 +4,8 @@
     "project": "tsconfig.json"
   },
   "rules": {
+    "@typescript-eslint/no-unsafe-assignment": 0,
+    "@typescript-eslint/no-unnecessary-type-assertion": 0,
     "@typescript-eslint/no-floating-promises": [
       "error",
       {

packages/content-fetch/Dockerfile

@@ -1,9 +1,10 @@
-FROM node:18.16
+FROM node:22.12
 LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
 
 # Installs latest Chromium package.
 RUN apt-get update && apt-get install -y \
     chromium \
+    firefox-esr \
     ca-certificates \
     nodejs \
     yarn \
@@ -14,6 +15,7 @@ RUN apt-get update && apt-get install -y \
 WORKDIR /app
 
 ENV CHROMIUM_PATH /usr/bin/chromium
+ENV FIREFOX_PATH /usr/bin/firefox
 ENV LAUNCH_HEADLESS=true
 
 COPY package.json .
@@ -45,5 +47,13 @@ RUN yarn install --pure-lockfile --production
 
 EXPOSE 8080
 
-CMD ["yarn", "workspace", "@omnivore/content-fetch", "start"]
+# In Firefox we can't use the adblocking sites. Adding them to the hosts file of the docker seems to work.
+RUN wget https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
+RUN echo "#!/bin/bash \n\
+cat hosts >> /etc/hosts \n\
+yarn workspace @omnivore/content-fetch start" >> ./start.sh
+
+RUN chmod +x ./start.sh
+
+CMD ["./start.sh"]
 

packages/content-fetch/src/request_handler.ts

@@ -305,7 +305,7 @@ export const processFetchContentJob = async (
 
     const savedDate = savedAt ? new Date(savedAt) : new Date()
     const { finalUrl, title, content, contentType } = fetchResult
-    if (content) {
+    if (content && process.env['SKIP_UPLOAD_ORIGINAL'] !== 'true') {
       await uploadOriginalContent(users, content, savedDate.getTime())
     }
 

packages/content-handler/src/index.ts

@@ -39,6 +39,7 @@ import { WikipediaHandler } from './websites/wikipedia-handler'
 import { YoutubeHandler } from './websites/youtube-handler'
 import { ZhihuHandler } from './websites/zhihu-handler'
 import { TikTokHandler } from './websites/tiktok-handler'
+import { RawContentHandler } from './websites/raw-handler'
 
 const validateUrlString = (url: string): boolean => {
   const u = new URL(url)
@@ -66,6 +67,7 @@ const contentHandlers: ContentHandler[] = [
   new DerstandardHandler(),
   new ImageHandler(),
   new MediumHandler(),
+  new RawContentHandler(),
   new PdfHandler(),
   new ScrapingBeeHandler(),
   new TDotCoHandler(),

packages/content-handler/src/websites/medium-handler.ts

@@ -1,4 +1,6 @@
 import { ContentHandler, PreHandleResult } from '../content-handler'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
 
 export class MediumHandler extends ContentHandler {
   constructor() {
@@ -11,13 +13,52 @@ export class MediumHandler extends ContentHandler {
     return u.hostname.endsWith('medium.com')
   }
 
+  addImages(document: Document): Document {
+    const pictures = document.querySelectorAll('picture')
+
+    pictures.forEach((pict) => {
+      const source = pict.querySelector('source')
+      if (source) {
+        const srcSet = source.getAttribute('srcSet')
+
+        const sources = (srcSet || '')
+          .split(', ')
+          .map((src) => src.split(' '))
+          .sort((a, b) =>
+            Number(a[1].replace('w', '')) > Number(b[1].replace('w', ''))
+              ? -1
+              : 1
+          )
+
+        // This should be the largest image in the source set.
+        if (sources && sources.length && Array.isArray(sources[0])) {
+          const url = sources[0][0]
+          const img = document.createElement('img')
+          img.src = url
+          pict.after(img)
+          pict.remove()
+        }
+      }
+    })
+
+    return document
+  }
+
   async preHandle(url: string): Promise<PreHandleResult> {
     console.log('prehandling medium url', url)
 
     try {
       const res = new URL(url)
       res.searchParams.delete('source')
-      return Promise.resolve({ url: res.toString() })
+
+      const response = await axios.get(res.toString())
+      const dom = parseHTML(response.data).document
+      const imageAddedDom = this.addImages(dom)
+      return {
+        title: dom.title,
+        content: imageAddedDom.body.outerHTML,
+        url: res.toString(),
+      }
     } catch (error) {
       console.error('error prehandling medium url', error)
       throw error

packages/content-handler/src/websites/raw-handler.ts

@@ -0,0 +1,33 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+export class RawContentHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'RawContentHandler'
+  }
+
+  shouldPreHandle(url: string): boolean {
+    const u = new URL(url)
+    const hostnames = [
+      'medium.com',
+      'fastcompany.com',
+      'fortelabs.com',
+      'theverge.com',
+    ]
+
+    return hostnames.some((h) => u.hostname.endsWith(h))
+  }
+
+  async preHandle(url: string): Promise<PreHandleResult> {
+    try {
+      const response = await axios.get(url)
+      const dom = parseHTML(response.data).document
+      return { title: dom.title, content: response.data as string, url: url }
+    } catch (error) {
+      console.error('error prehandling URL', error)
+      throw error
+    }
+  }
+}

packages/db/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.16
+FROM node:22.12
 
 WORKDIR /app
 

packages/discover/Dockerfile

@@ -1,4 +1,4 @@
- FROM node:18.16 as builder
+ FROM node:22.12 as builder
 
 WORKDIR /app
 
@@ -17,7 +17,7 @@ COPY /packages/discover/tsconfig.json ./packages/discover/tsconfig.json
 RUN yarn install --pure-lockfile
 RUN yarn workspace @omnivore/discover build
 
-FROM node:18.16 as runner
+FROM node:22.12 as runner
 
 WORKDIR /app
 

packages/discover/src/lib/inputSources/articles/rss/rssConverters/atom.ts

@@ -3,9 +3,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-return */
 /* eslint-disable @typescript-eslint/restrict-template-expressions */
 
-import { OmnivoreArticle } from '../../../../../types/OmnivoreArticle'
 import { slugify } from 'voca'
-import { Observable, tap } from 'rxjs'
 import { fromArrayLike } from 'rxjs/internal/observable/innerFrom'
 import { mapOrNull } from '../../../../utils/reactive'
 import {

packages/export-handler/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:22.12-alpine
 
 WORKDIR /app
 

packages/import-handler/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:22.12-alpine
 
 WORKDIR /app
 

packages/import-handler/Dockerfile-collector

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:22.12-alpine
 
 WORKDIR /app
 

packages/import-handler/package.json

@@ -9,7 +9,6 @@
   "keywords": [],
   "license": "Apache-2.0",
   "scripts": {
-    "test": "yarn mocha -r ts-node/register --config mocha-config.json",
     "test:typecheck": "tsc --noEmit",
     "lint": "eslint src --ext ts,js,tsx,jsx",
     "compile": "tsc",

packages/inbound-email-handler/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:22.12-alpine
 
 # Run everything after as non-privileged user.
 WORKDIR /app

packages/integration-handler/Dockerfile-exporter

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:22.12-alpine
 
 # Run everything after as non-privileged user.
 WORKDIR /app

packages/integration-handler/Dockerfile-importer

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:22.12-alpine
 
 # Run everything after as non-privileged user.
 WORKDIR /app

packages/local-mail-watcher/.eslintrc

@@ -0,0 +1,13 @@
+{
+  "extends": "../../.eslintrc",
+  "parserOptions": {
+    "project": "tsconfig.json"
+  },
+  "rules": {
+    "@typescript-eslint/no-unsafe-argument": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/strictNullChecks": "off",
+    "@typescript-eslint/no-unsafe-member-access": "off",
+    "@typescript-eslint/no-unsafe-assignment": "off"
+  }
+}

packages/local-mail-watcher/.gitignore

@@ -0,0 +1,131 @@
+.idea/
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*

packages/local-mail-watcher/Dockerfile

@@ -0,0 +1,40 @@
+ FROM node:22.12 as builder
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y g++ make python3
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/local-mail-watcher/src ./packages/local-mail-watcher/src
+COPY /packages/local-mail-watcher/package.json ./packages/local-mail-watcher/package.json
+COPY /packages/local-mail-watcher/tsconfig.json ./packages/local-mail-watcher/tsconfig.json
+COPY /packages/utils/package.json ./packages/utils/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/utils ./packages/utils
+
+RUN yarn workspace @omnivore/utils build
+RUN yarn workspace @omnivore/local-mail-watcher build
+
+FROM node:22.12 as runner
+
+WORKDIR /app
+
+ENV NODE_ENV production
+
+COPY --from=builder /app/packages/local-mail-watcher/dist /app/packages/local-mail-watcher/dist
+COPY --from=builder /app/packages/local-mail-watcher/package.json /app/packages/local-mail-watcher/package.json
+COPY --from=builder /app/packages/local-mail-watcher/node_modules /app/packages/local-mail-watcher/node_modules
+COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
+
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+
+CMD ["yarn", "workspace", "@omnivore/local-mail-watcher", "start"]

packages/local-mail-watcher/Dockerfile-watcher

@@ -0,0 +1,39 @@
+ FROM node:22.12 as builder
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y g++ make python3
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/local-mail-watcher/src ./packages/local-mail-watcher/src
+COPY /packages/local-mail-watcher/package.json ./packages/local-mail-watcher/package.json
+COPY /packages/local-mail-watcher/tsconfig.json ./packages/local-mail-watcher/tsconfig.json
+COPY /packages/utils/package.json ./packages/utils/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/utils ./packages/utils
+
+RUN yarn workspace @omnivore/utils build
+RUN yarn workspace @omnivore/local-mail-watcher build
+
+FROM node:22.12 as runner
+
+WORKDIR /app
+
+ENV NODE_ENV production
+
+COPY --from=builder /app/packages/local-mail-watcher/dist /app/packages/local-mail-watcher/dist
+COPY --from=builder /app/packages/local-mail-watcher/package.json /app/packages/local-mail-watcher/package.json
+COPY --from=builder /app/packages/local-mail-watcher/node_modules /app/packages/local-mail-watcher/node_modules
+COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+
+CMD ["yarn", "workspace", "@omnivore/local-mail-watcher", "start-watcher"]

packages/local-mail-watcher/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@omnivore/local-mail-watcher",
+  "version": "0.0.1",
+  "scripts": {
+    "build": "tsc",
+    "dev": "ts-node-dev --files src/index.ts",
+    "start": "node dist/index.js",
+    "start-watcher": "node dist/watcher.js",
+    "lint": "eslint src --ext ts,js,tsx,jsx",
+    "lint:fix": "eslint src --fix --ext ts,js,tsx,jsx",
+    "test:typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "chokidar": "^4.0.1",
+    "mailparser": "^3.7.1",
+    "axios": "^1.7.7",
+    "express": "^4.21.1",
+    "bullmq": "^5.22.0",
+    "@omnivore/utils": "1.0.0"
+  },
+  "devDependencies": {
+    "@types/html-to-text": "^9.0.2",
+    "@types/jsdom": "^21.1.3",
+    "@types/mailparser": "^3.4.5",
+    "@types/axios" : "^0.14.4",
+    "@types/node": "^20.8.4",
+    "@types/express": "^5.0.0",
+    "@types/pg": "^8.10.5",
+    "@types/pg-format": "^1.0.3",
+    "@types/urlsafe-base64": "^1.0.28",
+    "@types/uuid": "^9.0.1",
+    "@types/voca": "^1.4.3",
+    "ts-node": "^10.9.1",
+    "tslib": "^2.6.2"
+  },
+  "volta": {
+    "extends": "../../package.json"
+  }
+}

packages/local-mail-watcher/src/env.ts

@@ -0,0 +1,66 @@
+interface redisConfig {
+  url?: string
+  cert?: string
+}
+
+interface WatcherEnv {
+  filesystem: {
+    filePath: string
+  }
+  redis: {
+    mq: redisConfig
+    cache: redisConfig
+  }
+  sns: {
+    snsArn: string
+  }
+  apiKey: string
+  apiEndpoint: string
+}
+
+const envParser =
+  (env: { [key: string]: string | undefined }) =>
+  (varName: string, throwOnUndefined = false): string | undefined => {
+    const value = env[varName]
+    if (typeof value === 'string' && value) {
+      return value
+    }
+
+    if (throwOnUndefined) {
+      throw new Error(
+        `Missing ${varName} with a non-empty value in process environment`
+      )
+    }
+
+    return
+  }
+
+export function getEnv(): WatcherEnv {
+  const parse = envParser(process.env)
+  const filesystem = {
+    filePath: parse('MAIL_FILE_PATH')!,
+  }
+  const redis = {
+    mq: {
+      url: parse('MQ_REDIS_URL'),
+      cert: parse('MQ_REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
+    },
+    cache: {
+      url: parse('REDIS_URL'),
+      cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
+    },
+  }
+  const sns = {
+    snsArn: parse('SNS_ARN') || '',
+  }
+
+  return {
+    apiKey: parse('WATCHER_API_KEY')!,
+    apiEndpoint: parse('WATCHER_API_ENDPOINT')!,
+    sns,
+    filesystem,
+    redis,
+  }
+}
+
+export const env = getEnv()

packages/local-mail-watcher/src/index.ts

@@ -0,0 +1,122 @@
+import { RedisDataSource } from '@omnivore/utils'
+import express, { Express, Request, Response } from 'express'
+
+import { env } from './env'
+import { getQueue } from './lib/queue'
+import { SnsMessage } from './types/SNS'
+import { simpleParser } from 'mailparser'
+import axios from 'axios'
+import { convertToMailObject } from './lib/emailApi'
+
+console.log('Starting worker...')
+
+const app: Express = express()
+
+app.use(express.text({ limit: '50mb' }))
+// Force JSON for SNS
+app.use((req, res, next) => {
+  req.headers['content-type'] = 'application/json'
+  next()
+})
+
+app.use(express.json({ limit: '50mb' }))
+app.use(express.urlencoded({ limit: '50mb', extended: true }))
+
+// create redis source
+const redisDataSource = new RedisDataSource({
+  cache: {
+    url: process.env.REDIS_URL,
+    cert: process.env.REDIS_CERT,
+  },
+  mq: {
+    url: process.env.MQ_REDIS_URL,
+    cert: process.env.MQ_REDIS_CERT,
+  },
+})
+const queue = getQueue(redisDataSource.queueRedisClient)
+
+const addEmailEventToQueue = async (req: Request, res: Response) => {
+  const apiKey = req.headers['x-api-key']
+
+  if (!apiKey) {
+    res.status(401).send('Unauthorized: API key is missing')
+    return
+  }
+
+  if (apiKey != env.apiKey) {
+    res.status(401).send('Unauthorized: Invalid API Key')
+    return
+  }
+
+  await (
+    await queue
+  ).add('save-newsletter', req.body, {
+    priority: 1,
+    attempts: 1,
+    delay: 500,
+  })
+  res.sendStatus(200)
+}
+
+// respond healthy to auto-scaler.
+app.get('/_ah/health', (_req: Request, res: Response) => {
+  res.sendStatus(200)
+})
+
+app.post('/mail', addEmailEventToQueue)
+
+app.post('/sns', async (req, res) => {
+  const bodyString = req.body as string
+  const snsMessage = JSON.parse(bodyString) as SnsMessage
+
+  console.log(`Received SNS Message`, snsMessage)
+  console.log(`Sns Topic ARN ${snsMessage['TopicArn']}`)
+
+  if (snsMessage.TopicArn != env.sns.snsArn) {
+    console.log(
+      `Topic ARN: ${snsMessage.TopicArn} Doesnt Match ${env.sns.snsArn}, failing...`
+    )
+    res.status(401).send()
+    return
+  }
+
+  if (snsMessage.Type == 'SubscriptionConfirmation') {
+    console.log('Subscribing to topic')
+    await axios.get(snsMessage.SubscribeURL)
+    res.status(200).send()
+    return
+  }
+
+  if (snsMessage.Type == 'Notification') {
+    const message = JSON.parse(snsMessage.Message) as {
+      notificationType: string
+      content: string
+    }
+    if (message.notificationType != 'Received') {
+      console.log('Not an email, failing...')
+      res.status(400).send()
+    }
+
+    const mailContent = await simpleParser(message.content)
+    const mail = convertToMailObject(mailContent)
+    console.log(mail)
+    await (
+      await queue
+    ).add('save-newsletter', mail, {
+      priority: 1,
+      attempts: 1,
+      delay: 500,
+    })
+    res.sendStatus(200)
+
+    res.status(200).send()
+    return
+  }
+
+  res.status(400).send()
+})
+
+const port = process.env.PORT || 8080
+const server = app.listen(port, () => {
+  console.log('Mail Server started')
+})

packages/local-mail-watcher/src/lib/emailApi.ts

@@ -0,0 +1,25 @@
+import { EmailContents } from '../types/EmailContents'
+import axios from 'axios'
+import { env } from '../env'
+import { ParsedMail } from 'mailparser'
+
+export const sendToEmailApi = (data: EmailContents) => {
+  return axios.post(env.apiEndpoint, data, {
+    headers: {
+      ['x-api-key']: env.apiKey,
+      'Content-Type': 'application/json',
+    },
+    timeout: 5000,
+  })
+}
+
+export const convertToMailObject = (it: ParsedMail): EmailContents => {
+  return {
+    from: it.from?.value[0]?.address || '',
+    to: (Array.isArray(it.to) ? it.to[0].text : it.to?.text) || '',
+    subject: it.subject || '',
+    html: it.html || '',
+    text: it.text || '',
+    headers: it.headers,
+  }
+}

packages/local-mail-watcher/src/lib/queue.ts

@@ -0,0 +1,27 @@
+import { RedisDataSource } from '@omnivore/utils'
+import { Queue, RedisClient } from 'bullmq'
+
+export const QUEUE = 'omnivore-backend-queue'
+
+export const getQueue = async (
+  connection: RedisClient,
+  queueName = QUEUE
+): Promise<Queue> => {
+  const queue = new Queue(queueName, {
+    connection,
+    defaultJobOptions: {
+      backoff: {
+        type: 'exponential',
+        delay: 2000, // 2 seconds
+      },
+      removeOnComplete: {
+        age: 3600, // keep up to 1 hour
+      },
+      removeOnFail: {
+        age: 24 * 3600, // keep up to 1 day
+      },
+    },
+  })
+  await queue.waitUntilReady()
+  return queue
+}

packages/local-mail-watcher/src/types/EmailContents.ts

@@ -0,0 +1,20 @@
+import { HeaderValue } from 'mailparser'
+
+export type EmailContents = {
+  from: string
+  to: string
+  subject: string
+  html: string
+  text: string
+  headers: Map<string, HeaderValue>
+  unsubMailTo?: string
+  unsubHttpUrl?: string
+  forwardedFrom?: string
+  replyTo?: string
+  confirmationCode?: string
+  uploadFile?: {
+    fileName: string
+    contentType: string
+    id: string
+  }
+}

packages/local-mail-watcher/src/types/SNS.ts

@@ -0,0 +1,7 @@
+export type SnsMessage = {
+  Type: string
+  TopicArn: string
+  SubscribeURL: string
+  content: string
+  Message: string
+}

packages/local-mail-watcher/src/watcher.ts

@@ -0,0 +1,22 @@
+import chokidar from 'chokidar'
+import { simpleParser } from 'mailparser'
+import * as fs from 'node:fs'
+import { convertToMailObject, sendToEmailApi } from './lib/emailApi'
+import { env } from './env'
+
+chokidar.watch(env.filesystem.filePath).on('add', (path, _event) => {
+  console.log(path)
+  const contents = fs.readFileSync(path).toString()
+  void simpleParser(contents)
+    .then(convertToMailObject)
+    .then(async (emailData) => {
+      await sendToEmailApi(emailData)
+      console.log('Sent to email API')
+    })
+    .then(() => {
+      if (process.env['DELETE_FILE'] == 'true') {
+        fs.unlinkSync(path)
+      }
+      console.log('Deleted File')
+    })
+})

packages/local-mail-watcher/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./../../tsconfig.json",
+  "compileOnSave": false,
+  "include": ["./src/**/*"],
+  "compilerOptions": {
+    "outDir": "dist",
+    "typeRoots": ["./../../node_modules/pgvector/types"]
+  }
+}

packages/local-mail-watcher/tslint.json

@@ -0,0 +1,51 @@
+{
+  "extends": "tslint:recommended",
+  "rulesDirectory": ["codelyzer"],
+  "rules": {
+    "array-type": false,
+    "arrow-parens": false,
+    "deprecation": {
+      "severity": "warn"
+    },
+    "import-blacklist": [true, "rxjs/Rx"],
+    "interface-name": false,
+    "max-classes-per-file": false,
+    "max-line-length": [true, 140],
+    "member-access": false,
+    "member-ordering": [
+      true,
+      {
+        "order": [
+          "static-field",
+          "instance-field",
+          "static-method",
+          "instance-method"
+        ]
+      }
+    ],
+    "no-consecutive-blank-lines": false,
+    "no-console": [true, "debug", "info", "time", "timeEnd", "trace"],
+    "no-empty": false,
+    "no-inferrable-types": [true, "ignore-params"],
+    "no-non-null-assertion": true,
+    "no-redundant-jsdoc": true,
+    "no-switch-case-fall-through": true,
+    "no-use-before-declare": true,
+    "no-var-requires": false,
+    "object-literal-key-quotes": [true, "as-needed"],
+    "object-literal-sort-keys": false,
+    "ordered-imports": false,
+    "quotemark": [true, "single"],
+    "trailing-comma": false,
+    "no-output-on-prefix": true,
+    "no-inputs-metadata-property": true,
+    "no-outputs-metadata-property": true,
+    "no-host-metadata-property": true,
+    "no-input-rename": true,
+    "no-output-rename": true,
+    "use-life-cycle-interface": true,
+    "use-pipe-transform-interface": true,
+    "component-class-suffix": true,
+    "directive-class-suffix": true
+  }
+}

packages/pdf-handler/.eslintrc

@@ -2,5 +2,10 @@
   "extends": "../../.eslintrc",
   "parserOptions": {
     "project": "tsconfig.json"
+  },
+  "rules": {
+    "@typescript-eslint/no-unsafe-assignment": 0,
+    "@typescript-eslint/no-unsafe-argument": 0,
+    "@typescript-eslint/restrict-template-expressions": 0
   }
-}
+}

packages/pdf-handler/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:22.12-alpine
 
 # Run everything after as non-privileged user.
 WORKDIR /app

packages/pdf-handler/package.json

@@ -36,7 +36,7 @@
     "bullmq": "^5.1.4",
     "concurrently": "^7.0.0",
     "dotenv": "^8.2.0",
-    "pdfjs-dist": "^2.9.359"
+    "pdfjs-dist": "^2.16.105"
   },
   "volta": {
     "extends": "../../package.json"

packages/pdf-handler/src/pdf.ts

@@ -3,12 +3,13 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 /* eslint-disable @typescript-eslint/no-unsafe-argument */
-import { getDocument as _getDocument } from 'pdfjs-dist/legacy/build/pdf'
+
 import {
-  TextItem,
-  PDFPageProxy,
+  getDocument as _getDocument,
   PDFDocumentProxy,
-} from 'pdfjs-dist/types/display/api'
+  PDFPageProxy,
+} from 'pdfjs-dist/legacy/build/pdf'
+import { TextItem } from 'pdfjs-dist/types/src/display/api'
 
 interface Page {
   lines: string[]
@@ -72,7 +73,7 @@ const getMetadataItem = async (
     .getMetadata()
     .then((metadata) => metadata.info as MetadataInfo)
     .then((info) => {
-      return info[key]
+      return info[key] as string
     })
 }
 
@@ -122,7 +123,7 @@ export const readPdfText = async (
 const parsePage = async (pdfPage: PDFPageProxy): Promise<Page> => {
   const rawContent = await pdfPage.getTextContent()
   return parsePageItems(
-    rawContent.items.filter((item): item is TextItem => 'str' in item)
+    rawContent.items.filter((item: any): item is TextItem => 'str' in item)
   )
 }
 
@@ -156,6 +157,7 @@ const parsePageItems = (pdfItems: TextItem[]): Page => {
       if (nextY != undefined) {
         const currentLineHeight: number = lineData[currentY].reduce(
           (finalValue, current) =>
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-return
             finalValue > current.height ? finalValue : current.height,
           -1
         )

packages/pdf-handler/test/pdf/pdf.test.ts

@@ -21,7 +21,7 @@ describe('open a simple PDF with a set title', () => {
     const doc = await getDocument('./test/pdf/data/pdf-simple-test.pdf')
     const result = await getDocumentText(doc)
     expect(result).to.equal(
-      'This is the page title \n \nThis is some more text \n'
+      'This is the page title\n\nThis is some more text\n'
     )
   })
 })
@@ -30,8 +30,9 @@ describe('open a complex PDF with no title', () => {
   it('should return some initial content as the title', async () => {
     const doc = await getDocument('./test/pdf/data/pdf-complex-test.pdf')
     const result = await getDocumentTitle(doc)
+    console.log(result);
     expect(result).to.startWith(
-      'Improving communications around vaccine breakthrough and vaccine effectiveness'
+      'Improving communications'
     )
   })
 
@@ -47,6 +48,7 @@ describe('open a PDF with metadata set', () => {
     const parsed = await parsePdf(
       new URL('file://' + __dirname + '/data/welcome_to_your_library.pdf')
     )
+
     expect(parsed.title).to.eq('Welcome to your Omnivore Library')
     expect(parsed.author).to.eq('Jackson Harper')
     expect(parsed.description).to.eq('This is the description of my PDF')

packages/puppeteer-parse/package.json

@@ -9,7 +9,7 @@
   ],
   "dependencies": {
     "@omnivore/content-handler": "1.0.0",
-    "puppeteer-core": "^22.12.1",
+    "puppeteer-core": "^23.6.1",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-adblocker": "^2.13.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2"

packages/puppeteer-parse/src/browser.ts

@@ -3,8 +3,10 @@ import puppeteer from 'puppeteer-extra'
 import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker'
 import StealthPlugin from 'puppeteer-extra-plugin-stealth'
 
-puppeteer.use(StealthPlugin())
-puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
+if (process.env['USE_FIREFOX'] != 'true') {
+  puppeteer.use(StealthPlugin())
+  puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
+}
 
 let browserInstance: Browser | null = null
 
@@ -51,15 +53,22 @@ export const getBrowser = async (): Promise<Browser> => {
       isMobile: false,
       width: 1920,
     },
-    executablePath: process.env.CHROMIUM_PATH,
+    ignoreHTTPSErrors: true,
+    executablePath:
+      process.env.USE_FIREFOX == 'true'
+        ? process.env.FIREFOX_PATH
+        : process.env.CHROMIUM_PATH,
     // run in shell mode if headless
-    headless: process.env.LAUNCH_HEADLESS === 'true' ? 'shell' : false,
-    timeout: 10_000, // 10 seconds
-    dumpio: true, // show console logs in the terminal
+    headless: true,
+    browser: process.env['USE_FIREFOX'] == 'true' ? 'firefox' : 'chrome',
+    product: process.env['USE_FIREFOX'] == 'true' ? 'firefox' : 'chrome',
+    timeout: 30000,
+    dumpio: true,
+
     // filter out targets
     targetFilter: (target: Target) =>
       target.type() !== 'other' || !!target.url(),
-  })) as Browser
+  })) as unknown as Browser
 
   const version = await browserInstance.version()
   console.log('Browser started', version)

packages/puppeteer-parse/src/index.ts

@@ -144,6 +144,52 @@ function getUrl(urlStr: string) {
   return parsed.href
 }
 
+const waitForDOMToSettle = (page: Page, timeoutMs = 5000, debounceMs = 1000) =>
+  page.evaluate(
+    (timeoutMs, debounceMs) => {
+      const debounce = (func: (...args: unknown[]) => void, ms = 1000) => {
+        let timeout: NodeJS.Timeout
+        console.log(`Debouncing in  ${ms}`)
+        return (...args: unknown[]) => {
+          console.log('in debounce, clearing timeout again')
+          clearTimeout(timeout)
+          timeout = setTimeout(() => {
+            func.apply(this, args)
+          }, ms)
+        }
+      }
+      return new Promise<void>((resolve) => {
+        const mainTimeout = setTimeout(() => {
+          observer.disconnect()
+          console.log(
+            'Timed out whilst waiting for DOM to settle. Using what we have.'
+          )
+          resolve()
+        }, timeoutMs)
+
+        const debouncedResolve = debounce(() => {
+          observer.disconnect()
+          clearTimeout(mainTimeout)
+          resolve()
+        }, debounceMs)
+
+        const observer = new MutationObserver(() => {
+          debouncedResolve()
+        })
+
+        const config = {
+          attributes: true,
+          childList: true,
+          subtree: true,
+        }
+
+        observer.observe(document.body, config)
+      })
+    },
+    timeoutMs,
+    debounceMs
+  )
+
 async function retrievePage(
   url: string,
   logRecord: Record<string, any>,
@@ -177,105 +223,128 @@ async function retrievePage(
     }
 
     // set timezone for the page
-    if (timezone) {
-      await page.emulateTimezone(timezone)
-    }
-
-    const client = await page.createCDPSession()
-
-    const downloadPath = path.resolve('./download_dir/')
-    await client.send('Page.setDownloadBehavior', {
-      behavior: 'allow',
-      downloadPath,
-    })
-
-    // intercept request when response headers was received
-    await client.send('Network.setRequestInterception', {
-      patterns: [
-        {
-          urlPattern: '*',
-          resourceType: 'Document',
-          interceptionStage: 'HeadersReceived',
-        },
-      ],
-    })
-
-    client.on(
-      'Network.requestIntercepted',
-      (e: Protocol.Network.RequestInterceptedEvent) => {
-        ;(async () => {
-          const headers = e.responseHeaders || {}
-
-          const [contentType] = (
-            headers['content-type'] ||
-            headers['Content-Type'] ||
-            ''
-          )
-            .toLowerCase()
-            .split(';')
-          const obj: Protocol.Network.ContinueInterceptedRequestRequest = {
-            interceptionId: e.interceptionId,
-          }
-
-          if (
-            e.responseStatusCode &&
-            e.responseStatusCode >= 200 &&
-            e.responseStatusCode < 300
-          ) {
-            // We only check content-type on success responses
-            // as it doesn't matter what the content type is for things
-            // like redirects
-            if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) {
-              obj['errorReason'] = 'BlockedByClient'
-            }
-          }
-
-          try {
-            await client.send('Network.continueInterceptedRequest', obj)
-          } catch {
-            // ignore
-          }
-        })()
+    if (process.env['USE_FIREFOX'] !== 'true') {
+      if (timezone) {
+        await page.emulateTimezone(timezone)
       }
-    )
+
+      const client = await page.createCDPSession()
+
+      const downloadPath = path.resolve('./download_dir/')
+      await client.send('Page.setDownloadBehavior', {
+        behavior: 'allow',
+        downloadPath,
+      })
+
+      // intercept request when response headers was received
+      await client.send('Network.setRequestInterception', {
+        patterns: [
+          {
+            urlPattern: '*',
+            resourceType: 'Document',
+            interceptionStage: 'HeadersReceived',
+          },
+        ],
+      })
+
+      client.on(
+        'Network.requestIntercepted',
+        (e: Protocol.Network.RequestInterceptedEvent) => {
+          ;(async () => {
+            const headers = e.responseHeaders || {}
+
+            const [contentType] = (
+              headers['content-type'] ||
+              headers['Content-Type'] ||
+              ''
+            )
+              .toLowerCase()
+              .split(';')
+            const obj: Protocol.Network.ContinueInterceptedRequestRequest = {
+              interceptionId: e.interceptionId,
+            }
+
+            if (
+              e.responseStatusCode &&
+              e.responseStatusCode >= 200 &&
+              e.responseStatusCode < 300
+            ) {
+              // We only check content-type on success responses
+              // as it doesn't matter what the content type is for things
+              // like redirects
+              if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) {
+                obj['errorReason'] = 'BlockedByClient'
+              }
+            }
+
+            try {
+              await client.send('Network.continueInterceptedRequest', obj)
+            } catch {
+              // ignore
+            }
+          })()
+        }
+      )
+    }
 
     /*
      * Disallow MathJax from running in Puppeteer and modifying the document,
      * we shall instead run it in our frontend application to transform any
      * mathjax content when present.
      */
-    await page.setRequestInterception(true)
+
     let requestCount = 0
+    const failedRequests = new Set()
+    page.removeAllListeners('request')
     page.on('request', (request) => {
       ;(async () => {
-        if (request.resourceType() === 'font') {
+        if (request.isInterceptResolutionHandled()) return
+        // since .requestType() is not FF compatible, look for font files.
+        if (request.url().toLowerCase().includes('.woff2')) {
           // Disallow fonts from loading
           return request.abort()
         }
+
         if (requestCount++ > 100) {
           return request.abort()
         }
-        if (
-          request.resourceType() === 'script' &&
-          request.url().toLowerCase().indexOf('mathjax') > -1
-        ) {
+
+        if (failedRequests.has(request.url())) {
+          return request.abort()
+        }
+
+        if (request.url().toLowerCase().indexOf('mathjax') > -1) {
           return request.abort()
         }
 
         await request.continue()
       })()
     })
+    await page.setRequestInterception(true)
 
     page.on('response', (response) => {
+      if (!response.ok()) {
+        console.log('Failed request', response.url())
+        failedRequests.add(response.url())
+      }
+
       if (response.headers()['content-type'] === 'application/pdf') {
         lastPdfUrl = response.url()
       }
     })
 
+    console.log('Trying to load page, for 30 seconds')
+
     const response = await page.goto(url, {
       timeout: 30 * 1000,
-      waitUntil: ['networkidle0'],
+      waitUntil: ['load'],
     })
+
+    console.log('Waited for content to load, waiting for DOM to settle.')
+    await waitForDOMToSettle(page)
+    // Just wait for a few seconds to allow the dom to resolve.
+    // await new Promise((r) => setTimeout(r, 2500))
+
     if (!response) {
       throw new Error('No response from page')
     }

packages/rss-handler/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:22.12-alpine
 
 # Run everything after as non-privileged user.
 WORKDIR /app

packages/rule-handler/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:22.12-alpine
 
 # Run everything after as non-privileged user.
 WORKDIR /app

packages/text-to-speech/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.16
+FROM node:22.12
 
 # Run everything after as non-privileged user.
 WORKDIR /app

packages/thumbnail-handler/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:22.12-alpine
 
 # Run everything after as non-privileged user.
 WORKDIR /app

packages/web/.eslintrc

@@ -21,6 +21,7 @@
   "ignorePatterns": ["next.config.js", "jest.config.js"],
   "rules": {
     "functional/no-mixed-type": 0,
-    "react/react-in-jsx-scope": 0
+    "react/react-in-jsx-scope": 0,
+    "@typescript-eslint/ban-ts-comment" : 0
   }
 }

packages/web/Dockerfile

@@ -1,7 +1,7 @@
 # Note this docker file is meant for local testing
 # and not for production.
 
-FROM node:18.16-alpine as builder
+FROM node:22.12-alpine as builder
 ENV NODE_OPTIONS=--max-old-space-size=8192
 ARG APP_ENV
 ARG BASE_URL
@@ -12,7 +12,7 @@ ENV NEXT_PUBLIC_BASE_URL=$BASE_URL
 ENV NEXT_PUBLIC_SERVER_BASE_URL=$SERVER_BASE_URL
 ENV NEXT_PUBLIC_HIGHLIGHTS_BASE_URL=$HIGHLIGHTS_BASE_URL
 
-RUN apk add g++ make python3
+RUN apk add g++ make python3 py3-setuptools
 
 WORKDIR /app
 
@@ -32,7 +32,7 @@ RUN echo "module.exports = {}" > ./packages/web/next.config.js
 RUN yarn workspace @omnivore/web build
 
 
-FROM node:18.16-alpine as runner
+FROM node:22.12-alpine as builder
 LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
 
 ENV NODE_ENV production

packages/web/Dockerfile-self

@@ -0,0 +1,52 @@
+# Note this docker file is meant for local testing
+# and not for production.
+
+FROM node:22.12-alpine as builder
+ENV NODE_OPTIONS=--max-old-space-size=8192
+ARG APP_ENV
+ARG BASE_URL
+ARG SERVER_BASE_URL
+ARG HIGHLIGHTS_BASE_URL
+ENV NEXT_PUBLIC_APP_ENV=$APP_ENV
+ENV NEXT_PUBLIC_BASE_URL=$BASE_URL
+ENV NEXT_PUBLIC_SERVER_BASE_URL=$SERVER_BASE_URL
+ENV NEXT_PUBLIC_HIGHLIGHTS_BASE_URL=$HIGHLIGHTS_BASE_URL
+
+RUN apk add g++ make python3 py3-setuptools
+
+WORKDIR /app
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/web/package.json ./packages/web/package.json
+
+RUN yarn install --pure-lockfile
+ADD /packages/web ./packages/web
+
+COPY ./packages/web/next.config.self.js ./packages/web/next.config.js
+RUN yarn workspace @omnivore/web build
+
+
+FROM node:22.12-alpine as runner
+LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
+
+ENV NODE_ENV production
+ENV PORT=8080
+ENV NEXT_TELEMETRY_DISABLED 1
+
+WORKDIR /app
+
+COPY --from=builder /app/packages/web/next.config.js /app/packages/web/next.config.js
+COPY --from=builder /app/packages/web/public/ /app/packages/web/public/
+COPY --from=builder /app/packages/web/.next/ /app/packages/web/.next/
+COPY --from=builder /app/packages/web/package.json /app/packages/web/package.json
+COPY --from=builder /app/packages/web/node_modules /app/packages/web/node_modules
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+
+EXPOSE 8080
+CMD ["yarn", "workspace", "@omnivore/web", "start"]

packages/web/components/elements/ShutdownBanner.tsx

@@ -1,52 +0,0 @@
-import { usePersistedState } from '../../lib/hooks/usePersistedState'
-import { CloseButton } from './CloseButton'
-import { HStack, SpanBox } from './LayoutPrimitives'
-
-export const ShutdownBanner = () => {
-  const [
-    showMaintenanceMode,
-    setShowMaintenanceMode,
-    isLoadingShowMaintenanceMode,
-  ] = usePersistedState({
-    key: 'show-shutdown-mode',
-    isSessionStorage: true,
-    initialValue: true,
-  })
-  return (
-    <>
-      {!isLoadingShowMaintenanceMode && showMaintenanceMode && (
-        <HStack
-          css={{
-            p: '5px',
-            top: 0,
-            left: 0,
-            width: '100vw',
-            position: 'absolute',
-            bg: '#FF5733',
-            color: '#FFFFFF',
-            zIndex: '100',
-            font: '$inter',
-            gap: '10px',
-          }}
-          alignment="start"
-          distribution="center"
-        >
-          Omnivore is shutting down on Nov. 30th.
-          <a
-            href="https://blog.omnivore.app/p/details-on-omnivore-shutting-down"
-            target="_blank"
-            rel="noreferrer"
-          >
-            Read More
-          </a>
-          <SpanBox css={{ width: '50px' }} />
-          <CloseButton
-            close={() => {
-              setShowMaintenanceMode(false)
-            }}
-          />
-        </HStack>
-      )}
-    </>
-  )
-}

packages/web/components/patterns/HighlightBar.tsx

@@ -45,7 +45,7 @@ export function HighlightBar(props: HighlightBarProps): JSX.Element {
         borderRadius: '5px',
         border: '1px solid $thHighlightBar',
         boxShadow: `0px 4px 4px 0px rgba(0, 0, 0, 0.15)`,
-
+        zIndex: 999,
         ...(props.displayAtBottom && {
           bottom: 'calc(38px + env(safe-area-inset-bottom, 40px))',
         }),

packages/web/components/templates/AddLinkModal.tsx

@@ -62,17 +62,16 @@ export function AddLinkModal(props: AddLinkModalProps): JSX.Element {
         }}
       >
         <VStack distribution="start" css={{ gap: '20px' }}>
-          {/* <TabBar
+          <TabBar
             selectedTab={selectedTab}
             setSelectedTab={setSelectedTab}
             onOpenChange={props.onOpenChange}
-          /> */}
+          />
           <Box css={{ width: '100%' }}>
             {selectedTab == 'link' && <AddLinkTab {...props} />}
-            {/* {selectedTab == 'feed' && <AddFeedTab {...props} />}
+            {selectedTab == 'feed' && <AddFeedTab {...props} />}
             {selectedTab == 'opml' && <UploadOPMLTab />}
             {selectedTab == 'pdf' && <UploadPDFTab />}
-            {selectedTab == 'import' && <UploadImportTab {...props} />} */}
           </Box>
         </VStack>
       </ModalContent>
@@ -550,6 +549,7 @@ const UploadPad = (props: UploadPadProps): JSX.Element => {
               withCredentials: false,
               headers: {
                 'Content-Type': file.file.type,
+                'origin': 'http://localhost:3000'
               },
               onUploadProgress: (p) => {
                 if (!p.total) {

packages/web/components/templates/AuthLayout.tsx

@@ -1,5 +1,4 @@
 import { Box, VStack, HStack } from '../elements/LayoutPrimitives'
-import { ShutdownBanner } from '../elements/ShutdownBanner'
 import { OmnivoreNameLogo } from '../elements/images/OmnivoreNameLogo'
 import { theme } from '../tokens/stitches.config'
 import { GoogleReCaptchaProvider } from '@google-recaptcha/react'

packages/web/components/templates/LoginLayout.tsx

@@ -10,12 +10,10 @@ import type { LoginFormProps } from './LoginForm'
 import { OmnivoreNameLogo } from '../elements/images/OmnivoreNameLogo'
 
 import featureFullWidthImage from '../../public/static/images/login/login-feature-image-full.png'
-import { ShutdownBanner } from '../elements/ShutdownBanner'
 
 export function LoginLayout(props: LoginFormProps): JSX.Element {
   return (
     <>
-      <ShutdownBanner />
       <MediumBreakpointBox
         smallerLayoutNode={<MobileLoginLayout {...props} />}
         largerLayoutNode={<MediumLoginLayout {...props} />}

packages/web/components/templates/NavigationLayout.tsx

@@ -21,9 +21,6 @@ import useWindowDimensions from '../../lib/hooks/useGetWindowDimensions'
 import { useHandleAddUrl } from '../../lib/hooks/useHandleAddUrl'
 import { useGetViewer } from '../../lib/networking/viewer/useGetViewer'
 import { useQueryClient } from '@tanstack/react-query'
-import { usePersistedState } from '../../lib/hooks/usePersistedState'
-import { CloseButton } from '../elements/CloseButton'
-import { ShutdownBanner } from '../elements/ShutdownBanner'
 
 export type NavigationSection =
   | 'home'
@@ -207,7 +204,6 @@ const Header = (props: HeaderProps): JSX.Element => {
         height: '58px',
       }}
     >
-      <ShutdownBanner />
       <Button
         style="plainIcon"
         onClick={(event) => {

packages/web/components/templates/SettingsLayout.tsx

@@ -13,7 +13,6 @@ import { useVerifyAuth } from '../../lib/hooks/useVerifyAuth'
 import Link from 'next/link'
 import { CaretLeft } from '@phosphor-icons/react'
 import { DEFAULT_HOME_PATH } from '../../lib/navigations'
-import { ShutdownBanner } from '../elements/ShutdownBanner'
 
 type SettingsLayoutProps = {
   title?: string
@@ -83,7 +82,6 @@ export function SettingsLayout(props: SettingsLayoutProps): JSX.Element {
       css={{ width: '100%', height: '100%', minHeight: '100vh' }}
     >
       <PageMetaData path="settings" title="Settings" />
-      <ShutdownBanner />
       <VStack css={{ width: '100%', height: '100%' }}>
         <Box
           css={{

packages/web/components/templates/UploadModal.tsx

@@ -293,8 +293,7 @@ export function UploadModal(props: UploadModalProps): JSX.Element {
             title="Upload file"
             onOpenChange={props.onOpenChange}
           />
-          The uploader is currently disabled.
-          {/* <Dropzone
+          <Dropzone
             ref={dropzoneRef}
             onDragEnter={() => {
               setInDragOperation(true)
@@ -448,7 +447,7 @@ export function UploadModal(props: UploadModalProps): JSX.Element {
                 <input {...getInputProps()} />
               </div>
             )}
-          </Dropzone> */}
+          </Dropzone>
         </VStack>
       </ModalContent>
     </ModalRoot>

packages/web/components/templates/article/NativePdfArticleContainer.tsx

@@ -0,0 +1,91 @@
+import {
+  ArticleAttributes,
+} from '../../../lib/networking/library_items/useLibraryItems'
+import { Box } from '../../elements/LayoutPrimitives'
+import { useState, useRef } from 'react'
+import type { Highlight } from '../../../lib/networking/fragments/highlightFragment'
+import { HighlightNoteModal } from './HighlightNoteModal'
+import { DEFAULT_HEADER_HEIGHT } from '../homeFeed/HeaderSpacer'
+import { UserBasicData } from '../../../lib/networking/queries/useGetViewerQuery'
+import 'react-sliding-pane/dist/react-sliding-pane.css'
+import { NotebookContent } from './Notebook'
+import { NotebookHeader } from './NotebookHeader'
+import useWindowDimensions from '../../../lib/hooks/useGetWindowDimensions'
+import { ResizableSidebar } from './ResizableSidebar'
+
+
+export type PdfArticleContainerProps = {
+  viewer: UserBasicData
+  article: ArticleAttributes
+  showHighlightsModal: boolean
+  setShowHighlightsModal: React.Dispatch<React.SetStateAction<boolean>>
+}
+
+export default function NativePdfArticleContainer(
+  props: PdfArticleContainerProps
+): JSX.Element {
+  const containerRef = useRef<HTMLDivElement | null>(null)
+  const [noteTarget, setNoteTarget] = useState<Highlight | undefined>(undefined)
+    useState<number | undefined>(undefined)
+  const highlightsRef = useRef<Highlight[]>([])
+
+
+  const windowDimensions = useWindowDimensions()
+
+  return (
+    <Box
+      id="article-wrapper"
+      css={{
+        width: '100%',
+        height: `calc(100vh - ${DEFAULT_HEADER_HEIGHT})`,
+      }}
+    >
+      <div ref={containerRef} style={{ width: '100%', height: '100%' }}>
+        <embed src={props.article.url} width={windowDimensions.width} height={windowDimensions.height} />
+      </div>
+      {noteTarget && (
+        <HighlightNoteModal
+          highlight={noteTarget}
+          libraryItemId={props.article.id}
+          libraryItemSlug={props.article.slug}
+          onUpdate={(highlight: Highlight) => {
+            const savedHighlight = highlightsRef.current.find(
+              (other: Highlight) => {
+                return other.id == highlight.id
+              }
+            )
+
+            if (savedHighlight) {
+              savedHighlight.annotation = highlight.annotation
+            }
+          }}
+          onOpenChange={() => {
+            setNoteTarget(undefined)
+          }}
+        />
+      )}
+      <ResizableSidebar
+        isShow={props.showHighlightsModal}
+        onClose={() => {
+          props.setShowHighlightsModal(false)
+        }}
+      >
+        <NotebookHeader
+          viewer={props.viewer}
+          item={props.article}
+          setShowNotebook={props.setShowHighlightsModal}
+        />
+        <NotebookContent
+          viewer={props.viewer}
+          item={props.article}
+          viewInReader={(highlightId) => {
+            const event = new CustomEvent('scrollToHighlightId', {
+              detail: highlightId,
+            })
+            document.dispatchEvent(event)
+          }}
+        />
+      </ResizableSidebar>
+    </Box>
+  )
+}

packages/web/components/templates/article/PDFDisplaySettingsModal.tsx

@@ -59,6 +59,17 @@ function PDFSettings(props: SettingsProps): JSX.Element {
     initialValue: true,
     isSessionStorage: false,
   })
+  const [rememberLatestPage, setLatestPage] = usePersistedState({
+    key: 'reader-remember-latest-page',
+    initialValue: true,
+    isSessionStorage: false,
+  })
+
+  const [useNativeReader, setUseNativeReader] = usePersistedState({
+    key: 'reader-use-native-reader',
+    initialValue: false,
+    isSessionStorage: false,
+  })
 
   return (
     <VStack
@@ -86,6 +97,7 @@ function PDFSettings(props: SettingsProps): JSX.Element {
             Show Tool Bar
           </StyledText>
         </Label>
+
         <SwitchRoot
           id="show-menu-bar"
           checked={showPDFToolBar}
@@ -98,6 +110,72 @@ function PDFSettings(props: SettingsProps): JSX.Element {
         </SwitchRoot>
       </HStack>
 
+      <HStack
+        css={{
+          width: '100%',
+          pr: '30px',
+          alignItems: 'center',
+          '&:hover': {
+            opacity: 0.8,
+          },
+          '&[data-state="on"]': {
+            bg: '$thBackground',
+          },
+        }}
+        alignment="start"
+        distribution="between"
+      >
+        <Label htmlFor="remember-latest-page" css={{ width: '100%' }}>
+          <StyledText style="displaySettingsLabel" css={{ pl: '20px' }}>
+            Remember last page visited
+          </StyledText>
+        </Label>
+
+        <SwitchRoot
+          id="remember-latest-page"
+          checked={rememberLatestPage}
+          onCheckedChange={(checked: boolean) => {
+            setLatestPage(checked)
+            document.dispatchEvent(new Event('pdfReaderUpdateSettings'))
+          }}
+        >
+          <SwitchThumb />
+        </SwitchRoot>
+      </HStack>
+
+        <HStack
+          css={{
+            width: '100%',
+            pr: '30px',
+            alignItems: 'center',
+            '&:hover': {
+              opacity: 0.8,
+            },
+            '&[data-state="on"]': {
+              bg: '$thBackground',
+            },
+          }}
+          alignment="start"
+          distribution="between"
+        >
+          <Label htmlFor="use-native-reader" css={{ width: '100%' }}>
+            <StyledText style="displaySettingsLabel" css={{ pl: '20px' }}>
+              Use Browsers Native PDF Reader
+            </StyledText>
+          </Label>
+
+          <SwitchRoot
+            id="use-native-reader"
+            checked={useNativeReader}
+            onCheckedChange={(checked: boolean) => {
+              setUseNativeReader(checked)
+              document.dispatchEvent(new Event('pdfReaderUpdateSettings'))
+            }}
+          >
+            <SwitchThumb />
+          </SwitchRoot>
+        </HStack>
+
       {/* <HStack
         css={{
           width: '100%',

packages/web/components/templates/article/pdf.js/PdfArticleContainer.tsx

@@ -0,0 +1,290 @@
+import {
+  ArticleAttributes,
+  ArticleReadingProgressMutationInput,
+  useUpdateItemReadStatus,
+} from '../../../../lib/networking/library_items/useLibraryItems'
+import { HStack, VStack } from '../../../elements/LayoutPrimitives'
+import React, { useEffect, useRef, useState } from 'react'
+import { UserBasicData } from '../../../../lib/networking/queries/useGetViewerQuery'
+import 'react-sliding-pane/dist/react-sliding-pane.css'
+import {
+  CreateHighlightInput,
+  useCreateHighlight,
+  useDeleteHighlight,
+  useMergeHighlight,
+  useUpdateHighlight,
+} from '../../../../lib/networking/highlights/useItemHighlights'
+import 'pdfjs-dist/web/pdf_viewer.css'
+import { EventBus, PDFViewer } from 'pdfjs-dist/types/web/pdf_viewer'
+import PdfViewer from './PdfViewer'
+import PdfToolbar from './PdfToolbar'
+import PdfSearchBar from './PdfSearchBar'
+import { NotebookHeader } from '../NotebookHeader'
+import { NotebookContent } from '../Notebook'
+import { ResizableSidebar } from '../ResizableSidebar'
+import { PDFDocumentProxy } from 'pdfjs-dist'
+import { PDFLinkService } from 'pdfjs-dist/types/web/pdf_link_service'
+import PdfSideBar from './PdfSideBar'
+
+export type PdfArticleContainerProps = {
+  viewer: UserBasicData
+  article: ArticleAttributes
+  showHighlightsModal: boolean
+  setShowHighlightsModal: React.Dispatch<React.SetStateAction<boolean>>
+}
+
+export default function PdfArticleContainer(props: PdfArticleContainerProps) {
+  // @ts-ignore
+  const pdfJS = import('pdfjs-dist/build/pdf.min.mjs')
+  const containerRef = useRef<HTMLDivElement | null>(null)
+  const [pdfViewer, setPdfViewer] = useState<PDFViewer | null>(null)
+  const [eventBus, setEventBus] = useState<EventBus | null>(null)
+  const [pageNumber, setPageNumber] = useState<number>(1)
+  const [pageCount, setTotalPageCount] = useState<number>(0)
+  const [showSearch, setShowSearch] = useState(false)
+  const [showToolbar, setShowToolbar] = useState(true)
+  const [saveLatestPage, setSaveLatestPage] = useState(true);
+
+  const [sidebarActive, setSidebarActive] = useState<boolean>(false)
+
+  const createHighlight = useCreateHighlight()
+  const deleteHighlight = useDeleteHighlight()
+  const mergeHighlight = useMergeHighlight()
+  const updateHighlight = useUpdateHighlight()
+  const updateItemReadStatus = useUpdateItemReadStatus()
+
+  const createPdfViewer = async (): Promise<PDFViewer> => {
+    const pdfJSLib = await pdfJS
+    const pdfjsViewer = await import('pdfjs-dist/web/pdf_viewer.mjs')
+
+    pdfJSLib.GlobalWorkerOptions.workerSrc =
+      window.location.origin + '/pdfjs-dist/pdf.worker.min.mjs'
+
+    const eventBus = new pdfjsViewer.EventBus()
+    const pdfLinkService = new pdfjsViewer.PDFLinkService({
+      eventBus,
+    })
+    const pdfFindController = new pdfjsViewer.PDFFindController({
+      eventBus,
+      linkService: pdfLinkService,
+    })
+
+    const pdfScriptingManager = new pdfjsViewer.PDFScriptingManager({
+      eventBus,
+      sandboxBundleSrc: window.location.origin + '/pdfjs-dist/pdf.sandbox.mjs',
+    })
+
+    const pdfViewer = new pdfjsViewer.PDFViewer({
+      container: containerRef.current!,
+      eventBus,
+      linkService: pdfLinkService,
+      findController: pdfFindController,
+      scriptingManager: pdfScriptingManager,
+    })
+    pdfScriptingManager.setViewer(pdfViewer)
+    pdfLinkService.pdfViewer = pdfViewer
+
+    return pdfViewer
+  }
+
+  const loadPdfDocument = async (): Promise<PDFDocumentProxy> => {
+    const pdfJsLib = await pdfJS
+    const loadingTask = pdfJsLib.getDocument({
+      url: props.article.url,
+      cMapUrl: window.location.origin + '/pdfjs-dist/cmaps/',
+      cMapPacked: true,
+      enableXfa: true,
+    })
+
+    return loadingTask.promise
+  }
+
+  useEffect(() => {
+    // Uses the existing mechanism to hide the reader toolbar from pspdfkit
+    const updateReaderSettings = () => {
+      const show = localStorage.getItem('reader-show-pdf-tool-bar')
+      const showBar = show ? JSON.parse(show) == true : false
+      setShowToolbar(showBar)
+
+      const latestPage = localStorage.getItem('reader-remember-latest-page')
+      const latestPageSave = latestPage ? JSON.parse(latestPage) == true : false
+      setSaveLatestPage(latestPageSave)
+    }
+
+    document.addEventListener('pdfReaderUpdateSettings', updateReaderSettings)
+    updateReaderSettings();
+
+    ;(async () => {
+      const pdfViewer = await createPdfViewer()
+      const pdfDocument = await loadPdfDocument()
+
+      setTotalPageCount(pdfDocument.numPages)
+      pdfViewer.setDocument(pdfDocument)
+
+      const linkService = pdfViewer.linkService as PDFLinkService
+      linkService.setDocument(pdfDocument, null)
+
+      // Doesn't seem to get applied straight away, causing an issue where the scale would
+      // be set to 0. We do a 200 ms timeout to avoid this bug.
+      setTimeout(() => {
+        pdfViewer.currentScale = 1
+        pdfViewer.scrollPageIntoView({
+          pageNumber: props.article.readingProgressAnchorIndex,
+        })
+      }, 200)
+
+      setPdfViewer(pdfViewer)
+      setEventBus(pdfViewer.eventBus)
+
+      pdfViewer.eventBus.on(
+        'pagechanging',
+        (e: { previous: number; pageNumber: number }) => {
+          console.log('Page Changing....')
+          setPageNumber(e.pageNumber)
+        }
+      )
+    })()
+  }, [])
+
+  return (
+    <VStack css={{ width: '100%' }}>
+      {showSearch && <PdfSearchBar pdfViewer={pdfViewer} eventBus={eventBus} />}
+      {showToolbar && (
+        <PdfToolbar
+          setShowSidebar={setSidebarActive}
+          sidebarActive={sidebarActive}
+          viewer={props.viewer}
+          article={props.article}
+          pdfViewer={pdfViewer}
+          eventBus={eventBus}
+          pageNumber={pageNumber}
+          setPageNumber={setPageNumber}
+          totalPageNumbers={pageCount}
+          showSearch={showSearch}
+          setShowSearch={setShowSearch}
+        />
+      )}
+
+      <HStack>
+        <PdfSideBar
+          setPage={(page: number) => {
+            if (pdfViewer) {
+              pdfViewer.currentPageNumber = page
+            }
+          }}
+          pdfDocument={pdfViewer?.pdfDocument}
+          activePage={pageNumber}
+          sidebarActive={sidebarActive}
+          totalPages={pageCount}
+        />
+        <PdfViewer
+          viewer={props.viewer}
+          article={props.article}
+          containerRef={containerRef}
+          eventBus={eventBus}
+          sidebarActive={sidebarActive}
+          saveLatestPage={saveLatestPage}
+          pdfViewer={pdfViewer}
+          articleMutations={{
+            createHighlightMutation: async (input: CreateHighlightInput) => {
+              try {
+                return await createHighlight.mutateAsync({
+                  itemId: props.article.id,
+                  slug: props.article.slug,
+                  input,
+                })
+              } catch (err) {
+                console.log('error creating highlight', err)
+                return undefined
+              }
+            },
+            deleteHighlightMutation: async (
+              _libraryItemId: string,
+              highlightId: string
+            ) => {
+              try {
+                await deleteHighlight.mutateAsync({
+                  itemId: props.article.id,
+                  slug: props.article.slug,
+                  highlightId,
+                })
+                return true
+              } catch (err) {
+                console.log('error deleting highlight', err)
+                return false
+              }
+            },
+            mergeHighlightMutation: async (input) => {
+              try {
+                const result = await mergeHighlight.mutateAsync({
+                  itemId: props.article.id,
+                  slug: props.article.slug,
+                  input,
+                })
+                return result?.highlight
+              } catch (err) {
+                console.log('error merging highlight', err)
+                return undefined
+              }
+            },
+            updateHighlightMutation: async (input) => {
+              try {
+                const result = await updateHighlight.mutateAsync({
+                  itemId: props.article.id,
+                  slug: props.article.slug,
+                  input,
+                })
+                return result?.id
+              } catch (err) {
+                console.log('error updating highlight', err)
+                return undefined
+              }
+            },
+            articleReadingProgressMutation: async (
+              input: ArticleReadingProgressMutationInput
+            ) => {
+              try {
+                await updateItemReadStatus.mutateAsync({
+                  itemId: props.article.id,
+                  slug: props.article.slug,
+                  input,
+                })
+              } catch {
+                return false
+              }
+              return true
+            },
+          }}
+        />
+      </HStack>
+      <ResizableSidebar
+        isShow={props.showHighlightsModal}
+        onClose={() => {
+          props.setShowHighlightsModal(false)
+        }}
+      >
+        <NotebookHeader
+          viewer={props.viewer}
+          item={props.article}
+          setShowNotebook={props.setShowHighlightsModal}
+        />
+        <NotebookContent
+          viewer={props.viewer}
+          item={props.article}
+          viewInReader={(highlightId) => {
+            const highlight = props.article.highlights?.filter(
+              (it) => it.id == highlightId
+            )
+            if (highlight && highlight.length == 1) {
+              const pageId = highlight[0].highlightPositionAnchorIndex
+
+              if (pdfViewer) {
+                pdfViewer.currentPageNumber = pageId ?? 1
+              }
+            }
+          }}
+        />
+      </ResizableSidebar>
+    </VStack>
+  )
+}