Self-Hosting Changes

packages/api/package.json

@@ -118,7 +118,10 @@
     "voca": "^1.4.0",
     "winston": "^3.3.3",
     "yaml": "^2.4.1",
-    "youtubei": "^1.5.4"
+    "youtubei": "^1.5.4",
+    "@aws-sdk/client-s3": "^3.679.0",
+    "@aws-sdk/s3-request-presigner": "^3.679.0",
+    "@aws-sdk/lib-storage": "^3.679.0"
   },
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.2",

packages/api/queue-processor/Dockerfile

@@ -0,0 +1,61 @@
+FROM node:18.16 as builder
+
+WORKDIR /app
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+RUN apt-get update && apt-get install -y g++ make python3
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json
+COPY /packages/api/package.json ./packages/api/package.json
+COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json
+COPY /packages/content-handler/package.json ./packages/content-handler/package.json
+COPY /packages/liqe/package.json ./packages/liqe/package.json
+COPY /packages/utils/package.json ./packages/utils/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/readabilityjs ./packages/readabilityjs
+ADD /packages/api ./packages/api
+ADD /packages/text-to-speech ./packages/text-to-speech
+ADD /packages/content-handler ./packages/content-handler
+ADD /packages/liqe ./packages/liqe
+ADD /packages/utils ./packages/utils
+
+RUN yarn workspace @omnivore/utils build
+RUN yarn workspace @omnivore/text-to-speech-handler build
+RUN yarn workspace @omnivore/content-handler build
+RUN yarn workspace @omnivore/liqe build
+RUN yarn workspace @omnivore/api build
+
+# After building, fetch the production dependencies
+RUN rm -rf /app/packages/api/node_modules
+RUN rm -rf /app/node_modules
+RUN yarn install --pure-lockfile --production
+
+FROM node:18.16 as runner
+LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
+
+RUN apt-get update && apt-get install -y netcat-openbsd
+
+WORKDIR /app
+
+ENV NODE_ENV production
+
+COPY --from=builder /app/packages/api/dist /app/packages/api/dist
+COPY --from=builder /app/packages/readabilityjs/ /app/packages/readabilityjs/
+COPY --from=builder /app/packages/api/package.json /app/packages/api/package.json
+COPY --from=builder /app/packages/api/node_modules /app/packages/api/node_modules
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+COPY --from=builder /app/packages/text-to-speech/ /app/packages/text-to-speech/
+COPY --from=builder /app/packages/content-handler/ /app/packages/content-handler/
+COPY --from=builder /app/packages/liqe/ /app/packages/liqe/
+COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
+CMD ["yarn", "workspace", "@omnivore/api", "start_queue_processor"]

packages/api/src/jobs/export.ts

@@ -12,13 +12,24 @@ import { sendExportJobEmail } from '../services/send_emails'
 import { findActiveUser } from '../services/user'
 import { logger } from '../utils/logger'
 import { highlightToMarkdown } from '../utils/parser'
-import { contentFilePath, createGCSFile } from '../utils/uploads'
+import { contentFilePath } from '../utils/uploads'
+import { env } from '../env'
+import { File, Storage } from '@google-cloud/storage'
 
 export interface ExportJobData {
   userId: string
   exportId: string
 }
 
+export const storage = env.fileUpload?.gcsUploadSAKeyFilePath
+  ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath })
+  : new Storage()
+const bucketName = env.fileUpload.gcsUploadBucket
+
+const createGCSFile = (filename: string): File => {
+  return storage.bucket(bucketName).file(filename)
+}
+
 export const EXPORT_JOB_NAME = 'export'
 
 const itemStateMappping = (state: LibraryItemState) => {

packages/api/src/queue-processor.ts

@@ -56,7 +56,10 @@ import {
   PROCESS_YOUTUBE_VIDEO_JOB_NAME,
 } from './jobs/process-youtube-video'
 import { pruneTrashJob, PRUNE_TRASH_JOB } from './jobs/prune_trash'
-import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds'
+import {
+  REFRESH_ALL_FEEDS_JOB_NAME,
+  refreshAllFeeds,
+} from './jobs/rss/refreshAllFeeds'
 import { refreshFeed } from './jobs/rss/refreshFeed'
 import { savePageJob } from './jobs/save_page'
 import {
@@ -159,17 +162,17 @@ export const createWorker = (connection: ConnectionOptions) =>
     async (job: Job) => {
       const executeJob = async (job: Job) => {
         switch (job.name) {
-          // case 'refresh-all-feeds': {
-          //   const queue = await getQueue()
-          //   const counts = await queue?.getJobCounts('prioritized')
-          //   if (counts && counts.wait > 1000) {
-          //     return
-          //   }
-          //   return await refreshAllFeeds(appDataSource)
-          // }
-          // case 'refresh-feed': {
-          //   return await refreshFeed(job.data)
-          // }
+          case 'refresh-all-feeds': {
+            const queue = await getQueue()
+            const counts = await queue?.getJobCounts('prioritized')
+            if (counts && counts.wait > 1000) {
+              return
+            }
+            return await refreshAllFeeds(appDataSource)
+          }
+          case 'refresh-feed': {
+            return await refreshFeed(job.data)
+          }
           case 'save-page': {
             return savePageJob(job.data, job.attemptsMade)
           }
@@ -260,6 +263,17 @@ const setupCronJobs = async () => {
       },
     }
   )
+
+  await queue.add(
+    REFRESH_ALL_FEEDS_JOB_NAME,
+    {},
+    {
+      priority: getJobPriority(REFRESH_ALL_FEEDS_JOB_NAME),
+      repeat: {
+        every: 14_400_000, // 4 Hours
+      },
+    }
+  )
 }
 
 const main = async () => {

packages/api/src/repository/storage/GcsStorageClient.ts

@@ -0,0 +1,71 @@
+import { SignedUrlParameters, StorageClient, File } from './StorageClient'
+import { Storage, File as GCSFile } from '@google-cloud/storage'
+
+export class GcsStorageClient implements StorageClient {
+  private storage: Storage
+
+  constructor(keyFilename: string | undefined) {
+    this.storage = new Storage({
+      keyFilename,
+    })
+  }
+
+  private convertFileToGeneric(gcsFile: GCSFile): File {
+    return {
+      isPublic: async () => {
+        const [isPublic] = await gcsFile.isPublic()
+        return isPublic
+      },
+      exists: async () => (await gcsFile.exists())[0],
+      download: async () => (await gcsFile.download())[0],
+      bucket: gcsFile.bucket.name,
+      publicUrl: () => gcsFile.publicUrl(),
+      getMetadataMd5: async () => {
+        const [metadata] = await gcsFile.getMetadata()
+        return metadata.md5Hash
+      },
+    }
+  }
+
+  downloadFile(bucket: string, filePath: string): Promise<File> {
+    const file = this.storage.bucket(bucket).file(filePath)
+    return Promise.resolve(this.convertFileToGeneric(file))
+  }
+
+  async getFilesFromPrefix(bucket: string, prefix: string): Promise<File[]> {
+    const [filesWithPrefix] = await this.storage
+      .bucket(bucket)
+      .getFiles({ prefix })
+
+    return filesWithPrefix.map((it: GCSFile) => this.convertFileToGeneric(it))
+  }
+
+  async signedUrl(
+    bucket: string,
+    filePath: string,
+    options: SignedUrlParameters
+  ): Promise<string> {
+    const [url] = await this.storage
+      .bucket(bucket)
+      .file(filePath)
+      .getSignedUrl({ ...options, version: 'v4' })
+
+    return url
+  }
+
+  upload(
+    bucket: string,
+    filePath: string,
+    data: Buffer,
+    options: {
+      contentType?: string
+      public?: boolean
+      timeout?: number
+    }
+  ): Promise<void> {
+    return this.storage
+      .bucket(bucket)
+      .file(filePath)
+      .save(data, { timeout: 30000, ...options })
+  }
+}

packages/api/src/repository/storage/S3StorageClient.ts

@@ -0,0 +1,145 @@
+import { SignedUrlParameters, StorageClient, File } from './StorageClient'
+import {
+  GetObjectCommand,
+  GetObjectCommandOutput,
+  S3Client,
+  ListObjectsV2Command,
+  PutObjectCommand,
+} from '@aws-sdk/client-s3'
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
+import type { Readable } from 'stream'
+
+// While this is listed as S3, for self hosting we will use MinIO, which is
+// S3 Compatible.
+export class S3StorageClient implements StorageClient {
+  private s3Client: S3Client
+  private urlOverride: string | undefined
+
+  constructor(urlOverride: string | undefined) {
+    this.urlOverride = urlOverride
+    this.s3Client = new S3Client({
+      forcePathStyle: true,
+      endpoint: urlOverride,
+      region: 'us-east-1',
+      credentials: {
+        accessKeyId: 'minio',
+        secretAccessKey: 'miniominio',
+      },
+    })
+  }
+
+  private convertFileToGeneric(
+    s3File: GetObjectCommandOutput
+  ): Omit<File, 'bucket' | 'publicUrl'> {
+    return {
+      exists: () => {
+        return Promise.resolve(s3File.$metadata.httpStatusCode == 200)
+      },
+      isPublic: async () => Promise.resolve(true),
+      download: async () => this.getFileFromReadable(s3File.Body as Readable),
+      getMetadataMd5: () => Promise.resolve(s3File.ETag),
+    }
+  }
+
+  private getFileFromReadable(stream: Readable): Promise<Buffer> {
+    return new Promise<Buffer>((resolve, reject) => {
+      const chunks: Buffer[] = []
+      stream.on('data', (chunk) => chunks.push(chunk))
+      stream.once('end', () => resolve(Buffer.concat(chunks)))
+      stream.once('error', reject)
+    })
+  }
+
+  async downloadFile(bucket: string, filePath: string): Promise<File> {
+    const s3File = await this.s3Client.send(
+      new GetObjectCommand({
+        Bucket: bucket,
+        Key: filePath, // path to the file you want to download,
+      })
+    )
+
+    return {
+      ...this.convertFileToGeneric(s3File),
+      bucket: bucket,
+      publicUrl: () => `${this.urlOverride ?? ''}/${bucket}/${filePath}`,
+    }
+  }
+
+  async getFilesFromPrefix(bucket: string, prefix: string): Promise<File[]> {
+    const s3PrefixedFiles = await this.s3Client.send(
+      new ListObjectsV2Command({
+        Bucket: bucket,
+        Prefix: prefix, // path to the file you want to download,
+      })
+    )
+
+    const prefixKeys = s3PrefixedFiles.CommonPrefixes || []
+
+    return prefixKeys
+      .map(({ Prefix }) => Prefix)
+      .map((key) => {
+        return {
+          exists: () => Promise.resolve(true),
+          isPublic: async () => Promise.resolve(true),
+          download: async () => {
+            const s3File = await this.s3Client.send(
+              new GetObjectCommand({
+                Bucket: bucket,
+                Key: key, // path to the file you want to download,
+              })
+            )
+
+            return this.getFileFromReadable(s3File.Body as Readable)
+          },
+          getMetadataMd5: () => Promise.resolve(key),
+          bucket: bucket,
+          publicUrl: () => `${this.urlOverride ?? ''}/${bucket}/${key}`,
+        }
+      })
+  }
+
+  async signedUrl(
+    bucket: string,
+    filePath: string,
+    options: SignedUrlParameters
+  ): Promise<string> {
+    const command =
+      options.action == 'read'
+        ? new GetObjectCommand({
+            Bucket: bucket,
+            Key: filePath, // path to the file you want to download,
+          })
+        : new PutObjectCommand({
+            Bucket: bucket,
+            Key: filePath, // path to the file you want to download,
+          })
+
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-call
+    const url = await getSignedUrl(this.s3Client, command, {
+      expiresIn: 900,
+    })
+
+    return url
+  }
+
+  upload(
+    bucket: string,
+    filePath: string,
+    data: Buffer,
+    options: {
+      contentType?: string
+      public?: boolean
+      timeout?: number
+    }
+  ): Promise<void> {
+    return this.s3Client
+      .send(
+        new PutObjectCommand({
+          Bucket: bucket,
+          Key: filePath,
+          Body: data,
+        })
+      )
+      .then(() => {})
+  }
+}

packages/api/src/repository/storage/StorageClient.ts

@@ -0,0 +1,32 @@
+export type SignedUrlParameters = {
+  action: 'read' | 'write' | 'delete' | 'resumable'
+  expires: number
+}
+
+export type File = {
+  isPublic: () => Promise<boolean>
+  publicUrl: () => string
+  download: () => Promise<Buffer>
+  exists: () => Promise<boolean>
+  getMetadataMd5: () => Promise<string | undefined>
+  bucket: string
+}
+
+export interface StorageClient {
+  downloadFile(bucket: string, filePath: string): Promise<File>
+
+  getFilesFromPrefix(bucket: string, filePrefix: string): Promise<File[]>
+
+  upload(
+    bucket: string,
+    filePath: string,
+    data: Buffer,
+    options: { contentType?: string; public?: boolean; timeout?: number }
+  ): Promise<void>
+
+  signedUrl(
+    bucket: string,
+    filePath: string,
+    options: SignedUrlParameters
+  ): Promise<string>
+}

packages/api/src/repository/storage/storage.ts

@@ -0,0 +1,7 @@
+import { env } from '../../env'
+import { S3StorageClient } from './S3StorageClient'
+import { GcsStorageClient } from './GcsStorageClient'
+
+export const storage = env.fileUpload.useLocalStorage
+  ? new S3StorageClient('http://localhost:1010')
+  : new GcsStorageClient(env.fileUpload?.gcsUploadSAKeyFilePath)

packages/api/src/routers/auth/auth_router.ts

@@ -29,6 +29,7 @@ import {
 import { analytics } from '../../utils/analytics'
 import {
   comparePassword,
+  generateVerificationToken,
   hashPassword,
   setAuthInCookie,
   verifyToken,
@@ -544,7 +545,7 @@ export function authRouter() {
       try {
         // hash password
         const hashedPassword = await hashPassword(password)
-        await createUser({
+        const [user] = await createUser({
           email: trimmedEmail,
           provider: 'EMAIL',
           sourceUserId: trimmedEmail,
@@ -553,12 +554,17 @@ export function authRouter() {
           pictureUrl,
           bio,
           password: hashedPassword,
-          pendingConfirmation: true,
+          pendingConfirmation: !env.dev.autoVerify,
         })
 
-        res.redirect(
-          `${env.client.url}/auth/verify-email?message=SIGNUP_SUCCESS`
-        )
+        if (env.dev.autoVerify) {
+          const token = await generateVerificationToken({ id: user.id })
+          res.redirect(`${env.client.url}/auth/confirm-email/${token}`)
+        } else {
+          res.redirect(
+            `${env.client.url}/auth/verify-email?message=SIGNUP_SUCCESS`
+          )
+        }
       } catch (e) {
         logger.info('email-signup exception:', e)
         if (isErrorWithCode(e)) {

packages/api/src/routers/auth/mobile/sign_up.ts

@@ -13,6 +13,7 @@ import {
 } from '../auth_types'
 import { decodeGoogleToken } from '../google_auth'
 import { createPendingUserToken, suggestedUsername } from '../jwt_helpers'
+import { env } from '../../../env'
 
 export async function createMobileSignUpResponse(
   isAndroid: boolean,
@@ -66,7 +67,7 @@ export async function createMobileEmailSignUpResponse(
       name: name.trim(),
       username: username.trim().toLowerCase(),
       password: hashedPassword,
-      pendingConfirmation: true,
+      pendingConfirmation: !env.dev.autoVerify,
     })
 
     return {

packages/api/src/util.ts

@@ -73,6 +73,7 @@ export interface BackendEnv {
   }
   dev: {
     isLocal: boolean
+    autoVerify: boolean
   }
   queue: {
     location: string
@@ -94,6 +95,7 @@ export interface BackendEnv {
     gcsUploadSAKeyFilePath: string
     gcsUploadPrivateBucket: string
     dailyUploadLimit: number
+    useLocalStorage: boolean
   }
   sender: {
     message: string
@@ -197,6 +199,7 @@ const nullableEnvVars = [
   'PG_REPLICA_USER',
   'PG_REPLICA_PASSWORD',
   'PG_REPLICA_DB',
+  'AUTO_VERIFY',
   'INTERCOM_WEB_SECRET',
   'INTERCOM_IOS_SECRET',
   'INTERCOM_ANDROID_SECRET',
@@ -288,6 +291,7 @@ export function getEnv(): BackendEnv {
   }
   const dev = {
     isLocal: parse('API_ENV') == 'local',
+    autoVerify: parse('AUTO_VERIFY') === 'true',
   }
   const queue = {
     location: parse('PUPPETEER_QUEUE_LOCATION'),
@@ -318,6 +322,7 @@ export function getEnv(): BackendEnv {
     dailyUploadLimit: parse('GCS_UPLOAD_DAILY_LIMIT')
       ? parseInt(parse('GCS_UPLOAD_DAILY_LIMIT'), 10)
       : 5, // default to 5
+    useLocalStorage: parse('GCS_USE_LOCAL_HOST') == 'true',
   }
   const sender = {
     message: parse('SENDER_MESSAGE'),

packages/api/src/utils/uploads.ts

@@ -1,12 +1,12 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import { File, GetSignedUrlConfig, Storage } from '@google-cloud/storage'
 import axios from 'axios'
 import { ContentReaderType } from '../entity/library_item'
 import { env } from '../env'
 import { PageType } from '../generated/graphql'
 import { ContentFormat } from '../jobs/upload_content'
 import { logger } from './logger'
+import { storage } from '../repository/storage/storage'
 
 export const contentReaderForLibraryItem = (
   itemType: string,
@@ -31,14 +31,12 @@ export const contentReaderForLibraryItem = (
  * the default app engine service account on the IAM page. We also need to
  * enable IAM related APIs on the project.
  */
-export const storage = env.fileUpload?.gcsUploadSAKeyFilePath
-  ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath })
-  : new Storage()
+
 const bucketName = env.fileUpload.gcsUploadBucket
 const maxContentLength = 10 * 1024 * 1024 // 10MB
 
 export const countOfFilesWithPrefix = async (prefix: string) => {
-  const [files] = await storage.bucket(bucketName).getFiles({ prefix })
+  const files = await storage.getFilesFromPrefix(bucketName, prefix)
   return files.length
 }
 
@@ -48,40 +46,29 @@ export const generateUploadSignedUrl = async (
   selectedBucket?: string
 ): Promise<string> => {
   // These options will allow temporary uploading of file with requested content type
-  const options: GetSignedUrlConfig = {
+  const options = {
     version: 'v4',
-    action: 'write',
+    action: 'write' as const,
     expires: Date.now() + 15 * 60 * 1000, // 15 minutes
     contentType: contentType,
   }
   logger.info('signed url for: ', options)
 
-  // Get a v4 signed URL for uploading file
-  const [url] = await storage
-    .bucket(selectedBucket || bucketName)
-    .file(filePathName)
-    .getSignedUrl(options)
-  return url
+  return storage.signedUrl(selectedBucket || bucketName, filePathName, options)
 }
 
 export const generateDownloadSignedUrl = async (
   filePathName: string,
   config?: {
-    bucketName?: string
     expires?: number
   }
 ): Promise<string> => {
-  const options: GetSignedUrlConfig = {
-    version: 'v4',
-    action: 'read',
-    expires: config?.expires ?? Date.now() + 240 * 60 * 1000, // four hours
+  const options = {
+    action: 'read' as const,
+    expires: Date.now() + 240 * 60 * 1000, // four hours
+    ...config,
   }
-  const [url] = await storage
-    .bucket(config?.bucketName || bucketName)
-    .file(filePathName)
-    .getSignedUrl(options)
-  logger.info(`generating download signed url: ${url}`)
-  return url
+  return storage.signedUrl(bucketName, filePathName, options)
 }
 
 export const getStorageFileDetails = async (
@@ -89,10 +76,10 @@ export const getStorageFileDetails = async (
   fileName: string
 ): Promise<{ md5Hash: string; fileUrl: string }> => {
   const filePathName = generateUploadFilePathName(id, fileName)
-  const file = storage.bucket(bucketName).file(filePathName)
-  const [metadata] = await file.getMetadata()
+  const file = await storage.downloadFile(bucketName, filePathName)
+  const metadataMd5 = await file.getMetadataMd5()
   // GCS returns MD5 Hash in base64 encoding, we convert it here to hex string
-  const md5Hash = Buffer.from(metadata.md5Hash || '', 'base64').toString('hex')
+  const md5Hash = Buffer.from(metadataMd5 || '', 'base64').toString('hex')
 
   return { md5Hash, fileUrl: file.publicUrl() }
 }
@@ -110,17 +97,10 @@ export const uploadToBucket = async (
   options?: { contentType?: string; public?: boolean; timeout?: number },
   selectedBucket?: string
 ): Promise<void> => {
-  await storage
-    .bucket(selectedBucket || bucketName)
-    .file(filePath)
-    .save(data, { timeout: 30000, ...options }) // default timeout 30s
-}
-
-export const createGCSFile = (
-  filename: string,
-  selectedBucket = bucketName
-): File => {
-  return storage.bucket(selectedBucket).file(filename)
+  await storage.upload(selectedBucket || bucketName, filePath, data, {
+    timeout: 30000,
+    ...options,
+  })
 }
 
 export const downloadFromUrl = async (
@@ -154,16 +134,14 @@ export const uploadToSignedUrl = async (
 }
 
 export const isFileExists = async (filePath: string): Promise<boolean> => {
-  const [exists] = await storage.bucket(bucketName).file(filePath).exists()
+  const file = await storage.downloadFile(bucketName, filePath)
+  const exists = await file.exists()
   return exists
 }
 
 export const downloadFromBucket = async (filePath: string): Promise<Buffer> => {
-  const file = storage.bucket(bucketName).file(filePath)
-
-  // Download the file contents
-  const [data] = await file.download()
-  return data
+  const file = await storage.downloadFile(bucketName, filePath)
+  return file.download()
 }
 
 export const contentFilePath = ({

packages/api/test/hooks.ts

@@ -1,13 +1,13 @@
-import { Storage } from '@google-cloud/storage'
-import sinon from 'sinon'
-import * as uploads from '../src/utils/uploads'
-import { MockStorage } from './mock_storage'
-
-export const mochaHooks = {
-  beforeEach() {
-    // Mock cloud storage
-    sinon
-      .stub(uploads, 'storage')
-      .value(new MockStorage() as unknown as Storage)
-  },
-}
+// import { Storage } from '@google-cloud/storage'
+// import sinon from 'sinon'
+// import * as uploads from '../src/utils/uploads'
+// import { MockStorage } from './mock_storage'
+//
+// export const mochaHooks = {
+//   beforeEach() {
+//     // Mock cloud storage
+//     sinon
+//       .stub(uploads, 'storage')
+//       .value(new MockStorage() as unknown as Storage)
+//   },
+// }

packages/content-fetch/src/request_handler.ts

@@ -305,7 +305,7 @@ export const processFetchContentJob = async (
 
     const savedDate = savedAt ? new Date(savedAt) : new Date()
     const { finalUrl, title, content, contentType } = fetchResult
-    if (content) {
+    if (content && process.env['SKIP_UPLOAD_ORIGINAL'] === 'true') {
       await uploadOriginalContent(users, content, savedDate.getTime())
     }
 

packages/web/Dockerfile-self

@@ -0,0 +1,52 @@
+# Note this docker file is meant for local testing
+# and not for production.
+
+FROM node:18.16-alpine as builder
+ENV NODE_OPTIONS=--max-old-space-size=8192
+ARG APP_ENV
+ARG BASE_URL
+ARG SERVER_BASE_URL
+ARG HIGHLIGHTS_BASE_URL
+ENV NEXT_PUBLIC_APP_ENV=$APP_ENV
+ENV NEXT_PUBLIC_BASE_URL=$BASE_URL
+ENV NEXT_PUBLIC_SERVER_BASE_URL=$SERVER_BASE_URL
+ENV NEXT_PUBLIC_HIGHLIGHTS_BASE_URL=$HIGHLIGHTS_BASE_URL
+
+RUN apk add g++ make python3
+
+WORKDIR /app
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/web/package.json ./packages/web/package.json
+
+RUN yarn install --pure-lockfile
+ADD /packages/web ./packages/web
+
+COPY ./packages/web/next.config.self.js ./packages/web/next.config.js
+RUN yarn workspace @omnivore/web build
+
+
+FROM node:18.16-alpine as runner
+LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
+
+ENV NODE_ENV production
+ENV PORT=8080
+ENV NEXT_TELEMETRY_DISABLED 1
+
+WORKDIR /app
+
+COPY --from=builder /app/packages/web/next.config.js /app/packages/web/next.config.js
+COPY --from=builder /app/packages/web/public/ /app/packages/web/public/
+COPY --from=builder /app/packages/web/.next/ /app/packages/web/.next/
+COPY --from=builder /app/packages/web/package.json /app/packages/web/package.json
+COPY --from=builder /app/packages/web/node_modules /app/packages/web/node_modules
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+
+EXPOSE 8080
+CMD ["yarn", "workspace", "@omnivore/web", "start"]

packages/web/next.config.self.js

@@ -0,0 +1,43 @@
+const moduleExports = {
+  rewrites: () => {
+    const rewrites = []
+    rewrites.push({
+      source: '/home',
+      destination: '/l/home',
+    })
+    rewrites.push({
+      source: '/library',
+      destination: '/l/library',
+    })
+    rewrites.push({
+      source: '/subscriptions',
+      destination: '/l/subscriptions',
+    })
+    rewrites.push({
+      source: '/highlights',
+      destination: '/l/highlights',
+    })
+    rewrites.push({
+      source: '/subscriptions',
+      destination: '/l/subscriptions',
+    })
+    rewrites.push({
+      source: '/search',
+      destination: '/l/search',
+    })
+    rewrites.push({
+      source: '/archive',
+      destination: '/l/archive',
+    })
+    rewrites.push({
+      source: '/trash',
+      destination: '/l/trash',
+    })
+    return rewrites
+  }
+}
+
+const withBundleAnalyzer = require('@next/bundle-analyzer')({
+  enabled: process.env.ANALYZE === 'true',
+})
+module.exports = withBundleAnalyzer(moduleExports)

self-hosting/docker-compose/.env.example

@@ -0,0 +1,61 @@
+# Postgres & Migrate
+
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=omnivore
+PGPASSWORD=postgres
+POSTGRES_USER=postgres
+PG_HOST=postgres
+PG_PASSWORD=app_pass
+PG_DB=omnivore
+PG_USER=app_user
+PG_PORT=5432
+PG_POOL_MAX=20
+
+# API
+
+API_ENV=local
+JAEGER_HOST=jaeger # Is this needed?
+IMAGE_PROXY_SECRET=some-secret
+JWT_SECRET=some_secret
+SSO_JWT_SECRET=some_sso_secret
+GATEWAY_URL=http://api:8080/api
+CONTENT_FETCH_URL=http://content-fetch:8080/?token=some_token
+GCS_USE_LOCAL_HOST=true
+GCS_UPLOAD_BUCKET=omnivore
+AUTO_VERIFY=true
+AWS_ACCESS_KEY_ID=minio   # Used for Minio S3 Client
+AWS_SECRET_ACCESS_KEY=miniominio
+AWS_REGION=us-east-1
+INTERCOM_WEB_SECRET=unused
+INTERCOM_IOS_SECRET=unused
+INTERCOM_ANDROID_SECRET=unused
+CONTENT_FETCH_QUEUE_ENABLED=true
+
+IMAGE_PROXY_URL=http://localhost:7070 # Need to change this for NGINX
+CLIENT_URL=http://localhost:3000 # Need to change this when using NGINX
+
+# Redis
+REDIS_URL=redis://redis:6379/0
+
+
+# Web
+APP_ENV=prod
+NEXT_PUBLIC_APP_ENV=prod
+BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX
+SERVER_BASE_URL=http://localhost:4000 # API Server, need to change this when using NGINX
+HIGHLIGHTS_BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX
+NEXT_PUBLIC_BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX
+NEXT_PUBLIC_SERVER_BASE_URL=http://localhost:4000 # API Server, need to change this when using NGINX
+NEXT_PUBLIC_HIGHLIGHTS_BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX
+
+# Content Fetch
+
+VERIFICATION_TOKEN=some_token
+REST_BACKEND_ENDPOINT=http://api:8080/api
+SKIP_UPLOAD_ORIGINAL=true
+
+# Minio
+MINIO_ACCESS_KEY=minio
+MINIO_SECRET_KEY=miniominio
+AWS_S3_ENDPOINT_URL=http://minio:9000

self-hosting/docker-compose/docker-compose.yml

@@ -0,0 +1,161 @@
+version: '3'
+x-postgres:
+  &postgres-common
+  image: "ankane/pgvector:v0.5.1"
+  user: postgres
+  healthcheck:
+    test: "exit 0"
+    interval: 2s
+    timeout: 12s
+    retries: 3
+
+services:
+  postgres:
+    <<: *postgres-common
+    container_name: "omnivore-postgres"
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    env_file:
+      - .env
+
+  migrate:
+    build:
+      context: ../../
+      dockerfile: ./packages/db/Dockerfile
+    container_name: "omnivore-migrate"
+    command: '/bin/sh ./packages/db/setup.sh' # Also create a demo user with email: demo@omnivore.app, password: demo_password
+    env_file:
+      - .env
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  api:
+    build:
+      context: ../../
+      dockerfile: ./packages/api/Dockerfile
+    container_name: "omnivore-api"
+    ports:
+      - "4000:8080"
+    healthcheck:
+      test: ["CMD-SHELL", "nc -z 0.0.0.0 8080 || exit 1"]
+      interval: 15s
+      timeout: 90s
+    env_file:
+      - .env
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+
+  queue-processor:
+    build:
+      context: ../../
+      dockerfile: ./packages/api/queue-processor/Dockerfile
+    container_name: "omnivore-queue-processor"
+    env_file:
+      - .env
+    depends_on:
+      api:
+        condition: service_started
+
+  web:
+    build:
+      context: ../../
+      dockerfile: ./packages/web/Dockerfile-self
+      args:
+        - APP_ENV=prod
+        - BASE_URL=http://localhost:3000
+        - SERVER_BASE_URL=http://localhost:4000
+        - HIGHLIGHTS_BASE_URL=http://localhost:3000
+    container_name: "omnivore-web"
+    ports:
+      - "3000:8080"
+    environment:
+      - NEXT_PUBLIC_APP_ENV=prod
+      - NEXT_PUBLIC_BASE_URL=http://localhost:3000
+      - NEXT_PUBLIC_SERVER_BASE_URL=http://localhost:4000
+      - NEXT_PUBLIC_HIGHLIGHTS_BASE_URL=http://localhost:3000
+    depends_on:
+      api:
+        condition: service_healthy
+
+  image-proxy:
+    build:
+      context: ../../
+      dockerfile: ./imageproxy/Dockerfile
+    container_name: "omnivore-image-proxy"
+    ports:
+      - "7070:8080"
+    env_file:
+      - .env
+
+  content-fetch:
+    build:
+      context: ../../
+      dockerfile: ./packages/content-fetch/Dockerfile
+    container_name: "omnivore-content-fetch"
+    ports:
+      - "9090:8080"
+    env_file:
+      - .env
+    depends_on:
+      redis:
+        condition: service_healthy
+      api:
+        condition: service_healthy
+
+  redis:
+    image: "redis:7.2.4"
+    container_name: "omnivore-redis"
+    expose:
+      - 6379
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+    volumes:
+      - redis_data:/data
+
+  minio:
+    image: minio/minio
+    expose:
+      - 1010
+    ports:
+      - "1010:9000"
+    healthcheck:
+      test: [ "CMD", "mc", "ready", "local" ]
+      interval: 5s
+      timeout: 1s
+    environment:
+      - "MINIO_ACCESS_KEY=minio"
+      - "MINIO_SECRET_KEY=miniominio"
+      - "AWS_S3_ENDPOINT_URL=http://minio:1010"
+    command: server /data
+    volumes:
+      - minio_data:/data
+
+  createbuckets:
+    image: minio/mc
+    environment:
+      - MINIO_ACCESS_KEY=minio
+      - MINIO_SECRET_KEY=miniominio
+      - BUCKET_NAME=omnivore
+      - ENDPOINT=http://minio:9000
+      - AWS_S3_ENDPOINT_URL=http://minio:9000
+    depends_on:
+      - minio
+    entrypoint: >
+      /bin/sh -c "
+      sleep 5;
+      until (/usr/bin/mc config host add myminio http://minio:9000 minio miniominio) do echo '...waiting...' && sleep 1; done;
+      /usr/bin/mc mb myminio/omnivore;
+      /usr/bin/mc policy set public myminio/omnivore;
+      exit 0;
+      "
+volumes:
+  pgdata:
+  redis_data:
+  minio_data:
+