Merge da6ab7aa4a into 4cd5f2f02a

README.md

@@ -151,24 +151,7 @@ is done fetching your content you will see it in your library.
 
 ## How to deploy to your own server
 
-Omnivore was originally designed to be deployed on GCP and takes advantage
-of some of GCP's PaaS features. We are working to make Omnivore more portable
-so you can easily run the service on your own infrastructure. You can track
-progress here: <https://github.com/omnivore-app/omnivore/issues/25>
-
-To deploy Omnivore on your own hardware you will need to deploy three
-dockerized services and configure access to a postgres service. To handle
-PDF documents you will need to configure access to a Google Cloud Storage
-bucket.
-
-- `packages/api` - the backend API service
-- `packages/web` - the web frontend (can easily be deployed to vercel)
-- `packages/puppeteer-parse` - the content fetching service (can easily
-  be deployed as an AWS lambda or GCP Cloud Function)
-
-Additionally, you will need to run our database migrations to initialize
-your database. These are dockerized and can be run with the
-`packages/db` service.
+A guide for running a self hosted server can be found [here](./self-hosting/GUIDE.md)
 
 ## License
 

imageproxy/Dockerfile

@@ -1,4 +1,4 @@
-FROM willnorris/imageproxy:v0.10.0 as build
+FROM ghcr.io/willnorris/imageproxy:main as build
 
 # Above imageproxy image is built from scratch image and is barebones
 # Switching over to ubuntu base image to allow us to debug better.

packages/api/package.json

@@ -120,7 +120,10 @@
     "voca": "^1.4.0",
     "winston": "^3.3.3",
     "yaml": "^2.4.1",
-    "youtubei": "^1.5.4"
+    "youtubei": "^1.5.4",
+    "@aws-sdk/client-s3": "^3.679.0",
+    "@aws-sdk/s3-request-presigner": "^3.679.0",
+    "@aws-sdk/lib-storage": "^3.679.0"
   },
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.2",

packages/api/queue-processor/Dockerfile

@@ -0,0 +1,61 @@
+FROM node:18.16 as builder
+
+WORKDIR /app
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+RUN apt-get update && apt-get install -y g++ make python3
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json
+COPY /packages/api/package.json ./packages/api/package.json
+COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json
+COPY /packages/content-handler/package.json ./packages/content-handler/package.json
+COPY /packages/liqe/package.json ./packages/liqe/package.json
+COPY /packages/utils/package.json ./packages/utils/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/readabilityjs ./packages/readabilityjs
+ADD /packages/api ./packages/api
+ADD /packages/text-to-speech ./packages/text-to-speech
+ADD /packages/content-handler ./packages/content-handler
+ADD /packages/liqe ./packages/liqe
+ADD /packages/utils ./packages/utils
+
+RUN yarn workspace @omnivore/utils build
+RUN yarn workspace @omnivore/text-to-speech-handler build
+RUN yarn workspace @omnivore/content-handler build
+RUN yarn workspace @omnivore/liqe build
+RUN yarn workspace @omnivore/api build
+
+# After building, fetch the production dependencies
+RUN rm -rf /app/packages/api/node_modules
+RUN rm -rf /app/node_modules
+RUN yarn install --pure-lockfile --production
+
+FROM node:18.16 as runner
+LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
+
+RUN apt-get update && apt-get install -y netcat-openbsd
+
+WORKDIR /app
+
+ENV NODE_ENV production
+
+COPY --from=builder /app/packages/api/dist /app/packages/api/dist
+COPY --from=builder /app/packages/readabilityjs/ /app/packages/readabilityjs/
+COPY --from=builder /app/packages/api/package.json /app/packages/api/package.json
+COPY --from=builder /app/packages/api/node_modules /app/packages/api/node_modules
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+COPY --from=builder /app/packages/text-to-speech/ /app/packages/text-to-speech/
+COPY --from=builder /app/packages/content-handler/ /app/packages/content-handler/
+COPY --from=builder /app/packages/liqe/ /app/packages/liqe/
+COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
+CMD ["yarn", "workspace", "@omnivore/api", "start_queue_processor"]

packages/api/src/jobs/export.ts

@@ -1,10 +1,6 @@
 import archiver, { Archiver } from 'archiver'
 import { v4 as uuidv4 } from 'uuid'
-import {
-  ContentReaderType,
-  LibraryItem,
-  LibraryItemState,
-} from '../entity/library_item'
+import { LibraryItem, LibraryItemState } from '../entity/library_item'
 import { TaskState } from '../generated/graphql'
 import { findExportById, saveExport } from '../services/export'
 import { findHighlightsByLibraryItemId } from '../services/highlights'
@@ -17,12 +13,11 @@ import { sendExportJobEmail } from '../services/send_emails'
 import { findActiveUser } from '../services/user'
 import { logger } from '../utils/logger'
 import { highlightToMarkdown } from '../utils/parser'
-import {
-  contentFilePath,
-  createGCSFile,
-  generateUploadFilePathName,
-} from '../utils/uploads'
-import { batch } from 'googleapis/build/src/apis/batch'
+import { env } from '../env'
+import { storage } from '../repository/storage/storage'
+import { File } from '../repository/storage/StorageClient'
+import { Readable } from 'stream'
+import { contentFilePath, generateUploadFilePathName } from '../utils/uploads'
 import { getRepository } from '../repository'
 import { UploadFile } from '../entity/upload_file'
 
@@ -31,6 +26,12 @@ export interface ExportJobData {
   exportId: string
 }
 
+const bucketName = env.fileUpload.gcsUploadBucket
+
+const createGCSFile = (filename: string): File => {
+  return storage.createFile(bucketName, filename)
+}
+
 export const EXPORT_JOB_NAME = 'export'
 
 const itemStateMappping = (state: LibraryItemState) => {
@@ -61,7 +62,7 @@ const uploadContent = async (
   const file = createGCSFile(filePath)
 
   // check if file is already uploaded
-  const [exists] = await file.exists()
+  const exists = await file.exists()
   if (!exists) {
     logger.info(`File not found: ${filePath}`)
 
@@ -81,10 +82,14 @@ const uploadContent = async (
       contentType: 'text/html',
       private: true,
     })
+    archive.append(Readable.from(item.readableContent), {
+      name: `content/${libraryItem.slug}.html`,
+    })
   }
 
   // append the existing file to the archive
-  archive.append(file.createReadStream(), {
+  const content = await file.download()
+  archive.append(Readable.from(content.toString()), {
     name: `content/${libraryItem.slug}.html`,
   })
 }
@@ -97,17 +102,19 @@ const uploadPdfContent = async (
     id: libraryItem.uploadFileId,
   })
   if (!upload || !upload.fileName) {
-    console.log(`upload does not have a filename: ${upload}`)
+    console.log(
+      `upload does not have a filename: ${upload?.fileName ?? 'empty'}`
+    )
     return
   }
 
   const filePath = generateUploadFilePathName(upload.id, upload.fileName)
   const file = createGCSFile(filePath)
-  const [exists] = await file.exists()
+  const exists = await file.exists()
   if (exists) {
     console.log(`adding PDF file: ${filePath}`)
     // append the existing file to the archive
-    archive.append(file.createReadStream(), {
+    archive.append(await file.download(), {
       name: `content/${libraryItem.slug}.pdf`,
     })
   }
@@ -238,9 +245,18 @@ export const exportJob = async (jobData: ExportJobData) => {
 
     // Create a write stream
     const writeStream = file.createWriteStream({
-      metadata: {
-        contentType: 'application/zip',
-      },
+      contentType: 'application/zip',
+    })
+
+    const finishedPromise = new Promise<void>((resolve, reject) => {
+      if (writeStream.closed) {
+        resolve()
+      }
+      writeStream.on('finish', () => {
+        logger.info('File successfully written to GCS')
+        resolve()
+      })
+      writeStream.on('error', reject)
     })
 
     // Handle any errors in the streams
@@ -248,10 +264,6 @@ export const exportJob = async (jobData: ExportJobData) => {
       logger.error('Error writing to GCS:', err)
     })
 
-    writeStream.on('finish', () => {
-      logger.info('File successfully written to GCS')
-    })
-
     // Initialize archiver for zipping files
     const archive = archiver('zip', {
       zlib: { level: 9 }, // Compression level
@@ -264,7 +276,6 @@ export const exportJob = async (jobData: ExportJobData) => {
 
     // Pipe the archiver output to the write stream
     archive.pipe(writeStream)
-
     let cursor = 0
     try {
       // fetch data from the database
@@ -305,17 +316,14 @@ export const exportJob = async (jobData: ExportJobData) => {
     }
 
     // Ensure that the writeStream has finished
-    await new Promise((resolve, reject) => {
-      writeStream.on('finish', resolve)
-      writeStream.on('error', reject)
-    })
+    await finishedPromise
 
     logger.info(`export completed, exported ${cursor} items`, {
       userId,
     })
 
     // generate a temporary signed url for the zip file
-    const [signedUrl] = await file.getSignedUrl({
+    const signedUrl = await storage.signedUrl(bucketName, fullPath, {
       action: 'read',
       expires: Date.now() + 168 * 60 * 60 * 1000, // one week
     })

packages/api/src/queue-processor.ts

@@ -56,7 +56,10 @@ import {
   PROCESS_YOUTUBE_VIDEO_JOB_NAME,
 } from './jobs/process-youtube-video'
 import { pruneTrashJob, PRUNE_TRASH_JOB } from './jobs/prune_trash'
-import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds'
+import {
+  REFRESH_ALL_FEEDS_JOB_NAME,
+  refreshAllFeeds,
+} from './jobs/rss/refreshAllFeeds'
 import { refreshFeed } from './jobs/rss/refreshFeed'
 import { savePageJob } from './jobs/save_page'
 import {
@@ -159,25 +162,25 @@ export const createWorker = (connection: ConnectionOptions) =>
     async (job: Job) => {
       const executeJob = async (job: Job) => {
         switch (job.name) {
-          // case 'refresh-all-feeds': {
-          //   const queue = await getQueue()
-          //   const counts = await queue?.getJobCounts('prioritized')
-          //   if (counts && counts.wait > 1000) {
-          //     return
-          //   }
-          //   return await refreshAllFeeds(appDataSource)
-          // }
-          // case 'refresh-feed': {
-          //   return await refreshFeed(job.data)
-          // }
+          case 'refresh-all-feeds': {
+            const queue = await getQueue()
+            const counts = await queue?.getJobCounts('prioritized')
+            if (counts && counts.wait > 1000) {
+              return
+            }
+            return await refreshAllFeeds(appDataSource)
+          }
+          case 'refresh-feed': {
+            return await refreshFeed(job.data)
+          }
           case 'save-page': {
             return savePageJob(job.data, job.attemptsMade)
           }
           // case 'update-pdf-content': {
           //   return updatePDFContentJob(job.data)
           // }
-          // case THUMBNAIL_JOB:
-          //   return findThumbnail(job.data)
+          case THUMBNAIL_JOB:
+            return findThumbnail(job.data)
           case TRIGGER_RULE_JOB_NAME:
             return triggerRule(job.data)
           case UPDATE_LABELS_JOB:
@@ -218,8 +221,8 @@ export const createWorker = (connection: ConnectionOptions) =>
           //   return updateHome(job.data)
           // case SCORE_LIBRARY_ITEM_JOB:
           //   return scoreLibraryItem(job.data)
-          // case GENERATE_PREVIEW_CONTENT_JOB:
-          //   return generatePreviewContent(job.data)
+          case GENERATE_PREVIEW_CONTENT_JOB:
+            return generatePreviewContent(job.data)
           case PRUNE_TRASH_JOB:
             return pruneTrashJob(job.data)
           case EXPIRE_FOLDERS_JOB_NAME:
@@ -260,6 +263,17 @@ const setupCronJobs = async () => {
       },
     }
   )
+
+  await queue.add(
+    REFRESH_ALL_FEEDS_JOB_NAME,
+    {},
+    {
+      priority: getJobPriority(REFRESH_ALL_FEEDS_JOB_NAME),
+      repeat: {
+        every: 14_400_000, // 4 Hours
+      },
+    }
+  )
 }
 
 const main = async () => {

packages/api/src/repository/storage/GcsStorageClient.ts

@@ -0,0 +1,88 @@
+import {
+  SignedUrlParameters,
+  StorageClient,
+  File,
+  SaveOptions,
+  SaveData,
+} from './StorageClient'
+import { Storage, File as GCSFile } from '@google-cloud/storage'
+
+export class GcsStorageClient implements StorageClient {
+  private storage: Storage
+
+  constructor(keyFilename: string | undefined) {
+    this.storage = new Storage({
+      keyFilename,
+    })
+  }
+
+  private convertFileToGeneric(gcsFile: GCSFile): File {
+    return {
+      isPublic: async () => {
+        const [isPublic] = await gcsFile.isPublic()
+        return isPublic
+      },
+      exists: async () => (await gcsFile.exists())[0],
+      download: async () => (await gcsFile.download())[0],
+      bucket: gcsFile.bucket.name,
+      publicUrl: () => gcsFile.publicUrl(),
+      getMetadataMd5: async () => {
+        const [metadata] = await gcsFile.getMetadata()
+        return metadata.md5Hash
+      },
+      createWriteStream: (saveOptions: SaveOptions) =>
+        gcsFile.createWriteStream({
+          metadata: { contentType: saveOptions.contentType },
+        }),
+      save: (saveData: SaveData, saveOptions: SaveOptions) =>
+        gcsFile.save(saveData, saveOptions),
+      key: gcsFile.name,
+    }
+  }
+
+  downloadFile(bucket: string, filePath: string): Promise<File> {
+    const file = this.storage.bucket(bucket).file(filePath)
+    return Promise.resolve(this.convertFileToGeneric(file))
+  }
+
+  createFile(bucket: string, filePath: string): File {
+    return this.convertFileToGeneric(this.storage.bucket(bucket).file(filePath))
+  }
+
+  async getFilesFromPrefix(bucket: string, prefix: string): Promise<File[]> {
+    const [filesWithPrefix] = await this.storage
+      .bucket(bucket)
+      .getFiles({ prefix })
+
+    return filesWithPrefix.map((it: GCSFile) => this.convertFileToGeneric(it))
+  }
+
+  async signedUrl(
+    bucket: string,
+    filePath: string,
+    options: SignedUrlParameters
+  ): Promise<string> {
+    const [url] = await this.storage
+      .bucket(bucket)
+      .file(filePath)
+      .getSignedUrl({ ...options, version: 'v4' })
+
+    return url
+  }
+
+  upload(
+    bucket: string,
+    filePath: string,
+    data: Buffer,
+    options: {
+      contentType?: string
+      public?: boolean
+      timeout?: number
+    }
+  ): Promise<void> {
+    return this.storage
+      .bucket(bucket)
+      .file(filePath)
+      .save(data, { timeout: 30000, ...options })
+  }
+}

packages/api/src/repository/storage/S3StorageClient.ts

@@ -0,0 +1,252 @@
+import {
+  SignedUrlParameters,
+  StorageClient,
+  File,
+  SaveOptions,
+  SaveData,
+} from './StorageClient'
+import { Upload } from '@aws-sdk/lib-storage'
+import {
+  GetObjectCommand,
+  GetObjectCommandOutput,
+  S3Client,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  HeadObjectCommand,
+  S3ServiceException,
+} from '@aws-sdk/client-s3'
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
+import { Readable } from 'stream'
+import * as stream from 'node:stream'
+
+// While this is listed as S3, for self hosting we will use MinIO, which is
+// S3 Compatible.
+export class S3StorageClient implements StorageClient {
+  BlankFile = class implements File {
+    bucket: string
+    key: string
+    s3Client: S3StorageClient
+    downloadedFile: File | undefined
+
+    constructor(s3StorageClass: S3StorageClient, bucket: string, file: string) {
+      this.bucket = bucket
+      this.key = file
+      this.s3Client = s3StorageClass
+    }
+
+    isPublic() {
+      return Promise.resolve(true)
+    }
+
+    publicUrl() {
+      return `${this.s3Client.urlOverride ?? ''}/${this.bucket}/${this.key}`
+    }
+
+    async download(): Promise<Buffer> {
+      this.downloadedFile = await this.s3Client.downloadFile(
+        this.bucket,
+        this.key
+      )
+      return this.downloadedFile.download()
+    }
+
+    async exists() {
+      try {
+        await this.s3Client.s3Client.send(
+          new HeadObjectCommand({
+            Bucket: this.bucket,
+            Key: this.key,
+          })
+        )
+
+        return true
+      } catch (e) {
+        if (
+          e instanceof S3ServiceException &&
+          e.$metadata.httpStatusCode == 404
+        ) {
+          return false
+        }
+
+        throw e
+      }
+    }
+
+    save(saveData: SaveData, saveOptions: SaveOptions): Promise<void> {
+      return this.s3Client.upload(this.bucket, this.key, saveData, saveOptions)
+    }
+
+    createWriteStream(saveOptions: SaveOptions) {
+      return this.s3Client.createS3UploadStream(
+        this.bucket,
+        this.key,
+        saveOptions
+      )
+    }
+
+    getMetadataMd5() {
+      return this.downloadedFile?.getMetadataMd5() || Promise.resolve('')
+    }
+  }
+
+  private s3Client: S3Client
+  private urlOverride: string | undefined
+
+  constructor(urlOverride: string | undefined) {
+    this.urlOverride = urlOverride
+    this.s3Client = new S3Client({
+      forcePathStyle: true,
+      endpoint: urlOverride,
+    })
+  }
+
+  private createS3UploadStream = (
+    bucket: string,
+    key: string,
+    saveOptions: SaveOptions
+  ) => {
+    const passThroughStream = new stream.PassThrough()
+
+    const upload = new Upload({
+      client: this.s3Client,
+      params: {
+        Bucket: bucket,
+        Key: key,
+        Body: passThroughStream,
+        ContentType: saveOptions.contentType,
+      },
+    })
+
+    void upload.done().then((res) => {
+      console.log(`Successfully Uploaded File ${res.Key ?? ''}`)
+    })
+
+    return passThroughStream
+  }
+
+  private convertFileToGeneric(
+    s3File: GetObjectCommandOutput,
+    bucket: string,
+    key: string
+  ): File {
+    return {
+      exists: () => {
+        return Promise.resolve(s3File.$metadata.httpStatusCode == 200)
+      },
+      save: async () => Promise.resolve(),
+      isPublic: async () => Promise.resolve(true),
+      download: async () => this.getFileFromReadable(s3File.Body as Readable),
+      getMetadataMd5: () => Promise.resolve(s3File.ETag),
+      createWriteStream: (saveOptions: SaveOptions) =>
+        this.createS3UploadStream(bucket, key, saveOptions),
+      publicUrl: () => `${this.urlOverride ?? ''}/${bucket}/${key}`,
+      bucket,
+      key,
+    }
+  }
+
+  private getFileFromReadable(stream: Readable): Promise<Buffer> {
+    return new Promise<Buffer>((resolve, reject) => {
+      const chunks: Buffer[] = []
+      stream.on('data', (chunk) => chunks.push(chunk))
+      stream.once('end', () => resolve(Buffer.concat(chunks)))
+      stream.once('error', reject)
+    })
+  }
+
+  async downloadFile(bucket: string, filePath: string): Promise<File> {
+    const s3File = await this.s3Client.send(
+      new GetObjectCommand({
+        Bucket: bucket,
+        Key: filePath, // path to the file you want to download,
+      })
+    )
+
+    return this.convertFileToGeneric(s3File, bucket, filePath)
+  }
+
+  createFile(bucket: string, filePath: string): File {
+    return new this.BlankFile(this, bucket, filePath) as unknown as File
+  }
+
+  async getFilesFromPrefix(bucket: string, prefix: string): Promise<File[]> {
+    const s3PrefixedFiles = await this.s3Client.send(
+      new ListObjectsV2Command({
+        Bucket: bucket,
+        Prefix: prefix, // path to the file you want to download,
+      })
+    )
+
+    const prefixKeys = s3PrefixedFiles.CommonPrefixes || []
+
+    return prefixKeys
+      .map(({ Prefix }) => Prefix)
+      .map((key: string | undefined) => {
+        return {
+          key: key || '',
+          exists: () => Promise.resolve(true),
+          isPublic: async () => Promise.resolve(true),
+          download: async () => {
+            const s3File = await this.s3Client.send(
+              new GetObjectCommand({
+                Bucket: bucket,
+                Key: key, // path to the file you want to download,
+              })
+            )
+
+            return this.getFileFromReadable(s3File.Body as Readable)
+          },
+          save: () => Promise.resolve(),
+          createWriteStream: (saveOptions: SaveOptions) =>
+            new stream.PassThrough(),
+          getMetadataMd5: () => Promise.resolve(key),
+          bucket: bucket,
+          publicUrl: () => `${this.urlOverride ?? ''}/${bucket}/${key ?? ''}`,
+        }
+      })
+  }
+
+  async signedUrl(
+    bucket: string,
+    filePath: string,
+    options: SignedUrlParameters
+  ): Promise<string> {
+    const command =
+      options.action == 'read'
+        ? new GetObjectCommand({
+            Bucket: bucket,
+            Key: filePath, // path to the file you want to download,
+          })
+        : new PutObjectCommand({
+            Bucket: bucket,
+            Key: filePath, // path to the file you want to download,
+          })
+
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-call
+    const url = await getSignedUrl(this.s3Client, command, {
+      expiresIn: 900,
+    })
+
+    return url
+  }
+
+  async upload(
+    bucket: string,
+    filePath: string,
+    data: SaveData,
+    options: {
+      contentType?: string
+      public?: boolean
+      timeout?: number
+    }
+  ): Promise<void> {
+    await this.s3Client.send(
+      new PutObjectCommand({
+        Bucket: bucket,
+        Key: filePath,
+        Body: data.toString(),
+        ContentType: options.contentType,
+      })
+    )
+  }
+}

packages/api/src/repository/storage/StorageClient.ts

@@ -0,0 +1,49 @@
+import { PipelineSource, Writable } from 'stream'
+
+export type SignedUrlParameters = {
+  action: 'read' | 'write' | 'delete' | 'resumable'
+  expires: number
+}
+
+export type SaveData = string | Buffer | PipelineSource<string | Buffer>
+export type SaveOptions = {
+  contentType?: string
+  gzip?: string | boolean
+  resumable?: boolean
+  timeout?: number
+  validation?: string | boolean
+  private?: boolean | undefined
+}
+
+export type File = {
+  isPublic: () => Promise<boolean>
+  publicUrl: () => string
+  download: () => Promise<Buffer>
+  exists: () => Promise<boolean>
+  save: (saveData: SaveData, saveOptions: SaveOptions) => Promise<void>,
+  createWriteStream: (saveOptions: SaveOptions) => Writable
+  getMetadataMd5: () => Promise<string | undefined>
+  bucket: string
+  key: string
+}
+
+export interface StorageClient {
+  downloadFile(bucket: string, filePath: string): Promise<File>
+
+  createFile(bucket: string, filePath: string): File
+
+  getFilesFromPrefix(bucket: string, filePrefix: string): Promise<File[]>
+
+  upload(
+    bucket: string,
+    filePath: string,
+    data: Buffer,
+    options: { contentType?: string; public?: boolean; timeout?: number }
+  ): Promise<void>
+
+  signedUrl(
+    bucket: string,
+    filePath: string,
+    options: SignedUrlParameters
+  ): Promise<string>
+}

packages/api/src/repository/storage/storage.ts

@@ -0,0 +1,7 @@
+import { env } from '../../env'
+import { S3StorageClient } from './S3StorageClient'
+import { GcsStorageClient } from './GcsStorageClient'
+
+export const storage = env.fileUpload.useLocalStorage
+  ? new S3StorageClient(env.fileUpload.localMinioUrl)
+  : new GcsStorageClient(env.fileUpload?.gcsUploadSAKeyFilePath)

packages/api/src/routers/auth/auth_router.ts

@@ -29,6 +29,7 @@ import {
 import { analytics } from '../../utils/analytics'
 import {
   comparePassword,
+  generateVerificationToken,
   hashPassword,
   setAuthInCookie,
   verifyToken,
@@ -544,7 +545,7 @@ export function authRouter() {
       try {
         // hash password
         const hashedPassword = await hashPassword(password)
-        await createUser({
+        const [user] = await createUser({
           email: trimmedEmail,
           provider: 'EMAIL',
           sourceUserId: trimmedEmail,
@@ -553,12 +554,17 @@ export function authRouter() {
           pictureUrl,
           bio,
           password: hashedPassword,
-          pendingConfirmation: true,
+          pendingConfirmation: !env.dev.autoVerify,
         })
 
-        res.redirect(
-          `${env.client.url}/auth/verify-email?message=SIGNUP_SUCCESS`
-        )
+        if (env.dev.autoVerify) {
+          const token = await generateVerificationToken({ id: user.id })
+          res.redirect(`${env.client.url}/auth/confirm-email/${token}`)
+        } else {
+          res.redirect(
+            `${env.client.url}/auth/verify-email?message=SIGNUP_SUCCESS`
+          )
+        }
       } catch (e) {
         logger.info('email-signup exception:', e)
         if (isErrorWithCode(e)) {

packages/api/src/routers/auth/mobile/sign_up.ts

@@ -13,6 +13,7 @@ import {
 } from '../auth_types'
 import { decodeGoogleToken } from '../google_auth'
 import { createPendingUserToken, suggestedUsername } from '../jwt_helpers'
+import { env } from '../../../env'
 
 export async function createMobileSignUpResponse(
   isAndroid: boolean,
@@ -66,7 +67,7 @@ export async function createMobileEmailSignUpResponse(
       name: name.trim(),
       username: username.trim().toLowerCase(),
       password: hashedPassword,
-      pendingConfirmation: true,
+      pendingConfirmation: !env.dev.autoVerify,
     })
 
     return {

packages/api/src/services/newsletters.ts

@@ -113,7 +113,9 @@ const createRandomEmailAddress = (userName: string, length: number): string => {
   when rand is sdfsdf-: jacksonh-sdfsdf-e@inbox.omnivore.app
   when rand is abcdef: jacksonh-abcdefe@inbox.omnivore.app
    */
-  return `${userName}-${nanoid(length)}e@${inbox}.omnivore.app`
+  return `${userName}-${nanoid(length)}e@${
+    env.email.domain || `@${inbox}.omnivore.app`
+  }`
 }
 
 export const findNewsletterEmailById = async (

packages/api/src/services/upload_file.ts

@@ -137,7 +137,7 @@ export const uploadFile = async (
         itemType,
         uploadFile: { id: uploadFileData.id },
         slug: generateSlug(uploadFilePathName),
-        state: LibraryItemState.Processing,
+        state: LibraryItemState.Succeeded,
         contentReader: contentReaderForLibraryItem(itemType, uploadFileId),
       },
       uid

packages/api/src/util.ts

@@ -73,6 +73,7 @@ export interface BackendEnv {
   }
   dev: {
     isLocal: boolean
+    autoVerify: boolean
   }
   queue: {
     location: string
@@ -94,6 +95,11 @@ export interface BackendEnv {
     gcsUploadSAKeyFilePath: string
     gcsUploadPrivateBucket: string
     dailyUploadLimit: number
+    useLocalStorage: boolean
+    localMinioUrl: string
+  }
+  email: {
+    domain: string
   }
   sender: {
     message: string
@@ -197,10 +203,13 @@ const nullableEnvVars = [
   'PG_REPLICA_USER',
   'PG_REPLICA_PASSWORD',
   'PG_REPLICA_DB',
+  'AUTO_VERIFY',
   'INTERCOM_WEB_SECRET',
   'INTERCOM_IOS_SECRET',
   'INTERCOM_ANDROID_SECRET',
   'EXPORT_TASK_HANDLER_URL',
+  'LOCAL_MINIO_URL',
+  'LOCAL_EMAIL_DOMAIN',
 ] // Allow some vars to be null/empty
 
 const envParser =
@@ -240,6 +249,7 @@ export function getEnv(): BackendEnv {
     pool: {
       max: parseInt(parse('PG_POOL_MAX'), 10),
     },
+
     replication: parse('PG_REPLICATION') === 'true',
     replica: {
       host: parse('PG_REPLICA_HOST'),
@@ -249,6 +259,9 @@ export function getEnv(): BackendEnv {
       dbName: parse('PG_REPLICA_DB'),
     },
   }
+  const email = {
+    domain: parse('LOCAL_EMAIL_DOMAIN'),
+  }
   const server = {
     jwtSecret: parse('JWT_SECRET'),
     ssoJwtSecret: parse('SSO_JWT_SECRET'),
@@ -288,6 +301,7 @@ export function getEnv(): BackendEnv {
   }
   const dev = {
     isLocal: parse('API_ENV') == 'local',
+    autoVerify: parse('AUTO_VERIFY') === 'true',
   }
   const queue = {
     location: parse('PUPPETEER_QUEUE_LOCATION'),
@@ -318,6 +332,8 @@ export function getEnv(): BackendEnv {
     dailyUploadLimit: parse('GCS_UPLOAD_DAILY_LIMIT')
       ? parseInt(parse('GCS_UPLOAD_DAILY_LIMIT'), 10)
       : 5, // default to 5
+    useLocalStorage: parse('GCS_USE_LOCAL_HOST') == 'true',
+    localMinioUrl: parse('LOCAL_MINIO_URL'),
   }
   const sender = {
     message: parse('SENDER_MESSAGE'),
@@ -374,6 +390,7 @@ export function getEnv(): BackendEnv {
   return {
     pg,
     client,
+    email,
     server,
     google,
     posthog,

packages/api/src/utils/uploads.ts

@@ -1,12 +1,12 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import { File, GetSignedUrlConfig, Storage } from '@google-cloud/storage'
 import axios from 'axios'
 import { ContentReaderType } from '../entity/library_item'
 import { env } from '../env'
 import { PageType } from '../generated/graphql'
 import { ContentFormat } from '../jobs/upload_content'
 import { logger } from './logger'
+import { storage } from '../repository/storage/storage'
 
 export const contentReaderForLibraryItem = (
   itemType: string,
@@ -31,14 +31,12 @@ export const contentReaderForLibraryItem = (
  * the default app engine service account on the IAM page. We also need to
  * enable IAM related APIs on the project.
  */
-export const storage = env.fileUpload?.gcsUploadSAKeyFilePath
-  ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath })
-  : new Storage()
+
 const bucketName = env.fileUpload.gcsUploadBucket
 const maxContentLength = 10 * 1024 * 1024 // 10MB
 
 export const countOfFilesWithPrefix = async (prefix: string) => {
-  const [files] = await storage.bucket(bucketName).getFiles({ prefix })
+  const files = await storage.getFilesFromPrefix(bucketName, prefix)
   return files.length
 }
 
@@ -48,40 +46,29 @@ export const generateUploadSignedUrl = async (
   selectedBucket?: string
 ): Promise<string> => {
   // These options will allow temporary uploading of file with requested content type
-  const options: GetSignedUrlConfig = {
+  const options = {
     version: 'v4',
-    action: 'write',
+    action: 'write' as const,
     expires: Date.now() + 15 * 60 * 1000, // 15 minutes
     contentType: contentType,
   }
   logger.info('signed url for: ', options)
 
-  // Get a v4 signed URL for uploading file
-  const [url] = await storage
-    .bucket(selectedBucket || bucketName)
-    .file(filePathName)
-    .getSignedUrl(options)
-  return url
+  return storage.signedUrl(selectedBucket || bucketName, filePathName, options)
 }
 
 export const generateDownloadSignedUrl = async (
   filePathName: string,
   config?: {
-    bucketName?: string
     expires?: number
   }
 ): Promise<string> => {
-  const options: GetSignedUrlConfig = {
-    version: 'v4',
-    action: 'read',
-    expires: config?.expires ?? Date.now() + 240 * 60 * 1000, // four hours
+  const options = {
+    action: 'read' as const,
+    expires: Date.now() + 240 * 60 * 1000, // four hours
+    ...config,
   }
-  const [url] = await storage
-    .bucket(config?.bucketName || bucketName)
-    .file(filePathName)
-    .getSignedUrl(options)
-  logger.info(`generating download signed url: ${url}`)
-  return url
+  return storage.signedUrl(bucketName, filePathName, options)
 }
 
 export const getStorageFileDetails = async (
@@ -89,10 +76,10 @@ export const getStorageFileDetails = async (
   fileName: string
 ): Promise<{ md5Hash: string; fileUrl: string }> => {
   const filePathName = generateUploadFilePathName(id, fileName)
-  const file = storage.bucket(bucketName).file(filePathName)
-  const [metadata] = await file.getMetadata()
+  const file = await storage.downloadFile(bucketName, filePathName)
+  const metadataMd5 = await file.getMetadataMd5()
   // GCS returns MD5 Hash in base64 encoding, we convert it here to hex string
-  const md5Hash = Buffer.from(metadata.md5Hash || '', 'base64').toString('hex')
+  const md5Hash = Buffer.from(metadataMd5 || '', 'base64').toString('hex')
 
   return { md5Hash, fileUrl: file.publicUrl() }
 }
@@ -110,17 +97,10 @@ export const uploadToBucket = async (
   options?: { contentType?: string; public?: boolean; timeout?: number },
   selectedBucket?: string
 ): Promise<void> => {
-  await storage
-    .bucket(selectedBucket || bucketName)
-    .file(filePath)
-    .save(data, { timeout: 30000, ...options }) // default timeout 30s
-}
-
-export const createGCSFile = (
-  filename: string,
-  selectedBucket = bucketName
-): File => {
-  return storage.bucket(selectedBucket).file(filename)
+  await storage.upload(selectedBucket || bucketName, filePath, data, {
+    timeout: 30000,
+    ...options,
+  })
 }
 
 export const downloadFromUrl = async (
@@ -154,16 +134,14 @@ export const uploadToSignedUrl = async (
 }
 
 export const isFileExists = async (filePath: string): Promise<boolean> => {
-  const [exists] = await storage.bucket(bucketName).file(filePath).exists()
+  const file = await storage.downloadFile(bucketName, filePath)
+  const exists = await file.exists()
   return exists
 }
 
 export const downloadFromBucket = async (filePath: string): Promise<Buffer> => {
-  const file = storage.bucket(bucketName).file(filePath)
-
-  // Download the file contents
-  const [data] = await file.download()
-  return data
+  const file = await storage.downloadFile(bucketName, filePath)
+  return file.download()
 }
 
 export const contentFilePath = ({

packages/api/test/hooks.ts

@@ -1,13 +1,13 @@
-import { Storage } from '@google-cloud/storage'
-import sinon from 'sinon'
-import * as uploads from '../src/utils/uploads'
-import { MockStorage } from './mock_storage'
-
-export const mochaHooks = {
-  beforeEach() {
-    // Mock cloud storage
-    sinon
-      .stub(uploads, 'storage')
-      .value(new MockStorage() as unknown as Storage)
-  },
-}
+// import { Storage } from '@google-cloud/storage'
+// import sinon from 'sinon'
+// import * as uploads from '../src/utils/uploads'
+// import { MockStorage } from './mock_storage'
+//
+// export const mochaHooks = {
+//   beforeEach() {
+//     // Mock cloud storage
+//     sinon
+//       .stub(uploads, 'storage')
+//       .value(new MockStorage() as unknown as Storage)
+//   },
+// }

packages/content-fetch/Dockerfile

@@ -4,6 +4,7 @@ LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
 # Installs latest Chromium package.
 RUN apt-get update && apt-get install -y \
     chromium \
+    firefox-esr \
     ca-certificates \
     nodejs \
     yarn \
@@ -14,6 +15,7 @@ RUN apt-get update && apt-get install -y \
 WORKDIR /app
 
 ENV CHROMIUM_PATH /usr/bin/chromium
+ENV FIREFOX_PATH /usr/bin/firefox
 ENV LAUNCH_HEADLESS=true
 
 COPY package.json .
@@ -45,5 +47,13 @@ RUN yarn install --pure-lockfile --production
 
 EXPOSE 8080
 
-CMD ["yarn", "workspace", "@omnivore/content-fetch", "start"]
+# In Firefox we can't use the adblocking sites. Adding them to the hosts file of the docker seems to work.
+RUN wget https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
+RUN echo "#!/bin/bash \n\
+cat hosts >> /etc/hosts \n\
+yarn workspace @omnivore/content-fetch start" >> ./start.sh
+
+RUN chmod +x ./start.sh
+
+CMD ["./start.sh"]
 

packages/content-fetch/src/request_handler.ts

@@ -305,7 +305,7 @@ export const processFetchContentJob = async (
 
     const savedDate = savedAt ? new Date(savedAt) : new Date()
     const { finalUrl, title, content, contentType } = fetchResult
-    if (content) {
+    if (content && process.env['SKIP_UPLOAD_ORIGINAL'] !== 'true') {
       await uploadOriginalContent(users, content, savedDate.getTime())
     }
 

packages/content-handler/src/index.ts

@@ -39,6 +39,7 @@ import { WikipediaHandler } from './websites/wikipedia-handler'
 import { YoutubeHandler } from './websites/youtube-handler'
 import { ZhihuHandler } from './websites/zhihu-handler'
 import { TikTokHandler } from './websites/tiktok-handler'
+import { RawContentHandler } from './websites/raw-handler'
 
 const validateUrlString = (url: string): boolean => {
   const u = new URL(url)
@@ -66,6 +67,7 @@ const contentHandlers: ContentHandler[] = [
   new DerstandardHandler(),
   new ImageHandler(),
   new MediumHandler(),
+  new RawContentHandler(),
   new PdfHandler(),
   new ScrapingBeeHandler(),
   new TDotCoHandler(),

packages/content-handler/src/websites/medium-handler.ts

@@ -1,4 +1,6 @@
 import { ContentHandler, PreHandleResult } from '../content-handler'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
 
 export class MediumHandler extends ContentHandler {
   constructor() {
@@ -11,13 +13,52 @@ export class MediumHandler extends ContentHandler {
     return u.hostname.endsWith('medium.com')
   }
 
+  addImages(document: Document): Document {
+    const pictures = document.querySelectorAll('picture')
+
+    pictures.forEach((pict) => {
+      const source = pict.querySelector('source')
+      if (source) {
+        const srcSet = source.getAttribute('srcSet')
+
+        const sources = (srcSet || '')
+          .split(', ')
+          .map((src) => src.split(' '))
+          .sort((a, b) =>
+            Number(a[1].replace('w', '')) > Number(b[1].replace('w', ''))
+              ? -1
+              : 1
+          )
+
+        // This should be the largest image in the source set.
+        if (sources && sources.length && Array.isArray(sources[0])) {
+          const url = sources[0][0]
+          const img = document.createElement('img')
+          img.src = url
+          pict.after(img)
+          pict.remove()
+        }
+      }
+    })
+
+    return document
+  }
+
   async preHandle(url: string): Promise<PreHandleResult> {
     console.log('prehandling medium url', url)
 
     try {
       const res = new URL(url)
       res.searchParams.delete('source')
-      return Promise.resolve({ url: res.toString() })
+
+      const response = await axios.get(res.toString())
+      const dom = parseHTML(response.data).document
+      const imageAddedDom = this.addImages(dom)
+      return {
+        title: dom.title,
+        content: imageAddedDom.body.outerHTML,
+        url: res.toString(),
+      }
     } catch (error) {
       console.error('error prehandling medium url', error)
       throw error

packages/content-handler/src/websites/raw-handler.ts

@@ -0,0 +1,33 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+export class RawContentHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'RawContentHandler'
+  }
+
+  shouldPreHandle(url: string): boolean {
+    const u = new URL(url)
+    const hostnames = [
+      'medium.com',
+      'fastcompany.com',
+      'fortelabs.com',
+      'theverge.com',
+    ]
+
+    return hostnames.some((h) => u.hostname.endsWith(h))
+  }
+
+  async preHandle(url: string): Promise<PreHandleResult> {
+    try {
+      const response = await axios.get(url)
+      const dom = parseHTML(response.data).document
+      return { title: dom.title, content: response.data as string, url: url }
+    } catch (error) {
+      console.error('error prehandling URL', error)
+      throw error
+    }
+  }
+}

packages/discover/src/lib/inputSources/articles/rss/rssConverters/atom.ts

@@ -3,9 +3,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-return */
 /* eslint-disable @typescript-eslint/restrict-template-expressions */
 
-import { OmnivoreArticle } from '../../../../../types/OmnivoreArticle'
 import { slugify } from 'voca'
-import { Observable, tap } from 'rxjs'
 import { fromArrayLike } from 'rxjs/internal/observable/innerFrom'
 import { mapOrNull } from '../../../../utils/reactive'
 import {

packages/local-mail-watcher/.eslintrc

@@ -0,0 +1,13 @@
+{
+  "extends": "../../.eslintrc",
+  "parserOptions": {
+    "project": "tsconfig.json"
+  },
+  "rules": {
+    "@typescript-eslint/no-unsafe-argument": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/strictNullChecks": "off",
+    "@typescript-eslint/no-unsafe-member-access": "off",
+    "@typescript-eslint/no-unsafe-assignment": "off"
+  }
+}

packages/local-mail-watcher/.gitignore

@@ -0,0 +1,131 @@
+.idea/
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*

packages/local-mail-watcher/Dockerfile

@@ -0,0 +1,40 @@
+ FROM node:18.16 as builder
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y g++ make python3
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/local-mail-watcher/src ./packages/local-mail-watcher/src
+COPY /packages/local-mail-watcher/package.json ./packages/local-mail-watcher/package.json
+COPY /packages/local-mail-watcher/tsconfig.json ./packages/local-mail-watcher/tsconfig.json
+COPY /packages/utils/package.json ./packages/utils/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/utils ./packages/utils
+
+RUN yarn workspace @omnivore/utils build
+RUN yarn workspace @omnivore/local-mail-watcher build
+
+FROM node:18.16 as runner
+
+WORKDIR /app
+
+ENV NODE_ENV production
+
+COPY --from=builder /app/packages/local-mail-watcher/dist /app/packages/local-mail-watcher/dist
+COPY --from=builder /app/packages/local-mail-watcher/package.json /app/packages/local-mail-watcher/package.json
+COPY --from=builder /app/packages/local-mail-watcher/node_modules /app/packages/local-mail-watcher/node_modules
+COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
+
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+
+CMD ["yarn", "workspace", "@omnivore/local-mail-watcher", "start"]

packages/local-mail-watcher/Dockerfile-watcher

@@ -0,0 +1,39 @@
+ FROM node:18.16 as builder
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y g++ make python3
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/local-mail-watcher/src ./packages/local-mail-watcher/src
+COPY /packages/local-mail-watcher/package.json ./packages/local-mail-watcher/package.json
+COPY /packages/local-mail-watcher/tsconfig.json ./packages/local-mail-watcher/tsconfig.json
+COPY /packages/utils/package.json ./packages/utils/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/utils ./packages/utils
+
+RUN yarn workspace @omnivore/utils build
+RUN yarn workspace @omnivore/local-mail-watcher build
+
+FROM node:18.16 as runner
+
+WORKDIR /app
+
+ENV NODE_ENV production
+
+COPY --from=builder /app/packages/local-mail-watcher/dist /app/packages/local-mail-watcher/dist
+COPY --from=builder /app/packages/local-mail-watcher/package.json /app/packages/local-mail-watcher/package.json
+COPY --from=builder /app/packages/local-mail-watcher/node_modules /app/packages/local-mail-watcher/node_modules
+COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+
+CMD ["yarn", "workspace", "@omnivore/local-mail-watcher", "start-watcher"]

packages/local-mail-watcher/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@omnivore/local-mail-watcher",
+  "version": "0.0.1",
+  "scripts": {
+    "build": "tsc",
+    "dev": "ts-node-dev --files src/index.ts",
+    "start": "node dist/index.js",
+    "start-watcher": "node dist/watcher.js",
+    "lint": "eslint src --ext ts,js,tsx,jsx",
+    "lint:fix": "eslint src --fix --ext ts,js,tsx,jsx",
+    "test:typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "chokidar": "^4.0.1",
+    "mailparser": "^3.7.1",
+    "axios": "^1.7.7",
+    "express": "^4.21.1",
+    "bullmq": "^5.22.0",
+    "@omnivore/utils": "1.0.0"
+  },
+  "devDependencies": {
+    "@types/html-to-text": "^9.0.2",
+    "@types/jsdom": "^21.1.3",
+    "@types/mailparser": "^3.4.5",
+    "@types/axios" : "^0.14.4",
+    "@types/node": "^20.8.4",
+    "@types/express": "^5.0.0",
+    "@types/pg": "^8.10.5",
+    "@types/pg-format": "^1.0.3",
+    "@types/urlsafe-base64": "^1.0.28",
+    "@types/uuid": "^9.0.1",
+    "@types/voca": "^1.4.3",
+    "ts-node": "^10.9.1",
+    "tslib": "^2.6.2"
+  },
+  "volta": {
+    "extends": "../../package.json"
+  }
+}

packages/local-mail-watcher/src/env.ts

@@ -0,0 +1,66 @@
+interface redisConfig {
+  url?: string
+  cert?: string
+}
+
+interface WatcherEnv {
+  filesystem: {
+    filePath: string
+  }
+  redis: {
+    mq: redisConfig
+    cache: redisConfig
+  }
+  sns: {
+    snsArn: string
+  }
+  apiKey: string
+  apiEndpoint: string
+}
+
+const envParser =
+  (env: { [key: string]: string | undefined }) =>
+  (varName: string, throwOnUndefined = false): string | undefined => {
+    const value = env[varName]
+    if (typeof value === 'string' && value) {
+      return value
+    }
+
+    if (throwOnUndefined) {
+      throw new Error(
+        `Missing ${varName} with a non-empty value in process environment`
+      )
+    }
+
+    return
+  }
+
+export function getEnv(): WatcherEnv {
+  const parse = envParser(process.env)
+  const filesystem = {
+    filePath: parse('MAIL_FILE_PATH')!,
+  }
+  const redis = {
+    mq: {
+      url: parse('MQ_REDIS_URL'),
+      cert: parse('MQ_REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
+    },
+    cache: {
+      url: parse('REDIS_URL'),
+      cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
+    },
+  }
+  const sns = {
+    snsArn: parse('SNS_ARN') || '',
+  }
+
+  return {
+    apiKey: parse('WATCHER_API_KEY')!,
+    apiEndpoint: parse('WATCHER_API_ENDPOINT')!,
+    sns,
+    filesystem,
+    redis
+  }
+}
+
+export const env = getEnv()

packages/local-mail-watcher/src/index.ts

@@ -0,0 +1,120 @@
+import { RedisDataSource } from '@omnivore/utils'
+import express, { Express, Request, Response } from 'express'
+
+import { env } from './env'
+import { getQueue } from './lib/queue'
+import { SnsMessage } from './types/SNS'
+import { simpleParser } from 'mailparser'
+import axios from 'axios'
+import { convertToMailObject } from './lib/emailApi'
+
+console.log('Starting worker...')
+
+const app: Express = express()
+
+app.use(express.text({ limit: '50mb' }))
+// Force JSON for SNS
+app.use((req, res, next) => {
+  req.headers['content-type'] = 'application/json'
+  next()
+})
+
+app.use(express.json({ limit: '50mb' }))
+app.use(express.urlencoded({ limit: '50mb', extended: true }))
+
+// create redis source
+const redisDataSource = new RedisDataSource({
+  cache: {
+    url: process.env.REDIS_URL,
+    cert: process.env.REDIS_CERT,
+  },
+  mq: {
+    url: process.env.MQ_REDIS_URL,
+    cert: process.env.MQ_REDIS_CERT,
+  },
+})
+const queue = getQueue(redisDataSource.queueRedisClient)
+
+const addEmailEventToQueue = async (req: Request, res: Response) => {
+  const apiKey = req.headers['x-api-key']
+
+  if (!apiKey) {
+    res.status(401).send('Unauthorized: API key is missing')
+    return
+  }
+
+  if (apiKey != env.apiKey) {
+    res.status(401).send('Unauthorized: Invalid API Key')
+    return
+  }
+
+  await (
+    await queue
+  ).add('save-newsletter', req.body, {
+    priority: 1,
+    attempts: 1,
+    delay: 500,
+  })
+  res.sendStatus(200)
+}
+
+// respond healthy to auto-scaler.
+app.get('/_ah/health', (_req: Request, res: Response) => {
+  res.sendStatus(200)
+})
+
+app.post('/mail', addEmailEventToQueue)
+
+app.post('/sns', async (req, res) => {
+  const bodyString = req.body as string
+  const snsMessage = JSON.parse(bodyString) as SnsMessage
+
+  console.log(`Received SNS Message`, snsMessage)
+  console.log(`Sns Topic ARN ${snsMessage['TopicArn']}`)
+
+  if (snsMessage.TopicArn != env.sns.snsArn) {
+    console.log(`Topic ARN: ${snsMessage.TopicArn} Doesnt Match ${env.sns.snsArn}, failing...`)
+    res.status(401).send()
+    return
+  }
+
+  if (snsMessage.Type == 'SubscriptionConfirmation') {
+    console.log('Subscribing to topic')
+    await axios.get(snsMessage.SubscribeURL)
+    res.status(200).send()
+    return
+  }
+
+  if (snsMessage.Type == 'Notification') {
+    const message = JSON.parse(snsMessage.Message) as {
+      notificationType: string
+      content: string
+    }
+    if (message.notificationType != 'Received') {
+      console.log('Not an email, failing...')
+      res.status(400).send()
+    }
+
+    const mailContent = await simpleParser(message.content)
+    const mail = convertToMailObject(mailContent)
+    console.log(mail)
+    await (
+      await queue
+    ).add('save-newsletter', mail, {
+      priority: 1,
+      attempts: 1,
+      delay: 500,
+    })
+    res.sendStatus(200)
+
+    res.status(200).send()
+    return
+  }
+
+  res.status(400).send()
+})
+
+const port = process.env.PORT || 8080
+const server = app.listen(port, () => {
+  console.log('Mail Server started')
+})

packages/local-mail-watcher/src/lib/emailApi.ts

@@ -0,0 +1,25 @@
+import { EmailContents } from '../types/EmailContents'
+import axios from 'axios'
+import { env } from '../env'
+import { ParsedMail } from 'mailparser'
+
+export const sendToEmailApi = (data: EmailContents) => {
+  return axios.post(env.apiEndpoint, data, {
+    headers: {
+      ['x-api-key']: env.apiKey,
+      'Content-Type': 'application/json',
+    },
+    timeout: 5000,
+  })
+}
+
+export const convertToMailObject = (it: ParsedMail): EmailContents => {
+  return {
+    from: it.from?.value[0]?.address || '',
+    to: (Array.isArray(it.to) ? it.to[0].text : it.to?.text) || '',
+    subject: it.subject || '',
+    html: it.html || '',
+    text: it.text || '',
+    headers: it.headers,
+  }
+}

packages/local-mail-watcher/src/lib/queue.ts

@@ -0,0 +1,29 @@
+import { RedisDataSource } from '@omnivore/utils'
+import { Queue, RedisClient } from 'bullmq'
+
+export const QUEUE = 'omnivore-backend-queue'
+
+export const getQueue = async (
+  connection: RedisClient,
+  queueName = QUEUE
+): Promise<Queue> => {
+  const queue = new Queue(queueName, {
+    connection,
+    defaultJobOptions: {
+      backoff: {
+        type: 'exponential',
+        delay: 2000, // 2 seconds
+      },
+      removeOnComplete: {
+        age: 3600, // keep up to 1 hour
+      },
+      removeOnFail: {
+        age: 24 * 3600, // keep up to 1 day
+      },
+    },
+  })
+  await queue.waitUntilReady()
+  return queue
+}
+
+

packages/local-mail-watcher/src/types/EmailContents.ts

@@ -0,0 +1,20 @@
+import { HeaderValue } from 'mailparser'
+
+export type EmailContents = {
+  from: string
+  to: string
+  subject: string
+  html: string
+  text: string
+  headers: Map<string, HeaderValue>
+  unsubMailTo?: string
+  unsubHttpUrl?: string
+  forwardedFrom?: string
+  replyTo?: string
+  confirmationCode?: string
+  uploadFile?: {
+    fileName: string
+    contentType: string
+    id: string
+  }
+}

packages/local-mail-watcher/src/types/SNS.ts

@@ -0,0 +1,7 @@
+export type SnsMessage = {
+  Type: string
+  TopicArn: string
+  SubscribeURL: string
+  content: string
+  Message: string
+}

packages/local-mail-watcher/src/watcher.ts

@@ -0,0 +1,22 @@
+import chokidar from 'chokidar'
+import { simpleParser } from 'mailparser'
+import * as fs from 'node:fs'
+import { convertToMailObject, sendToEmailApi } from './lib/emailApi'
+import { env } from './env'
+
+chokidar.watch(env.filesystem.filePath).on('add', (path, _event) => {
+  console.log(path)
+  const contents = fs.readFileSync(path).toString()
+  simpleParser(contents)
+    .then(convertToMailObject)
+    .then(async (emailData) => {
+      await sendToEmailApi(emailData)
+      console.log('Sent to email API')
+    })
+    .then(() => {
+      if (process.env['DELETE_FILE'] == 'true') {
+        fs.unlinkSync(path)
+      }
+      console.log('Deleted File')
+    })
+})

packages/local-mail-watcher/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./../../tsconfig.json",
+  "compileOnSave": false,
+  "include": ["./src/**/*"],
+  "compilerOptions": {
+    "outDir": "dist",
+    "typeRoots": ["./../../node_modules/pgvector/types"]
+  }
+}

packages/local-mail-watcher/tslint.json

@@ -0,0 +1,51 @@
+{
+  "extends": "tslint:recommended",
+  "rulesDirectory": ["codelyzer"],
+  "rules": {
+    "array-type": false,
+    "arrow-parens": false,
+    "deprecation": {
+      "severity": "warn"
+    },
+    "import-blacklist": [true, "rxjs/Rx"],
+    "interface-name": false,
+    "max-classes-per-file": false,
+    "max-line-length": [true, 140],
+    "member-access": false,
+    "member-ordering": [
+      true,
+      {
+        "order": [
+          "static-field",
+          "instance-field",
+          "static-method",
+          "instance-method"
+        ]
+      }
+    ],
+    "no-consecutive-blank-lines": false,
+    "no-console": [true, "debug", "info", "time", "timeEnd", "trace"],
+    "no-empty": false,
+    "no-inferrable-types": [true, "ignore-params"],
+    "no-non-null-assertion": true,
+    "no-redundant-jsdoc": true,
+    "no-switch-case-fall-through": true,
+    "no-use-before-declare": true,
+    "no-var-requires": false,
+    "object-literal-key-quotes": [true, "as-needed"],
+    "object-literal-sort-keys": false,
+    "ordered-imports": false,
+    "quotemark": [true, "single"],
+    "trailing-comma": false,
+    "no-output-on-prefix": true,
+    "no-inputs-metadata-property": true,
+    "no-outputs-metadata-property": true,
+    "no-host-metadata-property": true,
+    "no-input-rename": true,
+    "no-output-rename": true,
+    "use-life-cycle-interface": true,
+    "use-pipe-transform-interface": true,
+    "component-class-suffix": true,
+    "directive-class-suffix": true
+  }
+}

packages/puppeteer-parse/package.json

@@ -9,7 +9,7 @@
   ],
   "dependencies": {
     "@omnivore/content-handler": "1.0.0",
-    "puppeteer-core": "^22.12.1",
+    "puppeteer-core": "^23.6.1",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-adblocker": "^2.13.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2"

packages/puppeteer-parse/src/browser.ts

@@ -3,8 +3,10 @@ import puppeteer from 'puppeteer-extra'
 import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker'
 import StealthPlugin from 'puppeteer-extra-plugin-stealth'
 
-puppeteer.use(StealthPlugin())
-puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
+if (process.env['USE_FIREFOX'] != 'true') {
+  puppeteer.use(StealthPlugin())
+  puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
+}
 
 let browserInstance: Browser | null = null
 
@@ -51,15 +53,22 @@ export const getBrowser = async (): Promise<Browser> => {
       isMobile: false,
       width: 1920,
     },
-    executablePath: process.env.CHROMIUM_PATH,
+    ignoreHTTPSErrors: true,
+    executablePath:
+      process.env.USE_FIREFOX == 'true'
+        ? process.env.FIREFOX_PATH
+        : process.env.CHROMIUM_PATH,
     // run in shell mode if headless
-    headless: process.env.LAUNCH_HEADLESS === 'true' ? 'shell' : false,
-    timeout: 10_000, // 10 seconds
-    dumpio: true, // show console logs in the terminal
+    headless: true,
+    browser: process.env['USE_FIREFOX'] == 'true' ? 'firefox' : 'chrome',
+    product: process.env['USE_FIREFOX'] == 'true' ? 'firefox' : 'chrome',
+    timeout: 30000,
+    dumpio: true,
+
     // filter out targets
     targetFilter: (target: Target) =>
       target.type() !== 'other' || !!target.url(),
-  })) as Browser
+  })) as unknown as Browser
 
   const version = await browserInstance.version()
   console.log('Browser started', version)

packages/puppeteer-parse/src/index.ts

@@ -144,6 +144,52 @@ function getUrl(urlStr: string) {
   return parsed.href
 }
 
+const waitForDOMToSettle = (page: Page, timeoutMs = 5000, debounceMs = 1000) =>
+  page.evaluate(
+    (timeoutMs, debounceMs) => {
+      const debounce = (func: (...args: unknown[]) => void, ms = 1000) => {
+        let timeout: NodeJS.Timeout
+        console.log(`Debouncing in  ${ms}`)
+        return (...args: unknown[]) => {
+          console.log('in debounce, clearing timeout again')
+          clearTimeout(timeout)
+          timeout = setTimeout(() => {
+            func.apply(this, args)
+          }, ms)
+        }
+      }
+      return new Promise<void>((resolve) => {
+        const mainTimeout = setTimeout(() => {
+          observer.disconnect()
+          console.log(
+            'Timed out whilst waiting for DOM to settle. Using what we have.'
+          )
+          resolve()
+        }, timeoutMs)
+
+        const debouncedResolve = debounce(() => {
+          observer.disconnect()
+          clearTimeout(mainTimeout)
+          resolve()
+        }, debounceMs)
+
+        const observer = new MutationObserver(() => {
+          debouncedResolve()
+        })
+
+        const config = {
+          attributes: true,
+          childList: true,
+          subtree: true,
+        }
+
+        observer.observe(document.body, config)
+      })
+    },
+    timeoutMs,
+    debounceMs
+  )
+
 async function retrievePage(
   url: string,
   logRecord: Record<string, any>,
@@ -177,86 +223,97 @@ async function retrievePage(
     }
 
     // set timezone for the page
-    if (timezone) {
-      await page.emulateTimezone(timezone)
-    }
-
-    const client = await page.createCDPSession()
-
-    const downloadPath = path.resolve('./download_dir/')
-    await client.send('Page.setDownloadBehavior', {
-      behavior: 'allow',
-      downloadPath,
-    })
-
-    // intercept request when response headers was received
-    await client.send('Network.setRequestInterception', {
-      patterns: [
-        {
-          urlPattern: '*',
-          resourceType: 'Document',
-          interceptionStage: 'HeadersReceived',
-        },
-      ],
-    })
-
-    client.on(
-      'Network.requestIntercepted',
-      (e: Protocol.Network.RequestInterceptedEvent) => {
-        ;(async () => {
-          const headers = e.responseHeaders || {}
-
-          const [contentType] = (
-            headers['content-type'] ||
-            headers['Content-Type'] ||
-            ''
-          )
-            .toLowerCase()
-            .split(';')
-          const obj: Protocol.Network.ContinueInterceptedRequestRequest = {
-            interceptionId: e.interceptionId,
-          }
-
-          if (
-            e.responseStatusCode &&
-            e.responseStatusCode >= 200 &&
-            e.responseStatusCode < 300
-          ) {
-            // We only check content-type on success responses
-            // as it doesn't matter what the content type is for things
-            // like redirects
-            if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) {
-              obj['errorReason'] = 'BlockedByClient'
-            }
-          }
-
-          try {
-            await client.send('Network.continueInterceptedRequest', obj)
-          } catch {
-            // ignore
-          }
-        })()
+    if (process.env['USE_FIREFOX'] !== 'true') {
+      if (timezone) {
+        await page.emulateTimezone(timezone)
       }
-    )
+
+      const client = await page.createCDPSession()
+
+      const downloadPath = path.resolve('./download_dir/')
+      await client.send('Page.setDownloadBehavior', {
+        behavior: 'allow',
+        downloadPath,
+      })
+
+      // intercept request when response headers was received
+      await client.send('Network.setRequestInterception', {
+        patterns: [
+          {
+            urlPattern: '*',
+            resourceType: 'Document',
+            interceptionStage: 'HeadersReceived',
+          },
+        ],
+      })
+
+      client.on(
+        'Network.requestIntercepted',
+        (e: Protocol.Network.RequestInterceptedEvent) => {
+          ;(async () => {
+            const headers = e.responseHeaders || {}
+
+            const [contentType] = (
+              headers['content-type'] ||
+              headers['Content-Type'] ||
+              ''
+            )
+              .toLowerCase()
+              .split(';')
+            const obj: Protocol.Network.ContinueInterceptedRequestRequest = {
+              interceptionId: e.interceptionId,
+            }
+
+            if (
+              e.responseStatusCode &&
+              e.responseStatusCode >= 200 &&
+              e.responseStatusCode < 300
+            ) {
+              // We only check content-type on success responses
+              // as it doesn't matter what the content type is for things
+              // like redirects
+              if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) {
+                obj['errorReason'] = 'BlockedByClient'
+              }
+            }
+
+            try {
+              await client.send('Network.continueInterceptedRequest', obj)
+            } catch {
+              // ignore
+            }
+          })()
+        }
+      )
+    }
 
     /*
      * Disallow MathJax from running in Puppeteer and modifying the document,
      * we shall instead run it in our frontend application to transform any
      * mathjax content when present.
      */
-    await page.setRequestInterception(true)
+
     let requestCount = 0
+    const failedRequests = new Set()
+    page.removeAllListeners('request')
     page.on('request', (request) => {
       ;(async () => {
-        if (request.resourceType() === 'font') {
+        if (request.isInterceptResolutionHandled()) return
+        // since .requestType() is not FF compatible, look for font files.
+        if (request.url().toLowerCase().includes('.woff2')) {
           // Disallow fonts from loading
           return request.abort()
         }
+
         if (requestCount++ > 100) {
           return request.abort()
         }
+
+        if (failedRequests.has(request.url())) {
+          return request.abort()
+        }
+
         if (
-          request.resourceType() === 'script' &&
           request.url().toLowerCase().indexOf('mathjax') > -1
         ) {
           return request.abort()
@@ -265,17 +322,32 @@ async function retrievePage(
         await request.continue()
       })()
     })
+    await page.setRequestInterception(true)
+
 
     page.on('response', (response) => {
+      if (!response.ok()) {
+        console.log('Failed request', response.url())
+        failedRequests.add(response.url())
+      }
+
       if (response.headers()['content-type'] === 'application/pdf') {
         lastPdfUrl = response.url()
       }
     })
 
+    console.log('Trying to load page, for 30 seconds')
+
     const response = await page.goto(url, {
       timeout: 30 * 1000,
-      waitUntil: ['networkidle0'],
+      waitUntil: ['load'],
     })
+
+    console.log('Waited for content to load, waiting for DOM to settle.')
+    await waitForDOMToSettle(page)
+    // Just wait for a few seconds to allow the dom to resolve.
+    // await new Promise((r) => setTimeout(r, 2500))
+
     if (!response) {
       throw new Error('No response from page')
     }

packages/web/Dockerfile-self

@@ -0,0 +1,52 @@
+# Note this docker file is meant for local testing
+# and not for production.
+
+FROM node:18.16-alpine as builder
+ENV NODE_OPTIONS=--max-old-space-size=8192
+ARG APP_ENV
+ARG BASE_URL
+ARG SERVER_BASE_URL
+ARG HIGHLIGHTS_BASE_URL
+ENV NEXT_PUBLIC_APP_ENV=$APP_ENV
+ENV NEXT_PUBLIC_BASE_URL=$BASE_URL
+ENV NEXT_PUBLIC_SERVER_BASE_URL=$SERVER_BASE_URL
+ENV NEXT_PUBLIC_HIGHLIGHTS_BASE_URL=$HIGHLIGHTS_BASE_URL
+
+RUN apk add g++ make python3
+
+WORKDIR /app
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/web/package.json ./packages/web/package.json
+
+RUN yarn install --pure-lockfile
+ADD /packages/web ./packages/web
+
+COPY ./packages/web/next.config.self.js ./packages/web/next.config.js
+RUN yarn workspace @omnivore/web build
+
+
+FROM node:18.16-alpine as runner
+LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
+
+ENV NODE_ENV production
+ENV PORT=8080
+ENV NEXT_TELEMETRY_DISABLED 1
+
+WORKDIR /app
+
+COPY --from=builder /app/packages/web/next.config.js /app/packages/web/next.config.js
+COPY --from=builder /app/packages/web/public/ /app/packages/web/public/
+COPY --from=builder /app/packages/web/.next/ /app/packages/web/.next/
+COPY --from=builder /app/packages/web/package.json /app/packages/web/package.json
+COPY --from=builder /app/packages/web/node_modules /app/packages/web/node_modules
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+
+EXPOSE 8080
+CMD ["yarn", "workspace", "@omnivore/web", "start"]

packages/web/components/elements/ShutdownBanner.tsx

@@ -1,52 +0,0 @@
-import { usePersistedState } from '../../lib/hooks/usePersistedState'
-import { CloseButton } from './CloseButton'
-import { HStack, SpanBox } from './LayoutPrimitives'
-
-export const ShutdownBanner = () => {
-  const [
-    showMaintenanceMode,
-    setShowMaintenanceMode,
-    isLoadingShowMaintenanceMode,
-  ] = usePersistedState({
-    key: 'show-shutdown-mode',
-    isSessionStorage: true,
-    initialValue: true,
-  })
-  return (
-    <>
-      {!isLoadingShowMaintenanceMode && showMaintenanceMode && (
-        <HStack
-          css={{
-            p: '5px',
-            top: 0,
-            left: 0,
-            width: '100vw',
-            position: 'absolute',
-            bg: '#FF5733',
-            color: '#FFFFFF',
-            zIndex: '100',
-            font: '$inter',
-            gap: '10px',
-          }}
-          alignment="start"
-          distribution="center"
-        >
-          Omnivore is shutting down on Nov. 30th.
-          <a
-            href="https://blog.omnivore.app/p/details-on-omnivore-shutting-down"
-            target="_blank"
-            rel="noreferrer"
-          >
-            Read More
-          </a>
-          <SpanBox css={{ width: '50px' }} />
-          <CloseButton
-            close={() => {
-              setShowMaintenanceMode(false)
-            }}
-          />
-        </HStack>
-      )}
-    </>
-  )
-}

packages/web/components/templates/AuthLayout.tsx

@@ -1,5 +1,4 @@
 import { Box, VStack, HStack } from '../elements/LayoutPrimitives'
-import { ShutdownBanner } from '../elements/ShutdownBanner'
 import { OmnivoreNameLogo } from '../elements/images/OmnivoreNameLogo'
 import { theme } from '../tokens/stitches.config'
 import { GoogleReCaptchaProvider } from '@google-recaptcha/react'

packages/web/components/templates/LoginLayout.tsx

@@ -10,12 +10,10 @@ import type { LoginFormProps } from './LoginForm'
 import { OmnivoreNameLogo } from '../elements/images/OmnivoreNameLogo'
 
 import featureFullWidthImage from '../../public/static/images/login/login-feature-image-full.png'
-import { ShutdownBanner } from '../elements/ShutdownBanner'
 
 export function LoginLayout(props: LoginFormProps): JSX.Element {
   return (
     <>
-      <ShutdownBanner />
       <MediumBreakpointBox
         smallerLayoutNode={<MobileLoginLayout {...props} />}
         largerLayoutNode={<MediumLoginLayout {...props} />}

packages/web/components/templates/NavigationLayout.tsx

@@ -21,9 +21,6 @@ import useWindowDimensions from '../../lib/hooks/useGetWindowDimensions'
 import { useHandleAddUrl } from '../../lib/hooks/useHandleAddUrl'
 import { useGetViewer } from '../../lib/networking/viewer/useGetViewer'
 import { useQueryClient } from '@tanstack/react-query'
-import { usePersistedState } from '../../lib/hooks/usePersistedState'
-import { CloseButton } from '../elements/CloseButton'
-import { ShutdownBanner } from '../elements/ShutdownBanner'
 
 export type NavigationSection =
   | 'home'
@@ -207,7 +204,6 @@ const Header = (props: HeaderProps): JSX.Element => {
         height: '58px',
       }}
     >
-      <ShutdownBanner />
       <Button
         style="plainIcon"
         onClick={(event) => {

packages/web/components/templates/SettingsLayout.tsx

@@ -13,7 +13,6 @@ import { useVerifyAuth } from '../../lib/hooks/useVerifyAuth'
 import Link from 'next/link'
 import { CaretLeft } from '@phosphor-icons/react'
 import { DEFAULT_HOME_PATH } from '../../lib/navigations'
-import { ShutdownBanner } from '../elements/ShutdownBanner'
 
 type SettingsLayoutProps = {
   title?: string
@@ -83,7 +82,6 @@ export function SettingsLayout(props: SettingsLayoutProps): JSX.Element {
       css={{ width: '100%', height: '100%', minHeight: '100vh' }}
     >
       <PageMetaData path="settings" title="Settings" />
-      <ShutdownBanner />
       <VStack css={{ width: '100%', height: '100%' }}>
         <Box
           css={{

packages/web/next.config.self.js

@@ -0,0 +1,43 @@
+const moduleExports = {
+  rewrites: () => {
+    const rewrites = []
+    rewrites.push({
+      source: '/home',
+      destination: '/l/home',
+    })
+    rewrites.push({
+      source: '/library',
+      destination: '/l/library',
+    })
+    rewrites.push({
+      source: '/subscriptions',
+      destination: '/l/subscriptions',
+    })
+    rewrites.push({
+      source: '/highlights',
+      destination: '/l/highlights',
+    })
+    rewrites.push({
+      source: '/subscriptions',
+      destination: '/l/subscriptions',
+    })
+    rewrites.push({
+      source: '/search',
+      destination: '/l/search',
+    })
+    rewrites.push({
+      source: '/archive',
+      destination: '/l/archive',
+    })
+    rewrites.push({
+      source: '/trash',
+      destination: '/l/trash',
+    })
+    return rewrites
+  }
+}
+
+const withBundleAnalyzer = require('@next/bundle-analyzer')({
+  enabled: process.env.ANALYZE === 'true',
+})
+module.exports = withBundleAnalyzer(moduleExports)

self-hosting/GUIDE.md

@@ -0,0 +1,375 @@
+# Self Hosting
+
+- [Docker Compose](#docker-compose)
+- [Nginx Reverse Proxy](#nginx-reverse-proxy)
+- [Cloudflare Tunnel](#cloudflare-tunnel)
+- [Email](#email)
+- - [Self Hosted Mail Server](#docker-mailserver-and-mail-watcher)
+- - [Third Party Services](#third-party-services)
+
+## Docker Compose
+
+We recommend using Docker-compose for the simplest way to deploy Omnivore. We have provided a configuration in the `self-hosting/docker-compose` folder. 
+
+All networking and persistent storage is handled by the docker-compose file.
+
+### Requirements
+* Docker 
+* Docker Compose 
+
+### 1. Clone the Repository
+
+Clone the repository at ``git@github.com:omnivore-app/omnivore.git``
+
+### 2. Change directory to self-hosting/docker-compose
+
+The Docker-compose file and necessary environment variables are found in the self-hosting folder under docker-compose. 
+
+These files provide all you need to get Omnivore up and running on your local environment. 
+
+### 3. Populate the .env file 
+
+There is a .env.example file located within the docker-compose folder that should give you the necessary environment variables to begin running. 
+You can use these by `mv .env.example .env`
+
+The following environment variables should be changed to reflect where you are running your application. 
+
+| Environment Variable            | Description                                    | Local Parameter       |
+|---------------------------------|------------------------------------------------|-----------------------|
+| BASE URL                        | The URL of the Front End of the Application.   | http://localhost:3000 |
+| SERVER_BASE_URL                 | The URL of the API Server.                     | http://localhost:4000 |
+| HIGHLIGHTS_BASE_URL             | The URL of the Front end of the Application    | http://localhost:3000 |
+| NEXT_PUBLIC_BASE_URL            | Same as above BASE URL, but for NEXT           | http://localhost:3000 |
+| NEXT_PUBLIC_SERVER_BASE_URL     | Same as above SERVER_BASE_URL, but for NEXT    | http://localhost:4000 |
+| NEXT_PUBLIC_HIGHLIGHTS_BASE_URL | Same as above HIGHLIGHTS_BASE_URL but for NEXT | http://localhost:3000 |
+| CLIENT_URL                      | The URL of the Front end of the Application    | http://localhost:3000 |
+| IMAGEPROXY_URL                  | Service that proxies images to avoid blocking  | http://localhost:7070 |
+
+
+Additionally, when doing a docker-compose build, if you are hosting this application you must change the args in the `docker-compose` file. 
+
+```yaml
+web:
+    build:
+    context: ../../
+    dockerfile: ./packages/web/Dockerfile-self
+args:
+  - APP_ENV=prod
+  - BASE_URL=http://localhost:3000
+  - SERVER_BASE_URL=http://localhost:4000
+  - HIGHLIGHTS_BASE_URL=http://localhost:3000
+```
+
+They are the same as the listed environment variables above. 
+
+### 4. Build the docker images. 
+
+Running `docker compose build` will go through and build all the necessary docker images. 
+
+### 5. Start the service. 
+
+Running `docker compose up` will start the services.
+
+During the first deployment omnivore-migrate will go through and set up the necessary Postgres tables. 
+This will also create a demo user with email: demo@omnivore.app, password: demo_password. 
+
+When the service is ready you can access the web-app by using localhost:3000
+
+With the default .env file you will be able to use Omnivore, add RSS Feeds, add stories etc. 
+
+
+### Additional Services used: 
+
+#### PGVector
+A PGVector image is used to provide Postgres functionality. If you have another postgres service running it is possible to remove 
+this from the docker-compose and provide the host, username and password of the Postgres instance. 
+
+#### Redis 
+Redis is used as a queueing system, and for caching. If you have a Redis Instance already it is possible to remove this from the docker-compose 
+and rely on the hosted Redis. You must replace the redis url for this. 
+
+#### Minio (Self-Host)
+Minio is an AWS S3 compatible Object storage service that you can self-host. It is included in the docker-compose file. 
+It allows you to use the S3 Storage API. 
+
+We also have a small client that creates the necessary buckets (createbuckets). See below: 
+```bash
+until (/usr/bin/mc config host add myminio http://minio:9000 minio miniominio) do echo '...waiting...' && sleep 1; done;
+/usr/bin/mc mb myminio/omnivore;
+/usr/bin/mc policy set public myminio/omnivore;
+```
+
+If you use GCS, or S3 buckets you can do the following:
+
+##### S3 (Optional): 
+S3 is an AWS Block Storage Service. You can also use S3 as your storage service, rather than the included MinIO self-host. In order to use S3, you must do the following.
+
+Replace the following with the correct parameters. 
+```env
+AWS_ACCESS_KEY_ID=minio   # Used for Minio S3 Client
+AWS_SECRET_ACCESS_KEY=miniominio
+AWS_REGION=us-east-1
+```
+
+Replace the following with an endpoint URL from [here](https://docs.aws.amazon.com/general/latest/gr/s3.html)
+```env
+LOCAL_MINIO_URL=http://localhost:1010
+
+```
+
+##### GCS (Optional): 
+Remove the following Environment Variable: 
+```env
+GCS_USE_LOCAL_HOST=true
+```
+and populate 
+```
+GCS_UPLOAD_SA_KEY_FILE_PATH
+```
+with the path of the JSON key file for the service account.
+
+## Nginx Reverse Proxy
+
+Nginx is a reverse proxy that receives requests, and directs them to the correct service internally. Omnivore runs 4 services we want to redirect to.
+
+* Omnivore Web
+* Omnivore API
+* Omnivore Bucket [Optional]
+* Omnivore Image Proxy [Optional]
+
+We have included an example Nginx Configuration that redirects traffic from http (80) to https (443), and then directs traffic to the correct service based on the request path. 
+
+```nginx
+events {}
+
+http {
+    sendfile    on;
+    keepalive_timeout 60;
+
+     upstream omnivore_web {
+    	ip_hash;
+    	server 127.0.0.1:3000;
+     }
+
+     upstream omnivore_backend {
+    	ip_hash;
+    	server 127.0.0.1:4000;
+     }
+
+     upstream omnivore_imageproxy {
+    	ip_hash;
+    	server 127.0.0.1:1010;
+     }
+
+    upstream omnivore_bucket {
+    	ip_hash;
+    	server 127.0.0.1:7070;
+    }
+
+    server {
+        listen 80;
+        return 301 https://$host$request_uri
+    }
+
+    server {
+        listen 443;
+        server_name  omnivore.domain.com;
+
+
+        ssl_certification   /path/to/cert.crt;
+        ssl_certificate_key /path/to/cert.key;
+
+        ssl on;
+        ssl_session_cache builtin:1000 shared:SSL:10m;
+        ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+        ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4;
+        ssl_prefer_server_ciphers on;
+
+        # Override for authentication on the frontend
+        location /api/client/auth {
+            proxy_pass http://omnivore_web;
+        }
+
+        # API
+        location /api {
+            proxy_pass http://omnivore_backend;
+        }
+
+        # Minio
+        location /bucket {
+            proxy_pass http://omnivore_bucket;
+        }
+
+        # ImageProxy
+        location /images {
+            proxy_pass http://omnivore_imageproxy;
+        }
+
+        # FrontEnd application
+        location / {
+            proxy_pass http://omnivore_web;
+        }
+        
+        location /mail {
+            proxy_pass http://localhost:4398/mail;
+        }
+    }
+}
+```
+
+## Cloudflare Tunnel
+Cloudflare tunnels is an easy way to expose a service running on a local machine to the internet without a publicly routable IP Address. 
+
+You run a daemon on your host machine, which creates outbound connections to the
+
+![Tunnels Config](../docs/guides/images/cloudflare-tunnel.png)
+
+Omnivore is no way affiliated with Cloudflare, it is just the method to which the person writing this guide used, and found pretty painless overall.
+
+[Read More](https://www.cloudflare.com/products/tunnel/)
+
+
+## Emails and Newsletters
+
+Another Feature of Omnivore is the ability to receive Newsletters directly into your Inbox using email. This feature is described more [here](#receiving-newsletter-subscriptions-via-email).
+
+This works by generating an email address, and subscribing to a newsletter using that email address. 
+
+In order to get this working in a self-hosted way we have created a new endpoint that allows you to send an API request with the emails contents.  
+
+We will go over 
+
+#### Receiving Newsletter Subscriptions via Email
+1. On the Omnivore website or app, tap your photo, initial, or avatar in the top right corner to access the profile menu. Select Emails from the menu.
+
+2. Tap Create a New Email Address to add a new email address (e.g. username-123abc@inbox.omnivore.app) to the list.
+
+3. Click the Copy icon next to the email address.
+
+4. Navigate to the signup page for the newsletter you wish to subscribe to.
+
+5. Paste the Omnivore email address into the signup form.
+
+6. New newsletters will be automatically delivered to your Omnivore inbox.
+
+### Docker-mailserver and mail-watcher
+
+One way to get this functionality back is to host your own mail server. In this example we will only be using this mail server as an incoming mailbox to receive emails. I would not recommend this method, as it's largely more effort  than it is worth.  
+
+We have used [Docker-mailserver](https://docker-mailserver.github.io) here. A guide on how to set this up is found [here](https://docker-mailserver.github.io/docker-mailserver/latest/examples/tutorials/basic-installation/). 
+
+We have included a docker file `self-hosting/docker-compose/mail/docker-compose-mail`. This file does a few things.
+
+* Setups Docker-mailserver with minimal settings. 
+* Creates a user `user@domain.tld` where `domain.tld` is your email servers domain. 
+* Reroutes all mail from `*@domain.tld` to `user@domain.tld`
+* Watches for any new mail incoming, converts it to a payload for the mail proxy, and forwards it on. 
+
+There are a few environment variables that need to be set. 
+
+```.env
+WATCHER_API_KEY=mail-api-key # The API Key that runs the mail-watcher-api 
+MAIL_FILE_PATH=/var/mail/domain.tld/user/new # where domain.tld is the name of your domain
+WATCHER_API_ENDPOINT=https://omnivore-watch.domain.tld # The hosted watcher api - where mail is proxied to and processed.
+```
+
+Additionally you need to change a few things in the docker-file. 
+
+```
+hostname: mail.domain.tld
+```
+```
+    environment:
+      - DOMAIN="domain.tld"
+```
+```
+docker exec -ti mailserver setup email add user@domain.tld pass123;
+echo '@domain.tld user@domain.tld' > /tmp/docker-mailserver/config/postfix-virtual.cf
+```
+replace domain.tld with your mail servers domain. 
+
+Additionally you need to replace the following environment variables for the API. 
+
+```
+WATCHER_API_KEY=mail-api-key # The same as the one in the mail server.
+LOCAL_EMAIL_DOMAIN=domain.tld # Your email domain.
+```
+
+### Third Party Services
+Setting up your own email server is a bit overkill for what we are trying to achieve. Below are some additional services that can be used to achieve the mail functionality. These are just a few examples, but others will also work.
+
+#### Amazon Simple Email Service and SNS 
+
+Amazon Simple Email Service (SES) has options for email receiving. We can use this to add the email functionality to Omnivore-self hosted. 
+
+##### Step 1. Create Identity
+Create your identity using Amazon SES. This will be your domain. 
+
+![create-identity](../docs/guides/images/ses-add-domain.png)
+
+##### Step 2. Verify the Domain using the CNAME Records.
+![Verify Domain](../docs/guides/images/ses-verify.png)
+
+#### Step 3. Add the MX Record 
+
+See instructions on how to do that [here](https://docs.aws.amazon.com/ses/latest/dg/receiving-email-mx-record.html)
+
+##### Step 4. Create Email-Receiving Ruleset
+![Create Ruleset](../docs/guides/images/ses-verify.png)
+![Create Ruleset](../docs/guides/images/sns-define-incoming-rule.png)
+
+
+##### Step 5. Create SNS Topic Target
+![SNS add action](../docs/guides/images/sns-add-actions-sns-menu.png)
+![SNS add action publish](../docs/guides/images/sns-add-action-publish.png)
+![SNS Create](../docs/guides/images/sns-create-topic.png)
+![SNS Topic Menu](../docs/guides/images/sns-topic-menu.png)
+![SNS publish](../docs/guides/images/sns-publish-menu.png)
+
+
+##### Step 6. Setup Subscription
+In SNS you must setup a subscription to your Omnivore Host.
+![Sns Subscription](../docs/guides/images/sns-create-subscription.png)
+
+
+##### Step 7. Test by sending email to Omnivore Email
+![Email](../docs/guides/images/create-new-email.png)
+![Incoming](../docs/guides/images/testing-incoming-email.png)
+![Received](../docs/guides/images/received-email.png)
+
+#### Zapier and other Webhook Services. 
+
+If you are just looking for a simple way to import emails into your Self Hosted Omnivore Account, you can use a service like Zapier to forward the email into the mail-proxy. 
+
+Below is a set of instructions to get this working. 
+
+##### Step 1. Create an Omnivore Email 
+![Email](../docs/guides/images/create-new-email.png)
+
+##### Step 2. Create a Zapier Integration, using Gmail or Equivalent
+You can either use your own email with a filter, or alternatively create a new gmail account exclusively for your Newsletters.
+![Zapier-Email](../docs/guides/images/zapier-email-webhook.png)
+
+##### Step 3. Convert Email into Payload for Webhook. 
+![Zapier-Javascript](../docs/guides/images/zapier-javascript-step.png)
+
+For the to object use the email provided in step 1. 
+
+```javascript
+return { data: JSON.stringify(inputData) }
+```
+
+##### Step 4. Send to Mail Proxy. 
+![Zapier-Proxy](../docs/guides/images/zapier-webhook-step.png)
+* POST Request
+* Use the x-api-key set in your .env file 
+* The data is the output from the previous step. 
+
+##### Email Imported
+
+Following these steps you should see your email imported into Omnivore. 
+
+![imported-email](../docs/guides/images/imported-email.png)
+
+
+

self-hosting/docker-compose/.env.example

@@ -0,0 +1,63 @@
+# Postgres & Migrate
+
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=omnivore
+PGPASSWORD=postgres
+POSTGRES_USER=postgres
+PG_HOST=postgres
+PG_PASSWORD=app_pass
+PG_DB=omnivore
+PG_USER=app_user
+PG_PORT=5432
+PG_POOL_MAX=20
+
+# API
+
+API_ENV=local
+IMAGE_PROXY_SECRET=some-secret
+JWT_SECRET=some_secret
+SSO_JWT_SECRET=some_sso_secret
+GATEWAY_URL=http://api:8080/api
+CONTENT_FETCH_URL=http://content-fetch:8080/?token=some_token
+GCS_USE_LOCAL_HOST=true
+GCS_UPLOAD_BUCKET=omnivore
+AUTO_VERIFY=true
+AWS_ACCESS_KEY_ID=minio   # Used for Minio S3 Client
+AWS_SECRET_ACCESS_KEY=miniominio
+AWS_REGION=us-east-1
+CONTENT_FETCH_QUEUE_ENABLED=true
+
+IMAGE_PROXY_URL=http://localhost:7070 # Need to change this for NGINX
+CLIENT_URL=http://localhost:3000 # Need to change this when using NGINX
+LOCAL_MINIO_URL=http://localhost:1010
+
+# Redis
+REDIS_URL=redis://redis:6379/0
+
+#MAIL
+WATCHER_API_KEY=mail-api-key
+LOCAL_EMAIL_DOMAIN=domain.tld
+LOCAL_EMAIL_DOMAIN=domain.tld
+SNS_ARN=arn_of_sns #for if you use SES and SNS for Email.
+
+# Web
+APP_ENV=prod
+NEXT_PUBLIC_APP_ENV=prod
+BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX
+SERVER_BASE_URL=http://localhost:4000 # API Server, need to change this when using NGINX
+HIGHLIGHTS_BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX
+NEXT_PUBLIC_BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX
+NEXT_PUBLIC_SERVER_BASE_URL=http://localhost:4000 # API Server, need to change this when using NGINX
+NEXT_PUBLIC_HIGHLIGHTS_BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX
+
+# Content Fetch
+
+VERIFICATION_TOKEN=some_token
+REST_BACKEND_ENDPOINT=http://api:8080/api
+SKIP_UPLOAD_ORIGINAL=true
+
+# Minio
+MINIO_ACCESS_KEY=minio
+MINIO_SECRET_KEY=miniominio
+AWS_S3_ENDPOINT_URL=http://minio:9000

self-hosting/docker-compose/docker-compose.yml

@@ -0,0 +1,173 @@
+version: '3'
+x-postgres:
+  &postgres-common
+  image: "ankane/pgvector:v0.5.1"
+  user: postgres
+  healthcheck:
+    test: "exit 0"
+    interval: 2s
+    timeout: 12s
+    retries: 3
+
+services:
+  postgres:
+    <<: *postgres-common
+    container_name: "omnivore-postgres"
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    env_file:
+      - .env
+
+  migrate:
+    build:
+      context: ../../
+      dockerfile: ./packages/db/Dockerfile
+    container_name: "omnivore-migrate"
+    command: '/bin/sh ./packages/db/setup.sh' # Also create a demo user with email: demo@omnivore.app, password: demo_password
+    env_file:
+      - .env
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  api:
+    build:
+      context: ../../
+      dockerfile: ./packages/api/Dockerfile
+    container_name: "omnivore-api"
+    ports:
+      - "4000:8080"
+    healthcheck:
+      test: ["CMD-SHELL", "nc -z 0.0.0.0 8080 || exit 1"]
+      interval: 15s
+      timeout: 90s
+      retries: 6
+    env_file:
+      - .env
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+
+  queue-processor:
+    build:
+      context: ../../
+      dockerfile: ./packages/api/queue-processor/Dockerfile
+    container_name: "omnivore-queue-processor"
+    env_file:
+      - .env
+    depends_on:
+      api:
+        condition: service_started
+
+  web:
+    build:
+      context: ../../
+      dockerfile: ./packages/web/Dockerfile-self
+      args:
+        - APP_ENV=prod
+        - BASE_URL=http://localhost:3000
+        - SERVER_BASE_URL=http://localhost:4000
+        - HIGHLIGHTS_BASE_URL=http://localhost:3000
+    container_name: "omnivore-web"
+    ports:
+      - "3000:8080"
+    env_file:
+      .env
+    depends_on:
+      api:
+        condition: service_healthy
+
+  image-proxy:
+    build:
+      context: ../../imageproxy
+      dockerfile: ./Dockerfile
+    container_name: "omnivore-image-proxy"
+    ports:
+      - "7070:8080"
+    env_file:
+      - .env
+
+  content-fetch:
+    build:
+      context: ../../
+      dockerfile: ./packages/content-fetch/Dockerfile
+    container_name: "omnivore-content-fetch"
+    ports:
+      - "9090:8080"
+    environment:
+      - USE_FIREFOX=true # Using Firefox here because the official chrome version seems to freeze a lot in Docker.
+    env_file:
+      - .env
+    depends_on:
+      redis:
+        condition: service_healthy
+      api:
+        condition: service_healthy
+
+  redis:
+    image: "redis:7.2.4"
+    container_name: "omnivore-redis"
+    expose:
+      - 6379
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+    volumes:
+      - redis_data:/data
+
+  minio:
+    image: minio/minio
+    expose:
+      - 1010
+    ports:
+      - "1010:9000"
+    healthcheck:
+      test: [ "CMD", "mc", "ready", "local" ]
+      interval: 5s
+      timeout: 1s
+    environment:
+      - "MINIO_ACCESS_KEY=minio"
+      - "MINIO_SECRET_KEY=miniominio"
+      - "AWS_S3_ENDPOINT_URL=http://minio:1010"
+    command: server /data
+    volumes:
+      - minio_data:/data
+
+  createbuckets:
+    image: minio/mc
+    environment:
+      - MINIO_ACCESS_KEY=minio
+      - MINIO_SECRET_KEY=miniominio
+      - BUCKET_NAME=omnivore
+      - ENDPOINT=http://minio:9000
+      - AWS_S3_ENDPOINT_URL=http://minio:9000
+    depends_on:
+      - minio
+    entrypoint: >
+      /bin/bash -c "
+      sleep 5;
+      until (/usr/bin/mc config host add myminio http://minio:9000 minio miniominio) do echo '...waiting...' && sleep 1; done;
+      /usr/bin/mc mb myminio/omnivore;
+      /usr/bin/mc policy set public myminio/omnivore;
+      exit 0;
+      "
+  mail-watch-server:
+    build:
+      context: ../../
+      dockerfile: ./packages/local-mail-watcher/Dockerfile
+    container_name: "omnivore-mail-watch-server"
+    ports:
+      - "4398:8080"
+    env_file:
+      - .env
+    depends_on:
+      redis:
+        condition: service_healthy
+volumes:
+  pgdata:
+  redis_data:
+  minio_data:
+

self-hosting/docker-compose/mail/.env.mail

@@ -0,0 +1,5 @@
+#MAIL
+WATCHER_API_KEY=mail-api-key
+MAIL_FILE_PATH=/var/mail/domain.tld/user/new
+WATCHER_API_ENDPOINT=https://omnivore-watch.domain.tld
+DELETE_FILE=true

self-hosting/docker-compose/mail/docker-compose-mail.yml

@@ -0,0 +1,62 @@
+services:
+  mailserver:
+    image: ghcr.io/docker-mailserver/docker-mailserver:latest
+    container_name: mailserver
+    # Provide the FQDN of your mail server here (Your DNS MX record should point to this value)
+    hostname: mail.domain.tld
+    env_file: mailserver.env
+    # More information about the mail-server ports:
+    # https://docker-mailserver.github.io/docker-mailserver/latest/config/security/understanding-the-ports/
+    ports:
+      - "25:25"    # SMTP  (explicit TLS => STARTTLS, Authentication is DISABLED => use port 465/587 instead)
+      - "143:143"  # IMAP4 (explicit TLS => STARTTLS)
+      - "465:465"  # ESMTP (implicit TLS)
+      - "587:587"  # ESMTP (explicit TLS => STARTTLS)
+      - "993:993"  # IMAP4 (implicit TLS)
+    volumes:
+      - ./docker-data/dms/mail-data/:/var/mail/
+      - ./docker-data/dms/mail-state/:/var/mail-state/
+      - ./docker-data/dms/mail-logs/:/var/log/mail/
+      - ./docker-data/dms/config/:/tmp/docker-mailserver/
+      - /etc/localtime:/etc/localtime:ro
+    restart: always
+    stop_grace_period: 1m
+    # Uncomment if using `ENABLE_FAIL2BAN=1`:
+    # cap_add:
+    #   - NET_ADMIN
+    healthcheck:
+      test: "ss --listening --tcp | grep -P 'LISTEN.+:smtp' || exit 1"
+      timeout: 3s
+      retries: 0
+  docker:
+    image: docker:latest
+    container_name: setup-email
+    environment:
+      - DOMAIN="domain.tld"
+    volumes:
+      - ./docker-data/dms/mail-data/:/var/mail/
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./docker-data/dms/config/:/tmp/docker-mailserver/
+    depends_on:
+      mailserver:
+        condition: service_started
+    tty: true
+    entrypoint: >
+      /bin/sh -c "
+      sleep 5; 
+      docker exec -ti mailserver setup email add user@domain.tld pass123;
+      echo '@domain.tld user@domain.tld' > /tmp/docker-mailserver/config/postfix-virtual.cf
+      exit 0
+      "
+  watcher:
+    build:
+      context: ../../../
+      dockerfile: ./packages/local-mail-watcher/Dockerfile-watcher
+    container_name: "omnivore-mail-watch"
+    volumes:
+      - ./docker-data/dms/mail-data/:/var/mail/
+    env_file:
+      - .env.mail
+    depends_on:
+      docker:
+        condition: service_completed_successfully

self-hosting/docker-compose/mail/mailserver.env

@@ -0,0 +1,661 @@
+# -----------------------------------------------
+# --- Mailserver Environment Variables ----------
+# -----------------------------------------------
+
+# DOCUMENTATION FOR THESE VARIABLES IS FOUND UNDER
+# https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/
+
+# -----------------------------------------------
+# --- General Section ---------------------------
+# -----------------------------------------------
+
+# empty => uses the `hostname` command to get the mail server's canonical hostname
+# => Specify a fully-qualified domainname to serve mail for.  This is used for many of the config features so if you can't set your hostname (e.g. you're in a container platform that doesn't let you) specify it in this environment variable.
+OVERRIDE_HOSTNAME=
+
+# REMOVED in version v11.0.0! Use LOG_LEVEL instead.
+DMS_DEBUG=0
+
+# Set the log level for DMS.
+# This is mostly relevant for container startup scripts and change detection event feedback.
+#
+# Valid values (in order of increasing verbosity) are: `error`, `warn`, `info`, `debug` and `trace`.
+# The default log level is `info`.
+LOG_LEVEL=info
+
+# critical => Only show critical messages
+# error => Only show erroneous output
+# **warn** => Show warnings
+# info => Normal informational output
+# debug => Also show debug messages
+SUPERVISOR_LOGLEVEL=
+
+# Support for deployment where these defaults are not compatible (eg: some NAS appliances):
+# /var/mail vmail User ID (default: 5000)
+DMS_VMAIL_UID=
+# /var/mail vmail Group ID (default: 5000)
+DMS_VMAIL_GID=
+
+# **empty** => use FILE
+# LDAP => use LDAP authentication
+# OIDC => use OIDC authentication (not yet implemented)
+# FILE => use local files (this is used as the default)
+ACCOUNT_PROVISIONER=
+
+# empty => postmaster@domain.com
+# => Specify the postmaster address
+POSTMASTER_ADDRESS=
+
+# Check for updates on container start and then once a day
+# If an update is available, a mail is sent to POSTMASTER_ADDRESS
+# 0 => Update check disabled
+# 1 => Update check enabled
+ENABLE_UPDATE_CHECK=1
+
+# Customize the update check interval.
+# Number + Suffix. Suffix must be 's' for seconds, 'm' for minutes, 'h' for hours or 'd' for days.
+UPDATE_CHECK_INTERVAL=1d
+
+# Set different options for mynetworks option (can be overwrite in postfix-main.cf)
+# **WARNING**: Adding the docker network's gateway to the list of trusted hosts, e.g. using the `network` or
+# `connected-networks` option, can create an open relay
+# https://github.com/docker-mailserver/docker-mailserver/issues/1405#issuecomment-590106498
+# The same can happen for rootless podman. To prevent this, set the value to "none" or configure slirp4netns
+# https://github.com/docker-mailserver/docker-mailserver/issues/2377
+#
+# none => Explicitly force authentication
+# container => Container IP address only
+# host => Add docker container network (ipv4 only)
+# network => Add all docker container networks (ipv4 only)
+# connected-networks => Add all connected docker networks (ipv4 only)
+PERMIT_DOCKER=none
+
+# Set the timezone. If this variable is unset, the container runtime will try to detect the time using
+# `/etc/localtime`, which you can alternatively mount into the container. The value of this variable
+# must follow the pattern `AREA/ZONE`, i.e. of you want to use Germany's time zone, use `Europe/Berlin`.
+# You can lookup all available timezones here: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List
+TZ=
+
+# In case you network interface differs from 'eth0', e.g. when you are using HostNetworking in Kubernetes,
+# you can set NETWORK_INTERFACE to whatever interface you want. This interface will then be used.
+#  - **empty** => eth0
+NETWORK_INTERFACE=
+
+# empty => modern
+# modern => Enables TLSv1.2 and modern ciphers only. (default)
+# intermediate => Enables TLSv1, TLSv1.1 and TLSv1.2 and broad compatibility ciphers.
+TLS_LEVEL=
+
+# Configures the handling of creating mails with forged sender addresses.
+#
+# **0** => (not recommended) Mail address spoofing allowed. Any logged in user may create email messages with a forged sender address (see also https://en.wikipedia.org/wiki/Email_spoofing).
+# 1 => Mail spoofing denied. Each user may only send with his own or his alias addresses. Addresses with extension delimiters(http://www.postfix.org/postconf.5.html#recipient_delimiter) are not able to send messages.
+SPOOF_PROTECTION=
+
+# Enables the Sender Rewriting Scheme. SRS is needed if your mail server acts as forwarder. See [postsrsd](https://github.com/roehling/postsrsd/blob/master/README.md#sender-rewriting-scheme-crash-course) for further explanation.
+#  - **0** => Disabled
+#  - 1 => Enabled
+ENABLE_SRS=0
+
+# Enables the OpenDKIM service.
+# **1** => Enabled
+#   0   => Disabled
+ENABLE_OPENDKIM=1
+
+# Enables the OpenDMARC service.
+# **1** => Enabled
+#   0   => Disabled
+ENABLE_OPENDMARC=1
+
+
+# Enabled `policyd-spf` in Postfix's configuration. You will likely want to set this
+# to `0` in case you're using Rspamd (`ENABLE_RSPAMD=1`).
+#
+# - 0     => Disabled
+# - **1** => Enabled
+ENABLE_POLICYD_SPF=1
+
+# Enables POP3 service
+# - **0** => Disabled
+# - 1     => Enabled
+ENABLE_POP3=
+
+# Enables IMAP service
+# - 0     => Disabled
+# - **1** => Enabled
+ENABLE_IMAP=1
+
+# Enables ClamAV, and anti-virus scanner.
+#   1   => Enabled
+# **0** => Disabled
+ENABLE_CLAMAV=0
+
+# Add the value of this ENV as a prefix to the mail subject when spam is detected.
+# NOTE: This subject prefix may be redundant (by default spam is delivered to a junk folder).
+#       It provides value when your junk mail is stored alongside legitimate mail instead of a separate location (like with `SPAMASSASSIN_SPAM_TO_INBOX=1` or `MOVE_SPAM_TO_JUNK=0` or a POP3 only setup, without IMAP).
+# NOTE: When not using Docker Compose, other CRI may not support quote-wrapping the value here to preserve any trailing white-space.
+SPAM_SUBJECT=
+
+# Enables Rspamd
+# **0** => Disabled
+#   1   => Enabled
+ENABLE_RSPAMD=0
+
+# When `ENABLE_RSPAMD=1`, an internal Redis instance is enabled implicitly.
+# This setting provides an opt-out to allow using an external instance instead.
+# 0 => Disabled
+# 1 => Enabled
+ENABLE_RSPAMD_REDIS=
+
+# When enabled,
+#
+# 1. the "[autolearning][rspamd-autolearn]" feature is turned on;
+# 2. the Bayes classifier will be trained when moving mails from or to the Junk folder (with the help of Sieve scripts).
+#
+# **0** => disabled
+# 1     => enabled
+RSPAMD_LEARN=0
+
+# This settings controls whether checks should be performed on emails coming
+# from authenticated users (i.e. most likely outgoing emails). The default value
+# is `0` in order to align better with SpamAssassin. We recommend reading
+# through https://rspamd.com/doc/tutorials/scanning_outbound.html though to
+# decide for yourself whether you need and want this feature.
+#
+# Note that DKIM signing of e-mails will still happen.
+RSPAMD_CHECK_AUTHENTICATED=0
+
+# Controls whether the Rspamd Greylisting module is enabled.
+# This module can further assist in avoiding spam emails by greylisting
+# e-mails with a certain spam score.
+#
+# **0** => disabled
+# 1     => enabled
+RSPAMD_GREYLISTING=0
+
+# Can be used to enable or disable the Hfilter group module.
+#
+# - 0     => Disabled
+# - **1** => Enabled
+RSPAMD_HFILTER=1
+
+# Can be used to control the score when the HFILTER_HOSTNAME_UNKNOWN symbol applies. A higher score is more punishing. Setting it to 15 is equivalent to rejecting the email when the check fails.
+#
+# Default: 6
+RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE=6
+
+# Can be used to enable or disable the (still experimental) neural module.
+#
+# - **0** => Disabled
+# - 1     => Enabled
+RSPAMD_NEURAL=0
+
+# Amavis content filter (used for ClamAV & SpamAssassin)
+# 0 => Disabled
+# 1 => Enabled
+ENABLE_AMAVIS=1
+
+# -1/-2/-3 => Only show errors
+# **0**    => Show warnings
+# 1/2      => Show default informational output
+# 3/4/5    => log debug information (very verbose)
+AMAVIS_LOGLEVEL=0
+
+# This enables DNS block lists in Postscreen.
+# Note: Emails will be rejected, if they don't pass the block list checks!
+# **0** => DNS block lists are disabled
+# 1     => DNS block lists are enabled
+ENABLE_DNSBL=0
+
+# If you enable Fail2Ban, don't forget to add the following lines to your `compose.yaml`:
+#    cap_add:
+#      - NET_ADMIN
+# Otherwise, `nftables` won't be able to ban IPs.
+ENABLE_FAIL2BAN=0
+
+# Fail2Ban blocktype
+# drop   => drop packet (send NO reply)
+# reject => reject packet (send ICMP unreachable)
+FAIL2BAN_BLOCKTYPE=drop
+
+# 1 => Enables Managesieve on port 4190
+# empty => disables Managesieve
+ENABLE_MANAGESIEVE=
+
+# **enforce** => Allow other tests to complete. Reject attempts to deliver mail with a 550 SMTP reply, and log the helo/sender/recipient information. Repeat this test the next time the client connects.
+# drop => Drop the connection immediately with a 521 SMTP reply. Repeat this test the next time the client connects.
+# ignore => Ignore the failure of this test. Allow other tests to complete. Repeat this test the next time the client connects. This option is useful for testing and collecting statistics without blocking mail.
+POSTSCREEN_ACTION=enforce
+
+# empty => all daemons start
+# 1 => only launch postfix smtp
+SMTP_ONLY=
+
+# Please read [the SSL page in the documentation](https://docker-mailserver.github.io/docker-mailserver/latest/config/security/ssl) for more information.
+#
+# empty => SSL disabled
+# letsencrypt => Enables Let's Encrypt certificates
+# custom => Enables custom certificates
+# manual => Let's you manually specify locations of your SSL certificates for non-standard cases
+# self-signed => Enables self-signed certificates
+SSL_TYPE=
+
+# These are only supported with `SSL_TYPE=manual`.
+# Provide the path to your cert and key files that you've mounted access to within the container.
+SSL_CERT_PATH=
+SSL_KEY_PATH=
+# Optional: A 2nd certificate can be supported as fallback (dual cert support), eg ECDSA with an RSA fallback.
+# Useful for additional compatibility with older MTA and MUA (eg pre-2015).
+SSL_ALT_CERT_PATH=
+SSL_ALT_KEY_PATH=
+
+# Set how many days a virusmail will stay on the server before being deleted
+# empty => 7 days
+VIRUSMAILS_DELETE_DELAY=
+
+# Configure Postfix `virtual_transport` to deliver mail to a different LMTP client (default is a dovecot socket).
+# Provide any valid URI. Examples:
+#
+# empty => `lmtp:unix:/var/run/dovecot/lmtp` (default, configured in Postfix main.cf)
+# `lmtp:unix:private/dovecot-lmtp` (use socket)
+# `lmtps:inet:<host>:<port>` (secure lmtp with starttls)
+# `lmtp:<kopano-host>:2003` (use kopano as mailstore)
+POSTFIX_DAGENT=
+
+# Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). Size is in bytes.
+#
+# empty => 0
+POSTFIX_MAILBOX_SIZE_LIMIT=
+
+# See https://docker-mailserver.github.io/docker-mailserver/latest/config/account-management/overview/#quotas
+# 0 => Dovecot quota is disabled
+# 1 => Dovecot quota is enabled
+ENABLE_QUOTAS=1
+
+# Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!). Size is in bytes.
+#
+# empty => 10240000 (~10 MB)
+POSTFIX_MESSAGE_SIZE_LIMIT=
+
+# Mails larger than this limit won't be scanned.
+# ClamAV must be enabled (ENABLE_CLAMAV=1) for this.
+#
+# empty => 25M (25 MB)
+CLAMAV_MESSAGE_SIZE_LIMIT=
+
+# Enables regular pflogsumm mail reports.
+# This is a new option. The old REPORT options are still supported for backwards compatibility. If this is not set and reports are enabled with the old options, logrotate will be used.
+#
+# not set => No report
+# daily_cron => Daily report for the previous day
+# logrotate => Full report based on the mail log when it is rotated
+PFLOGSUMM_TRIGGER=
+
+# Recipient address for pflogsumm reports.
+#
+# not set => Use REPORT_RECIPIENT or POSTMASTER_ADDRESS
+# => Specify the recipient address(es)
+PFLOGSUMM_RECIPIENT=
+
+# Sender address (`FROM`) for pflogsumm reports if pflogsumm reports are enabled.
+#
+# not set => Use REPORT_SENDER
+# => Specify the sender address
+PFLOGSUMM_SENDER=
+
+# Interval for logwatch report.
+#
+# none => No report is generated
+# daily => Send a daily report
+# weekly => Send a report every week
+LOGWATCH_INTERVAL=
+
+# Recipient address for logwatch reports if they are enabled.
+#
+# not set => Use REPORT_RECIPIENT or POSTMASTER_ADDRESS
+# => Specify the recipient address(es)
+LOGWATCH_RECIPIENT=
+
+# Sender address (`FROM`) for logwatch reports if logwatch reports are enabled.
+#
+# not set => Use REPORT_SENDER
+# => Specify the sender address
+LOGWATCH_SENDER=
+
+# Defines who receives reports if they are enabled.
+# **empty** => ${POSTMASTER_ADDRESS}
+# => Specify the recipient address
+REPORT_RECIPIENT=
+
+# Defines who sends reports if they are enabled.
+# **empty** => mailserver-report@${DOMAINNAME}
+# => Specify the sender address
+REPORT_SENDER=
+
+# Changes the interval in which log files are rotated
+# **weekly** => Rotate log files weekly
+# daily => Rotate log files daily
+# monthly => Rotate log files monthly
+#
+# Note: This Variable actually controls logrotate inside the container
+# and rotates the log files depending on this setting. The main log output is
+# still available in its entirety via `docker logs mail` (Or your
+# respective container name). If you want to control logrotation for
+# the Docker-generated logfile see:
+# https://docs.docker.com/config/containers/logging/configure/
+#
+# Note: This variable can also determine the interval for Postfix's log summary reports, see [`PFLOGSUMM_TRIGGER`](#pflogsumm_trigger).
+LOGROTATE_INTERVAL=weekly
+
+# Defines how many log files are kept by logrorate
+LOGROTATE_COUNT=4
+
+
+# If enabled, employs `reject_unknown_client_hostname` to sender restrictions in Postfix's configuration.
+#
+# - **0** => Disabled
+# - 1 => Enabled
+POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME=0
+
+# Choose TCP/IP protocols for postfix to use
+# **all** => All possible protocols.
+# ipv4 => Use only IPv4 traffic. Most likely you want this behind Docker.
+# ipv6 => Use only IPv6 traffic.
+#
+# Note: More details at http://www.postfix.org/postconf.5.html#inet_protocols
+POSTFIX_INET_PROTOCOLS=all
+
+# Enables MTA-STS support for outbound mail.
+# More details: https://docker-mailserver.github.io/docker-mailserver/v13.3/config/best-practices/mta-sts/
+# - **0** ==> MTA-STS disabled
+# - 1 => MTA-STS enabled
+ENABLE_MTA_STS=0
+
+# Choose TCP/IP protocols for dovecot to use
+# **all** => Listen on all interfaces
+# ipv4 => Listen only on IPv4 interfaces. Most likely you want this behind Docker.
+# ipv6 => Listen only on IPv6 interfaces.
+#
+# Note: More information at https://dovecot.org/doc/dovecot-example.conf
+DOVECOT_INET_PROTOCOLS=all
+
+# -----------------------------------------------
+# --- SpamAssassin Section ----------------------
+# -----------------------------------------------
+
+ENABLE_SPAMASSASSIN=0
+
+# KAM is a 3rd party SpamAssassin ruleset, provided by the McGrail Foundation.
+# If SpamAssassin is enabled, KAM can be used in addition to the default ruleset.
+# - **0** => KAM disabled
+# - 1 => KAM enabled
+#
+# Note: only has an effect if `ENABLE_SPAMASSASSIN=1`
+ENABLE_SPAMASSASSIN_KAM=0
+
+# deliver spam messages to the inbox (tagged using SPAM_SUBJECT)
+SPAMASSASSIN_SPAM_TO_INBOX=1
+
+# spam messages will be moved in the Junk folder (SPAMASSASSIN_SPAM_TO_INBOX=1 required)
+MOVE_SPAM_TO_JUNK=1
+
+# spam messages will be marked as read
+MARK_SPAM_AS_READ=0
+
+# add 'spam info' headers at, or above this level
+SA_TAG=2.0
+
+# add 'spam detected' headers at, or above this level
+SA_TAG2=6.31
+
+# triggers spam evasive actions
+SA_KILL=10.0
+
+# -----------------------------------------------
+# --- Fetchmail Section -------------------------
+# -----------------------------------------------
+
+ENABLE_FETCHMAIL=0
+
+# The interval to fetch mail in seconds
+FETCHMAIL_POLL=300
+# Use multiple fetchmail instances (1 per poll entry in fetchmail.cf)
+# Supports multiple IMAP IDLE connections when a server is used across multiple poll entries
+# https://otremba.net/wiki/Fetchmail_(Debian)#Immediate_Download_via_IMAP_IDLE
+FETCHMAIL_PARALLEL=0
+
+# Enable or disable `getmail`.
+#
+# - **0** => Disabled
+# - 1 => Enabled
+ENABLE_GETMAIL=0
+
+# The number of minutes for the interval. Min: 1; Default: 5.
+GETMAIL_POLL=5
+
+# -----------------------------------------------
+# --- OAUTH2 Section ----------------------------
+# -----------------------------------------------
+
+# empty => OAUTH2 authentication is disabled
+# 1 => OAUTH2 authentication is enabled
+ENABLE_OAUTH2=
+
+# Specify the user info endpoint URL of the oauth2 provider
+# Example: https://oauth2.example.com/userinfo/
+OAUTH2_INTROSPECTION_URL=
+
+# -----------------------------------------------
+# --- LDAP Section ------------------------------
+# -----------------------------------------------
+
+# A second container for the ldap service is necessary (i.e. https://hub.docker.com/r/bitnami/openldap/)
+
+# empty => no
+# yes => LDAP over TLS enabled for Postfix
+LDAP_START_TLS=
+
+# empty => mail.example.com
+# Specify the `<dns-name>` / `<ip-address>` where the LDAP server is reachable via a URI like: `ldaps://mail.example.com`.
+# Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`).
+LDAP_SERVER_HOST=
+
+# empty => ou=people,dc=domain,dc=com
+# => e.g. LDAP_SEARCH_BASE=dc=mydomain,dc=local
+LDAP_SEARCH_BASE=
+
+# empty => cn=admin,dc=domain,dc=com
+# => take a look at examples of SASL_LDAP_BIND_DN
+LDAP_BIND_DN=
+
+# empty** => admin
+# => Specify the password to bind against ldap
+LDAP_BIND_PW=
+
+# e.g. `"(&(mail=%s)(mailEnabled=TRUE))"`
+# => Specify how ldap should be asked for users
+LDAP_QUERY_FILTER_USER=
+
+# e.g. `"(&(mailGroupMember=%s)(mailEnabled=TRUE))"`
+# => Specify how ldap should be asked for groups
+LDAP_QUERY_FILTER_GROUP=
+
+# e.g. `"(&(mailAlias=%s)(mailEnabled=TRUE))"`
+# => Specify how ldap should be asked for aliases
+LDAP_QUERY_FILTER_ALIAS=
+
+# e.g. `"(&(|(mail=*@%s)(mailalias=*@%s)(mailGroupMember=*@%s))(mailEnabled=TRUE))"`
+# => Specify how ldap should be asked for domains
+LDAP_QUERY_FILTER_DOMAIN=
+
+# -----------------------------------------------
+# --- Dovecot Section ---------------------------
+# -----------------------------------------------
+
+# empty => no
+# yes => LDAP over TLS enabled for Dovecot
+DOVECOT_TLS=
+
+# e.g. `"(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))"`
+DOVECOT_USER_FILTER=
+
+# e.g. `"(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))"`
+DOVECOT_PASS_FILTER=
+
+# Define the mailbox format to be used
+# default is maildir, supported values are: sdbox, mdbox, maildir
+DOVECOT_MAILBOX_FORMAT=maildir
+
+# empty => no
+# yes => Allow bind authentication for LDAP
+# https://wiki.dovecot.org/AuthDatabase/LDAP/AuthBinds
+DOVECOT_AUTH_BIND=
+
+# -----------------------------------------------
+# --- Postgrey Section --------------------------
+# -----------------------------------------------
+
+ENABLE_POSTGREY=0
+# greylist for N seconds
+POSTGREY_DELAY=300
+# delete entries older than N days since the last time that they have been seen
+POSTGREY_MAX_AGE=35
+# response when a mail is greylisted
+POSTGREY_TEXT="Delayed by Postgrey"
+# whitelist host after N successful deliveries (N=0 to disable whitelisting)
+POSTGREY_AUTO_WHITELIST_CLIENTS=5
+
+# -----------------------------------------------
+# --- SASL Section ------------------------------
+# -----------------------------------------------
+
+ENABLE_SASLAUTHD=0
+
+# empty => pam
+# `ldap` => authenticate against ldap server
+# `shadow` => authenticate against local user db
+# `mysql` => authenticate against mysql db
+# `rimap` => authenticate against imap server
+# Note: can be a list of mechanisms like pam ldap shadow
+SASLAUTHD_MECHANISMS=
+
+# empty => None
+# e.g. with SASLAUTHD_MECHANISMS rimap you need to specify the ip-address/servername of the imap server  ==> xxx.xxx.xxx.xxx
+SASLAUTHD_MECH_OPTIONS=
+
+# empty => Use value of LDAP_SERVER_HOST
+# Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`).
+SASLAUTHD_LDAP_SERVER=
+
+# empty => Use value of LDAP_BIND_DN
+# specify an object with privileges to search the directory tree
+# e.g. active directory: SASLAUTHD_LDAP_BIND_DN=cn=Administrator,cn=Users,dc=mydomain,dc=net
+# e.g. openldap: SASLAUTHD_LDAP_BIND_DN=cn=admin,dc=mydomain,dc=net
+SASLAUTHD_LDAP_BIND_DN=
+
+# empty => Use value of LDAP_BIND_PW
+SASLAUTHD_LDAP_PASSWORD=
+
+# empty => Use value of LDAP_SEARCH_BASE
+# specify the search base
+SASLAUTHD_LDAP_SEARCH_BASE=
+
+# empty => default filter `(&(uniqueIdentifier=%u)(mailEnabled=TRUE))`
+# e.g. for active directory: `(&(sAMAccountName=%U)(objectClass=person))`
+# e.g. for openldap: `(&(uid=%U)(objectClass=person))`
+SASLAUTHD_LDAP_FILTER=
+
+# empty => no
+# yes => LDAP over TLS enabled for SASL
+# If set to yes, the protocol in SASLAUTHD_LDAP_SERVER must be ldap:// or missing.
+SASLAUTHD_LDAP_START_TLS=
+
+# empty => no
+# yes => Require and verify server certificate
+# If yes you must/could specify SASLAUTHD_LDAP_TLS_CACERT_FILE or SASLAUTHD_LDAP_TLS_CACERT_DIR.
+SASLAUTHD_LDAP_TLS_CHECK_PEER=
+
+# File containing CA (Certificate Authority) certificate(s).
+# empty => Nothing is added to the configuration
+# Any value => Fills the `ldap_tls_cacert_file` option
+SASLAUTHD_LDAP_TLS_CACERT_FILE=
+
+# Path to directory with CA (Certificate Authority) certificates.
+# empty => Nothing is added to the configuration
+# Any value => Fills the `ldap_tls_cacert_dir` option
+SASLAUTHD_LDAP_TLS_CACERT_DIR=
+
+# Specify what password attribute to use for password verification.
+# empty => Nothing is added to the configuration but the documentation says it is `userPassword` by default.
+# Any value => Fills the `ldap_password_attr` option
+SASLAUTHD_LDAP_PASSWORD_ATTR=
+
+# empty => `bind` will be used as a default value
+# `fastbind` => The fastbind method is used
+# `custom` => The custom method uses userPassword attribute to verify the password
+SASLAUTHD_LDAP_AUTH_METHOD=
+
+# Specify the authentication mechanism for SASL bind
+# empty => Nothing is added to the configuration
+# Any value => Fills the `ldap_mech` option
+SASLAUTHD_LDAP_MECH=
+
+# -----------------------------------------------
+# --- SRS Section -------------------------------
+# -----------------------------------------------
+
+# envelope_sender => Rewrite only envelope sender address (default)
+# header_sender => Rewrite only header sender (not recommended)
+# envelope_sender,header_sender => Rewrite both senders
+# An email has an "envelope" sender (indicating the sending server) and a
+# "header" sender (indicating who sent it). More strict SPF policies may require
+# you to replace both instead of just the envelope sender.
+SRS_SENDER_CLASSES=envelope_sender
+
+# empty => Envelope sender will be rewritten for all domains
+# provide comma separated list of domains to exclude from rewriting
+SRS_EXCLUDE_DOMAINS=
+
+# empty => generated when the image is built
+# provide a secret to use in base64
+# you may specify multiple keys, comma separated. the first one is used for
+# signing and the remaining will be used for verification. this is how you
+# rotate and expire keys
+SRS_SECRET=
+
+# -----------------------------------------------
+# --- Default Relay Host Section ----------------
+# -----------------------------------------------
+
+# Setup relaying all mail through a default relay host
+#
+# Set a default host to relay all mail through (optionally include a port)
+# Example: [mail.example.com]:587
+DEFAULT_RELAY_HOST=
+
+# -----------------------------------------------
+# --- Multi-Domain Relay Section ----------------
+# -----------------------------------------------
+
+# Setup relaying for multiple domains based on the domain name of the sender
+# optionally uses usernames and passwords in postfix-sasl-password.cf and relay host mappings in postfix-relaymap.cf
+#
+# Set a default host to relay mail through
+# Example: mail.example.com
+RELAY_HOST=
+
+# empty => 25
+# default port to relay mail
+RELAY_PORT=25
+
+# -----------------------------------------------
+# --- Relay Host Credentials Section ------------
+# -----------------------------------------------
+
+# Configure a relay user and password to use with RELAY_HOST / DEFAULT_RELAY_HOST
+
+# empty => no default
+RELAY_USER=
+
+# empty => no default
+RELAY_PASSWORD=

self-hosting/nginx/nginx.conf

@@ -0,0 +1,75 @@
+events {}
+
+http {
+    sendfile    on;
+    keepalive_timeout 60;
+
+     upstream omnivore_web {
+    	ip_hash;
+    	server 127.0.0.1:3000;
+     }
+
+     upstream omnivore_backend {
+    	ip_hash;
+    	server 127.0.0.1:4000;
+     }
+
+     upstream omnivore_imageproxy {
+    	ip_hash;
+    	server 127.0.0.1:7070;
+     }
+
+    upstream omnivore_bucket {
+    	ip_hash;
+    	server 127.0.0.1:1010;
+    }
+
+    server {
+        listen 80;
+        return 301 https://$host$request_uri;
+    }
+
+    server {
+        listen 443 ssl;
+        server_name  omnivore.domain.com;
+
+
+        ssl_certificate   /path/to/cert.crt;
+        ssl_certificate_key /path/to/cert.key;
+        ssl_session_cache builtin:1000 shared:SSL:10m;
+        ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+        ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4;
+        ssl_prefer_server_ciphers on;
+
+        # Override for authentication on the frontend
+        location /api/client/auth {
+            proxy_pass http://omnivore_web;
+        }
+
+        # API
+        location /api {
+            proxy_pass http://omnivore_backend;
+        }
+
+        # Minio
+        location /bucket {
+            proxy_pass http://omnivore_bucket;
+        }
+
+        # ImageProxy
+        location /images {
+            rewrite ^/images/(.*)$ /$1 break;
+            proxy_pass http://omnivore_imageproxy;
+        }
+
+        # FrontEnd application
+        location / {
+            proxy_pass http://omnivore_web;
+        }
+
+        # Mail Proxy
+        location /mail {
+            proxy_pass http://localhost:4398/mail;
+        }
+    }
+}