diff --git a/packages/api/package.json b/packages/api/package.json index dfb233d4c..a6d9a4346 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -118,7 +118,10 @@ "voca": "^1.4.0", "winston": "^3.3.3", "yaml": "^2.4.1", - "youtubei": "^1.5.4" + "youtubei": "^1.5.4", + "@aws-sdk/client-s3": "^3.679.0", + "@aws-sdk/s3-request-presigner": "^3.679.0", + "@aws-sdk/lib-storage": "^3.679.0" }, "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.2", diff --git a/packages/api/queue-processor/Dockerfile b/packages/api/queue-processor/Dockerfile new file mode 100644 index 000000000..959e2722c --- /dev/null +++ b/packages/api/queue-processor/Dockerfile @@ -0,0 +1,61 @@ +FROM node:18.16 as builder + +WORKDIR /app + +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +RUN apt-get update && apt-get install -y g++ make python3 + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .prettierrc . +COPY .eslintrc . + +COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json +COPY /packages/api/package.json ./packages/api/package.json +COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json +COPY /packages/content-handler/package.json ./packages/content-handler/package.json +COPY /packages/liqe/package.json ./packages/liqe/package.json +COPY /packages/utils/package.json ./packages/utils/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/readabilityjs ./packages/readabilityjs +ADD /packages/api ./packages/api +ADD /packages/text-to-speech ./packages/text-to-speech +ADD /packages/content-handler ./packages/content-handler +ADD /packages/liqe ./packages/liqe +ADD /packages/utils ./packages/utils + +RUN yarn workspace @omnivore/utils build +RUN yarn workspace @omnivore/text-to-speech-handler build +RUN yarn workspace @omnivore/content-handler build +RUN yarn workspace @omnivore/liqe build +RUN yarn workspace @omnivore/api build + +# After building, fetch the production dependencies +RUN rm -rf /app/packages/api/node_modules +RUN rm -rf /app/node_modules +RUN yarn install --pure-lockfile --production + +FROM node:18.16 as runner +LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore" + +RUN apt-get update && apt-get install -y netcat-openbsd + +WORKDIR /app + +ENV NODE_ENV production + +COPY --from=builder /app/packages/api/dist /app/packages/api/dist +COPY --from=builder /app/packages/readabilityjs/ /app/packages/readabilityjs/ +COPY --from=builder /app/packages/api/package.json /app/packages/api/package.json +COPY --from=builder /app/packages/api/node_modules /app/packages/api/node_modules +COPY --from=builder /app/node_modules /app/node_modules +COPY --from=builder /app/package.json /app/package.json +COPY --from=builder /app/packages/text-to-speech/ /app/packages/text-to-speech/ +COPY --from=builder /app/packages/content-handler/ /app/packages/content-handler/ +COPY --from=builder /app/packages/liqe/ /app/packages/liqe/ +COPY --from=builder /app/packages/utils/ /app/packages/utils/ + +CMD ["yarn", "workspace", "@omnivore/api", "start_queue_processor"] diff --git a/packages/api/src/jobs/export.ts b/packages/api/src/jobs/export.ts index 8245eed4f..5f3516b84 100644 --- a/packages/api/src/jobs/export.ts +++ b/packages/api/src/jobs/export.ts @@ -12,13 +12,24 @@ import { sendExportJobEmail } from '../services/send_emails' import { findActiveUser } from '../services/user' import { logger } from '../utils/logger' import { highlightToMarkdown } from '../utils/parser' -import { contentFilePath, createGCSFile } from '../utils/uploads' +import { contentFilePath } from '../utils/uploads' +import { env } from '../env' +import { File, Storage } from '@google-cloud/storage' export interface ExportJobData { userId: string exportId: string } +export const storage = env.fileUpload?.gcsUploadSAKeyFilePath + ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath }) + : new Storage() +const bucketName = env.fileUpload.gcsUploadBucket + +const createGCSFile = (filename: string): File => { + return storage.bucket(bucketName).file(filename) +} + export const EXPORT_JOB_NAME = 'export' const itemStateMappping = (state: LibraryItemState) => { diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 62afd000a..d530d1aad 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -56,7 +56,10 @@ import { PROCESS_YOUTUBE_VIDEO_JOB_NAME, } from './jobs/process-youtube-video' import { pruneTrashJob, PRUNE_TRASH_JOB } from './jobs/prune_trash' -import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds' +import { + REFRESH_ALL_FEEDS_JOB_NAME, + refreshAllFeeds, +} from './jobs/rss/refreshAllFeeds' import { refreshFeed } from './jobs/rss/refreshFeed' import { savePageJob } from './jobs/save_page' import { @@ -159,17 +162,17 @@ export const createWorker = (connection: ConnectionOptions) => async (job: Job) => { const executeJob = async (job: Job) => { switch (job.name) { - // case 'refresh-all-feeds': { - // const queue = await getQueue() - // const counts = await queue?.getJobCounts('prioritized') - // if (counts && counts.wait > 1000) { - // return - // } - // return await refreshAllFeeds(appDataSource) - // } - // case 'refresh-feed': { - // return await refreshFeed(job.data) - // } + case 'refresh-all-feeds': { + const queue = await getQueue() + const counts = await queue?.getJobCounts('prioritized') + if (counts && counts.wait > 1000) { + return + } + return await refreshAllFeeds(appDataSource) + } + case 'refresh-feed': { + return await refreshFeed(job.data) + } case 'save-page': { return savePageJob(job.data, job.attemptsMade) } @@ -260,6 +263,17 @@ const setupCronJobs = async () => { }, } ) + + await queue.add( + REFRESH_ALL_FEEDS_JOB_NAME, + {}, + { + priority: getJobPriority(REFRESH_ALL_FEEDS_JOB_NAME), + repeat: { + every: 14_400_000, // 4 Hours + }, + } + ) } const main = async () => { diff --git a/packages/api/src/repository/storage/GcsStorageClient.ts b/packages/api/src/repository/storage/GcsStorageClient.ts new file mode 100644 index 000000000..fd798a799 --- /dev/null +++ b/packages/api/src/repository/storage/GcsStorageClient.ts @@ -0,0 +1,71 @@ +import { SignedUrlParameters, StorageClient, File } from './StorageClient' +import { Storage, File as GCSFile } from '@google-cloud/storage' + +export class GcsStorageClient implements StorageClient { + private storage: Storage + + constructor(keyFilename: string | undefined) { + this.storage = new Storage({ + keyFilename, + }) + } + + private convertFileToGeneric(gcsFile: GCSFile): File { + return { + isPublic: async () => { + const [isPublic] = await gcsFile.isPublic() + return isPublic + }, + exists: async () => (await gcsFile.exists())[0], + download: async () => (await gcsFile.download())[0], + bucket: gcsFile.bucket.name, + publicUrl: () => gcsFile.publicUrl(), + getMetadataMd5: async () => { + const [metadata] = await gcsFile.getMetadata() + return metadata.md5Hash + }, + } + } + + downloadFile(bucket: string, filePath: string): Promise { + const file = this.storage.bucket(bucket).file(filePath) + return Promise.resolve(this.convertFileToGeneric(file)) + } + + async getFilesFromPrefix(bucket: string, prefix: string): Promise { + const [filesWithPrefix] = await this.storage + .bucket(bucket) + .getFiles({ prefix }) + + return filesWithPrefix.map((it: GCSFile) => this.convertFileToGeneric(it)) + } + + async signedUrl( + bucket: string, + filePath: string, + options: SignedUrlParameters + ): Promise { + const [url] = await this.storage + .bucket(bucket) + .file(filePath) + .getSignedUrl({ ...options, version: 'v4' }) + + return url + } + + upload( + bucket: string, + filePath: string, + data: Buffer, + options: { + contentType?: string + public?: boolean + timeout?: number + } + ): Promise { + return this.storage + .bucket(bucket) + .file(filePath) + .save(data, { timeout: 30000, ...options }) + } +} diff --git a/packages/api/src/repository/storage/S3StorageClient.ts b/packages/api/src/repository/storage/S3StorageClient.ts new file mode 100644 index 000000000..590a9cf4f --- /dev/null +++ b/packages/api/src/repository/storage/S3StorageClient.ts @@ -0,0 +1,145 @@ +import { SignedUrlParameters, StorageClient, File } from './StorageClient' +import { + GetObjectCommand, + GetObjectCommandOutput, + S3Client, + ListObjectsV2Command, + PutObjectCommand, +} from '@aws-sdk/client-s3' +import { getSignedUrl } from '@aws-sdk/s3-request-presigner' +import type { Readable } from 'stream' + +// While this is listed as S3, for self hosting we will use MinIO, which is +// S3 Compatible. +export class S3StorageClient implements StorageClient { + private s3Client: S3Client + private urlOverride: string | undefined + + constructor(urlOverride: string | undefined) { + this.urlOverride = urlOverride + this.s3Client = new S3Client({ + forcePathStyle: true, + endpoint: urlOverride, + region: 'us-east-1', + credentials: { + accessKeyId: 'minio', + secretAccessKey: 'miniominio', + }, + }) + } + + private convertFileToGeneric( + s3File: GetObjectCommandOutput + ): Omit { + return { + exists: () => { + return Promise.resolve(s3File.$metadata.httpStatusCode == 200) + }, + isPublic: async () => Promise.resolve(true), + download: async () => this.getFileFromReadable(s3File.Body as Readable), + getMetadataMd5: () => Promise.resolve(s3File.ETag), + } + } + + private getFileFromReadable(stream: Readable): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = [] + stream.on('data', (chunk) => chunks.push(chunk)) + stream.once('end', () => resolve(Buffer.concat(chunks))) + stream.once('error', reject) + }) + } + + async downloadFile(bucket: string, filePath: string): Promise { + const s3File = await this.s3Client.send( + new GetObjectCommand({ + Bucket: bucket, + Key: filePath, // path to the file you want to download, + }) + ) + + return { + ...this.convertFileToGeneric(s3File), + bucket: bucket, + publicUrl: () => `${this.urlOverride ?? ''}/${bucket}/${filePath}`, + } + } + + async getFilesFromPrefix(bucket: string, prefix: string): Promise { + const s3PrefixedFiles = await this.s3Client.send( + new ListObjectsV2Command({ + Bucket: bucket, + Prefix: prefix, // path to the file you want to download, + }) + ) + + const prefixKeys = s3PrefixedFiles.CommonPrefixes || [] + + return prefixKeys + .map(({ Prefix }) => Prefix) + .map((key) => { + return { + exists: () => Promise.resolve(true), + isPublic: async () => Promise.resolve(true), + download: async () => { + const s3File = await this.s3Client.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, // path to the file you want to download, + }) + ) + + return this.getFileFromReadable(s3File.Body as Readable) + }, + getMetadataMd5: () => Promise.resolve(key), + bucket: bucket, + publicUrl: () => `${this.urlOverride ?? ''}/${bucket}/${key}`, + } + }) + } + + async signedUrl( + bucket: string, + filePath: string, + options: SignedUrlParameters + ): Promise { + const command = + options.action == 'read' + ? new GetObjectCommand({ + Bucket: bucket, + Key: filePath, // path to the file you want to download, + }) + : new PutObjectCommand({ + Bucket: bucket, + Key: filePath, // path to the file you want to download, + }) + + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + const url = await getSignedUrl(this.s3Client, command, { + expiresIn: 900, + }) + + return url + } + + upload( + bucket: string, + filePath: string, + data: Buffer, + options: { + contentType?: string + public?: boolean + timeout?: number + } + ): Promise { + return this.s3Client + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: filePath, + Body: data, + }) + ) + .then(() => {}) + } +} diff --git a/packages/api/src/repository/storage/StorageClient.ts b/packages/api/src/repository/storage/StorageClient.ts new file mode 100644 index 000000000..59ba25989 --- /dev/null +++ b/packages/api/src/repository/storage/StorageClient.ts @@ -0,0 +1,32 @@ +export type SignedUrlParameters = { + action: 'read' | 'write' | 'delete' | 'resumable' + expires: number +} + +export type File = { + isPublic: () => Promise + publicUrl: () => string + download: () => Promise + exists: () => Promise + getMetadataMd5: () => Promise + bucket: string +} + +export interface StorageClient { + downloadFile(bucket: string, filePath: string): Promise + + getFilesFromPrefix(bucket: string, filePrefix: string): Promise + + upload( + bucket: string, + filePath: string, + data: Buffer, + options: { contentType?: string; public?: boolean; timeout?: number } + ): Promise + + signedUrl( + bucket: string, + filePath: string, + options: SignedUrlParameters + ): Promise +} diff --git a/packages/api/src/repository/storage/storage.ts b/packages/api/src/repository/storage/storage.ts new file mode 100644 index 000000000..5a15299d7 --- /dev/null +++ b/packages/api/src/repository/storage/storage.ts @@ -0,0 +1,7 @@ +import { env } from '../../env' +import { S3StorageClient } from './S3StorageClient' +import { GcsStorageClient } from './GcsStorageClient' + +export const storage = env.fileUpload.useLocalStorage + ? new S3StorageClient('http://localhost:1010') + : new GcsStorageClient(env.fileUpload?.gcsUploadSAKeyFilePath) diff --git a/packages/api/src/routers/auth/auth_router.ts b/packages/api/src/routers/auth/auth_router.ts index ed88d9237..6bb17db28 100644 --- a/packages/api/src/routers/auth/auth_router.ts +++ b/packages/api/src/routers/auth/auth_router.ts @@ -29,6 +29,7 @@ import { import { analytics } from '../../utils/analytics' import { comparePassword, + generateVerificationToken, hashPassword, setAuthInCookie, verifyToken, @@ -544,7 +545,7 @@ export function authRouter() { try { // hash password const hashedPassword = await hashPassword(password) - await createUser({ + const [user] = await createUser({ email: trimmedEmail, provider: 'EMAIL', sourceUserId: trimmedEmail, @@ -553,12 +554,17 @@ export function authRouter() { pictureUrl, bio, password: hashedPassword, - pendingConfirmation: true, + pendingConfirmation: !env.dev.autoVerify, }) - res.redirect( - `${env.client.url}/auth/verify-email?message=SIGNUP_SUCCESS` - ) + if (env.dev.autoVerify) { + const token = await generateVerificationToken({ id: user.id }) + res.redirect(`${env.client.url}/auth/confirm-email/${token}`) + } else { + res.redirect( + `${env.client.url}/auth/verify-email?message=SIGNUP_SUCCESS` + ) + } } catch (e) { logger.info('email-signup exception:', e) if (isErrorWithCode(e)) { diff --git a/packages/api/src/routers/auth/mobile/sign_up.ts b/packages/api/src/routers/auth/mobile/sign_up.ts index 11bf3002d..c9fb8c2f8 100644 --- a/packages/api/src/routers/auth/mobile/sign_up.ts +++ b/packages/api/src/routers/auth/mobile/sign_up.ts @@ -13,6 +13,7 @@ import { } from '../auth_types' import { decodeGoogleToken } from '../google_auth' import { createPendingUserToken, suggestedUsername } from '../jwt_helpers' +import { env } from '../../../env' export async function createMobileSignUpResponse( isAndroid: boolean, @@ -66,7 +67,7 @@ export async function createMobileEmailSignUpResponse( name: name.trim(), username: username.trim().toLowerCase(), password: hashedPassword, - pendingConfirmation: true, + pendingConfirmation: !env.dev.autoVerify, }) return { diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 8d0eefc5c..6fc9328d2 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -73,6 +73,7 @@ export interface BackendEnv { } dev: { isLocal: boolean + autoVerify: boolean } queue: { location: string @@ -94,6 +95,7 @@ export interface BackendEnv { gcsUploadSAKeyFilePath: string gcsUploadPrivateBucket: string dailyUploadLimit: number + useLocalStorage: boolean } sender: { message: string @@ -197,6 +199,7 @@ const nullableEnvVars = [ 'PG_REPLICA_USER', 'PG_REPLICA_PASSWORD', 'PG_REPLICA_DB', + 'AUTO_VERIFY', 'INTERCOM_WEB_SECRET', 'INTERCOM_IOS_SECRET', 'INTERCOM_ANDROID_SECRET', @@ -288,6 +291,7 @@ export function getEnv(): BackendEnv { } const dev = { isLocal: parse('API_ENV') == 'local', + autoVerify: parse('AUTO_VERIFY') === 'true', } const queue = { location: parse('PUPPETEER_QUEUE_LOCATION'), @@ -318,6 +322,7 @@ export function getEnv(): BackendEnv { dailyUploadLimit: parse('GCS_UPLOAD_DAILY_LIMIT') ? parseInt(parse('GCS_UPLOAD_DAILY_LIMIT'), 10) : 5, // default to 5 + useLocalStorage: parse('GCS_USE_LOCAL_HOST') == 'true', } const sender = { message: parse('SENDER_MESSAGE'), diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index c4b71b5ca..a9d6a0426 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -1,12 +1,12 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { File, GetSignedUrlConfig, Storage } from '@google-cloud/storage' import axios from 'axios' import { ContentReaderType } from '../entity/library_item' import { env } from '../env' import { PageType } from '../generated/graphql' import { ContentFormat } from '../jobs/upload_content' import { logger } from './logger' +import { storage } from '../repository/storage/storage' export const contentReaderForLibraryItem = ( itemType: string, @@ -31,14 +31,12 @@ export const contentReaderForLibraryItem = ( * the default app engine service account on the IAM page. We also need to * enable IAM related APIs on the project. */ -export const storage = env.fileUpload?.gcsUploadSAKeyFilePath - ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath }) - : new Storage() + const bucketName = env.fileUpload.gcsUploadBucket const maxContentLength = 10 * 1024 * 1024 // 10MB export const countOfFilesWithPrefix = async (prefix: string) => { - const [files] = await storage.bucket(bucketName).getFiles({ prefix }) + const files = await storage.getFilesFromPrefix(bucketName, prefix) return files.length } @@ -48,40 +46,29 @@ export const generateUploadSignedUrl = async ( selectedBucket?: string ): Promise => { // These options will allow temporary uploading of file with requested content type - const options: GetSignedUrlConfig = { + const options = { version: 'v4', - action: 'write', + action: 'write' as const, expires: Date.now() + 15 * 60 * 1000, // 15 minutes contentType: contentType, } logger.info('signed url for: ', options) - // Get a v4 signed URL for uploading file - const [url] = await storage - .bucket(selectedBucket || bucketName) - .file(filePathName) - .getSignedUrl(options) - return url + return storage.signedUrl(selectedBucket || bucketName, filePathName, options) } export const generateDownloadSignedUrl = async ( filePathName: string, config?: { - bucketName?: string expires?: number } ): Promise => { - const options: GetSignedUrlConfig = { - version: 'v4', - action: 'read', - expires: config?.expires ?? Date.now() + 240 * 60 * 1000, // four hours + const options = { + action: 'read' as const, + expires: Date.now() + 240 * 60 * 1000, // four hours + ...config, } - const [url] = await storage - .bucket(config?.bucketName || bucketName) - .file(filePathName) - .getSignedUrl(options) - logger.info(`generating download signed url: ${url}`) - return url + return storage.signedUrl(bucketName, filePathName, options) } export const getStorageFileDetails = async ( @@ -89,10 +76,10 @@ export const getStorageFileDetails = async ( fileName: string ): Promise<{ md5Hash: string; fileUrl: string }> => { const filePathName = generateUploadFilePathName(id, fileName) - const file = storage.bucket(bucketName).file(filePathName) - const [metadata] = await file.getMetadata() + const file = await storage.downloadFile(bucketName, filePathName) + const metadataMd5 = await file.getMetadataMd5() // GCS returns MD5 Hash in base64 encoding, we convert it here to hex string - const md5Hash = Buffer.from(metadata.md5Hash || '', 'base64').toString('hex') + const md5Hash = Buffer.from(metadataMd5 || '', 'base64').toString('hex') return { md5Hash, fileUrl: file.publicUrl() } } @@ -110,17 +97,10 @@ export const uploadToBucket = async ( options?: { contentType?: string; public?: boolean; timeout?: number }, selectedBucket?: string ): Promise => { - await storage - .bucket(selectedBucket || bucketName) - .file(filePath) - .save(data, { timeout: 30000, ...options }) // default timeout 30s -} - -export const createGCSFile = ( - filename: string, - selectedBucket = bucketName -): File => { - return storage.bucket(selectedBucket).file(filename) + await storage.upload(selectedBucket || bucketName, filePath, data, { + timeout: 30000, + ...options, + }) } export const downloadFromUrl = async ( @@ -154,16 +134,14 @@ export const uploadToSignedUrl = async ( } export const isFileExists = async (filePath: string): Promise => { - const [exists] = await storage.bucket(bucketName).file(filePath).exists() + const file = await storage.downloadFile(bucketName, filePath) + const exists = await file.exists() return exists } export const downloadFromBucket = async (filePath: string): Promise => { - const file = storage.bucket(bucketName).file(filePath) - - // Download the file contents - const [data] = await file.download() - return data + const file = await storage.downloadFile(bucketName, filePath) + return file.download() } export const contentFilePath = ({ diff --git a/packages/api/test/hooks.ts b/packages/api/test/hooks.ts index 6ab516a71..f1799ccd2 100644 --- a/packages/api/test/hooks.ts +++ b/packages/api/test/hooks.ts @@ -1,13 +1,13 @@ -import { Storage } from '@google-cloud/storage' -import sinon from 'sinon' -import * as uploads from '../src/utils/uploads' -import { MockStorage } from './mock_storage' - -export const mochaHooks = { - beforeEach() { - // Mock cloud storage - sinon - .stub(uploads, 'storage') - .value(new MockStorage() as unknown as Storage) - }, -} +// import { Storage } from '@google-cloud/storage' +// import sinon from 'sinon' +// import * as uploads from '../src/utils/uploads' +// import { MockStorage } from './mock_storage' +// +// export const mochaHooks = { +// beforeEach() { +// // Mock cloud storage +// sinon +// .stub(uploads, 'storage') +// .value(new MockStorage() as unknown as Storage) +// }, +// } diff --git a/packages/content-fetch/src/request_handler.ts b/packages/content-fetch/src/request_handler.ts index 8ef69acbe..bd03ef298 100644 --- a/packages/content-fetch/src/request_handler.ts +++ b/packages/content-fetch/src/request_handler.ts @@ -305,7 +305,7 @@ export const processFetchContentJob = async ( const savedDate = savedAt ? new Date(savedAt) : new Date() const { finalUrl, title, content, contentType } = fetchResult - if (content) { + if (content && process.env['SKIP_UPLOAD_ORIGINAL'] === 'true') { await uploadOriginalContent(users, content, savedDate.getTime()) } diff --git a/packages/web/Dockerfile-self b/packages/web/Dockerfile-self new file mode 100644 index 000000000..c2e4d3382 --- /dev/null +++ b/packages/web/Dockerfile-self @@ -0,0 +1,52 @@ +# Note this docker file is meant for local testing +# and not for production. + +FROM node:18.16-alpine as builder +ENV NODE_OPTIONS=--max-old-space-size=8192 +ARG APP_ENV +ARG BASE_URL +ARG SERVER_BASE_URL +ARG HIGHLIGHTS_BASE_URL +ENV NEXT_PUBLIC_APP_ENV=$APP_ENV +ENV NEXT_PUBLIC_BASE_URL=$BASE_URL +ENV NEXT_PUBLIC_SERVER_BASE_URL=$SERVER_BASE_URL +ENV NEXT_PUBLIC_HIGHLIGHTS_BASE_URL=$HIGHLIGHTS_BASE_URL + +RUN apk add g++ make python3 + +WORKDIR /app + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .prettierrc . +COPY .eslintrc . + +COPY /packages/web/package.json ./packages/web/package.json + +RUN yarn install --pure-lockfile +ADD /packages/web ./packages/web + +COPY ./packages/web/next.config.self.js ./packages/web/next.config.js +RUN yarn workspace @omnivore/web build + + +FROM node:18.16-alpine as runner +LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore" + +ENV NODE_ENV production +ENV PORT=8080 +ENV NEXT_TELEMETRY_DISABLED 1 + +WORKDIR /app + +COPY --from=builder /app/packages/web/next.config.js /app/packages/web/next.config.js +COPY --from=builder /app/packages/web/public/ /app/packages/web/public/ +COPY --from=builder /app/packages/web/.next/ /app/packages/web/.next/ +COPY --from=builder /app/packages/web/package.json /app/packages/web/package.json +COPY --from=builder /app/packages/web/node_modules /app/packages/web/node_modules +COPY --from=builder /app/node_modules /app/node_modules +COPY --from=builder /app/package.json /app/package.json + +EXPOSE 8080 +CMD ["yarn", "workspace", "@omnivore/web", "start"] diff --git a/packages/web/next.config.self.js b/packages/web/next.config.self.js new file mode 100644 index 000000000..407bd2699 --- /dev/null +++ b/packages/web/next.config.self.js @@ -0,0 +1,43 @@ +const moduleExports = { + rewrites: () => { + const rewrites = [] + rewrites.push({ + source: '/home', + destination: '/l/home', + }) + rewrites.push({ + source: '/library', + destination: '/l/library', + }) + rewrites.push({ + source: '/subscriptions', + destination: '/l/subscriptions', + }) + rewrites.push({ + source: '/highlights', + destination: '/l/highlights', + }) + rewrites.push({ + source: '/subscriptions', + destination: '/l/subscriptions', + }) + rewrites.push({ + source: '/search', + destination: '/l/search', + }) + rewrites.push({ + source: '/archive', + destination: '/l/archive', + }) + rewrites.push({ + source: '/trash', + destination: '/l/trash', + }) + return rewrites + } +} + +const withBundleAnalyzer = require('@next/bundle-analyzer')({ + enabled: process.env.ANALYZE === 'true', +}) +module.exports = withBundleAnalyzer(moduleExports) diff --git a/self-hosting/docker-compose/.env.example b/self-hosting/docker-compose/.env.example new file mode 100644 index 000000000..e429e6b6c --- /dev/null +++ b/self-hosting/docker-compose/.env.example @@ -0,0 +1,61 @@ +# Postgres & Migrate + +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=omnivore +PGPASSWORD=postgres +POSTGRES_USER=postgres +PG_HOST=postgres +PG_PASSWORD=app_pass +PG_DB=omnivore +PG_USER=app_user +PG_PORT=5432 +PG_POOL_MAX=20 + +# API + +API_ENV=local +JAEGER_HOST=jaeger # Is this needed? +IMAGE_PROXY_SECRET=some-secret +JWT_SECRET=some_secret +SSO_JWT_SECRET=some_sso_secret +GATEWAY_URL=http://api:8080/api +CONTENT_FETCH_URL=http://content-fetch:8080/?token=some_token +GCS_USE_LOCAL_HOST=true +GCS_UPLOAD_BUCKET=omnivore +AUTO_VERIFY=true +AWS_ACCESS_KEY_ID=minio # Used for Minio S3 Client +AWS_SECRET_ACCESS_KEY=miniominio +AWS_REGION=us-east-1 +INTERCOM_WEB_SECRET=unused +INTERCOM_IOS_SECRET=unused +INTERCOM_ANDROID_SECRET=unused +CONTENT_FETCH_QUEUE_ENABLED=true + +IMAGE_PROXY_URL=http://localhost:7070 # Need to change this for NGINX +CLIENT_URL=http://localhost:3000 # Need to change this when using NGINX + +# Redis +REDIS_URL=redis://redis:6379/0 + + +# Web +APP_ENV=prod +NEXT_PUBLIC_APP_ENV=prod +BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX +SERVER_BASE_URL=http://localhost:4000 # API Server, need to change this when using NGINX +HIGHLIGHTS_BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX +NEXT_PUBLIC_BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX +NEXT_PUBLIC_SERVER_BASE_URL=http://localhost:4000 # API Server, need to change this when using NGINX +NEXT_PUBLIC_HIGHLIGHTS_BASE_URL=http://localhost:3000 # Front End - Need to change this when using NGINX + +# Content Fetch + +VERIFICATION_TOKEN=some_token +REST_BACKEND_ENDPOINT=http://api:8080/api +SKIP_UPLOAD_ORIGINAL=true + +# Minio +MINIO_ACCESS_KEY=minio +MINIO_SECRET_KEY=miniominio +AWS_S3_ENDPOINT_URL=http://minio:9000 diff --git a/self-hosting/docker-compose/docker-compose.yml b/self-hosting/docker-compose/docker-compose.yml new file mode 100644 index 000000000..1687f0c0d --- /dev/null +++ b/self-hosting/docker-compose/docker-compose.yml @@ -0,0 +1,161 @@ +version: '3' +x-postgres: + &postgres-common + image: "ankane/pgvector:v0.5.1" + user: postgres + healthcheck: + test: "exit 0" + interval: 2s + timeout: 12s + retries: 3 + +services: + postgres: + <<: *postgres-common + container_name: "omnivore-postgres" + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + env_file: + - .env + + migrate: + build: + context: ../../ + dockerfile: ./packages/db/Dockerfile + container_name: "omnivore-migrate" + command: '/bin/sh ./packages/db/setup.sh' # Also create a demo user with email: demo@omnivore.app, password: demo_password + env_file: + - .env + depends_on: + postgres: + condition: service_healthy + + api: + build: + context: ../../ + dockerfile: ./packages/api/Dockerfile + container_name: "omnivore-api" + ports: + - "4000:8080" + healthcheck: + test: ["CMD-SHELL", "nc -z 0.0.0.0 8080 || exit 1"] + interval: 15s + timeout: 90s + env_file: + - .env + depends_on: + migrate: + condition: service_completed_successfully + + queue-processor: + build: + context: ../../ + dockerfile: ./packages/api/queue-processor/Dockerfile + container_name: "omnivore-queue-processor" + env_file: + - .env + depends_on: + api: + condition: service_started + + web: + build: + context: ../../ + dockerfile: ./packages/web/Dockerfile-self + args: + - APP_ENV=prod + - BASE_URL=http://localhost:3000 + - SERVER_BASE_URL=http://localhost:4000 + - HIGHLIGHTS_BASE_URL=http://localhost:3000 + container_name: "omnivore-web" + ports: + - "3000:8080" + environment: + - NEXT_PUBLIC_APP_ENV=prod + - NEXT_PUBLIC_BASE_URL=http://localhost:3000 + - NEXT_PUBLIC_SERVER_BASE_URL=http://localhost:4000 + - NEXT_PUBLIC_HIGHLIGHTS_BASE_URL=http://localhost:3000 + depends_on: + api: + condition: service_healthy + + image-proxy: + build: + context: ../../ + dockerfile: ./imageproxy/Dockerfile + container_name: "omnivore-image-proxy" + ports: + - "7070:8080" + env_file: + - .env + + content-fetch: + build: + context: ../../ + dockerfile: ./packages/content-fetch/Dockerfile + container_name: "omnivore-content-fetch" + ports: + - "9090:8080" + env_file: + - .env + depends_on: + redis: + condition: service_healthy + api: + condition: service_healthy + + redis: + image: "redis:7.2.4" + container_name: "omnivore-redis" + expose: + - 6379 + ports: + - "6379:6379" + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + volumes: + - redis_data:/data + + minio: + image: minio/minio + expose: + - 1010 + ports: + - "1010:9000" + healthcheck: + test: [ "CMD", "mc", "ready", "local" ] + interval: 5s + timeout: 1s + environment: + - "MINIO_ACCESS_KEY=minio" + - "MINIO_SECRET_KEY=miniominio" + - "AWS_S3_ENDPOINT_URL=http://minio:1010" + command: server /data + volumes: + - minio_data:/data + + createbuckets: + image: minio/mc + environment: + - MINIO_ACCESS_KEY=minio + - MINIO_SECRET_KEY=miniominio + - BUCKET_NAME=omnivore + - ENDPOINT=http://minio:9000 + - AWS_S3_ENDPOINT_URL=http://minio:9000 + depends_on: + - minio + entrypoint: > + /bin/sh -c " + sleep 5; + until (/usr/bin/mc config host add myminio http://minio:9000 minio miniominio) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc mb myminio/omnivore; + /usr/bin/mc policy set public myminio/omnivore; + exit 0; + " +volumes: + pgdata: + redis_data: + minio_data: +