diff --git a/docker-compose.yml b/docker-compose.yml index ed3d27b48..8bf1ee02f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,7 @@ services: - CLIENT_URL=http://localhost:3000 - GATEWAY_URL=http://localhost:8080/api - CONTENT_FETCH_URL=http://content-fetch:8080/?token=some_token - - REDIS_URL='redis://redis:6379' + - REDIS_URL=redis://redis:6379 depends_on: migrate: condition: service_completed_successfully diff --git a/packages/db/migrate.ts b/packages/db/migrate.ts index f4867b3fa..1073decc7 100755 --- a/packages/db/migrate.ts +++ b/packages/db/migrate.ts @@ -89,6 +89,10 @@ const postgresMigration = postgrator .migrate(targetMigration) .then(logAppliedMigrations) .catch((error) => { + if (error.message.includes('already exists')) { + log('Schema already exists, continuing...', chalk.yellow) + return [] + } log( `${chalk.red('Postgres migration failed: ')}${error.message}`, chalk.red diff --git a/self-hosting/docker-compose/.env.example b/self-hosting/docker-compose/.env.example index f414eb7d0..7a298dbb0 100644 --- a/self-hosting/docker-compose/.env.example +++ b/self-hosting/docker-compose/.env.example @@ -66,3 +66,8 @@ EXPORT_QUEUE_NAME=omnivore-backend-queue # Youtube YOUTUBE_TRANSCRIPT_PROMPT=Reformat the supplied transcript data adding formatting, puncuation, and paragraphs, but do not summarize or change the content. Format the output as markdown. YOUTUBE_MAXIMUM_VIDEO_DURATION_TRANSCRIPT=1801 + +DEMO_EMAIL=demo@omnivore.app +DEMO_USERNAME=demo_user +DEMO_PASSWORD=demo_password # Only used in test/dev +ENVIRONMENT=development \ No newline at end of file diff --git a/self-hosting/docker-compose/docker-compose.yml b/self-hosting/docker-compose/docker-compose.yml index d36e12f9a..345904f26 100644 --- a/self-hosting/docker-compose/docker-compose.yml +++ b/self-hosting/docker-compose/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' x-postgres: &postgres-common image: "ankane/pgvector:v0.5.1" @@ -17,32 +16,48 @@ services: - pgdata:/var/lib/postgresql/data env_file: - .env + restart: unless-stopped + networks: + - omnivore-network migrate: image: "ghcr.io/omnivore-app/sh-migrate:latest" + build: + context: . + dockerfile: ./packages/db/Dockerfile container_name: "omnivore-migrate" - command: '/bin/sh ./packages/db/setup.sh' # Also create a demo user with email: demo@omnivore.app, password: demo_password + command: '/bin/sh ./packages/db/setup.sh' env_file: - .env depends_on: postgres: condition: service_healthy + restart: "no" + networks: + - omnivore-network api: image: "ghcr.io/omnivore-app/sh-backend:latest" container_name: "omnivore-api" - ports: - - "4000:8080" + expose: + - 8080 healthcheck: - test: ["CMD-SHELL", "nc -z 0.0.0.0 8080 || exit 1"] + test: ["CMD-SHELL", "nc -z localhost 8080 || exit 1"] interval: 15s timeout: 90s retries: 6 env_file: - .env depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy migrate: condition: service_completed_successfully + restart: unless-stopped + networks: + - omnivore-network queue-processor: image: "ghcr.io/omnivore-app/sh-queue-processor:latest" @@ -52,33 +67,33 @@ services: depends_on: api: condition: service_started + redis: + condition: service_healthy + restart: unless-stopped + networks: + - omnivore-network web: image: "ghcr.io/omnivore-app/sh-web:latest" container_name: "omnivore-web" - ports: - - "3000:8080" + expose: + - 8080 env_file: - .env + - .env depends_on: api: condition: service_healthy - - image-proxy: - image: "ghcr.io/omnivore-app/sh-image-proxy:latest" - container_name: "omnivore-image-proxy" - ports: - - "7070:8080" - env_file: - - .env + restart: unless-stopped + networks: + - omnivore-network content-fetch: image: "ghcr.io/omnivore-app/sh-content-fetch:latest" container_name: "omnivore-content-fetch" - ports: - - "9090:8080" + expose: + - 8080 environment: - - USE_FIREFOX=true # Using Firefox here because the official chrome version seems to freeze a lot in Docker. + - USE_FIREFOX=true env_file: - .env depends_on: @@ -86,67 +101,56 @@ services: condition: service_healthy api: condition: service_healthy + restart: unless-stopped + networks: + - omnivore-network redis: image: "redis:7.2.4" container_name: "omnivore-redis" expose: - 6379 - ports: - - "6379:6379" healthcheck: - test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + test: ["CMD", "redis-cli", "--raw", "incr", "ping"] + interval: 5s + timeout: 5s + retries: 5 volumes: - redis_data:/data - - minio: - image: minio/minio - expose: - - 1010 - ports: - - "1010:9000" - healthcheck: - test: [ "CMD", "mc", "ready", "local" ] - interval: 5s - timeout: 1s - environment: - - "MINIO_ACCESS_KEY=minio" - - "MINIO_SECRET_KEY=miniominio" - - "AWS_S3_ENDPOINT_URL=http://minio:1010" - command: server /data + restart: unless-stopped + networks: + - omnivore-network + nginx: + image: nginx:alpine + container_name: "omnivore-nginx" volumes: - - minio_data:/data - - createbuckets: - image: minio/mc - environment: - - MINIO_ACCESS_KEY=minio - - MINIO_SECRET_KEY=miniominio - - BUCKET_NAME=omnivore - - ENDPOINT=http://minio:9000 - - AWS_S3_ENDPOINT_URL=http://minio:9000 - depends_on: - - minio - entrypoint: > - /bin/bash -c " - sleep 5; - until (/usr/bin/mc config host add myminio http://minio:9000 minio miniominio) do echo '...waiting...' && sleep 1; done; - /usr/bin/mc mb myminio/omnivore; - /usr/bin/mc anonymous set public myminio/omnivore; - exit 0; - " - mail-watch-server: - image: "ghcr.io/omnivore-app/sh-local-mail-watcher:latest" - container_name: "omnivore-mail-watch-server" + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./nginx/conf.d:/etc/nginx/conf.d:ro ports: - - "4398:8080" - env_file: - - .env + - "80:80" + - "443:443" depends_on: - redis: - condition: service_healthy + - web + - api + networks: + - omnivore-network + + cloudflared: + image: cloudflare/cloudflared:latest + container_name: omnivore-cloudflared + command: tunnel run + environment: + - TUNNEL_TOKEN=${CLOUDFLARE_TUNNEL_TOKEN} + restart: unless-stopped + networks: + - omnivore-network + depends_on: + - nginx + +networks: + omnivore-network: + driver: bridge + volumes: pgdata: redis_data: - minio_data: - diff --git a/self-hosting/docker-compose/nginx/conf.d/default.conf b/self-hosting/docker-compose/nginx/conf.d/default.conf new file mode 100644 index 000000000..aad7df88e --- /dev/null +++ b/self-hosting/docker-compose/nginx/conf.d/default.conf @@ -0,0 +1,6 @@ +# Either remove this file or update it to: +server { + listen 80 default_server; + server_name _; + return 444; # Reject requests with no server_name match +} \ No newline at end of file diff --git a/self-hosting/docker-compose/nginx/nginx.conf b/self-hosting/docker-compose/nginx/nginx.conf new file mode 100644 index 000000000..7cdac391c --- /dev/null +++ b/self-hosting/docker-compose/nginx/nginx.conf @@ -0,0 +1,80 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + + include /etc/nginx/conf.d/*.conf; + client_max_body_size 50M; + + upstream omnivore_web { + server web:8080; + keepalive 32; + } + + upstream omnivore_backend { + server api:8080; + keepalive 32; + } + + server { + listen 80; + server_name omnivore.lepq.io; + + location /api/auth/ { + proxy_pass http://omnivore_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /api/client/auth { + proxy_pass http://omnivore_web; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + location /api { + proxy_pass http://omnivore_backend; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + location / { + proxy_pass http://omnivore_web; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-XSS-Protection "1; mode=block" always; + add_header X-Content-Type-Options "nosniff" always; + + gzip on; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml text/javascript; + } +} \ No newline at end of file diff --git a/self-hosting/nginx/nginx.conf b/self-hosting/nginx/nginx.conf deleted file mode 100644 index 7cbc813c0..000000000 --- a/self-hosting/nginx/nginx.conf +++ /dev/null @@ -1,80 +0,0 @@ -events {} - -http { - sendfile on; - keepalive_timeout 60; - - upstream omnivore_web { - ip_hash; - server 127.0.0.1:3000; - } - - upstream omnivore_backend { - ip_hash; - server 127.0.0.1:4000; - } - - upstream omnivore_imageproxy { - ip_hash; - server 127.0.0.1:7070; - } - - upstream omnivore_bucket { - ip_hash; - server 127.0.0.1:1010; - } - - server { - listen 80; - return 301 https://$host$request_uri; - } - - server { - listen 443 ssl; - server_name omnivore.domain.com; - - - ssl_certificate /path/to/cert.crt; - ssl_certificate_key /path/to/cert.key; - ssl_session_cache builtin:1000 shared:SSL:10m; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4; - ssl_prefer_server_ciphers on; - - # Override for authentication on the frontend - location /api/client/auth { - proxy_pass http://omnivore_web; - } - - # Save Route (Frontend) - location /api/save { - proxy_pass http://omnivore_web; - } - - # API - location /api { - proxy_pass http://omnivore_backend; - } - - # Minio - location /bucket { - proxy_pass http://omnivore_bucket; - } - - # ImageProxy - location /images { - rewrite ^/images/(.*)$ /$1 break; - proxy_pass http://omnivore_imageproxy; - } - - # FrontEnd application - location / { - proxy_pass http://omnivore_web; - } - - # Mail Proxy - location /mail { - proxy_pass http://localhost:4398/mail; - } - } -}