From e40c2b4ce7267e0d78f328399cd334bf883341a0 Mon Sep 17 00:00:00 2001 From: Aliaksei Karneyeu Date: Thu, 26 Feb 2026 15:00:43 +0100 Subject: [PATCH 01/10] Allow creating custom user --- packages/db/setup.sh | 9 +++++++++ self-hosting/docker-compose/.env.example | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/packages/db/setup.sh b/packages/db/setup.sh index 718912f46..0a6253fd0 100755 --- a/packages/db/setup.sh +++ b/packages/db/setup.sh @@ -24,3 +24,12 @@ if [ -z "${NO_DEMO_USER}" ]; then psql --host $PG_HOST --username $POSTGRES_USER --dbname $PG_DB --command "INSERT INTO omnivore.user (id, source, email, source_user_id, name, password) VALUES ('$USER_ID', 'EMAIL', 'demo@omnivore.work', 'demo@omnivore.work', 'Demo User', '$PASSWORD'); INSERT INTO omnivore.user_profile (user_id, username) VALUES ('$USER_ID', 'demo_user');" echo "created demo user with email: demo@omnivore.work, password: demo_password" fi + +# create a custom user if USER_EMAIL and USER_PASSWORD are set +if [ -n "${USER_EMAIL}" ] && [ -n "${USER_PASSWORD}" ]; then + USER_ID=$(uuidgen) + HASHED_PASSWORD=$(node -e "const bcrypt = require('bcryptjs'); console.log(bcrypt.hashSync(process.env.USER_PASSWORD, 10));") + USERNAME=$(echo "${USER_EMAIL}" | sed 's/@.*//' | tr -cd '[:alnum:]_') + psql --host $PG_HOST --username $POSTGRES_USER --dbname $PG_DB --command "INSERT INTO omnivore.user (id, source, email, source_user_id, name, password) VALUES ('$USER_ID', 'EMAIL', '${USER_EMAIL}', '${USER_EMAIL}', '${USER_NAME:-$USERNAME}', '$HASHED_PASSWORD'); INSERT INTO omnivore.user_profile (user_id, username) VALUES ('$USER_ID', '${USERNAME}');" + echo "created user with email: ${USER_EMAIL}" +fi diff --git a/self-hosting/docker-compose/.env.example b/self-hosting/docker-compose/.env.example index f414eb7d0..0ee353134 100644 --- a/self-hosting/docker-compose/.env.example +++ b/self-hosting/docker-compose/.env.example @@ -14,6 +14,16 @@ PG_USER=app_user PG_PORT=5432 PG_POOL_MAX=20 +# User creation + +# If you don't want to create demo user set this variable to any non-empty value +# so would be created user with email 'demo@omnivore.work' and password 'demo_password' +NO_DEMO_USER="" + +# If you need to create custom user fill in those variables +USER_EMAIL="" +USER_PASSWORD="" + # API API_ENV=local From 06474845979191cc557c9ee2c5997dff5541ed6a Mon Sep 17 00:00:00 2001 From: Aliaksei Karneyeu Date: Wed, 4 Mar 2026 14:41:08 +0100 Subject: [PATCH 02/10] Add Go replacement for content-fetch service and CLAUDE.md - Add packages/content-fetch-go: a Go rewrite of packages/content-fetch that is fully API-compatible (same HTTP endpoints, env vars, Redis key schema, and BullMQ v5 job format) but produces a significantly smaller Docker image (Alpine + Chromium only, no Node.js runtime) - Internal packages: - bullmq: BullMQ v5-compatible producer and consumer (Lua moveToActive) - queue/worker: concurrent job worker (4 goroutines, 500 ms poll) - fetch: Chromium page fetch via chromedp replacing puppeteer-parse - handler: processFetchContentJob logic (cache, GCS upload, job queuing) - gcs: Google Cloud Storage upload - analytics: PostHog failure event capture - server: HTTP endpoints (/_ah/health, /metrics, /lifecycle/prestop, /) - redisutil: dual Redis connections (cache + BullMQ MQ) - config: all env var loading - Add CLAUDE.md with build/test/lint commands and architecture overview Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 133 +++++ packages/content-fetch-go/.dockerignore | 3 + packages/content-fetch-go/Dockerfile | 49 ++ packages/content-fetch-go/go.mod | 70 +++ packages/content-fetch-go/go.sum | 166 +++++++ .../internal/analytics/analytics.go | 73 +++ .../internal/browser/browser.go | 106 ++++ .../internal/bullmq/bullmq.go | 371 ++++++++++++++ .../internal/config/config.go | 92 ++++ .../content-fetch-go/internal/fetch/fetch.go | 450 +++++++++++++++++ packages/content-fetch-go/internal/gcs/gcs.go | 73 +++ .../internal/handler/handler.go | 457 ++++++++++++++++++ .../content-fetch-go/internal/queue/worker.go | 110 +++++ .../internal/redisutil/redisutil.go | 88 ++++ .../internal/server/server.go | 132 +++++ packages/content-fetch-go/main.go | 80 +++ 16 files changed, 2453 insertions(+) create mode 100644 CLAUDE.md create mode 100644 packages/content-fetch-go/.dockerignore create mode 100644 packages/content-fetch-go/Dockerfile create mode 100644 packages/content-fetch-go/go.mod create mode 100644 packages/content-fetch-go/go.sum create mode 100644 packages/content-fetch-go/internal/analytics/analytics.go create mode 100644 packages/content-fetch-go/internal/browser/browser.go create mode 100644 packages/content-fetch-go/internal/bullmq/bullmq.go create mode 100644 packages/content-fetch-go/internal/config/config.go create mode 100644 packages/content-fetch-go/internal/fetch/fetch.go create mode 100644 packages/content-fetch-go/internal/gcs/gcs.go create mode 100644 packages/content-fetch-go/internal/handler/handler.go create mode 100644 packages/content-fetch-go/internal/queue/worker.go create mode 100644 packages/content-fetch-go/internal/redisutil/redisutil.go create mode 100644 packages/content-fetch-go/internal/server/server.go create mode 100644 packages/content-fetch-go/main.go diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..55e7b2535 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,133 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Omnivore is an open-source, self-hosted read-it-later application. The monorepo contains 24+ packages managed with **Yarn 1.22.19** and **Lerna**. Primary languages are TypeScript/JavaScript, with Swift (iOS), Kotlin (Android), and Go (imageproxy). + +## Commands + +### Development Setup +```bash +# Start all backend services (PostgreSQL, Redis, API, Content-Fetch) +docker compose up + +# Frontend development against dockerized backend +cd packages/web && cp .env.template .env.local && yarn dev + +# Run API dev server with hot-reload +make api # or: yarn workspace @omnivore/api dev + +# Run web dev server +make web # or: yarn workspace @omnivore/web dev + +# Run queue processor +make qp # or: yarn workspace @omnivore/api dev_qp +``` + +### Build +```bash +yarn build # Build all packages via Lerna + +# Build a specific package +yarn workspace @omnivore/api build +yarn workspace @omnivore/web build + +# Build content-fetch service (includes dependencies) +make content_fetch # builds content-handler, puppeteer-parse, content-fetch in order +``` + +### Testing +```bash +yarn test # Run all tests via Lerna (streaming output) + +# Run tests for a specific package +lerna run test --scope=@omnivore/api + +# Run a single test file (from within a package directory) +yarn mocha -r ts-node/register --config mocha-config.json test/path/to/file.test.ts + +# Type checking +yarn workspace @omnivore/api test:typecheck +``` + +Tests require PostgreSQL and Redis — use `docker compose -f docker-compose-test.yml up` for a test environment. + +### Linting +```bash +yarn lint # Lint all packages (ESLint, parallel) +yarn workspace @omnivore/api lint +yarn workspace @omnivore/api lint:fix # Auto-fix +``` + +Code style: no semicolons, single quotes (enforced by Prettier + ESLint with `@typescript-eslint`). + +## Architecture + +### Core Services + +**`packages/api`** — The central backend. GraphQL API (Apollo Server 3 on Express), running on port 4000 (8080 in Docker). Connects to PostgreSQL via TypeORM and Redis via ioredis/BullMQ. Exposes both a `/graphql` endpoint and REST routers under `/api/` and `/svc/` prefixes. + +**`packages/web`** — Next.js 14 frontend (React 18, TypeScript). Runs on port 3000. Uses SWR + TanStack Query for data fetching, Stitches for CSS-in-JS, and Radix UI components. Communicates with the API exclusively via GraphQL. + +**`packages/content-fetch`** — Async microservice for fetching and processing web content. Picks jobs off a BullMQ/Redis queue. Depends on `puppeteer-parse` and `content-handler`. + +**`packages/puppeteer-parse`** — Headless Chromium via Puppeteer for rendering and extracting page content. Called by `content-fetch`. + +**`packages/content-handler`** — Shared utilities for HTML parsing, sanitization, and Mozilla Readability integration. Used by both `api` and `puppeteer-parse`. + +**`packages/db`** — Database schema and migrations via Postgrator. Migration files live here; TypeORM entities live in `packages/api/src/entity/`. + +### Specialized Handlers (Cloud Function-style) + +These packages run independently and are invoked via HTTP or queue: +- `rss-handler` — RSS feed polling +- `inbound-email-handler` — Email ingestion +- `import-handler` / `export-handler` — Data migration/export jobs +- `pdf-handler` — PDF processing +- `thumbnail-handler` — Thumbnail generation +- `text-to-speech` — TTS audio generation +- `integration-handler` — Third-party integrations (Logseq, Obsidian, Readwise, etc.) +- `rule-handler` — User-defined automation rules + +### Shared Libraries + +- `packages/liqe` — Query parser/filter engine for Omnivore's search syntax +- `packages/readabilityjs` — Fork of Mozilla Readability +- `packages/utils` — Shared TypeScript utilities +- `packages/appreader` — WebView bundle (Next.js app) embedded in iOS and Android apps + +### Mobile and Native + +- `apple/` — iOS/macOS app in Swift/SwiftUI; uses `packages/appreader` bundle +- `android/` — Android app in Kotlin; uses `packages/appreader` bundle +- `imageproxy/` — Go service for proxying and caching images + +### Data Flow + +``` +Web/Mobile client + → GraphQL API (Apollo/Express) → PostgreSQL (TypeORM) + → Redis (cache + BullMQ queues) + → BullMQ queue workers + → content-fetch → puppeteer-parse → content-handler → stored article HTML +``` + +### Key Patterns + +- **GraphQL schema** is defined in `packages/api/src/schema/` and generated into `packages/api/src/generated/schema.graphql` +- **Resolvers** are in `packages/api/src/resolvers/` +- **REST service routes** (internal service-to-service) are under `packages/api/src/routers/svc/` +- **TypeORM entities** mirror the PostgreSQL schema in `packages/api/src/entity/` +- **Queue jobs** are defined in `packages/api/src/queue-processor.ts` and processed by BullMQ workers + +### Environment + +Key env vars (see `packages/api/.env.example` and `packages/web/.env.template`): +- `PG_HOST/PG_USER/PG_PASSWORD/PG_DB` — PostgreSQL connection +- `REDIS_URL` — Redis connection (used for both cache and queues) +- `JWT_SECRET` — Token signing +- `CONTENT_FETCH_URL` — URL of the content-fetch service +- `CLIENT_URL` — Frontend URL (for CORS and redirects) +- `IMAGE_PROXY_SECRET` / `IMAGE_PROXY_URL` — Image proxy config diff --git a/packages/content-fetch-go/.dockerignore b/packages/content-fetch-go/.dockerignore new file mode 100644 index 000000000..afdd9d8e7 --- /dev/null +++ b/packages/content-fetch-go/.dockerignore @@ -0,0 +1,3 @@ +Dockerfile +.dockerignore +*.md diff --git a/packages/content-fetch-go/Dockerfile b/packages/content-fetch-go/Dockerfile new file mode 100644 index 000000000..20d08d4e6 --- /dev/null +++ b/packages/content-fetch-go/Dockerfile @@ -0,0 +1,49 @@ +FROM golang:1.24-alpine AS build +LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore" + +RUN apk add --no-cache git ca-certificates + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o content-fetch-go . + +# ─── Runtime image ──────────────────────────────────────────────────────────── +FROM alpine:3.20 + +LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore" + +# Add Chromium and required fonts/libs from Alpine edge (mirrors the original Dockerfile) +RUN echo "@edge https://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ + && echo "@edge https://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories \ + && echo "@edge https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ + && apk -U upgrade \ + && apk add --no-cache \ + chromium@edge \ + freetype@edge \ + ttf-freefont@edge \ + nss@edge \ + libstdc++@edge \ + sqlite-libs@edge \ + ca-certificates@edge \ + && rm -rf /var/cache/apk/* + +WORKDIR /app + +ENV CHROMIUM_PATH=/usr/bin/chromium +ENV LAUNCH_HEADLESS=true +ENV PORT=8080 + +# Download ad/tracker block-list for the hosts file (matches original start.sh) +RUN wget -q https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts \ + && cat hosts >> /etc/hosts \ + && rm hosts + +COPY --from=build /app/content-fetch-go . + +EXPOSE 8080 + +CMD ["./content-fetch-go"] diff --git a/packages/content-fetch-go/go.mod b/packages/content-fetch-go/go.mod new file mode 100644 index 000000000..63e90053b --- /dev/null +++ b/packages/content-fetch-go/go.mod @@ -0,0 +1,70 @@ +module github.com/omnivore-app/omnivore/content-fetch-go + +go 1.25.0 + +require ( + cloud.google.com/go/storage v1.60.0 + github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 + github.com/chromedp/chromedp v0.14.2 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/posthog/posthog-go v1.10.0 + github.com/redis/go-redis/v9 v9.18.0 + google.golang.org/api v0.265.0 +) + +require ( + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.18.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chromedp/sysutil v1.1.0 // indirect + github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect + github.com/googleapis/gax-go/v2 v2.17.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/otel/sdk v1.39.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect + go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/crypto v0.47.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.35.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/text v0.33.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/packages/content-fetch-go/go.sum b/packages/content-fetch-go/go.sum new file mode 100644 index 000000000..faf93d16d --- /dev/null +++ b/packages/content-fetch-go/go.sum @@ -0,0 +1,166 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs= +cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY= +cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8= +cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 h1:UQ4AU+BGti3Sy/aLU8KVseYKNALcX9UXY6DfpwQ6J8E= +github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k= +github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM= +github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs= +github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= +github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= +github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8= +github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= +github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posthog/posthog-go v1.10.0 h1:wfoy7Jfb4LigCoHYyMZoiJmmEoCLOkSaYfDxM/NtCqY= +github.com/posthog/posthog-go v1.10.0/go.mod h1:wB3/9Q7d9gGb1P/yf/Wri9VBlbP8oA8z++prRzL5OcY= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.265.0 h1:FZvfUdI8nfmuNrE34aOWFPmLC+qRBEiNm3JdivTvAAU= +google.golang.org/api v0.265.0/go.mod h1:uAvfEl3SLUj/7n6k+lJutcswVojHPp2Sp08jWCu8hLY= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM= +google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0= +google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/content-fetch-go/internal/analytics/analytics.go b/packages/content-fetch-go/internal/analytics/analytics.go new file mode 100644 index 000000000..e4ea392aa --- /dev/null +++ b/packages/content-fetch-go/internal/analytics/analytics.go @@ -0,0 +1,73 @@ +// Package analytics wraps PostHog event capture, matching the analytics.ts behaviour. +package analytics + +import ( + "log" + + "github.com/omnivore-app/omnivore/content-fetch-go/internal/config" + "github.com/posthog/posthog-go" +) + +// Client sends analytics events to PostHog. +type Client struct { + ph posthog.Client + cfg *config.Config +} + +// Event carries data for an analytics capture call. +type Event struct { + Result string // "success" or "failure" + URL string + Source string + TotalTime int64 // milliseconds + ErrorMessage string +} + +// New creates a PostHog analytics client. +func New(cfg *config.Config) *Client { + ph, err := posthog.NewWithConfig(cfg.PostHogAPIKey, posthog.Config{}) + if err != nil { + log.Printf("Failed to create PostHog client: %v", err) + return &Client{cfg: cfg} + } + return &Client{ph: ph, cfg: cfg} +} + +// Capture sends a content_fetch_result event. +// Only failure events are sent when SEND_ANALYTICS is set, matching the TS behaviour. +func (c *Client) Capture(userIDs []string, ev Event) { + if c.ph == nil { + return + } + if !c.cfg.SendAnalytics || ev.Result != "failure" { + return + } + + for _, uid := range userIDs { + props := posthog.NewProperties(). + Set("url", ev.URL). + Set("source", ev.Source). + Set("totalTime", ev.TotalTime). + Set("env", c.cfg.APIEnv) + if ev.ErrorMessage != "" { + props.Set("errorMessage", ev.ErrorMessage) + } + + if err := c.ph.Enqueue(posthog.Capture{ + DistinctId: uid, + Event: "content_fetch_" + ev.Result, + Properties: props, + }); err != nil { + log.Printf("PostHog enqueue error: %v", err) + } + } +} + +// Close flushes and closes the PostHog client. +func (c *Client) Close() { + if c.ph != nil { + if err := c.ph.Close(); err != nil { + log.Printf("PostHog close error: %v", err) + } + } +} diff --git a/packages/content-fetch-go/internal/browser/browser.go b/packages/content-fetch-go/internal/browser/browser.go new file mode 100644 index 000000000..a33e93b75 --- /dev/null +++ b/packages/content-fetch-go/internal/browser/browser.go @@ -0,0 +1,106 @@ +// Package browser manages a shared headless Chromium instance via chromedp. +package browser + +import ( + "context" + "fmt" + "log" + "sync" + + "github.com/chromedp/chromedp" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/config" +) + +// Browser wraps a persistent chromedp browser allocator. +type Browser struct { + cfg *config.Config + allocCtx context.Context + allocCancel context.CancelFunc + mu sync.Mutex +} + +func New(cfg *config.Config) *Browser { + return &Browser{cfg: cfg} +} + +// allocatorOpts returns the chromedp ExecAllocator options matching the original Puppeteer args. +func (b *Browser) allocatorOpts() []chromedp.ExecAllocatorOption { + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("autoplay-policy", "user-gesture-required"), + chromedp.Flag("disable-component-update", true), + chromedp.Flag("disable-domain-reliability", true), + chromedp.Flag("disable-print-preview", true), + chromedp.Flag("disable-setuid-sandbox", true), + chromedp.Flag("disable-speech-api", true), + chromedp.Flag("enable-features", "SharedArrayBuffer"), + chromedp.Flag("hide-scrollbars", true), + chromedp.Flag("mute-audio", true), + chromedp.Flag("no-default-browser-check", true), + chromedp.Flag("no-pings", true), + chromedp.Flag("no-sandbox", true), + chromedp.Flag("no-zygote", true), + chromedp.Flag("disable-extensions", true), + chromedp.Flag("disable-dev-shm-usage", true), + chromedp.Flag("no-first-run", true), + chromedp.Flag("disable-background-networking", true), + chromedp.Flag("disable-gpu", true), + chromedp.Flag("disable-software-rasterizer", true), + chromedp.Flag("ignore-certificate-errors", true), + chromedp.WindowSize(1920, 1080), + chromedp.Headless, + ) + + if b.cfg.ChromiumPath != "" && !b.cfg.UseFirefox { + opts = append(opts, chromedp.ExecPath(b.cfg.ChromiumPath)) + } + + return opts +} + +// getAllocator returns the shared browser allocator context, creating it if needed. +func (b *Browser) getAllocator() (context.Context, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if b.allocCtx != nil { + // Check if still alive + select { + case <-b.allocCtx.Done(): + // Allocator died, recreate + b.allocCtx = nil + b.allocCancel = nil + default: + return b.allocCtx, nil + } + } + + log.Println("Starting chromedp browser allocator") + ctx, cancel := chromedp.NewExecAllocator(context.Background(), b.allocatorOpts()...) + b.allocCtx = ctx + b.allocCancel = cancel + return ctx, nil +} + +// NewContext creates a new browser tab context using the shared allocator. +func (b *Browser) NewContext() (context.Context, context.CancelFunc, error) { + allocCtx, err := b.getAllocator() + if err != nil { + return nil, nil, fmt.Errorf("get allocator: %w", err) + } + + ctx, cancel := chromedp.NewContext(allocCtx) + return ctx, cancel, nil +} + +// Close shuts down the browser. +func (b *Browser) Close() { + b.mu.Lock() + defer b.mu.Unlock() + + if b.allocCancel != nil { + b.allocCancel() + b.allocCancel = nil + b.allocCtx = nil + log.Println("Browser closed") + } +} diff --git a/packages/content-fetch-go/internal/bullmq/bullmq.go b/packages/content-fetch-go/internal/bullmq/bullmq.go new file mode 100644 index 000000000..63e61df5e --- /dev/null +++ b/packages/content-fetch-go/internal/bullmq/bullmq.go @@ -0,0 +1,371 @@ +// Package queue implements BullMQ-compatible Redis queue operations. +// +// BullMQ v5 stores jobs as Redis hashes at bull:{queue}:{jobId} +// and manages state via lists/sorted-sets: +// - bull:{queue}:wait LIST (pending, LIFO insertion LPUSH, LPOP consumption) +// - bull:{queue}:active LIST (currently processing) +// - bull:{queue}:completed ZSET (done, score=timestamp) +// - bull:{queue}:failed ZSET (failed, score=timestamp) +// - bull:{queue}:prioritized ZSET (priority queue, score encodes priority+counter) +// - bull:{queue}:id STRING (atomic ID counter) +// +// Workers use a Lua moveToActive script. For the consumer side we replicate +// a simplified version that is compatible with existing BullMQ producers and +// consumers (i.e. jobs added here can be consumed by the TS worker, and vice-versa). +package bullmq + +import ( + "context" + "encoding/json" + "fmt" + "log" + "strconv" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + bullPrefix = "bull" + + // Queue names + ContentFetchQueue = "omnivore-content-fetch-queue" + BackendQueue = "omnivore-backend-queue" + + // Job names + SavePageJob = "save-page" + + // removeOnComplete/removeOnFail ages (seconds) – match TS defaults + completeAge = 3600 + failAge = 86400 +) + +// queueKey returns the base key prefix for a BullMQ queue. +func queueKey(queueName string) string { + return fmt.Sprintf("%s:%s", bullPrefix, queueName) +} + +func waitKey(q string) string { return queueKey(q) + ":wait" } +func activeKey(q string) string { return queueKey(q) + ":active" } +func completedKey(q string) string { return queueKey(q) + ":completed" } +func failedKey(q string) string { return queueKey(q) + ":failed" } +func idKey(q string) string { return queueKey(q) + ":id" } +func metaKey(q string) string { return queueKey(q) + ":meta" } +func jobKey(q, id string) string { return queueKey(q) + ":" + id } +func prioritizedKey(q string) string { return queueKey(q) + ":prioritized" } +func eventsKey(q string) string { return queueKey(q) + ":events" } + +// JobOpts mirrors BullMQ BulkJobOptions. +type JobOpts struct { + Attempts int `json:"attempts"` + Priority int `json:"priority"` + Backoff BackoffOpt `json:"backoff"` + // removeOnComplete and removeOnFail are not stored in opts hash but handled by cleanup +} + +type BackoffOpt struct { + Type string `json:"type"` + Delay int `json:"delay"` +} + +// RawJob is a job as stored in Redis. +type RawJob struct { + ID string + Name string + Data json.RawMessage + Opts JobOpts + Timestamp int64 + AttemptsMade int +} + +// nextJobID atomically increments and returns a new job ID. +func nextJobID(ctx context.Context, rdb *redis.Client, queueName string) (string, error) { + id, err := rdb.Incr(ctx, idKey(queueName)).Result() + if err != nil { + return "", err + } + return strconv.FormatInt(id, 10), nil +} + +// AddJobOpts carries parameters for adding a single job. +type AddJobOpts struct { + Name string + Data interface{} + Opts JobOpts +} + +// AddBulk adds multiple jobs to a BullMQ queue, replicating addBulk() semantics. +// Each job is stored as a hash and its ID appended to the appropriate list/zset. +func AddBulk(ctx context.Context, rdb *redis.Client, queueName string, jobs []AddJobOpts) error { + for _, j := range jobs { + jobID, err := nextJobID(ctx, rdb, queueName) + if err != nil { + return fmt.Errorf("get next job id: %w", err) + } + + dataBytes, err := json.Marshal(j.Data) + if err != nil { + return fmt.Errorf("marshal job data: %w", err) + } + + optsBytes, err := json.Marshal(j.Opts) + if err != nil { + return fmt.Errorf("marshal job opts: %w", err) + } + + now := time.Now().UnixMilli() + key := jobKey(queueName, jobID) + + // Store the job hash + pipe := rdb.Pipeline() + pipe.HSet(ctx, key, + "name", j.Name, + "data", string(dataBytes), + "opts", string(optsBytes), + "timestamp", now, + "attemptsMade", 0, + "attemptsStarted", 0, + "stalledCounter", 0, + ) + + if j.Opts.Priority > 0 { + // Priority jobs go into the prioritized sorted set. + // BullMQ score = priority * 0x100000000 + counter (counter increments per priority) + // For simplicity we use: score = priority * 1e12 + now which preserves ordering. + score := float64(j.Opts.Priority)*1e12 + float64(now) + pipe.ZAdd(ctx, prioritizedKey(queueName), redis.Z{ + Score: score, + Member: jobID, + }) + } else { + pipe.LPush(ctx, waitKey(queueName), jobID) + } + + // Publish event for BullMQ dashboard/metrics compatibility + eventPayload, _ := json.Marshal(map[string]interface{}{ + "jobId": jobID, + "prev": "waiting", + }) + pipe.XAdd(ctx, &redis.XAddArgs{ + Stream: eventsKey(queueName), + Values: map[string]interface{}{ + "event": "waiting", + "data": string(eventPayload), + }, + MaxLen: 10000, + }) + + if _, err := pipe.Exec(ctx); err != nil { + return fmt.Errorf("add job %s: %w", jobID, err) + } + + log.Printf("Queued job %s/%s id=%s", queueName, j.Name, jobID) + } + return nil +} + +// moveToActive implements a simplified version of BullMQ's moveToActive Lua script. +// It atomically moves the next job from wait (or prioritized) → active. +// Returns the job ID and whether a job was found. +// +// The Lua script below is compatible with BullMQ v5 in that jobs added by the +// TS BullMQ library can be consumed here and vice-versa. It is intentionally +// simplified – it does not handle stalled job detection or rate limiting, which +// are handled by the BullMQ scheduler in the TS layer if both run simultaneously. +var moveToActiveScript = redis.NewScript(` +local waitKey = KEYS[1] +local prioritizedKey = KEYS[2] +local activeKey = KEYS[3] +local jobKeyPrefix = ARGV[1] -- e.g. "bull:omnivore-content-fetch-queue:" + +-- Try prioritized first (lowest score = highest priority) +local jobId = redis.call("ZPOPMIN", prioritizedKey, 1) +if #jobId > 0 then + jobId = jobId[1] +else + -- Fall back to regular wait list (RPOP = FIFO from the end, matching BullMQ) + jobId = redis.call("RPOP", waitKey) +end + +if jobId == false or jobId == nil then + return nil +end + +-- Move to active +redis.call("LPUSH", activeKey, jobId) + +return jobId +`) + +// PopJob atomically moves the next available job to the active list and returns it. +// Returns nil job if no job is available (non-blocking). +func PopJob(ctx context.Context, rdb *redis.Client, queueName string) (*RawJob, error) { + keys := []string{ + waitKey(queueName), + prioritizedKey(queueName), + activeKey(queueName), + } + prefix := queueKey(queueName) + ":" + + result, err := moveToActiveScript.Run(ctx, rdb, keys, prefix).Result() + if err == redis.Nil { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("moveToActive: %w", err) + } + + jobID, ok := result.(string) + if !ok || jobID == "" { + return nil, nil + } + + return getJob(ctx, rdb, queueName, jobID) +} + +func getJob(ctx context.Context, rdb *redis.Client, queueName, jobID string) (*RawJob, error) { + fields, err := rdb.HGetAll(ctx, jobKey(queueName, jobID)).Result() + if err != nil { + return nil, fmt.Errorf("hgetall job %s: %w", jobID, err) + } + if len(fields) == 0 { + return nil, fmt.Errorf("job %s not found", jobID) + } + + var opts JobOpts + if o := fields["opts"]; o != "" { + _ = json.Unmarshal([]byte(o), &opts) + } + + var attempts int + if a := fields["attemptsMade"]; a != "" { + attempts, _ = strconv.Atoi(a) + } + + return &RawJob{ + ID: jobID, + Name: fields["name"], + Data: json.RawMessage(fields["data"]), + Opts: opts, + AttemptsMade: attempts, + }, nil +} + +// CompleteJob moves a job from active to completed. +func CompleteJob(ctx context.Context, rdb *redis.Client, queueName, jobID string) error { + now := time.Now().UnixMilli() + pipe := rdb.Pipeline() + pipe.LRem(ctx, activeKey(queueName), 0, jobID) + pipe.ZAdd(ctx, completedKey(queueName), redis.Z{Score: float64(now), Member: jobID}) + pipe.HSet(ctx, jobKey(queueName, jobID), "finishedOn", now) + // Schedule cleanup (approximate removeOnComplete age) + pipe.Expire(ctx, jobKey(queueName, jobID), completeAge*time.Second) + _, err := pipe.Exec(ctx) + return err +} + +// FailJob moves a job from active to failed (or re-queues it for retry). +func FailJob(ctx context.Context, rdb *redis.Client, queueName, jobID string, reason string, opts JobOpts) error { + now := time.Now().UnixMilli() + + // Increment attemptsMade + newAttempts, err := rdb.HIncrBy(ctx, jobKey(queueName, jobID), "attemptsMade", 1).Result() + if err != nil { + return err + } + + if int(newAttempts) < opts.Attempts { + // Retry: calculate exponential backoff delay + delay := exponentialDelay(opts.Backoff.Delay, int(newAttempts)-1) + retryAt := now + int64(delay) + + pipe := rdb.Pipeline() + pipe.LRem(ctx, activeKey(queueName), 0, jobID) + pipe.ZAdd(ctx, fmt.Sprintf("%s:delayed", queueKey(queueName)), redis.Z{ + Score: float64(retryAt), + Member: jobID, + }) + pipe.HSet(ctx, jobKey(queueName, jobID), "failedReason", reason) + _, err = pipe.Exec(ctx) + return err + } + + // Max attempts reached → move to failed + pipe := rdb.Pipeline() + pipe.LRem(ctx, activeKey(queueName), 0, jobID) + pipe.ZAdd(ctx, failedKey(queueName), redis.Z{Score: float64(now), Member: jobID}) + pipe.HSet(ctx, jobKey(queueName, jobID), "failedReason", reason, "finishedOn", now) + pipe.Expire(ctx, jobKey(queueName, jobID), failAge*time.Second) + _, err = pipe.Exec(ctx) + return err +} + +// exponentialDelay returns the next retry delay in milliseconds (matches BullMQ's exponential). +func exponentialDelay(baseDelay, attempt int) int { + d := baseDelay + for i := 0; i < attempt; i++ { + d *= 2 + } + return d +} + +// GetQueueCounts returns job counts for the metrics endpoint. +func GetQueueCounts(ctx context.Context, rdb *redis.Client, queueName string) (map[string]int64, error) { + pipe := rdb.Pipeline() + activeCmd := pipe.LLen(ctx, activeKey(queueName)) + failedCmd := pipe.ZCard(ctx, failedKey(queueName)) + completedCmd := pipe.ZCard(ctx, completedKey(queueName)) + prioritizedCmd := pipe.ZCard(ctx, prioritizedKey(queueName)) + waitCmd := pipe.LLen(ctx, waitKey(queueName)) + + if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil { + return nil, err + } + + return map[string]int64{ + "active": activeCmd.Val(), + "failed": failedCmd.Val(), + "completed": completedCmd.Val(), + "prioritized": prioritizedCmd.Val() + waitCmd.Val(), + }, nil +} + +// OldestPrioritizedJobAge returns the age in seconds of the oldest prioritized job, or 0. +func OldestPrioritizedJobAge(ctx context.Context, rdb *redis.Client, queueName string) (float64, error) { + // Check both prioritized zset and wait list + results, err := rdb.ZRangeWithScores(ctx, prioritizedKey(queueName), 0, 0).Result() + if err != nil { + return 0, err + } + + if len(results) > 0 { + // score encodes priority+counter, so we need the job's timestamp field + jobID := results[0].Member.(string) + ts, err := rdb.HGet(ctx, jobKey(queueName, jobID), "timestamp").Result() + if err == nil { + if tsMs, err := strconv.ParseInt(ts, 10, 64); err == nil { + return float64(time.Now().UnixMilli()-tsMs) / 1000.0, nil + } + } + } + + // Fall back to wait list + waitIDs, err := rdb.LRange(ctx, waitKey(queueName), -1, -1).Result() // oldest = tail + if err != nil || len(waitIDs) == 0 { + return 0, nil + } + ts, err := rdb.HGet(ctx, jobKey(queueName, waitIDs[0]), "timestamp").Result() + if err != nil { + return 0, nil + } + tsMs, err := strconv.ParseInt(ts, 10, 64) + if err != nil { + return 0, nil + } + return float64(time.Now().UnixMilli()-tsMs) / 1000.0, nil +} + +// EnsureQueueMeta ensures the queue metadata key exists (BullMQ creates this on queue init). +func EnsureQueueMeta(ctx context.Context, rdb *redis.Client, queueName string) error { + return rdb.HSetNX(ctx, metaKey(queueName), "version", "5").Err() +} diff --git a/packages/content-fetch-go/internal/config/config.go b/packages/content-fetch-go/internal/config/config.go new file mode 100644 index 000000000..464298f09 --- /dev/null +++ b/packages/content-fetch-go/internal/config/config.go @@ -0,0 +1,92 @@ +package config + +import ( + "os" + "strconv" +) + +type Config struct { + // HTTP + Port int + VerificationToken string + + // Redis - cache + RedisURL string + RedisCert string + + // Redis - queue (BullMQ) + MQRedisURL string + MQRedisCert string + + // Browser + ChromiumPath string + FirefoxPath string + UseFirefox bool + LaunchHeadless bool + + // GCS + GCSUploadBucket string + GCSKeyFilePath string + SkipUploadOriginal bool + + // Analytics (PostHog) + PostHogAPIKey string + SendAnalytics bool + APIEnv string + + // Import metrics + ImporterMetricsCollectorURL string + JWTSecret string + + // Domain blocking + MaxFeedFetchFailures int +} + +func Load() *Config { + cfg := &Config{ + Port: envInt("PORT", 3002), + VerificationToken: os.Getenv("VERIFICATION_TOKEN"), + + RedisURL: os.Getenv("REDIS_URL"), + RedisCert: os.Getenv("REDIS_CERT"), + + MQRedisURL: os.Getenv("MQ_REDIS_URL"), + MQRedisCert: os.Getenv("MQ_REDIS_CERT"), + + ChromiumPath: envDefault("CHROMIUM_PATH", "/usr/bin/chromium"), + FirefoxPath: envDefault("FIREFOX_PATH", "/usr/bin/firefox"), + UseFirefox: os.Getenv("USE_FIREFOX") == "true", + LaunchHeadless: os.Getenv("LAUNCH_HEADLESS") == "true", + + GCSUploadBucket: envDefault("GCS_UPLOAD_BUCKET", "omnivore-files"), + GCSKeyFilePath: os.Getenv("GCS_UPLOAD_SA_KEY_FILE_PATH"), + SkipUploadOriginal: os.Getenv("SKIP_UPLOAD_ORIGINAL") == "true", + + PostHogAPIKey: envDefault("POSTHOG_API_KEY", "test"), + SendAnalytics: os.Getenv("SEND_ANALYTICS") != "", + APIEnv: os.Getenv("API_ENV"), + + ImporterMetricsCollectorURL: os.Getenv("IMPORTER_METRICS_COLLECTOR_URL"), + JWTSecret: os.Getenv("JWT_SECRET"), + + MaxFeedFetchFailures: envInt("MAX_FEED_FETCH_FAILURES", 10), + } + + return cfg +} + +func envDefault(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func envInt(key string, def int) int { + if v := os.Getenv(key); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return def +} diff --git a/packages/content-fetch-go/internal/fetch/fetch.go b/packages/content-fetch-go/internal/fetch/fetch.go new file mode 100644 index 000000000..bb079e569 --- /dev/null +++ b/packages/content-fetch-go/internal/fetch/fetch.go @@ -0,0 +1,450 @@ +// Package fetch implements content retrieval using chromedp (Chromium DevTools Protocol). +// It replicates the behaviour of packages/puppeteer-parse with equivalent Go logic. +package fetch + +import ( + "bytes" + "context" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/chromedp/cdproto/emulation" + "github.com/chromedp/cdproto/network" + "github.com/chromedp/chromedp" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/browser" +) + +// Result mirrors FetchResult from the TS implementation. +type Result struct { + FinalURL string + Title string + Content string + ContentType string +} + +// nonScriptHosts mirrors NON_SCRIPT_HOSTS – JS disabled for these. +var nonScriptHosts = []string{"medium.com", "fastcompany.com", "fortelabs.com"} + +// allowedContentTypes mirrors ALLOWED_CONTENT_TYPES. +var allowedContentTypes = map[string]bool{ + "text/html": true, + "application/octet-stream": true, + "text/plain": true, + "application/pdf": true, +} + +// noCacheURLs mirrors NO_CACHE_URLS from request_handler.ts (checked externally, listed here for reference). +var NoCacheURLs = map[string]bool{ + "https://deviceandbrowserinfo.com/are_you_a_bot": true, + "https://deviceandbrowserinfo.com/info_device": true, + "https://jacksonh.org": true, +} + +// FetchContent is the Go equivalent of fetchContent() in puppeteer-parse/src/index.ts. +func FetchContent(ctx context.Context, br *browser.Browser, rawURL, locale, timezone string) (*Result, error) { + start := time.Now() + log.Printf("content-fetch request url=%s locale=%s timezone=%s", rawURL, locale, timezone) + + parsedURL, err := parseAndValidateURL(rawURL) + if err != nil { + return nil, err + } + targetURL := parsedURL.String() + + // Pre-handle: detect PDFs, images, and other special content types. + preResult, err := preHandle(ctx, targetURL) + if err != nil { + log.Printf("pre-handle warning for %s: %v", targetURL, err) + // Non-fatal; continue with browser fetch + } + + if preResult != nil { + if preResult.URL != "" { + if p, err := parseAndValidateURL(preResult.URL); err == nil { + targetURL = p.String() + } + } + if preResult.ContentType == "application/pdf" || (preResult.Content != "" && preResult.Title != "") { + return &Result{ + FinalURL: targetURL, + Title: preResult.Title, + Content: preResult.Content, + ContentType: preResult.ContentType, + }, nil + } + } + + // Fall through to browser fetch + result, err := retrievePage(ctx, br, targetURL, locale, timezone) + if err != nil { + return nil, err + } + + log.Printf("content-fetch done url=%s duration=%s", targetURL, time.Since(start)) + return result, nil +} + +// preHandleResult carries the output of pre-handle checks. +type preHandleResult struct { + URL string + Title string + Content string + ContentType string +} + +// preHandle performs lightweight checks before launching the browser: +// - Detects PDF URLs (by extension or HEAD request) +// - Detects image content types +// Returns nil if normal browser rendering should proceed. +func preHandle(ctx context.Context, rawURL string) (*preHandleResult, error) { + lower := strings.ToLower(rawURL) + + // PDF by extension + if strings.HasSuffix(lower, ".pdf") || strings.Contains(lower, ".pdf?") { + return &preHandleResult{URL: rawURL, ContentType: "application/pdf"}, nil + } + + // Quick HEAD to detect content-type without full page load + client := &http.Client{Timeout: 5 * time.Second, CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 5 { + return fmt.Errorf("too many redirects") + } + return nil + }} + + req, err := http.NewRequestWithContext(ctx, http.MethodHead, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Omnivore/1.0)") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + ct := strings.ToLower(resp.Header.Get("Content-Type")) + if ct != "" { + ct = strings.SplitN(ct, ";", 2)[0] + ct = strings.TrimSpace(ct) + } + + if ct == "application/pdf" { + return &preHandleResult{URL: resp.Request.URL.String(), ContentType: "application/pdf"}, nil + } + + return nil, nil +} + +// retrievePage navigates to the URL using the browser, waits for load and DOM settle, +// then captures the full document HTML. +func retrievePage(ctx context.Context, br *browser.Browser, targetURL, locale, timezone string) (*Result, error) { + tabCtx, cancel, err := br.NewContext() + if err != nil { + return nil, fmt.Errorf("new browser context: %w", err) + } + defer cancel() + + // Timeout for the entire page load + DOM settle + tabCtx, timeoutCancel := context.WithTimeout(tabCtx, 60*time.Second) + defer timeoutCancel() + + result := &Result{FinalURL: targetURL} + + disableJS := shouldDisableJS(targetURL) + + // Track final URL (after redirects) and content-type via network events + var finalURL string + var contentType string + var lastPDFURL string + + chromedp.ListenTarget(tabCtx, func(ev interface{}) { + switch e := ev.(type) { + case *network.EventResponseReceived: + if e.Type == network.ResourceTypeDocument { + ct := strings.ToLower(e.Response.MimeType) + if ct == "application/pdf" { + lastPDFURL = e.Response.URL + } + if finalURL == "" { + finalURL = e.Response.URL + if ct, ok := e.Response.Headers["content-type"].(string); ok { + contentType = ct + } + if contentType == "" { + contentType = e.Response.MimeType + } + } + } + } + }) + + var actions []chromedp.Action + + // Set locale header + if locale != "" { + actions = append(actions, network.SetExtraHTTPHeaders(network.Headers{ + "Accept-Language": locale, + })) + } + + if disableJS { + actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error { + return emulation.SetScriptExecutionDisabled(true).Do(ctx) + })) + } + + // Navigate + actions = append(actions, chromedp.Navigate(targetURL)) + + // Wait for DOM to settle (equivalent to waitForDOMToSettle) + actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error { + return waitForDOMSettle(ctx, 5*time.Second, 1*time.Second) + })) + + // Scroll the page + actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error { + return scrollPage(ctx, 5*time.Second) + })) + + // Capture title + actions = append(actions, chromedp.Title(&result.Title)) + + // Capture and clean DOM + var domContent string + actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error { + return captureAndCleanDOM(ctx, &domContent) + })) + + if err := chromedp.Run(tabCtx, actions...); err != nil { + if lastPDFURL != "" { + return &Result{FinalURL: lastPDFURL, ContentType: "application/pdf"}, nil + } + return nil, fmt.Errorf("chromedp run: %w", err) + } + + if domContent == "IS_BLOCKED" { + return nil, fmt.Errorf("page is blocked by anti-bot protection") + } + + if finalURL != "" { + result.FinalURL = finalURL + } + result.Content = domContent + result.ContentType = contentType + + return result, nil +} + +// waitForDOMSettle runs a MutationObserver via JS to wait until DOM mutations settle. +// Mirrors waitForDOMToSettle from puppeteer-parse. +func waitForDOMSettle(ctx context.Context, timeout, debounce time.Duration) error { + timeoutMs := int(timeout.Milliseconds()) + debounceMs := int(debounce.Milliseconds()) + + script := fmt.Sprintf(` + new Promise((resolve) => { + const timeoutMs = %d; + const debounceMs = %d; + let debounceTimer; + const mainTimeout = setTimeout(() => { + observer.disconnect(); + resolve(); + }, timeoutMs); + const debouncedResolve = () => { + clearTimeout(debounceTimer); + debounceTimer = setTimeout(() => { + observer.disconnect(); + clearTimeout(mainTimeout); + resolve(); + }, debounceMs); + }; + const observer = new MutationObserver(debouncedResolve); + observer.observe(document.body, { attributes: true, childList: true, subtree: true }); + }) + `, timeoutMs, debounceMs) + + return chromedp.Evaluate(script, nil).Do(ctx) +} + +// scrollPage scrolls the entire page with a timeout, mirroring the Puppeteer scroll logic. +func scrollPage(ctx context.Context, timeout time.Duration) error { + scrollCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + script := ` + new Promise((resolve) => { + let scrollHeight = document.body.scrollHeight; + let totalHeight = 0; + let distance = 500; + let timer = setInterval(() => { + window.scrollBy(0, distance); + totalHeight += distance; + if (totalHeight >= scrollHeight) { + clearInterval(timer); + resolve(true); + } + }, 10); + }) + ` + err := chromedp.Evaluate(script, nil).Do(scrollCtx) + if err != nil && scrollCtx.Err() != nil { + return nil // Timeout during scroll is acceptable + } + return err +} + +// captureAndCleanDOM evaluates JS to clean the DOM and return the outerHTML. +// Mirrors the page.evaluate block in retrieveHtml() from puppeteer-parse/src/index.ts. +func captureAndCleanDOM(ctx context.Context, result *string) error { + script := ` + (function() { + const BI_SRC_REGEXP = /url\("(.+?)"\)/gi; + + Array.from(document.body.getElementsByTagName('*')).forEach((el) => { + const style = window.getComputedStyle(el); + const src = el.getAttribute('src'); + + try { + if (el.tagName && ['img', 'image'].includes(el.tagName.toLowerCase())) { + const filter = style.getPropertyValue('filter'); + if (filter && filter.startsWith('blur')) { + el.parentNode && el.parentNode.removeChild(el); + return; + } + } + } catch(err) {} + + const bgImage = style.getPropertyValue('background-image'); + if (bgImage && !['', 'none'].includes(bgImage)) { + const filter = style.getPropertyValue('filter'); + if (filter && filter.startsWith('blur')) { + el && el.parentNode && el.parentNode.removeChild(el); + } else { + BI_SRC_REGEXP.lastIndex = 0; + const matchedSRC = BI_SRC_REGEXP.exec(bgImage); + BI_SRC_REGEXP.lastIndex = 0; + if (matchedSRC && matchedSRC[1] && !src) { + if (!el.textContent) { + const img = document.createElement('img'); + img.src = matchedSRC[1]; + el && el.parentNode && el.parentNode.replaceChild(img, el); + } + } + } + } + + if (el.tagName === 'IFRAME') { + // Instagram iframe handling omitted (requires cross-frame access unavailable in CDP) + } + }); + + if ( + document.querySelector('[data-translate="managed_checking_msg"]') || + document.getElementById('px-block-form-wrapper') + ) { + return 'IS_BLOCKED'; + } + + return document.documentElement.outerHTML; + })() + ` + return chromedp.Evaluate(script, result).Do(ctx) +} + +// shouldDisableJS returns true for domains where JS should be disabled (mirrors NON_SCRIPT_HOSTS). +func shouldDisableJS(rawURL string) bool { + u, err := url.Parse(rawURL) + if err != nil { + return false + } + hostname := u.Hostname() + for _, host := range nonScriptHosts { + if strings.HasSuffix(hostname, host) { + return true + } + } + return false +} + +// parseAndValidateURL validates and normalises the URL, mirroring validateUrlString + getUrl. +func parseAndValidateURL(rawURL string) (*url.URL, error) { + // Extract first URL if embedded in a string + re := regexp.MustCompile(`(https?://[^\s]+)`) + matches := re.FindStringSubmatch(rawURL) + if len(matches) == 0 { + return nil, fmt.Errorf("no URL found in: %s", rawURL) + } + rawURL = matches[1] + + u, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("invalid URL: %w", err) + } + + if u.Scheme != "http" && u.Scheme != "https" { + return nil, fmt.Errorf("invalid URL protocol: %s", u.Scheme) + } + + hostname := u.Hostname() + if hostname == "localhost" || hostname == "0.0.0.0" { + return nil, fmt.Errorf("URL points to localhost") + } + + if isPrivateIP(hostname) { + return nil, fmt.Errorf("URL points to private IP: %s", hostname) + } + + return u, nil +} + +var privateIPRanges = []*regexp.Regexp{ + regexp.MustCompile(`^10\.`), + regexp.MustCompile(`^172\.(1[6-9]|2[0-9]|3[0-1])\.`), + regexp.MustCompile(`^192\.168\.`), +} + +func isPrivateIP(hostname string) bool { + // Try resolving if it's a hostname + ip := net.ParseIP(hostname) + if ip == nil { + return false + } + s := ip.String() + for _, re := range privateIPRanges { + if re.MatchString(s) { + return true + } + } + return false +} + +// FetchPDF downloads a PDF directly and returns its bytes (for GCS upload). +func FetchPDF(ctx context.Context, rawURL string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Omnivore/1.0)") + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var buf bytes.Buffer + if _, err := io.Copy(&buf, resp.Body); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/packages/content-fetch-go/internal/gcs/gcs.go b/packages/content-fetch-go/internal/gcs/gcs.go new file mode 100644 index 000000000..ed1dd66ce --- /dev/null +++ b/packages/content-fetch-go/internal/gcs/gcs.go @@ -0,0 +1,73 @@ +// Package gcs handles Google Cloud Storage uploads. +package gcs + +import ( + "context" + "fmt" + "io" + "log" + "strings" + "time" + + "cloud.google.com/go/storage" + "google.golang.org/api/option" +) + +// Client wraps GCS operations. +type Client struct { + bucket string + gcsClient *storage.Client +} + +// New creates a GCS client. If keyFilePath is empty, Application Default Credentials are used. +func New(ctx context.Context, bucketName, keyFilePath string) (*Client, error) { + var opts []option.ClientOption + if keyFilePath != "" { + opts = append(opts, option.WithCredentialsFile(keyFilePath)) + } + + gcsClient, err := storage.NewClient(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("new GCS client: %w", err) + } + + return &Client{bucket: bucketName, gcsClient: gcsClient}, nil +} + +// UploadContent uploads content string to GCS at filePath (non-public, 5s write timeout). +func (c *Client) UploadContent(ctx context.Context, filePath, content string) error { + writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + wc := c.gcsClient.Bucket(c.bucket).Object(filePath).NewWriter(writeCtx) + wc.ContentType = "text/html" + + if _, err := io.Copy(wc, strings.NewReader(content)); err != nil { + return fmt.Errorf("write to GCS %s: %w", filePath, err) + } + if err := wc.Close(); err != nil { + return fmt.Errorf("close GCS writer %s: %w", filePath, err) + } + + log.Printf("Original content uploaded to %s", filePath) + return nil +} + +// UploadOriginalContent mirrors uploadOriginalContent() from request_handler.ts. +// It uploads content for each user using the pattern content/{userId}/{libraryItemId}.{timestamp}.original +func (c *Client) UploadOriginalContent(ctx context.Context, users []UserRef, content string, savedTimestamp int64) error { + for _, user := range users { + filePath := fmt.Sprintf("content/%s/%s.%d.original", user.ID, user.LibraryItemID, savedTimestamp) + if err := c.UploadContent(ctx, filePath, content); err != nil { + // Log but don't fail the whole job + log.Printf("Failed to upload original content for user %s: %v", user.ID, err) + } + } + return nil +} + +// UserRef holds the minimal user info needed for uploads. +type UserRef struct { + ID string + LibraryItemID string +} diff --git a/packages/content-fetch-go/internal/handler/handler.go b/packages/content-fetch-go/internal/handler/handler.go new file mode 100644 index 000000000..a82eeb539 --- /dev/null +++ b/packages/content-fetch-go/internal/handler/handler.go @@ -0,0 +1,457 @@ +// Package handler implements processFetchContentJob — the core job processing logic. +// It mirrors packages/content-fetch/src/request_handler.ts exactly. +package handler + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/analytics" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/browser" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/bullmq" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/config" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/fetch" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/gcs" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/redisutil" + "github.com/redis/go-redis/v9" +) + +// UserConfig mirrors the TS UserConfig interface. +type UserConfig struct { + ID string `json:"id"` + LibraryItemID string `json:"libraryItemId"` + Folder *string `json:"folder,omitempty"` +} + +// JobData mirrors the TS JobData interface exactly. +type JobData struct { + URL string `json:"url"` + UserID *string `json:"userId,omitempty"` + SaveRequestID string `json:"saveRequestId"` + State *string `json:"state,omitempty"` + Labels []string `json:"labels,omitempty"` + Source *string `json:"source,omitempty"` + TaskID *string `json:"taskId,omitempty"` + Locale *string `json:"locale,omitempty"` + Timezone *string `json:"timezone,omitempty"` + RSSFeedURL *string `json:"rssFeedUrl,omitempty"` + SavedAt *string `json:"savedAt,omitempty"` + PublishedAt *string `json:"publishedAt,omitempty"` + Folder *string `json:"folder,omitempty"` + Users []UserConfig `json:"users,omitempty"` + Priority string `json:"priority"` // "high" | "low" +} + +// savePageJobData mirrors SavePageJobData from job.ts. +type savePageJobData struct { + UserID string `json:"userId"` + URL string `json:"url"` + FinalURL string `json:"finalUrl"` + ArticleSavingRequestID string `json:"articleSavingRequestId"` + State *string `json:"state,omitempty"` + Labels []string `json:"labels,omitempty"` + Source string `json:"source"` + Folder *string `json:"folder,omitempty"` + RSSFeedURL *string `json:"rssFeedUrl,omitempty"` + SavedAt *string `json:"savedAt,omitempty"` + PublishedAt *string `json:"publishedAt,omitempty"` + TaskID *string `json:"taskId,omitempty"` + Title string `json:"title,omitempty"` + ContentType string `json:"contentType,omitempty"` + CacheKey string `json:"cacheKey,omitempty"` +} + +const maxImportAttempts = 1 + +// ProcessFetchContentJob is the Go equivalent of processFetchContentJob() from request_handler.ts. +func ProcessFetchContentJob( + ctx context.Context, + cfg *config.Config, + rds *redisutil.RedisDataSource, + br *browser.Browser, + data *JobData, + attemptsMade int, +) error { + functionStartTime := time.Now() + + // Build user list (mirrors the TS logic) + users := make([]UserConfig, 0) + if data.UserID != nil && *data.UserID != "" { + folder := data.Folder + users = append(users, UserConfig{ + ID: *data.UserID, + LibraryItemID: data.SaveRequestID, + Folder: folder, + }) + } else { + users = data.Users + } + + source := "puppeteer-parse" + if data.Source != nil && *data.Source != "" { + source = *data.Source + } + + locale := "" + if data.Locale != nil { + locale = *data.Locale + } + timezone := "" + if data.Timezone != nil { + timezone = *data.Timezone + } + + logFields := map[string]interface{}{ + "url": data.URL, + "articleSavingRequestId": data.SaveRequestID, + "source": source, + "users": users, + } + log.Printf("Article parsing request %+v", logFields) + + var processErr error + defer func() { + totalTime := time.Since(functionStartTime).Milliseconds() + log.Printf("parse-page result url=%s totalTime=%dms error=%v", data.URL, totalTime, processErr) + + // Analytics + userIDs := make([]string, len(users)) + for i, u := range users { + userIDs[i] = u.ID + } + result := "success" + var errMsg string + if processErr != nil { + result = "failure" + errMsg = processErr.Error() + } + analyticsClient := analytics.New(cfg) + analyticsClient.Capture(userIDs, analytics.Event{ + Result: result, + URL: data.URL, + Source: source, + TotalTime: totalTime, + ErrorMessage: errMsg, + }) + analyticsClient.Close() + + // Import status update on final failure attempt + lastAttempt := attemptsMade+1 >= maxImportAttempts + if processErr != nil && data.TaskID != nil && *data.TaskID != "" && lastAttempt { + log.Println("Sending import status update (failure)") + if len(users) > 0 { + sendImportStatusUpdate(ctx, cfg, users[0].ID, *data.TaskID, false) + } + } + }() + + // Check domain block + domain, err := extractDomain(data.URL) + if err != nil { + processErr = fmt.Errorf("invalid URL: %w", err) + return processErr + } + + blocked, err := isDomainBlocked(ctx, rds, domain, cfg.MaxFeedFetchFailures) + if err != nil { + log.Printf("Error checking domain block: %v", err) + } + if blocked { + log.Printf("Domain is blocked: %s", domain) + // Return nil (not an error) — mirroring the TS behaviour of silently dropping + return nil + } + + // Try cache + cacheKey := buildCacheKey(data.URL, locale, timezone) + fetchResult, err := getCachedResult(ctx, rds, cacheKey) + if err != nil { + log.Printf("Cache read error: %v", err) + } + + if fetchResult == nil { + log.Printf("Fetch result not in cache, fetching now: %s", data.URL) + fetchResult, err = fetch.FetchContent(ctx, br, data.URL, locale, timezone) + if err != nil { + _ = incrementDomainFailure(ctx, rds, domain) + processErr = fmt.Errorf("fetchContent: %w", err) + return processErr + } + log.Println("Content fetched successfully") + + // Cache result (skip NO_CACHE_URLS) + if fetchResult.Content != "" && !fetch.NoCacheURLs[data.URL] { + if err := cacheResult(ctx, rds, cacheKey, fetchResult); err != nil { + log.Printf("Cache write error: %v", err) + } + } + } + + savedDate := time.Now() + if data.SavedAt != nil && *data.SavedAt != "" { + if t, err := time.Parse(time.RFC3339, *data.SavedAt); err == nil { + savedDate = t + } + } + + // Upload original content to GCS + if fetchResult.Content != "" && !cfg.SkipUploadOriginal { + gcsClient, err := gcs.New(ctx, cfg.GCSUploadBucket, cfg.GCSKeyFilePath) + if err != nil { + log.Printf("GCS client init error: %v", err) + } else { + refs := make([]gcs.UserRef, len(users)) + for i, u := range users { + refs[i] = gcs.UserRef{ID: u.ID, LibraryItemID: u.LibraryItemID} + } + if err := gcsClient.UploadOriginalContent(ctx, refs, fetchResult.Content, savedDate.UnixMilli()); err != nil { + log.Printf("GCS upload error: %v", err) + } + } + } + + // Build save-page jobs and queue them + savedAtStr := savedDate.Format(time.RFC3339) + savePageJobs := make([]bullmq.AddJobOpts, 0, len(users)) + for _, user := range users { + folder := user.Folder + jobData := savePageJobData{ + UserID: user.ID, + URL: data.URL, + FinalURL: fetchResult.FinalURL, + ArticleSavingRequestID: user.LibraryItemID, + State: data.State, + Labels: data.Labels, + Source: source, + Folder: folder, + RSSFeedURL: data.RSSFeedURL, + SavedAt: &savedAtStr, + PublishedAt: data.PublishedAt, + TaskID: data.TaskID, + Title: fetchResult.Title, + ContentType: fetchResult.ContentType, + CacheKey: cacheKey, + } + + isRSS := data.RSSFeedURL != nil && *data.RSSFeedURL != "" + isImport := data.TaskID != nil && *data.TaskID != "" + + priority := getBullMQPriority(isRSS, isImport, data.Priority) + attempts := getAttempts(isRSS, isImport) + backoffDelay := 2000 + + savePageJobs = append(savePageJobs, bullmq.AddJobOpts{ + Name: bullmq.SavePageJob, + Data: jobData, + Opts: bullmq.JobOpts{ + Attempts: attempts, + Priority: priority, + Backoff: bullmq.BackoffOpt{ + Type: "exponential", + Delay: backoffDelay, + }, + }, + }) + } + + if err := bullmq.AddBulk(ctx, rds.MQClient, bullmq.BackendQueue, savePageJobs); err != nil { + processErr = fmt.Errorf("queue save-page jobs: %w", err) + return processErr + } + + log.Printf("save-page jobs queued: %d", len(savePageJobs)) + return nil +} + +// getBullMQPriority mirrors getPriority() from job.ts. +func getBullMQPriority(isRSS, isImport bool, priority string) int { + if isImport { + return 100 + } + if isRSS { + if priority == "low" { + return 10 + } + return 5 + } + if priority == "low" { + return 5 + } + return 1 +} + +// getAttempts mirrors getAttempts() from job.ts. +func getAttempts(isRSS, isImport bool) int { + if isImport { + return 1 + } + if isRSS { + return 2 + } + return 3 +} + +// buildCacheKey mirrors cacheKey() from request_handler.ts. +func buildCacheKey(rawURL, locale, timezone string) string { + return fmt.Sprintf("fetch-result:%s:%s:%s", rawURL, locale, timezone) +} + +// cachedFetchResult is the JSON structure stored in Redis. +type cachedFetchResult struct { + FinalURL string `json:"finalUrl"` + Title string `json:"title,omitempty"` + Content string `json:"content,omitempty"` + ContentType string `json:"contentType,omitempty"` +} + +// getCachedResult attempts to get a cached fetch result from Redis. +func getCachedResult(ctx context.Context, rds *redisutil.RedisDataSource, key string) (*fetch.Result, error) { + val, err := rds.CacheClient.Get(ctx, key).Result() + if err == redis.Nil { + log.Printf("Fetch result not cached: %s", key) + return nil, nil + } + if err != nil { + return nil, err + } + + var cached cachedFetchResult + if err := json.Unmarshal([]byte(val), &cached); err != nil { + log.Printf("Invalid cache entry for key %s: %v", key, err) + return nil, nil + } + + if cached.FinalURL == "" { + return nil, nil + } + + log.Printf("Fetch result is cached: %s", key) + return &fetch.Result{ + FinalURL: cached.FinalURL, + Title: cached.Title, + Content: cached.Content, + ContentType: cached.ContentType, + }, nil +} + +// cacheResult stores a fetch result in Redis with a 24-hour TTL (NX = only if not exists). +func cacheResult(ctx context.Context, rds *redisutil.RedisDataSource, key string, r *fetch.Result) error { + val, err := json.Marshal(cachedFetchResult{ + FinalURL: r.FinalURL, + Title: r.Title, + Content: r.Content, + ContentType: r.ContentType, + }) + if err != nil { + return err + } + return rds.CacheClient.SetNX(ctx, key, string(val), 24*time.Hour).Err() +} + +// failureRedisKey mirrors failureRedisKey() from request_handler.ts. +func failureRedisKey(domain string) string { + return "fetch-failure:" + domain +} + +// isDomainBlocked mirrors isDomainBlocked() from request_handler.ts. +func isDomainBlocked(ctx context.Context, rds *redisutil.RedisDataSource, domain string, maxFailures int) (bool, error) { + blockedDomains := map[string]bool{ + "localhost": true, + "weibo.com": true, + } + if blockedDomains[domain] { + return true, nil + } + + key := failureRedisKey(domain) + val, err := rds.CacheClient.Get(ctx, key).Result() + if err == redis.Nil { + return false, nil + } + if err != nil { + return false, err + } + + var count int + if _, err := fmt.Sscanf(val, "%d", &count); err != nil { + return false, nil + } + + if count > maxFailures { + log.Printf("Domain is blocked (failure count=%d): %s", count, domain) + return true, nil + } + + return false, nil +} + +// incrementDomainFailure mirrors incrementContentFetchFailure() from request_handler.ts. +func incrementDomainFailure(ctx context.Context, rds *redisutil.RedisDataSource, domain string) error { + key := failureRedisKey(domain) + if err := rds.CacheClient.Incr(ctx, key).Err(); err != nil { + return err + } + return rds.CacheClient.Expire(ctx, key, time.Hour).Err() +} + +// sendImportStatusUpdate mirrors sendImportStatusUpdate() from request_handler.ts. +func sendImportStatusUpdate(ctx context.Context, cfg *config.Config, userID, taskID string, isImported bool) { + if cfg.JWTSecret == "" || cfg.ImporterMetricsCollectorURL == "" { + log.Println("JWT_SECRET or IMPORTER_METRICS_COLLECTOR_URL not set, skipping import status update") + return + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ + "uid": userID, + }) + tokenStr, err := token.SignedString([]byte(cfg.JWTSecret)) + if err != nil { + log.Printf("Failed to sign JWT: %v", err) + return + } + + status := "failed" + if isImported { + status = "imported" + } + + body, _ := json.Marshal(map[string]string{ + "taskId": taskID, + "status": status, + }) + + reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, cfg.ImporterMetricsCollectorURL, bytes.NewReader(body)) + if err != nil { + log.Printf("Failed to create import status request: %v", err) + return + } + req.Header.Set("Authorization", tokenStr) + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Printf("Failed to send import status update: %v", err) + return + } + defer resp.Body.Close() + log.Printf("Import status update sent: status=%s code=%d", status, resp.StatusCode) +} + +// extractDomain extracts the hostname from a URL. +func extractDomain(rawURL string) (string, error) { + u, err := url.Parse(rawURL) + if err != nil { + return "", err + } + return u.Hostname(), nil +} diff --git a/packages/content-fetch-go/internal/queue/worker.go b/packages/content-fetch-go/internal/queue/worker.go new file mode 100644 index 000000000..251d5f494 --- /dev/null +++ b/packages/content-fetch-go/internal/queue/worker.go @@ -0,0 +1,110 @@ +package queue + +import ( + "context" + "encoding/json" + "log" + "sync" + "time" + + "github.com/omnivore-app/omnivore/content-fetch-go/internal/browser" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/bullmq" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/config" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/handler" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/redisutil" +) + +const ( + workerConcurrency = 4 + workerPollInterval = 500 * time.Millisecond +) + +// Worker processes jobs from the content-fetch BullMQ queue. +type Worker struct { + ctx context.Context + cfg *config.Config + rds *redisutil.RedisDataSource + br *browser.Browser + wg sync.WaitGroup + sem chan struct{} +} + +func NewWorker(ctx context.Context, cfg *config.Config, rds *redisutil.RedisDataSource, br *browser.Browser) *Worker { + return &Worker{ + ctx: ctx, + cfg: cfg, + rds: rds, + br: br, + sem: make(chan struct{}, workerConcurrency), + } +} + +func (w *Worker) Start() { + w.wg.Add(1) + go w.run() +} + +func (w *Worker) Wait() { + w.wg.Wait() +} + +func (w *Worker) run() { + defer w.wg.Done() + + log.Println("Queue worker started") + + // Ensure queue meta exists + _ = bullmq.EnsureQueueMeta(w.ctx, w.rds.MQClient, bullmq.ContentFetchQueue) + + for { + select { + case <-w.ctx.Done(): + log.Println("Queue worker stopping, draining active slots...") + for i := 0; i < workerConcurrency; i++ { + w.sem <- struct{}{} + } + log.Println("Queue worker stopped") + return + default: + } + + job, err := bullmq.PopJob(w.ctx, w.rds.MQClient, bullmq.ContentFetchQueue) + if err != nil { + log.Printf("Error popping job: %v", err) + time.Sleep(workerPollInterval) + continue + } + if job == nil { + time.Sleep(workerPollInterval) + continue + } + + w.sem <- struct{}{} + w.wg.Add(1) + go func(j *bullmq.RawJob) { + defer func() { <-w.sem }() + defer w.wg.Done() + w.processJob(j) + }(job) + } +} + +func (w *Worker) processJob(job *bullmq.RawJob) { + log.Printf("Processing job id=%s name=%s", job.ID, job.Name) + + var data handler.JobData + if err := json.Unmarshal(job.Data, &data); err != nil { + log.Printf("Failed to unmarshal job data id=%s: %v", job.ID, err) + _ = bullmq.FailJob(w.ctx, w.rds.MQClient, bullmq.ContentFetchQueue, job.ID, err.Error(), job.Opts) + return + } + + if err := handler.ProcessFetchContentJob(w.ctx, w.cfg, w.rds, w.br, &data, job.AttemptsMade); err != nil { + log.Printf("Job id=%s failed: %v", job.ID, err) + _ = bullmq.FailJob(w.ctx, w.rds.MQClient, bullmq.ContentFetchQueue, job.ID, err.Error(), job.Opts) + return + } + + _ = bullmq.CompleteJob(w.ctx, w.rds.MQClient, bullmq.ContentFetchQueue, job.ID) + log.Printf("Job id=%s completed", job.ID) +} diff --git a/packages/content-fetch-go/internal/redisutil/redisutil.go b/packages/content-fetch-go/internal/redisutil/redisutil.go new file mode 100644 index 000000000..4e3e815d2 --- /dev/null +++ b/packages/content-fetch-go/internal/redisutil/redisutil.go @@ -0,0 +1,88 @@ +package redisutil + +import ( + "context" + "crypto/tls" + "fmt" + "log" + "strings" + "time" + + "github.com/omnivore-app/omnivore/content-fetch-go/internal/config" + "github.com/redis/go-redis/v9" +) + +// RedisDataSource holds two Redis clients: one for caching and one for BullMQ queues. +type RedisDataSource struct { + CacheClient *redis.Client + MQClient *redis.Client +} + +func New(cfg *config.Config) (*RedisDataSource, error) { + cacheClient, err := newClient(cfg.RedisURL, cfg.RedisCert) + if err != nil { + return nil, fmt.Errorf("cache redis: %w", err) + } + + mqClient := cacheClient + if cfg.MQRedisURL != "" { + mqClient, err = newClient(cfg.MQRedisURL, cfg.MQRedisCert) + if err != nil { + return nil, fmt.Errorf("mq redis: %w", err) + } + } + + // Ping both + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := cacheClient.Ping(ctx).Err(); err != nil { + return nil, fmt.Errorf("cache redis ping: %w", err) + } + if mqClient != cacheClient { + if err := mqClient.Ping(ctx).Err(); err != nil { + return nil, fmt.Errorf("mq redis ping: %w", err) + } + } + + log.Println("Redis connected") + return &RedisDataSource{ + CacheClient: cacheClient, + MQClient: mqClient, + }, nil +} + +func newClient(redisURL, cert string) (*redis.Client, error) { + if redisURL == "" { + return nil, fmt.Errorf("redis URL is empty") + } + + opt, err := redis.ParseURL(redisURL) + if err != nil { + return nil, fmt.Errorf("parse redis URL: %w", err) + } + + // TLS with custom cert for rediss:// URLs + if strings.HasPrefix(redisURL, "rediss://") && cert != "" { + opt.TLSConfig = &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // matches original TS behaviour + RootCAs: nil, + } + } + + // Match ioredis settings from original + opt.DialTimeout = 10 * time.Second + + return redis.NewClient(opt), nil +} + +func (r *RedisDataSource) Shutdown() { + if err := r.CacheClient.Close(); err != nil { + log.Printf("Error closing cache Redis: %v", err) + } + if r.MQClient != r.CacheClient { + if err := r.MQClient.Close(); err != nil { + log.Printf("Error closing MQ Redis: %v", err) + } + } + log.Println("Redis shutdown complete") +} diff --git a/packages/content-fetch-go/internal/server/server.go b/packages/content-fetch-go/internal/server/server.go new file mode 100644 index 000000000..ad7499758 --- /dev/null +++ b/packages/content-fetch-go/internal/server/server.go @@ -0,0 +1,132 @@ +// Package server implements the HTTP endpoints matching the original content-fetch Express app. +package server + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "strconv" + + "github.com/omnivore-app/omnivore/content-fetch-go/internal/browser" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/config" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/handler" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/bullmq" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/redisutil" +) + +// Worker is the minimal interface the server needs from the queue worker. +type Worker interface { + Wait() +} + +type mux struct { + cfg *config.Config + rds *redisutil.RedisDataSource + br *browser.Browser + worker Worker + http.ServeMux +} + +// New returns an http.Handler with all routes registered. +func New(cfg *config.Config, rds *redisutil.RedisDataSource, br *browser.Browser, w Worker) http.Handler { + m := &mux{cfg: cfg, rds: rds, br: br, worker: w} + m.HandleFunc("GET /_ah/health", m.health) + m.HandleFunc("GET /lifecycle/prestop", m.prestop) + m.HandleFunc("GET /metrics", m.metrics) + m.HandleFunc("/", m.root) // GET and POST + return m +} + +// health responds to Google Cloud autoscaler health checks. +func (m *mux) health(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + +// prestop implements the Kubernetes/Cloud Run prestop lifecycle hook. +// It signals the worker to stop and waits for in-flight jobs to finish. +func (m *mux) prestop(w http.ResponseWriter, r *http.Request) { + log.Println("Prestop lifecycle hook called.") + // Worker shutdown is handled by the caller (main) via context cancellation; + // here we just wait until it's done. + m.worker.Wait() + log.Println("Worker drained on prestop") + w.WriteHeader(http.StatusOK) +} + +// metrics returns Prometheus-format queue metrics, matching the original /metrics endpoint. +func (m *mux) metrics(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + counts, err := bullmq.GetQueueCounts(ctx, m.rds.MQClient, bullmq.ContentFetchQueue) + if err != nil { + log.Printf("Error getting queue counts: %v", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + age, err := bullmq.OldestPrioritizedJobAge(ctx, m.rds.MQClient, bullmq.ContentFetchQueue) + if err != nil { + log.Printf("Error getting oldest job age: %v", err) + } + + output := "" + for _, metric := range []string{"active", "failed", "completed", "prioritized"} { + val, _ := counts[metric] + output += fmt.Sprintf("# TYPE omnivore_queue_messages_%s gauge\n", metric) + output += fmt.Sprintf("omnivore_queue_messages_%s{queue=%q} %d\n", metric, bullmq.ContentFetchQueue, val) + } + output += "# TYPE omnivore_queue_messages_oldest_job_age_seconds gauge\n" + output += fmt.Sprintf("omnivore_queue_messages_oldest_job_age_seconds{queue=%q} %s\n", + bullmq.ContentFetchQueue, strconv.FormatFloat(age, 'f', -1, 64)) + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(output)) +} + +// root handles the primary job-processing endpoint (GET or POST /?token=...). +func (m *mux) root(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + + if r.Method != http.MethodGet && r.Method != http.MethodPost { + log.Printf("Request method is not GET or POST: %s", r.Method) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + if r.URL.Query().Get("token") != m.cfg.VerificationToken { + log.Println("Query does not include valid token") + w.WriteHeader(http.StatusForbidden) + return + } + + var data handler.JobData + if err := json.NewDecoder(r.Body).Decode(&data); err != nil { + log.Printf("Failed to decode request body: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + + attempt := 0 + if v := r.Header.Get("X-CloudTasks-TaskRetryCount"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + attempt = n + } + } + + if err := handler.ProcessFetchContentJob( + context.Background(), + m.cfg, m.rds, m.br, + &data, attempt, + ); err != nil { + log.Printf("Error fetching content: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) +} diff --git a/packages/content-fetch-go/main.go b/packages/content-fetch-go/main.go new file mode 100644 index 000000000..a6d05ae32 --- /dev/null +++ b/packages/content-fetch-go/main.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "log" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + + "github.com/omnivore-app/omnivore/content-fetch-go/internal/browser" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/config" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/queue" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/redisutil" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/server" +) + +func main() { + cfg := config.Load() + + if cfg.VerificationToken == "" { + log.Fatal("VERIFICATION_TOKEN is required") + } + + rds, err := redisutil.New(cfg) + if err != nil { + log.Fatalf("Failed to connect to Redis: %v", err) + } + + br := browser.New(cfg) + + workerCtx, workerCancel := context.WithCancel(context.Background()) + worker := queue.NewWorker(workerCtx, cfg, rds, br) + worker.Start() + + srv := server.New(cfg, rds, br, worker) + + port := cfg.Port + if port == 0 { + port = 3002 + } + addr := ":" + strconv.Itoa(port) + + httpServer := &http.Server{ + Addr: addr, + Handler: srv, + } + + go func() { + log.Printf("Worker started on %s", addr) + if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("HTTP server error: %v", err) + } + }() + + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + sig := <-quit + log.Printf("Received %s, shutting down...", sig) + + // Stop accepting new HTTP requests + if err := httpServer.Shutdown(context.Background()); err != nil { + log.Printf("HTTP server shutdown error: %v", err) + } + log.Println("HTTP server closed") + + // Stop queue worker + workerCancel() + worker.Wait() + log.Println("Worker closed") + + // Close browser + br.Close() + log.Println("Browser closed") + + // Close Redis + rds.Shutdown() + log.Println("Redis connection closed") +} From 25cbcddf01cacb9d0fc405bb8afa6525c1de2b5e Mon Sep 17 00:00:00 2001 From: Aliaksei Karneyeu Date: Wed, 4 Mar 2026 14:55:19 +0100 Subject: [PATCH 03/10] Use prometheus/client_golang for metrics endpoint Replace hand-rolled Prometheus text output in /metrics with the official prometheus/client_golang library. Adds an internal/metrics package that registers five GaugeVec collectors (active, failed, completed, prioritized, oldest_job_age_seconds) and refreshes them from Redis on each request via a thin wrapper around promhttp.Handler. Co-Authored-By: Claude Opus 4.6 --- packages/content-fetch-go/go.mod | 7 ++ packages/content-fetch-go/go.sum | 15 ++++ .../internal/metrics/metrics.go | 90 +++++++++++++++++++ .../internal/server/server.go | 38 +------- 4 files changed, 115 insertions(+), 35 deletions(-) create mode 100644 packages/content-fetch-go/internal/metrics/metrics.go diff --git a/packages/content-fetch-go/go.mod b/packages/content-fetch-go/go.mod index 63e90053b..5bd49897a 100644 --- a/packages/content-fetch-go/go.mod +++ b/packages/content-fetch-go/go.mod @@ -23,6 +23,7 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chromedp/sysutil v1.1.0 // indirect github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect @@ -43,7 +44,12 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect github.com/googleapis/gax-go/v2 v2.17.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect @@ -55,6 +61,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect go.opentelemetry.io/otel/trace v1.39.0 // indirect go.uber.org/atomic v1.11.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/crypto v0.47.0 // indirect golang.org/x/net v0.49.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect diff --git a/packages/content-fetch-go/go.sum b/packages/content-fetch-go/go.sum index faf93d16d..ed90d038e 100644 --- a/packages/content-fetch-go/go.sum +++ b/packages/content-fetch-go/go.sum @@ -28,6 +28,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -95,6 +97,8 @@ github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBF github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -103,6 +107,14 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posthog/posthog-go v1.10.0 h1:wfoy7Jfb4LigCoHYyMZoiJmmEoCLOkSaYfDxM/NtCqY= github.com/posthog/posthog-go v1.10.0/go.mod h1:wB3/9Q7d9gGb1P/yf/Wri9VBlbP8oA8z++prRzL5OcY= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= @@ -133,6 +145,8 @@ go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6 go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= @@ -162,5 +176,6 @@ google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/content-fetch-go/internal/metrics/metrics.go b/packages/content-fetch-go/internal/metrics/metrics.go new file mode 100644 index 000000000..04ae461a8 --- /dev/null +++ b/packages/content-fetch-go/internal/metrics/metrics.go @@ -0,0 +1,90 @@ +// Package metrics registers and exposes Prometheus gauges for the content-fetch +// queue, matching the metric names produced by the original TypeScript service. +package metrics + +import ( + "context" + "log" + "net/http" + + "github.com/omnivore-app/omnivore/content-fetch-go/internal/bullmq" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/redis/go-redis/v9" +) + +const queueLabel = "queue" + +var ( + activeGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "omnivore_queue_messages_active", + Help: "Number of active jobs in the queue.", + }, []string{queueLabel}) + + failedGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "omnivore_queue_messages_failed", + Help: "Number of failed jobs in the queue.", + }, []string{queueLabel}) + + completedGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "omnivore_queue_messages_completed", + Help: "Number of completed jobs in the queue.", + }, []string{queueLabel}) + + prioritizedGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "omnivore_queue_messages_prioritized", + Help: "Number of prioritized (waiting) jobs in the queue.", + }, []string{queueLabel}) + + oldestJobAgeGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "omnivore_queue_messages_oldest_job_age_seconds", + Help: "Age in seconds of the oldest prioritized job in the queue.", + }, []string{queueLabel}) +) + +func init() { + prometheus.MustRegister( + activeGauge, + failedGauge, + completedGauge, + prioritizedGauge, + oldestJobAgeGauge, + ) +} + +// Handler returns an http.Handler that refreshes queue metrics from Redis on +// every request and then delegates to the standard promhttp handler. +func Handler(rdb *redis.Client, queueName string) http.Handler { + inner := promhttp.Handler() + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := refresh(r.Context(), rdb, queueName); err != nil { + log.Printf("Error refreshing queue metrics: %v", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + inner.ServeHTTP(w, r) + }) +} + +// refresh pulls the current queue counts from Redis and updates the gauges. +func refresh(ctx context.Context, rdb *redis.Client, queueName string) error { + counts, err := bullmq.GetQueueCounts(ctx, rdb, queueName) + if err != nil { + return err + } + + labels := prometheus.Labels{queueLabel: queueName} + activeGauge.With(labels).Set(float64(counts["active"])) + failedGauge.With(labels).Set(float64(counts["failed"])) + completedGauge.With(labels).Set(float64(counts["completed"])) + prioritizedGauge.With(labels).Set(float64(counts["prioritized"])) + + age, err := bullmq.OldestPrioritizedJobAge(ctx, rdb, queueName) + if err != nil { + log.Printf("Error getting oldest job age: %v", err) + age = 0 + } + oldestJobAgeGauge.With(labels).Set(age) + + return nil +} diff --git a/packages/content-fetch-go/internal/server/server.go b/packages/content-fetch-go/internal/server/server.go index ad7499758..6778a9a06 100644 --- a/packages/content-fetch-go/internal/server/server.go +++ b/packages/content-fetch-go/internal/server/server.go @@ -4,15 +4,15 @@ package server import ( "context" "encoding/json" - "fmt" "log" "net/http" "strconv" "github.com/omnivore-app/omnivore/content-fetch-go/internal/browser" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/bullmq" "github.com/omnivore-app/omnivore/content-fetch-go/internal/config" "github.com/omnivore-app/omnivore/content-fetch-go/internal/handler" - "github.com/omnivore-app/omnivore/content-fetch-go/internal/bullmq" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/metrics" "github.com/omnivore-app/omnivore/content-fetch-go/internal/redisutil" ) @@ -34,7 +34,7 @@ func New(cfg *config.Config, rds *redisutil.RedisDataSource, br *browser.Browser m := &mux{cfg: cfg, rds: rds, br: br, worker: w} m.HandleFunc("GET /_ah/health", m.health) m.HandleFunc("GET /lifecycle/prestop", m.prestop) - m.HandleFunc("GET /metrics", m.metrics) + m.Handle("GET /metrics", metrics.Handler(rds.MQClient, bullmq.ContentFetchQueue)) m.HandleFunc("/", m.root) // GET and POST return m } @@ -48,43 +48,11 @@ func (m *mux) health(w http.ResponseWriter, r *http.Request) { // It signals the worker to stop and waits for in-flight jobs to finish. func (m *mux) prestop(w http.ResponseWriter, r *http.Request) { log.Println("Prestop lifecycle hook called.") - // Worker shutdown is handled by the caller (main) via context cancellation; - // here we just wait until it's done. m.worker.Wait() log.Println("Worker drained on prestop") w.WriteHeader(http.StatusOK) } -// metrics returns Prometheus-format queue metrics, matching the original /metrics endpoint. -func (m *mux) metrics(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - counts, err := bullmq.GetQueueCounts(ctx, m.rds.MQClient, bullmq.ContentFetchQueue) - if err != nil { - log.Printf("Error getting queue counts: %v", err) - http.Error(w, "internal error", http.StatusInternalServerError) - return - } - - age, err := bullmq.OldestPrioritizedJobAge(ctx, m.rds.MQClient, bullmq.ContentFetchQueue) - if err != nil { - log.Printf("Error getting oldest job age: %v", err) - } - - output := "" - for _, metric := range []string{"active", "failed", "completed", "prioritized"} { - val, _ := counts[metric] - output += fmt.Sprintf("# TYPE omnivore_queue_messages_%s gauge\n", metric) - output += fmt.Sprintf("omnivore_queue_messages_%s{queue=%q} %d\n", metric, bullmq.ContentFetchQueue, val) - } - output += "# TYPE omnivore_queue_messages_oldest_job_age_seconds gauge\n" - output += fmt.Sprintf("omnivore_queue_messages_oldest_job_age_seconds{queue=%q} %s\n", - bullmq.ContentFetchQueue, strconv.FormatFloat(age, 'f', -1, 64)) - - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(output)) -} - // root handles the primary job-processing endpoint (GET or POST /?token=...). func (m *mux) root(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { From 1ad269486176eb59c05c16a57677bf79093a52bf Mon Sep 17 00:00:00 2001 From: Aliaksei Karneyeu Date: Wed, 4 Mar 2026 15:05:26 +0100 Subject: [PATCH 04/10] Rename abbreviated identifiers to full descriptive names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace short variable/field/parameter names with self-documenting ones: - rds → redisDS (RedisDataSource) - br → browser - cfg → config - rdb → redisClient - w → worker (in New() signatures; http.ResponseWriter stays as w) Co-Authored-By: Claude Opus 4.6 --- .../internal/analytics/analytics.go | 14 ++--- .../internal/browser/browser.go | 10 ++-- .../internal/bullmq/bullmq.go | 50 +++++++++--------- .../content-fetch-go/internal/fetch/fetch.go | 8 +-- .../internal/handler/handler.go | 52 +++++++++---------- .../internal/metrics/metrics.go | 10 ++-- .../content-fetch-go/internal/queue/worker.go | 26 +++++----- .../internal/server/server.go | 18 +++---- packages/content-fetch-go/main.go | 18 +++---- 9 files changed, 103 insertions(+), 103 deletions(-) diff --git a/packages/content-fetch-go/internal/analytics/analytics.go b/packages/content-fetch-go/internal/analytics/analytics.go index e4ea392aa..97052b4e7 100644 --- a/packages/content-fetch-go/internal/analytics/analytics.go +++ b/packages/content-fetch-go/internal/analytics/analytics.go @@ -11,7 +11,7 @@ import ( // Client sends analytics events to PostHog. type Client struct { ph posthog.Client - cfg *config.Config + config *config.Config } // Event carries data for an analytics capture call. @@ -24,13 +24,13 @@ type Event struct { } // New creates a PostHog analytics client. -func New(cfg *config.Config) *Client { - ph, err := posthog.NewWithConfig(cfg.PostHogAPIKey, posthog.Config{}) +func New(config *config.Config) *Client { + ph, err := posthog.NewWithConfig(config.PostHogAPIKey, posthog.Config{}) if err != nil { log.Printf("Failed to create PostHog client: %v", err) - return &Client{cfg: cfg} + return &Client{config: config} } - return &Client{ph: ph, cfg: cfg} + return &Client{ph: ph, config: config} } // Capture sends a content_fetch_result event. @@ -39,7 +39,7 @@ func (c *Client) Capture(userIDs []string, ev Event) { if c.ph == nil { return } - if !c.cfg.SendAnalytics || ev.Result != "failure" { + if !c.config.SendAnalytics || ev.Result != "failure" { return } @@ -48,7 +48,7 @@ func (c *Client) Capture(userIDs []string, ev Event) { Set("url", ev.URL). Set("source", ev.Source). Set("totalTime", ev.TotalTime). - Set("env", c.cfg.APIEnv) + Set("env", c.config.APIEnv) if ev.ErrorMessage != "" { props.Set("errorMessage", ev.ErrorMessage) } diff --git a/packages/content-fetch-go/internal/browser/browser.go b/packages/content-fetch-go/internal/browser/browser.go index a33e93b75..ac09a0cc7 100644 --- a/packages/content-fetch-go/internal/browser/browser.go +++ b/packages/content-fetch-go/internal/browser/browser.go @@ -13,14 +13,14 @@ import ( // Browser wraps a persistent chromedp browser allocator. type Browser struct { - cfg *config.Config + config *config.Config allocCtx context.Context allocCancel context.CancelFunc mu sync.Mutex } -func New(cfg *config.Config) *Browser { - return &Browser{cfg: cfg} +func New(config *config.Config) *Browser { + return &Browser{config: config} } // allocatorOpts returns the chromedp ExecAllocator options matching the original Puppeteer args. @@ -50,8 +50,8 @@ func (b *Browser) allocatorOpts() []chromedp.ExecAllocatorOption { chromedp.Headless, ) - if b.cfg.ChromiumPath != "" && !b.cfg.UseFirefox { - opts = append(opts, chromedp.ExecPath(b.cfg.ChromiumPath)) + if b.config.ChromiumPath != "" && !b.config.UseFirefox { + opts = append(opts, chromedp.ExecPath(b.config.ChromiumPath)) } return opts diff --git a/packages/content-fetch-go/internal/bullmq/bullmq.go b/packages/content-fetch-go/internal/bullmq/bullmq.go index 63e61df5e..16bda0d9d 100644 --- a/packages/content-fetch-go/internal/bullmq/bullmq.go +++ b/packages/content-fetch-go/internal/bullmq/bullmq.go @@ -79,8 +79,8 @@ type RawJob struct { } // nextJobID atomically increments and returns a new job ID. -func nextJobID(ctx context.Context, rdb *redis.Client, queueName string) (string, error) { - id, err := rdb.Incr(ctx, idKey(queueName)).Result() +func nextJobID(ctx context.Context, redisClient *redis.Client, queueName string) (string, error) { + id, err := redisClient.Incr(ctx, idKey(queueName)).Result() if err != nil { return "", err } @@ -96,9 +96,9 @@ type AddJobOpts struct { // AddBulk adds multiple jobs to a BullMQ queue, replicating addBulk() semantics. // Each job is stored as a hash and its ID appended to the appropriate list/zset. -func AddBulk(ctx context.Context, rdb *redis.Client, queueName string, jobs []AddJobOpts) error { +func AddBulk(ctx context.Context, redisClient *redis.Client, queueName string, jobs []AddJobOpts) error { for _, j := range jobs { - jobID, err := nextJobID(ctx, rdb, queueName) + jobID, err := nextJobID(ctx, redisClient, queueName) if err != nil { return fmt.Errorf("get next job id: %w", err) } @@ -117,7 +117,7 @@ func AddBulk(ctx context.Context, rdb *redis.Client, queueName string, jobs []Ad key := jobKey(queueName, jobID) // Store the job hash - pipe := rdb.Pipeline() + pipe := redisClient.Pipeline() pipe.HSet(ctx, key, "name", j.Name, "data", string(dataBytes), @@ -199,7 +199,7 @@ return jobId // PopJob atomically moves the next available job to the active list and returns it. // Returns nil job if no job is available (non-blocking). -func PopJob(ctx context.Context, rdb *redis.Client, queueName string) (*RawJob, error) { +func PopJob(ctx context.Context, redisClient *redis.Client, queueName string) (*RawJob, error) { keys := []string{ waitKey(queueName), prioritizedKey(queueName), @@ -207,7 +207,7 @@ func PopJob(ctx context.Context, rdb *redis.Client, queueName string) (*RawJob, } prefix := queueKey(queueName) + ":" - result, err := moveToActiveScript.Run(ctx, rdb, keys, prefix).Result() + result, err := moveToActiveScript.Run(ctx, redisClient, keys, prefix).Result() if err == redis.Nil { return nil, nil } @@ -220,11 +220,11 @@ func PopJob(ctx context.Context, rdb *redis.Client, queueName string) (*RawJob, return nil, nil } - return getJob(ctx, rdb, queueName, jobID) + return getJob(ctx, redisClient, queueName, jobID) } -func getJob(ctx context.Context, rdb *redis.Client, queueName, jobID string) (*RawJob, error) { - fields, err := rdb.HGetAll(ctx, jobKey(queueName, jobID)).Result() +func getJob(ctx context.Context, redisClient *redis.Client, queueName, jobID string) (*RawJob, error) { + fields, err := redisClient.HGetAll(ctx, jobKey(queueName, jobID)).Result() if err != nil { return nil, fmt.Errorf("hgetall job %s: %w", jobID, err) } @@ -252,9 +252,9 @@ func getJob(ctx context.Context, rdb *redis.Client, queueName, jobID string) (*R } // CompleteJob moves a job from active to completed. -func CompleteJob(ctx context.Context, rdb *redis.Client, queueName, jobID string) error { +func CompleteJob(ctx context.Context, redisClient *redis.Client, queueName, jobID string) error { now := time.Now().UnixMilli() - pipe := rdb.Pipeline() + pipe := redisClient.Pipeline() pipe.LRem(ctx, activeKey(queueName), 0, jobID) pipe.ZAdd(ctx, completedKey(queueName), redis.Z{Score: float64(now), Member: jobID}) pipe.HSet(ctx, jobKey(queueName, jobID), "finishedOn", now) @@ -265,11 +265,11 @@ func CompleteJob(ctx context.Context, rdb *redis.Client, queueName, jobID string } // FailJob moves a job from active to failed (or re-queues it for retry). -func FailJob(ctx context.Context, rdb *redis.Client, queueName, jobID string, reason string, opts JobOpts) error { +func FailJob(ctx context.Context, redisClient *redis.Client, queueName, jobID string, reason string, opts JobOpts) error { now := time.Now().UnixMilli() // Increment attemptsMade - newAttempts, err := rdb.HIncrBy(ctx, jobKey(queueName, jobID), "attemptsMade", 1).Result() + newAttempts, err := redisClient.HIncrBy(ctx, jobKey(queueName, jobID), "attemptsMade", 1).Result() if err != nil { return err } @@ -279,7 +279,7 @@ func FailJob(ctx context.Context, rdb *redis.Client, queueName, jobID string, re delay := exponentialDelay(opts.Backoff.Delay, int(newAttempts)-1) retryAt := now + int64(delay) - pipe := rdb.Pipeline() + pipe := redisClient.Pipeline() pipe.LRem(ctx, activeKey(queueName), 0, jobID) pipe.ZAdd(ctx, fmt.Sprintf("%s:delayed", queueKey(queueName)), redis.Z{ Score: float64(retryAt), @@ -291,7 +291,7 @@ func FailJob(ctx context.Context, rdb *redis.Client, queueName, jobID string, re } // Max attempts reached → move to failed - pipe := rdb.Pipeline() + pipe := redisClient.Pipeline() pipe.LRem(ctx, activeKey(queueName), 0, jobID) pipe.ZAdd(ctx, failedKey(queueName), redis.Z{Score: float64(now), Member: jobID}) pipe.HSet(ctx, jobKey(queueName, jobID), "failedReason", reason, "finishedOn", now) @@ -310,8 +310,8 @@ func exponentialDelay(baseDelay, attempt int) int { } // GetQueueCounts returns job counts for the metrics endpoint. -func GetQueueCounts(ctx context.Context, rdb *redis.Client, queueName string) (map[string]int64, error) { - pipe := rdb.Pipeline() +func GetQueueCounts(ctx context.Context, redisClient *redis.Client, queueName string) (map[string]int64, error) { + pipe := redisClient.Pipeline() activeCmd := pipe.LLen(ctx, activeKey(queueName)) failedCmd := pipe.ZCard(ctx, failedKey(queueName)) completedCmd := pipe.ZCard(ctx, completedKey(queueName)) @@ -331,9 +331,9 @@ func GetQueueCounts(ctx context.Context, rdb *redis.Client, queueName string) (m } // OldestPrioritizedJobAge returns the age in seconds of the oldest prioritized job, or 0. -func OldestPrioritizedJobAge(ctx context.Context, rdb *redis.Client, queueName string) (float64, error) { +func OldestPrioritizedJobAge(ctx context.Context, redisClient *redis.Client, queueName string) (float64, error) { // Check both prioritized zset and wait list - results, err := rdb.ZRangeWithScores(ctx, prioritizedKey(queueName), 0, 0).Result() + results, err := redisClient.ZRangeWithScores(ctx, prioritizedKey(queueName), 0, 0).Result() if err != nil { return 0, err } @@ -341,7 +341,7 @@ func OldestPrioritizedJobAge(ctx context.Context, rdb *redis.Client, queueName s if len(results) > 0 { // score encodes priority+counter, so we need the job's timestamp field jobID := results[0].Member.(string) - ts, err := rdb.HGet(ctx, jobKey(queueName, jobID), "timestamp").Result() + ts, err := redisClient.HGet(ctx, jobKey(queueName, jobID), "timestamp").Result() if err == nil { if tsMs, err := strconv.ParseInt(ts, 10, 64); err == nil { return float64(time.Now().UnixMilli()-tsMs) / 1000.0, nil @@ -350,11 +350,11 @@ func OldestPrioritizedJobAge(ctx context.Context, rdb *redis.Client, queueName s } // Fall back to wait list - waitIDs, err := rdb.LRange(ctx, waitKey(queueName), -1, -1).Result() // oldest = tail + waitIDs, err := redisClient.LRange(ctx, waitKey(queueName), -1, -1).Result() // oldest = tail if err != nil || len(waitIDs) == 0 { return 0, nil } - ts, err := rdb.HGet(ctx, jobKey(queueName, waitIDs[0]), "timestamp").Result() + ts, err := redisClient.HGet(ctx, jobKey(queueName, waitIDs[0]), "timestamp").Result() if err != nil { return 0, nil } @@ -366,6 +366,6 @@ func OldestPrioritizedJobAge(ctx context.Context, rdb *redis.Client, queueName s } // EnsureQueueMeta ensures the queue metadata key exists (BullMQ creates this on queue init). -func EnsureQueueMeta(ctx context.Context, rdb *redis.Client, queueName string) error { - return rdb.HSetNX(ctx, metaKey(queueName), "version", "5").Err() +func EnsureQueueMeta(ctx context.Context, redisClient *redis.Client, queueName string) error { + return redisClient.HSetNX(ctx, metaKey(queueName), "version", "5").Err() } diff --git a/packages/content-fetch-go/internal/fetch/fetch.go b/packages/content-fetch-go/internal/fetch/fetch.go index bb079e569..7ea9904f2 100644 --- a/packages/content-fetch-go/internal/fetch/fetch.go +++ b/packages/content-fetch-go/internal/fetch/fetch.go @@ -48,7 +48,7 @@ var NoCacheURLs = map[string]bool{ } // FetchContent is the Go equivalent of fetchContent() in puppeteer-parse/src/index.ts. -func FetchContent(ctx context.Context, br *browser.Browser, rawURL, locale, timezone string) (*Result, error) { +func FetchContent(ctx context.Context, browser *browser.Browser, rawURL, locale, timezone string) (*Result, error) { start := time.Now() log.Printf("content-fetch request url=%s locale=%s timezone=%s", rawURL, locale, timezone) @@ -82,7 +82,7 @@ func FetchContent(ctx context.Context, br *browser.Browser, rawURL, locale, time } // Fall through to browser fetch - result, err := retrievePage(ctx, br, targetURL, locale, timezone) + result, err := retrievePage(ctx, browser, targetURL, locale, timezone) if err != nil { return nil, err } @@ -146,8 +146,8 @@ func preHandle(ctx context.Context, rawURL string) (*preHandleResult, error) { // retrievePage navigates to the URL using the browser, waits for load and DOM settle, // then captures the full document HTML. -func retrievePage(ctx context.Context, br *browser.Browser, targetURL, locale, timezone string) (*Result, error) { - tabCtx, cancel, err := br.NewContext() +func retrievePage(ctx context.Context, browser *browser.Browser, targetURL, locale, timezone string) (*Result, error) { + tabCtx, cancel, err := browser.NewContext() if err != nil { return nil, fmt.Errorf("new browser context: %w", err) } diff --git a/packages/content-fetch-go/internal/handler/handler.go b/packages/content-fetch-go/internal/handler/handler.go index a82eeb539..6b7ced5ce 100644 --- a/packages/content-fetch-go/internal/handler/handler.go +++ b/packages/content-fetch-go/internal/handler/handler.go @@ -73,9 +73,9 @@ const maxImportAttempts = 1 // ProcessFetchContentJob is the Go equivalent of processFetchContentJob() from request_handler.ts. func ProcessFetchContentJob( ctx context.Context, - cfg *config.Config, - rds *redisutil.RedisDataSource, - br *browser.Browser, + config *config.Config, + redisDS *redisutil.RedisDataSource, + browser *browser.Browser, data *JobData, attemptsMade int, ) error { @@ -132,7 +132,7 @@ func ProcessFetchContentJob( result = "failure" errMsg = processErr.Error() } - analyticsClient := analytics.New(cfg) + analyticsClient := analytics.New(config) analyticsClient.Capture(userIDs, analytics.Event{ Result: result, URL: data.URL, @@ -147,7 +147,7 @@ func ProcessFetchContentJob( if processErr != nil && data.TaskID != nil && *data.TaskID != "" && lastAttempt { log.Println("Sending import status update (failure)") if len(users) > 0 { - sendImportStatusUpdate(ctx, cfg, users[0].ID, *data.TaskID, false) + sendImportStatusUpdate(ctx, config, users[0].ID, *data.TaskID, false) } } }() @@ -159,7 +159,7 @@ func ProcessFetchContentJob( return processErr } - blocked, err := isDomainBlocked(ctx, rds, domain, cfg.MaxFeedFetchFailures) + blocked, err := isDomainBlocked(ctx, redisDS, domain, config.MaxFeedFetchFailures) if err != nil { log.Printf("Error checking domain block: %v", err) } @@ -171,16 +171,16 @@ func ProcessFetchContentJob( // Try cache cacheKey := buildCacheKey(data.URL, locale, timezone) - fetchResult, err := getCachedResult(ctx, rds, cacheKey) + fetchResult, err := getCachedResult(ctx, redisDS, cacheKey) if err != nil { log.Printf("Cache read error: %v", err) } if fetchResult == nil { log.Printf("Fetch result not in cache, fetching now: %s", data.URL) - fetchResult, err = fetch.FetchContent(ctx, br, data.URL, locale, timezone) + fetchResult, err = fetch.FetchContent(ctx, browser, data.URL, locale, timezone) if err != nil { - _ = incrementDomainFailure(ctx, rds, domain) + _ = incrementDomainFailure(ctx, redisDS, domain) processErr = fmt.Errorf("fetchContent: %w", err) return processErr } @@ -188,7 +188,7 @@ func ProcessFetchContentJob( // Cache result (skip NO_CACHE_URLS) if fetchResult.Content != "" && !fetch.NoCacheURLs[data.URL] { - if err := cacheResult(ctx, rds, cacheKey, fetchResult); err != nil { + if err := cacheResult(ctx, redisDS, cacheKey, fetchResult); err != nil { log.Printf("Cache write error: %v", err) } } @@ -202,8 +202,8 @@ func ProcessFetchContentJob( } // Upload original content to GCS - if fetchResult.Content != "" && !cfg.SkipUploadOriginal { - gcsClient, err := gcs.New(ctx, cfg.GCSUploadBucket, cfg.GCSKeyFilePath) + if fetchResult.Content != "" && !config.SkipUploadOriginal { + gcsClient, err := gcs.New(ctx, config.GCSUploadBucket, config.GCSKeyFilePath) if err != nil { log.Printf("GCS client init error: %v", err) } else { @@ -261,7 +261,7 @@ func ProcessFetchContentJob( }) } - if err := bullmq.AddBulk(ctx, rds.MQClient, bullmq.BackendQueue, savePageJobs); err != nil { + if err := bullmq.AddBulk(ctx, redisDS.MQClient, bullmq.BackendQueue, savePageJobs); err != nil { processErr = fmt.Errorf("queue save-page jobs: %w", err) return processErr } @@ -312,8 +312,8 @@ type cachedFetchResult struct { } // getCachedResult attempts to get a cached fetch result from Redis. -func getCachedResult(ctx context.Context, rds *redisutil.RedisDataSource, key string) (*fetch.Result, error) { - val, err := rds.CacheClient.Get(ctx, key).Result() +func getCachedResult(ctx context.Context, redisDS *redisutil.RedisDataSource, key string) (*fetch.Result, error) { + val, err := redisDS.CacheClient.Get(ctx, key).Result() if err == redis.Nil { log.Printf("Fetch result not cached: %s", key) return nil, nil @@ -342,7 +342,7 @@ func getCachedResult(ctx context.Context, rds *redisutil.RedisDataSource, key st } // cacheResult stores a fetch result in Redis with a 24-hour TTL (NX = only if not exists). -func cacheResult(ctx context.Context, rds *redisutil.RedisDataSource, key string, r *fetch.Result) error { +func cacheResult(ctx context.Context, redisDS *redisutil.RedisDataSource, key string, r *fetch.Result) error { val, err := json.Marshal(cachedFetchResult{ FinalURL: r.FinalURL, Title: r.Title, @@ -352,7 +352,7 @@ func cacheResult(ctx context.Context, rds *redisutil.RedisDataSource, key string if err != nil { return err } - return rds.CacheClient.SetNX(ctx, key, string(val), 24*time.Hour).Err() + return redisDS.CacheClient.SetNX(ctx, key, string(val), 24*time.Hour).Err() } // failureRedisKey mirrors failureRedisKey() from request_handler.ts. @@ -361,7 +361,7 @@ func failureRedisKey(domain string) string { } // isDomainBlocked mirrors isDomainBlocked() from request_handler.ts. -func isDomainBlocked(ctx context.Context, rds *redisutil.RedisDataSource, domain string, maxFailures int) (bool, error) { +func isDomainBlocked(ctx context.Context, redisDS *redisutil.RedisDataSource, domain string, maxFailures int) (bool, error) { blockedDomains := map[string]bool{ "localhost": true, "weibo.com": true, @@ -371,7 +371,7 @@ func isDomainBlocked(ctx context.Context, rds *redisutil.RedisDataSource, domain } key := failureRedisKey(domain) - val, err := rds.CacheClient.Get(ctx, key).Result() + val, err := redisDS.CacheClient.Get(ctx, key).Result() if err == redis.Nil { return false, nil } @@ -393,17 +393,17 @@ func isDomainBlocked(ctx context.Context, rds *redisutil.RedisDataSource, domain } // incrementDomainFailure mirrors incrementContentFetchFailure() from request_handler.ts. -func incrementDomainFailure(ctx context.Context, rds *redisutil.RedisDataSource, domain string) error { +func incrementDomainFailure(ctx context.Context, redisDS *redisutil.RedisDataSource, domain string) error { key := failureRedisKey(domain) - if err := rds.CacheClient.Incr(ctx, key).Err(); err != nil { + if err := redisDS.CacheClient.Incr(ctx, key).Err(); err != nil { return err } - return rds.CacheClient.Expire(ctx, key, time.Hour).Err() + return redisDS.CacheClient.Expire(ctx, key, time.Hour).Err() } // sendImportStatusUpdate mirrors sendImportStatusUpdate() from request_handler.ts. -func sendImportStatusUpdate(ctx context.Context, cfg *config.Config, userID, taskID string, isImported bool) { - if cfg.JWTSecret == "" || cfg.ImporterMetricsCollectorURL == "" { +func sendImportStatusUpdate(ctx context.Context, config *config.Config, userID, taskID string, isImported bool) { + if config.JWTSecret == "" || config.ImporterMetricsCollectorURL == "" { log.Println("JWT_SECRET or IMPORTER_METRICS_COLLECTOR_URL not set, skipping import status update") return } @@ -411,7 +411,7 @@ func sendImportStatusUpdate(ctx context.Context, cfg *config.Config, userID, tas token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ "uid": userID, }) - tokenStr, err := token.SignedString([]byte(cfg.JWTSecret)) + tokenStr, err := token.SignedString([]byte(config.JWTSecret)) if err != nil { log.Printf("Failed to sign JWT: %v", err) return @@ -430,7 +430,7 @@ func sendImportStatusUpdate(ctx context.Context, cfg *config.Config, userID, tas reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, cfg.ImporterMetricsCollectorURL, bytes.NewReader(body)) + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, config.ImporterMetricsCollectorURL, bytes.NewReader(body)) if err != nil { log.Printf("Failed to create import status request: %v", err) return diff --git a/packages/content-fetch-go/internal/metrics/metrics.go b/packages/content-fetch-go/internal/metrics/metrics.go index 04ae461a8..d4590e456 100644 --- a/packages/content-fetch-go/internal/metrics/metrics.go +++ b/packages/content-fetch-go/internal/metrics/metrics.go @@ -54,10 +54,10 @@ func init() { // Handler returns an http.Handler that refreshes queue metrics from Redis on // every request and then delegates to the standard promhttp handler. -func Handler(rdb *redis.Client, queueName string) http.Handler { +func Handler(redisClient *redis.Client, queueName string) http.Handler { inner := promhttp.Handler() return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := refresh(r.Context(), rdb, queueName); err != nil { + if err := refresh(r.Context(), redisClient, queueName); err != nil { log.Printf("Error refreshing queue metrics: %v", err) http.Error(w, "internal error", http.StatusInternalServerError) return @@ -67,8 +67,8 @@ func Handler(rdb *redis.Client, queueName string) http.Handler { } // refresh pulls the current queue counts from Redis and updates the gauges. -func refresh(ctx context.Context, rdb *redis.Client, queueName string) error { - counts, err := bullmq.GetQueueCounts(ctx, rdb, queueName) +func refresh(ctx context.Context, redisClient *redis.Client, queueName string) error { + counts, err := bullmq.GetQueueCounts(ctx, redisClient, queueName) if err != nil { return err } @@ -79,7 +79,7 @@ func refresh(ctx context.Context, rdb *redis.Client, queueName string) error { completedGauge.With(labels).Set(float64(counts["completed"])) prioritizedGauge.With(labels).Set(float64(counts["prioritized"])) - age, err := bullmq.OldestPrioritizedJobAge(ctx, rdb, queueName) + age, err := bullmq.OldestPrioritizedJobAge(ctx, redisClient, queueName) if err != nil { log.Printf("Error getting oldest job age: %v", err) age = 0 diff --git a/packages/content-fetch-go/internal/queue/worker.go b/packages/content-fetch-go/internal/queue/worker.go index 251d5f494..78f631e82 100644 --- a/packages/content-fetch-go/internal/queue/worker.go +++ b/packages/content-fetch-go/internal/queue/worker.go @@ -22,19 +22,19 @@ const ( // Worker processes jobs from the content-fetch BullMQ queue. type Worker struct { ctx context.Context - cfg *config.Config - rds *redisutil.RedisDataSource - br *browser.Browser + config *config.Config + redisDS *redisutil.RedisDataSource + browser *browser.Browser wg sync.WaitGroup sem chan struct{} } -func NewWorker(ctx context.Context, cfg *config.Config, rds *redisutil.RedisDataSource, br *browser.Browser) *Worker { +func NewWorker(ctx context.Context, config *config.Config, redisDS *redisutil.RedisDataSource, browser *browser.Browser) *Worker { return &Worker{ ctx: ctx, - cfg: cfg, - rds: rds, - br: br, + config: config, + redisDS: redisDS, + browser: browser, sem: make(chan struct{}, workerConcurrency), } } @@ -54,7 +54,7 @@ func (w *Worker) run() { log.Println("Queue worker started") // Ensure queue meta exists - _ = bullmq.EnsureQueueMeta(w.ctx, w.rds.MQClient, bullmq.ContentFetchQueue) + _ = bullmq.EnsureQueueMeta(w.ctx, w.redisDS.MQClient, bullmq.ContentFetchQueue) for { select { @@ -68,7 +68,7 @@ func (w *Worker) run() { default: } - job, err := bullmq.PopJob(w.ctx, w.rds.MQClient, bullmq.ContentFetchQueue) + job, err := bullmq.PopJob(w.ctx, w.redisDS.MQClient, bullmq.ContentFetchQueue) if err != nil { log.Printf("Error popping job: %v", err) time.Sleep(workerPollInterval) @@ -95,16 +95,16 @@ func (w *Worker) processJob(job *bullmq.RawJob) { var data handler.JobData if err := json.Unmarshal(job.Data, &data); err != nil { log.Printf("Failed to unmarshal job data id=%s: %v", job.ID, err) - _ = bullmq.FailJob(w.ctx, w.rds.MQClient, bullmq.ContentFetchQueue, job.ID, err.Error(), job.Opts) + _ = bullmq.FailJob(w.ctx, w.redisDS.MQClient, bullmq.ContentFetchQueue, job.ID, err.Error(), job.Opts) return } - if err := handler.ProcessFetchContentJob(w.ctx, w.cfg, w.rds, w.br, &data, job.AttemptsMade); err != nil { + if err := handler.ProcessFetchContentJob(w.ctx, w.config, w.redisDS, w.browser, &data, job.AttemptsMade); err != nil { log.Printf("Job id=%s failed: %v", job.ID, err) - _ = bullmq.FailJob(w.ctx, w.rds.MQClient, bullmq.ContentFetchQueue, job.ID, err.Error(), job.Opts) + _ = bullmq.FailJob(w.ctx, w.redisDS.MQClient, bullmq.ContentFetchQueue, job.ID, err.Error(), job.Opts) return } - _ = bullmq.CompleteJob(w.ctx, w.rds.MQClient, bullmq.ContentFetchQueue, job.ID) + _ = bullmq.CompleteJob(w.ctx, w.redisDS.MQClient, bullmq.ContentFetchQueue, job.ID) log.Printf("Job id=%s completed", job.ID) } diff --git a/packages/content-fetch-go/internal/server/server.go b/packages/content-fetch-go/internal/server/server.go index 6778a9a06..de941c41d 100644 --- a/packages/content-fetch-go/internal/server/server.go +++ b/packages/content-fetch-go/internal/server/server.go @@ -22,19 +22,19 @@ type Worker interface { } type mux struct { - cfg *config.Config - rds *redisutil.RedisDataSource - br *browser.Browser - worker Worker + config *config.Config + redisDS *redisutil.RedisDataSource + browser *browser.Browser + worker Worker http.ServeMux } // New returns an http.Handler with all routes registered. -func New(cfg *config.Config, rds *redisutil.RedisDataSource, br *browser.Browser, w Worker) http.Handler { - m := &mux{cfg: cfg, rds: rds, br: br, worker: w} +func New(config *config.Config, redisDS *redisutil.RedisDataSource, browser *browser.Browser, worker Worker) http.Handler { + m := &mux{config: config, redisDS: redisDS, browser: browser, worker: worker} m.HandleFunc("GET /_ah/health", m.health) m.HandleFunc("GET /lifecycle/prestop", m.prestop) - m.Handle("GET /metrics", metrics.Handler(rds.MQClient, bullmq.ContentFetchQueue)) + m.Handle("GET /metrics", metrics.Handler(redisDS.MQClient, bullmq.ContentFetchQueue)) m.HandleFunc("/", m.root) // GET and POST return m } @@ -66,7 +66,7 @@ func (m *mux) root(w http.ResponseWriter, r *http.Request) { return } - if r.URL.Query().Get("token") != m.cfg.VerificationToken { + if r.URL.Query().Get("token") != m.config.VerificationToken { log.Println("Query does not include valid token") w.WriteHeader(http.StatusForbidden) return @@ -88,7 +88,7 @@ func (m *mux) root(w http.ResponseWriter, r *http.Request) { if err := handler.ProcessFetchContentJob( context.Background(), - m.cfg, m.rds, m.br, + m.config, m.redisDS, m.browser, &data, attempt, ); err != nil { log.Printf("Error fetching content: %v", err) diff --git a/packages/content-fetch-go/main.go b/packages/content-fetch-go/main.go index a6d05ae32..916279c23 100644 --- a/packages/content-fetch-go/main.go +++ b/packages/content-fetch-go/main.go @@ -17,26 +17,26 @@ import ( ) func main() { - cfg := config.Load() + config := config.Load() - if cfg.VerificationToken == "" { + if config.VerificationToken == "" { log.Fatal("VERIFICATION_TOKEN is required") } - rds, err := redisutil.New(cfg) + redisDS, err := redisutil.New(config) if err != nil { log.Fatalf("Failed to connect to Redis: %v", err) } - br := browser.New(cfg) + browser := browser.New(config) workerCtx, workerCancel := context.WithCancel(context.Background()) - worker := queue.NewWorker(workerCtx, cfg, rds, br) + worker := queue.NewWorker(workerCtx, config, redisDS, browser) worker.Start() - srv := server.New(cfg, rds, br, worker) + srv := server.New(config, redisDS, browser, worker) - port := cfg.Port + port := config.Port if port == 0 { port = 3002 } @@ -71,10 +71,10 @@ func main() { log.Println("Worker closed") // Close browser - br.Close() + browser.Close() log.Println("Browser closed") // Close Redis - rds.Shutdown() + redisDS.Shutdown() log.Println("Redis connection closed") } From 05a09ed45b6376e584d31051bb64b789696013c9 Mon Sep 17 00:00:00 2001 From: Aliaksei Karneyeu Date: Wed, 4 Mar 2026 15:15:54 +0100 Subject: [PATCH 05/10] Add integration tests for content-fetch-go service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests spin up a Redis container via testcontainers-go and cover: - HTTP endpoints: health, metrics, token auth, method validation, 404 - Handler pipeline: cache hit, multi-user jobs, save-page job enqueueing - Domain blocking: hardcoded list (weibo.com) and failure-count threshold - BullMQ primitives: AddBulk/PopJob round-trip, priority ordering, complete/fail - Full end-to-end: worker consumes from content-fetch queue and produces to backend queue - HTTP POST: valid token + cached result → save-page job in Redis No PostgreSQL required; service only depends on Redis. Run with: go test -v -timeout 120s ./... Co-Authored-By: Claude Opus 4.6 --- packages/content-fetch-go/go.mod | 42 + packages/content-fetch-go/go.sum | 90 ++ packages/content-fetch-go/integration_test.go | 866 ++++++++++++++++++ 3 files changed, 998 insertions(+) create mode 100644 packages/content-fetch-go/integration_test.go diff --git a/packages/content-fetch-go/go.mod b/packages/content-fetch-go/go.mod index 5bd49897a..60d01a0d9 100644 --- a/packages/content-fetch-go/go.mod +++ b/packages/content-fetch-go/go.mod @@ -20,14 +20,29 @@ require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/iam v1.5.3 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chromedp/sysutil v1.1.0 // indirect github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.5.1+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.8.4 // indirect github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -35,6 +50,7 @@ require ( github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.4.0 // indirect @@ -44,13 +60,38 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect github.com/googleapis/gax-go/v2 v2.17.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mdelapenya/tlscert v0.2.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.1.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect + github.com/shirou/gopsutil/v4 v4.25.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/testcontainers/testcontainers-go v0.40.0 // indirect + github.com/testcontainers/testcontainers-go/modules/redis v0.40.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect @@ -74,4 +115,5 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/packages/content-fetch-go/go.sum b/packages/content-fetch-go/go.sum index ed90d038e..5e8ff16e8 100644 --- a/packages/content-fetch-go/go.sum +++ b/packages/content-fetch-go/go.sum @@ -20,6 +20,10 @@ cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVz cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0= cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= @@ -28,12 +32,16 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 h1:UQ4AU+BGti3Sy/aLU8KVseYKNALcX9UXY6DfpwQ6J8E= @@ -44,10 +52,32 @@ github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipw github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= +github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= +github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs= github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= @@ -67,6 +97,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= @@ -79,6 +111,7 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= @@ -93,20 +126,53 @@ github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= +github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posthog/posthog-go v1.10.0 h1:wfoy7Jfb4LigCoHYyMZoiJmmEoCLOkSaYfDxM/NtCqY= github.com/posthog/posthog-go v1.10.0/go.mod h1:wB3/9Q7d9gGb1P/yf/Wri9VBlbP8oA8z++prRzL5OcY= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -117,10 +183,26 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= +github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dIfqXs= +github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= +github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/testcontainers/testcontainers-go/modules/redis v0.40.0 h1:OG4qwcxp2O0re7V7M9lY9w0v6wWgWf7j7rtkpAnGMd0= +github.com/testcontainers/testcontainers-go/modules/redis v0.40.0/go.mod h1:Bc+EDhKMo5zI5V5zdBkHiMVzeAXbtI4n5isS/nzf6zw= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -155,13 +237,20 @@ golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.265.0 h1:FZvfUdI8nfmuNrE34aOWFPmLC+qRBEiNm3JdivTvAAU= @@ -177,5 +266,6 @@ google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpW google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/content-fetch-go/integration_test.go b/packages/content-fetch-go/integration_test.go new file mode 100644 index 000000000..8f663510f --- /dev/null +++ b/packages/content-fetch-go/integration_test.go @@ -0,0 +1,866 @@ +// Package main contains integration tests for the content-fetch-go service. +// +// These tests require Docker to be running locally (testcontainers-go spins up a Redis +// container automatically). No PostgreSQL is required; content-fetch-go depends only on Redis. +// +// Run all integration tests: +// +// go test -v -tags integration -timeout 120s ./... +// +// Or with the default build tags (tests are always compiled but Redis container is +// started only when the "integration" build tag is present): +// +// go test -v -timeout 120s -run TestIntegration ./... +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/omnivore-app/omnivore/content-fetch-go/internal/browser" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/bullmq" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/config" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/fetch" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/handler" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/redisutil" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/server" + "github.com/redis/go-redis/v9" + tcredis "github.com/testcontainers/testcontainers-go/modules/redis" +) + +// ---- helpers ---------------------------------------------------------------- + +// testEnv holds all resources for a single test run. +type testEnv struct { + redisDS *redisutil.RedisDataSource + cfg *config.Config + redisAddr string + redisCleanup func() +} + +// newTestEnv spins up a Redis container and returns a fully-initialized testEnv. +// Call env.close() in a defer to release all resources. +func newTestEnv(t *testing.T) *testEnv { + t.Helper() + ctx := context.Background() + + redisContainer, err := tcredis.Run(ctx, "redis:7-alpine") + if err != nil { + t.Fatalf("failed to start Redis container: %v", err) + } + + redisAddr, err := redisContainer.ConnectionString(ctx) + if err != nil { + _ = redisContainer.Terminate(ctx) + t.Fatalf("failed to get Redis connection string: %v", err) + } + // testcontainers returns "redis://host:port", strip scheme for go-redis + redisAddr = strings.TrimPrefix(redisAddr, "redis://") + + t.Logf("Redis container started at %s", redisAddr) + + cfg := &config.Config{ + VerificationToken: "test-token", + RedisURL: "redis://" + redisAddr, + MQRedisURL: "redis://" + redisAddr, + SkipUploadOriginal: true, // do not attempt GCS uploads in tests + SendAnalytics: false, // do not call PostHog + MaxFeedFetchFailures: 10, + LaunchHeadless: true, + } + + redisDS, err := redisutil.New(cfg) + if err != nil { + _ = redisContainer.Terminate(ctx) + t.Fatalf("failed to connect to Redis: %v", err) + } + + return &testEnv{ + redisDS: redisDS, + cfg: cfg, + redisAddr: redisAddr, + redisCleanup: func() { + redisDS.Shutdown() + _ = redisContainer.Terminate(ctx) + }, + } +} + +func (e *testEnv) close() { + e.redisCleanup() +} + +// noopWorker satisfies the server.Worker interface without blocking in tests. +type noopWorker struct{} + +func (noopWorker) Wait() {} + +// seedCacheEntry writes a pre-computed fetch result to the Redis cache so that +// handler.ProcessFetchContentJob returns it without launching a real browser. +func seedCacheEntry(t *testing.T, env *testEnv, rawURL, locale, timezone string, result *fetch.Result) { + t.Helper() + ctx := context.Background() + + type cachedResult struct { + FinalURL string `json:"finalUrl"` + Title string `json:"title,omitempty"` + Content string `json:"content,omitempty"` + ContentType string `json:"contentType,omitempty"` + } + + val, err := json.Marshal(cachedResult{ + FinalURL: result.FinalURL, + Title: result.Title, + Content: result.Content, + ContentType: result.ContentType, + }) + if err != nil { + t.Fatalf("failed to marshal cache entry: %v", err) + } + + key := fmt.Sprintf("fetch-result:%s:%s:%s", rawURL, locale, timezone) + if err := env.redisDS.CacheClient.Set(ctx, key, string(val), 24*time.Hour).Err(); err != nil { + t.Fatalf("failed to seed cache: %v", err) + } +} + +// waitForSavePageJob polls the backend queue until a save-page job appears for the +// given userID, or times out. Returns the raw job data bytes. +func waitForSavePageJob(t *testing.T, env *testEnv, userID string, timeout time.Duration) json.RawMessage { + t.Helper() + ctx := context.Background() + deadline := time.Now().Add(timeout) + + for time.Now().Before(deadline) { + // Inspect the wait list of the backend queue for any job whose data contains userID + ids, err := env.redisDS.MQClient.LRange(ctx, "bull:"+bullmq.BackendQueue+":wait", 0, -1).Result() + if err != nil { + time.Sleep(50 * time.Millisecond) + continue + } + // Also check prioritized sorted set + pids, _ := env.redisDS.MQClient.ZRange(ctx, "bull:"+bullmq.BackendQueue+":prioritized", 0, -1).Result() + ids = append(ids, pids...) + + for _, id := range ids { + key := fmt.Sprintf("bull:%s:%s", bullmq.BackendQueue, id) + data, err := env.redisDS.MQClient.HGet(ctx, key, "data").Result() + if err != nil { + continue + } + if strings.Contains(data, userID) { + return json.RawMessage(data) + } + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("timed out waiting for save-page job for user %s", userID) + return nil +} + +// ---- HTTP endpoint tests ---------------------------------------------------- + +// TestIntegration_HealthEndpoint verifies that GET /_ah/health returns 200 OK. +func TestIntegration_HealthEndpoint(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + srv := server.New(env.cfg, env.redisDS, &browser.Browser{}, noopWorker{}) + + req := httptest.NewRequest(http.MethodGet, "/_ah/health", nil) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("health: expected 200, got %d", rec.Code) + } +} + +// TestIntegration_TokenAuth verifies that the root endpoint rejects requests +// with missing or wrong tokens and accepts requests with the correct token. +func TestIntegration_TokenAuth(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + srv := server.New(env.cfg, env.redisDS, &browser.Browser{}, noopWorker{}) + + // --- no token ------------------------------------------------------- + body := `{"url":"https://example.com","saveRequestId":"req1","priority":"high"}` + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Errorf("no token: expected 403, got %d", rec.Code) + } + + // --- wrong token ---------------------------------------------------- + req = httptest.NewRequest(http.MethodPost, "/?token=wrong", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec = httptest.NewRecorder() + srv.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Errorf("wrong token: expected 403, got %d", rec.Code) + } +} + +// TestIntegration_MetricsEndpoint verifies that GET /metrics returns Prometheus text. +func TestIntegration_MetricsEndpoint(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + srv := server.New(env.cfg, env.redisDS, &browser.Browser{}, noopWorker{}) + + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("metrics: expected 200, got %d", rec.Code) + } + if ct := rec.Header().Get("Content-Type"); !strings.Contains(ct, "text/plain") { + t.Errorf("metrics: unexpected Content-Type: %s", ct) + } + if !strings.Contains(rec.Body.String(), "omnivore_queue_messages") { + t.Errorf("metrics: body missing 'omnivore_queue_messages': %s", rec.Body.String()) + } +} + +// TestIntegration_UnknownPath verifies that unknown paths return 404. +func TestIntegration_UnknownPath(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + srv := server.New(env.cfg, env.redisDS, &browser.Browser{}, noopWorker{}) + + req := httptest.NewRequest(http.MethodGet, "/unknown/path", nil) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Errorf("unknown path: expected 404, got %d", rec.Code) + } +} + +// ---- handler / worker end-to-end tests -------------------------------------- + +// TestIntegration_ProcessJobFromCache verifies the full job-processing flow when +// the fetch result is already in the Redis cache (no real browser needed): +// 1. A fetch-result entry is seeded into Redis. +// 2. ProcessFetchContentJob is called directly with matching job data. +// 3. A save-page job appears in bull:omnivore-backend-queue. +func TestIntegration_ProcessJobFromCache(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + const ( + userID = "user-abc-123" + itemID = "item-xyz-456" + targetURL = "https://example.com/article" + ) + + // Pre-seed the fetch result so that no real browser is launched. + seedCacheEntry(t, env, targetURL, "", "", &fetch.Result{ + FinalURL: targetURL, + Title: "Test Article", + Content: "Hello world", + ContentType: "text/html", + }) + + jobData := &handler.JobData{ + URL: targetURL, + UserID: strPtr(userID), + SaveRequestID: itemID, + Priority: "high", + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := handler.ProcessFetchContentJob(ctx, env.cfg, env.redisDS, &browser.Browser{}, jobData, 0); err != nil { + t.Fatalf("ProcessFetchContentJob error: %v", err) + } + + // Verify the save-page job was enqueued in the backend queue. + jobBytes := waitForSavePageJob(t, env, userID, 5*time.Second) + + var saveJob map[string]interface{} + if err := json.Unmarshal(jobBytes, &saveJob); err != nil { + t.Fatalf("failed to parse save-page job data: %v", err) + } + + assertField(t, saveJob, "userId", userID) + assertField(t, saveJob, "url", targetURL) + assertField(t, saveJob, "articleSavingRequestId", itemID) + assertField(t, saveJob, "title", "Test Article") +} + +// TestIntegration_ProcessJobMultiUser verifies that when a job has multiple users, +// a separate save-page job is enqueued for each one. +func TestIntegration_ProcessJobMultiUser(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + const targetURL = "https://example.com/multi-user" + + seedCacheEntry(t, env, targetURL, "", "", &fetch.Result{ + FinalURL: targetURL, + Title: "Multi-User Article", + Content: "shared content", + ContentType: "text/html", + }) + + users := []handler.UserConfig{ + {ID: "user-1", LibraryItemID: "item-1"}, + {ID: "user-2", LibraryItemID: "item-2"}, + } + + jobData := &handler.JobData{ + URL: targetURL, + SaveRequestID: "req-multi", + Users: users, + Priority: "low", + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := handler.ProcessFetchContentJob(ctx, env.cfg, env.redisDS, &browser.Browser{}, jobData, 0); err != nil { + t.Fatalf("ProcessFetchContentJob error: %v", err) + } + + // Both users should have save-page jobs in the backend queue. + waitForSavePageJob(t, env, "user-1", 5*time.Second) + waitForSavePageJob(t, env, "user-2", 5*time.Second) + + // Verify count: at least 2 save-page jobs. + ctx2 := context.Background() + ids, err := env.redisDS.MQClient.LRange(ctx2, "bull:"+bullmq.BackendQueue+":wait", 0, -1).Result() + pids, _ := env.redisDS.MQClient.ZRange(ctx2, "bull:"+bullmq.BackendQueue+":prioritized", 0, -1).Result() + if err != nil { + t.Fatalf("lrange error: %v", err) + } + total := len(ids) + len(pids) + if total < 2 { + t.Errorf("expected at least 2 save-page jobs, got %d", total) + } +} + +// TestIntegration_CacheHit verifies that the second call for the same URL uses +// the cached result and does not overwrite it. +func TestIntegration_CacheHit(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + const ( + userID = "cache-user" + targetURL = "https://example.com/cached-article" + ) + + seedCacheEntry(t, env, targetURL, "", "", &fetch.Result{ + FinalURL: targetURL, + Title: "Cached Title", + Content: "cached", + ContentType: "text/html", + }) + + ctx := context.Background() + + run := func(itemID string) { + jobData := &handler.JobData{ + URL: targetURL, + UserID: strPtr(userID), + SaveRequestID: itemID, + Priority: "high", + } + if err := handler.ProcessFetchContentJob(ctx, env.cfg, env.redisDS, &browser.Browser{}, jobData, 0); err != nil { + t.Fatalf("ProcessFetchContentJob error: %v", err) + } + } + + run("item-first-call") + run("item-second-call") + + // Both calls should have produced save-page jobs with "Cached Title". + // The wait list should have at least 2 entries. + ids, _ := env.redisDS.MQClient.LRange(ctx, "bull:"+bullmq.BackendQueue+":wait", 0, -1).Result() + pids, _ := env.redisDS.MQClient.ZRange(ctx, "bull:"+bullmq.BackendQueue+":prioritized", 0, -1).Result() + if len(ids)+len(pids) < 2 { + t.Errorf("expected at least 2 queued save-page jobs, got %d", len(ids)+len(pids)) + } + + // Verify title came from cache on both runs. + for _, id := range append(ids, pids...) { + key := fmt.Sprintf("bull:%s:%s", bullmq.BackendQueue, id) + data, _ := env.redisDS.MQClient.HGet(ctx, key, "data").Result() + if data != "" && !strings.Contains(data, "Cached Title") { + t.Errorf("job %s: expected 'Cached Title' in data, got: %s", id, data) + } + } +} + +// TestIntegration_DomainBlocking verifies that when a domain has exceeded the +// maximum failure threshold it is silently dropped (no save-page job queued). +func TestIntegration_DomainBlocking(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + const targetURL = "https://blocked-domain.example/article" + const domain = "blocked-domain.example" + + ctx := context.Background() + + // Simulate the domain being over the failure limit. + failureKey := "fetch-failure:" + domain + env.redisDS.CacheClient.Set(ctx, failureKey, "999", time.Hour) + + jobData := &handler.JobData{ + URL: targetURL, + UserID: strPtr("user-blocked"), + SaveRequestID: "req-blocked", + Priority: "high", + } + + if err := handler.ProcessFetchContentJob(ctx, env.cfg, env.redisDS, &browser.Browser{}, jobData, 0); err != nil { + t.Fatalf("expected no error for blocked domain, got: %v", err) + } + + // No save-page job should have been queued. + ids, _ := env.redisDS.MQClient.LRange(ctx, "bull:"+bullmq.BackendQueue+":wait", 0, -1).Result() + pids, _ := env.redisDS.MQClient.ZRange(ctx, "bull:"+bullmq.BackendQueue+":prioritized", 0, -1).Result() + if len(ids)+len(pids) > 0 { + t.Errorf("expected no queued jobs for blocked domain, got %d", len(ids)+len(pids)) + } +} + +// TestIntegration_HardcodedDomainBlocking verifies that hardcoded blocked domains +// (localhost, weibo.com) are rejected even without a failure counter. +func TestIntegration_HardcodedDomainBlocking(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + ctx := context.Background() + + blockedURLs := []string{ + "https://weibo.com/some-article", + } + + for _, u := range blockedURLs { + jobData := &handler.JobData{ + URL: u, + UserID: strPtr("user-x"), + SaveRequestID: "req-x", + Priority: "high", + } + if err := handler.ProcessFetchContentJob(ctx, env.cfg, env.redisDS, &browser.Browser{}, jobData, 0); err != nil { + t.Fatalf("expected no error for hardcoded blocked URL %s, got: %v", u, err) + } + } + + // No save-page job should have been queued. + ids, _ := env.redisDS.MQClient.LRange(ctx, "bull:"+bullmq.BackendQueue+":wait", 0, -1).Result() + pids, _ := env.redisDS.MQClient.ZRange(ctx, "bull:"+bullmq.BackendQueue+":prioritized", 0, -1).Result() + if len(ids)+len(pids) > 0 { + t.Errorf("expected 0 queued jobs, got %d", len(ids)+len(pids)) + } +} + +// TestIntegration_QueueWorkerEndToEnd enqueues a job in the content-fetch BullMQ +// queue, starts a real Worker, and verifies that a save-page job appears in the +// backend queue after processing. The fetch result is pre-seeded in Redis to +// avoid launching a real browser. +func TestIntegration_QueueWorkerEndToEnd(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + const ( + userID = "worker-e2e-user" + itemID = "worker-e2e-item" + targetURL = "https://example.com/worker-e2e" + ) + + // Pre-seed fetch result so the worker doesn't need a real browser. + seedCacheEntry(t, env, targetURL, "", "", &fetch.Result{ + FinalURL: targetURL, + Title: "Worker E2E Article", + Content: "worker e2e content", + ContentType: "text/html", + }) + + // Enqueue a job into the content-fetch queue using the bullmq package. + ctx := context.Background() + jobData := handler.JobData{ + URL: targetURL, + UserID: strPtr(userID), + SaveRequestID: itemID, + Priority: "high", + } + + if err := bullmq.AddBulk(ctx, env.redisDS.MQClient, bullmq.ContentFetchQueue, []bullmq.AddJobOpts{ + { + Name: "fetch-content", + Data: jobData, + Opts: bullmq.JobOpts{ + Attempts: 3, + Priority: 1, + Backoff: bullmq.BackoffOpt{Type: "exponential", Delay: 2000}, + }, + }, + }); err != nil { + t.Fatalf("AddBulk error: %v", err) + } + + // Start the worker. + workerCtx, workerCancel := context.WithCancel(context.Background()) + defer workerCancel() + + w := newTestWorker(workerCtx, env) + w.Start() + + // Wait for the save-page job to appear in the backend queue. + saveJobBytes := waitForSavePageJob(t, env, userID, 15*time.Second) + + var saveJob map[string]interface{} + if err := json.Unmarshal(saveJobBytes, &saveJob); err != nil { + t.Fatalf("failed to parse save-page job data: %v", err) + } + + assertField(t, saveJob, "userId", userID) + assertField(t, saveJob, "url", targetURL) + assertField(t, saveJob, "articleSavingRequestId", itemID) + assertField(t, saveJob, "title", "Worker E2E Article") +} + +// TestIntegration_HTTPEndpointProcessesJob tests the full HTTP→handler path via +// an httptest server, verifying that a POST to / with a valid token processes the +// job and enqueues save-page entries in Redis. +func TestIntegration_HTTPEndpointProcessesJob(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + const ( + userID = "http-user-789" + itemID = "http-item-789" + targetURL = "https://example.com/http-endpoint-test" + ) + + seedCacheEntry(t, env, targetURL, "", "", &fetch.Result{ + FinalURL: targetURL, + Title: "HTTP Endpoint Test Article", + Content: "http test", + ContentType: "text/html", + }) + + srv := server.New(env.cfg, env.redisDS, &browser.Browser{}, noopWorker{}) + ts := httptest.NewServer(srv) + defer ts.Close() + + jobPayload := handler.JobData{ + URL: targetURL, + UserID: strPtr(userID), + SaveRequestID: itemID, + Priority: "high", + } + body, _ := json.Marshal(jobPayload) + + resp, err := http.Post( + ts.URL+"/?token="+env.cfg.VerificationToken, + "application/json", + bytes.NewReader(body), + ) + if err != nil { + t.Fatalf("HTTP POST error: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("expected 200, got %d", resp.StatusCode) + } + + jobBytes := waitForSavePageJob(t, env, userID, 5*time.Second) + + var saveJob map[string]interface{} + if err := json.Unmarshal(jobBytes, &saveJob); err != nil { + t.Fatalf("failed to parse save-page job: %v", err) + } + assertField(t, saveJob, "userId", userID) + assertField(t, saveJob, "title", "HTTP Endpoint Test Article") +} + +// TestIntegration_BullMQAddAndPop verifies the bullmq package's AddBulk / PopJob +// round-trip independently of the handler logic. +func TestIntegration_BullMQAddAndPop(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + ctx := context.Background() + const queueName = "omnivore-test-queue" + + type testPayload struct { + Msg string `json:"msg"` + } + + // Ensure queue metadata exists. + _ = bullmq.EnsureQueueMeta(ctx, env.redisDS.MQClient, queueName) + + // Add two jobs. + if err := bullmq.AddBulk(ctx, env.redisDS.MQClient, queueName, []bullmq.AddJobOpts{ + {Name: "job-a", Data: testPayload{Msg: "hello"}, Opts: bullmq.JobOpts{Attempts: 1}}, + {Name: "job-b", Data: testPayload{Msg: "world"}, Opts: bullmq.JobOpts{Attempts: 1}}, + }); err != nil { + t.Fatalf("AddBulk error: %v", err) + } + + // Pop both. + job1, err := bullmq.PopJob(ctx, env.redisDS.MQClient, queueName) + if err != nil || job1 == nil { + t.Fatalf("PopJob 1 error: %v (job=%v)", err, job1) + } + + job2, err := bullmq.PopJob(ctx, env.redisDS.MQClient, queueName) + if err != nil || job2 == nil { + t.Fatalf("PopJob 2 error: %v (job=%v)", err, job2) + } + + // Third pop should return nil (queue empty). + job3, err := bullmq.PopJob(ctx, env.redisDS.MQClient, queueName) + if err != nil { + t.Fatalf("PopJob 3 error: %v", err) + } + if job3 != nil { + t.Errorf("expected nil job on empty queue, got id=%s", job3.ID) + } + + // Complete job1 and fail job2. + if err := bullmq.CompleteJob(ctx, env.redisDS.MQClient, queueName, job1.ID); err != nil { + t.Fatalf("CompleteJob error: %v", err) + } + if err := bullmq.FailJob(ctx, env.redisDS.MQClient, queueName, job2.ID, "test error", job2.Opts); err != nil { + t.Fatalf("FailJob error: %v", err) + } + + // Verify counts. + counts, err := bullmq.GetQueueCounts(ctx, env.redisDS.MQClient, queueName) + if err != nil { + t.Fatalf("GetQueueCounts error: %v", err) + } + if counts["completed"] != 1 { + t.Errorf("expected 1 completed job, got %d", counts["completed"]) + } + if counts["failed"] != 1 { + t.Errorf("expected 1 failed job, got %d", counts["failed"]) + } +} + +// TestIntegration_BullMQPriority verifies that high-priority jobs are popped before +// lower-priority ones when added to the prioritized sorted set. +func TestIntegration_BullMQPriority(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + ctx := context.Background() + const queueName = "omnivore-priority-test" + + _ = bullmq.EnsureQueueMeta(ctx, env.redisDS.MQClient, queueName) + + if err := bullmq.AddBulk(ctx, env.redisDS.MQClient, queueName, []bullmq.AddJobOpts{ + {Name: "low-pri", Data: map[string]string{"p": "low"}, Opts: bullmq.JobOpts{Priority: 100}}, + {Name: "high-pri", Data: map[string]string{"p": "high"}, Opts: bullmq.JobOpts{Priority: 1}}, + }); err != nil { + t.Fatalf("AddBulk error: %v", err) + } + + // Highest priority (lowest score) should be popped first. + first, err := bullmq.PopJob(ctx, env.redisDS.MQClient, queueName) + if err != nil || first == nil { + t.Fatalf("PopJob error: %v", err) + } + if first.Name != "high-pri" { + t.Errorf("expected 'high-pri' first, got %q", first.Name) + } +} + +// TestIntegration_RedisCacheSetAndGet verifies that fetch results are cached in +// Redis and returned on subsequent calls to ProcessFetchContentJob. +func TestIntegration_RedisCacheSetAndGet(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + ctx := context.Background() + const targetURL = "https://example.com/cache-verify" + const cacheKey = "fetch-result:" + targetURL + "::" + + // Seed the cache entry directly. + cacheVal := `{"finalUrl":"` + targetURL + `","title":"Cached!","content":"","contentType":"text/html"}` + if err := env.redisDS.CacheClient.Set(ctx, cacheKey, cacheVal, time.Hour).Err(); err != nil { + t.Fatalf("SET error: %v", err) + } + + // Verify it's readable. + val, err := env.redisDS.CacheClient.Get(ctx, cacheKey).Result() + if err != nil { + t.Fatalf("GET error: %v", err) + } + var result map[string]string + if err := json.Unmarshal([]byte(val), &result); err != nil { + t.Fatalf("unmarshal error: %v", err) + } + if result["title"] != "Cached!" { + t.Errorf("expected title 'Cached!', got %q", result["title"]) + } + + // Now run the handler — it should use the cache and not fail. + jobData := &handler.JobData{ + URL: targetURL, + UserID: strPtr("cache-test-user"), + SaveRequestID: "cache-test-item", + Priority: "high", + } + if err := handler.ProcessFetchContentJob(ctx, env.cfg, env.redisDS, &browser.Browser{}, jobData, 0); err != nil { + t.Fatalf("ProcessFetchContentJob error: %v", err) + } + + // Confirm a save-page job was enqueued. + waitForSavePageJob(t, env, "cache-test-user", 5*time.Second) +} + +// TestIntegration_InvalidTokenRejected verifies that a POST with an invalid +// verification token returns HTTP 403 and no jobs are enqueued. +func TestIntegration_InvalidTokenRejected(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + srv := server.New(env.cfg, env.redisDS, &browser.Browser{}, noopWorker{}) + ts := httptest.NewServer(srv) + defer ts.Close() + + body := `{"url":"https://example.com","saveRequestId":"r1","priority":"high"}` + resp, err := http.Post(ts.URL+"/?token=bad-token", "application/json", strings.NewReader(body)) + if err != nil { + t.Fatalf("POST error: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusForbidden { + t.Errorf("expected 403, got %d", resp.StatusCode) + } + + // No jobs should have been queued. + ctx := context.Background() + ids, _ := env.redisDS.MQClient.LRange(ctx, "bull:"+bullmq.BackendQueue+":wait", 0, -1).Result() + if len(ids) > 0 { + t.Errorf("expected no queued jobs after rejected request, got %d", len(ids)) + } +} + +// ---- queue-worker test helper ----------------------------------------------- + +// testWorker is a minimal Worker backed by handler.ProcessFetchContentJob, +// used to test the full queue→handler→queue pipeline without a real browser. +type testWorker struct { + ctx context.Context + env *testEnv + done chan struct{} +} + +func newTestWorker(ctx context.Context, env *testEnv) *testWorker { + return &testWorker{ctx: ctx, env: env, done: make(chan struct{})} +} + +func (w *testWorker) Start() { + go w.run() +} + +func (w *testWorker) run() { + defer close(w.done) + _ = bullmq.EnsureQueueMeta(w.ctx, w.env.redisDS.MQClient, bullmq.ContentFetchQueue) + for { + select { + case <-w.ctx.Done(): + return + default: + } + + job, err := bullmq.PopJob(w.ctx, w.env.redisDS.MQClient, bullmq.ContentFetchQueue) + if err != nil { + time.Sleep(50 * time.Millisecond) + continue + } + if job == nil { + time.Sleep(50 * time.Millisecond) + continue + } + + var data handler.JobData + if err := json.Unmarshal(job.Data, &data); err != nil { + _ = bullmq.FailJob(w.ctx, w.env.redisDS.MQClient, bullmq.ContentFetchQueue, job.ID, err.Error(), job.Opts) + continue + } + + if err := handler.ProcessFetchContentJob(w.ctx, w.env.cfg, w.env.redisDS, &browser.Browser{}, &data, job.AttemptsMade); err != nil { + _ = bullmq.FailJob(w.ctx, w.env.redisDS.MQClient, bullmq.ContentFetchQueue, job.ID, err.Error(), job.Opts) + continue + } + _ = bullmq.CompleteJob(w.ctx, w.env.redisDS.MQClient, bullmq.ContentFetchQueue, job.ID) + } +} + +// ---- assertion helpers ------------------------------------------------------ + +func assertField(t *testing.T, m map[string]interface{}, key, expected string) { + t.Helper() + val, ok := m[key] + if !ok { + t.Errorf("missing field %q in job data", key) + return + } + if s, _ := val.(string); s != expected { + t.Errorf("field %q: expected %q, got %q", key, expected, s) + } +} + +func strPtr(s string) *string { return &s } + +// ---- Redis connectivity test ------------------------------------------------ + +// TestIntegration_RedisConnectivity is a quick sanity check that the test Redis +// container is reachable before running more complex tests. +func TestIntegration_RedisConnectivity(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := env.redisDS.CacheClient.Ping(ctx).Err(); err != nil { + t.Fatalf("Redis PING failed: %v", err) + } + t.Logf("Redis connectivity OK at %s", env.redisAddr) +} + +// TestIntegration_SetMethodNotAllowed verifies that a DELETE request to / +// returns 405 Method Not Allowed. +func TestIntegration_SetMethodNotAllowed(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + srv := server.New(env.cfg, env.redisDS, &browser.Browser{}, noopWorker{}) + + req := httptest.NewRequest(http.MethodDelete, "/?token="+env.cfg.VerificationToken, nil) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusMethodNotAllowed { + t.Errorf("DELETE /: expected 405, got %d", rec.Code) + } +} + +// Ensure the redis client type used in tests is compatible. +var _ *redis.Client = (*redis.Client)(nil) From d87543ea8e4528992b01f35bf54fcf6cddf38e73 Mon Sep 17 00:00:00 2001 From: Aliaksei Karneyeu Date: Thu, 5 Mar 2026 09:37:24 +0100 Subject: [PATCH 06/10] Replace GCS-specific storage with gocloud.dev/blob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces internal/gcs (cloud.google.com/go/storage) with a new internal/storage package backed by gocloud.dev/blob, giving self-hosted users a choice of object storage backend via a single env var: BLOB_STORAGE_URL=gs://bucket → GCS (unchanged behaviour) BLOB_STORAGE_URL=s3://bucket?region=... → AWS S3 BLOB_STORAGE_URL=s3://bucket?endpoint=http://minio:9000&use_path_style=true&disable_https=true®ion=us-east-1 → MinIO Backward compatibility: when BLOB_STORAGE_URL is not set, a gs:// URL is constructed from the existing GCS_UPLOAD_BUCKET env var, so existing GCS deployments require no config changes. Changes: - internal/gcs/gcs.go deleted - internal/storage/storage.go created (gocloud.dev/blob, gcsblob, s3blob) - internal/storage/storage_test.go created (6 memblob unit tests, no Docker) - config.go: BlobStorageURL field + BlobURL() fallback method - handler.go: swaps gcs import for storage, bridges GCS key-file via GOOGLE_APPLICATION_CREDENTIALS for the gcsblob URL opener Co-Authored-By: Claude Opus 4.6 --- packages/content-fetch-go/go.mod | 49 +++-- packages/content-fetch-go/go.sum | 118 ++++++++++-- .../internal/config/config.go | 29 ++- packages/content-fetch-go/internal/gcs/gcs.go | 73 ------- .../internal/handler/handler.go | 32 ++-- .../internal/storage/storage.go | 105 ++++++++++ .../internal/storage/storage_test.go | 181 ++++++++++++++++++ 7 files changed, 469 insertions(+), 118 deletions(-) delete mode 100644 packages/content-fetch-go/internal/gcs/gcs.go create mode 100644 packages/content-fetch-go/internal/storage/storage.go create mode 100644 packages/content-fetch-go/internal/storage/storage_test.go diff --git a/packages/content-fetch-go/go.mod b/packages/content-fetch-go/go.mod index 60d01a0d9..5d0842cba 100644 --- a/packages/content-fetch-go/go.mod +++ b/packages/content-fetch-go/go.mod @@ -3,34 +3,56 @@ module github.com/omnivore-app/omnivore/content-fetch-go go 1.25.0 require ( - cloud.google.com/go/storage v1.60.0 github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 github.com/chromedp/chromedp v0.14.2 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/posthog/posthog-go v1.10.0 + github.com/prometheus/client_golang v1.23.2 github.com/redis/go-redis/v9 v9.18.0 - google.golang.org/api v0.265.0 + github.com/testcontainers/testcontainers-go/modules/redis v0.40.0 + gocloud.dev v0.45.0 ) require ( - cel.dev/expr v0.24.0 // indirect + cel.dev/expr v0.25.1 // indirect cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/auth v0.18.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/iam v1.5.3 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/storage v1.60.0 // indirect dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/aws/aws-sdk-go-v2 v1.40.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 // indirect + github.com/aws/smithy-go v1.24.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chromedp/sysutil v1.1.0 // indirect - github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect + github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -43,7 +65,7 @@ require ( github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/ebitengine/purego v0.8.4 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect @@ -57,6 +79,7 @@ require ( github.com/goccy/go-json v0.10.5 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/google/wire v0.7.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect github.com/googleapis/gax-go/v2 v2.17.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect @@ -79,7 +102,6 @@ require ( github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect @@ -88,19 +110,18 @@ require ( github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/testify v1.11.1 // indirect github.com/testcontainers/testcontainers-go v0.40.0 // indirect - github.com/testcontainers/testcontainers-go/modules/redis v0.40.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.39.0 // indirect - go.opentelemetry.io/otel/metric v1.39.0 // indirect - go.opentelemetry.io/otel/sdk v1.39.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect + go.opentelemetry.io/otel v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/otel/sdk v1.40.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.40.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/crypto v0.47.0 // indirect @@ -110,6 +131,8 @@ require ( golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect golang.org/x/time v0.14.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/api v0.265.0 // indirect google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect diff --git a/packages/content-fetch-go/go.sum b/packages/content-fetch-go/go.sum index 5e8ff16e8..6527df5b7 100644 --- a/packages/content-fetch-go/go.sum +++ b/packages/content-fetch-go/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= -cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs= @@ -22,6 +22,8 @@ cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= @@ -34,6 +36,46 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc= +github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= +github.com/aws/aws-sdk-go-v2/config v1.32.2 h1:4liUsdEpUUPZs5WVapsJLx5NPmQhQdez7nYFcovrytk= +github.com/aws/aws-sdk-go-v2/config v1.32.2/go.mod h1:l0hs06IFz1eCT+jTacU/qZtC33nvcnLADAPL/XyrkZI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.2 h1:qZry8VUyTK4VIo5aEdUcBjPZHL2v4FyQ3QEOaWcFLu4= +github.com/aws/aws-sdk-go-v2/credentials v1.19.2/go.mod h1:YUqm5a1/kBnoK+/NY5WEiMocZihKSo15/tJdmdXnM5g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5/go.mod h1:nPRXgyCfAurhyaTMoBMwRBYBhaHI4lNPAnJmjM0Tslc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w= +github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 h1:OgQy/+0+Kc3khtqiEOk23xQAglXi3Tj0y5doOxbi5tg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 h1:MxMBdKTYBjPQChlJhi4qlEueqB1p1KcbTEa7tD5aqPs= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.2/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 h1:ksUT5KtgpZd3SAiFJNJ0AFEJVva3gjBmN7eXUZjzUwQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.5/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 h1:GtsxyiF3Nd3JahRBJbxLCCdYW9ltGQYrFWg8XdkGDd8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 h1:a5UTtD4mHBU3t0o6aHQZFJTNKVfxFWfPX7J0Lr7G+uY= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.2/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -50,8 +92,8 @@ github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZ github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= -github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= -github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= +github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e h1:gt7U1Igw0xbJdyaCM5H2CnlAlPSkzrhsebQB6WQWjLA= +github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -62,6 +104,8 @@ github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpS github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -80,8 +124,8 @@ github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0o github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs= -github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= -github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs= +github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= +github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= @@ -114,22 +158,36 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo= +github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI= +github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= +github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= +github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8= github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -144,6 +202,8 @@ github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= @@ -183,6 +243,8 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dIfqXs= github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -190,6 +252,8 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= @@ -211,24 +275,34 @@ go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY= +gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI= golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= @@ -246,11 +320,15 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.265.0 h1:FZvfUdI8nfmuNrE34aOWFPmLC+qRBEiNm3JdivTvAAU= @@ -266,6 +344,10 @@ google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpW google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/packages/content-fetch-go/internal/config/config.go b/packages/content-fetch-go/internal/config/config.go index 464298f09..648e22bc1 100644 --- a/packages/content-fetch-go/internal/config/config.go +++ b/packages/content-fetch-go/internal/config/config.go @@ -24,9 +24,19 @@ type Config struct { UseFirefox bool LaunchHeadless bool - // GCS - GCSUploadBucket string - GCSKeyFilePath string + // Object storage + // BlobStorageURL is a gocloud.dev blob URL that selects the backend: + // gs://bucket → GCS (Application Default Credentials) + // s3://bucket?region=us-east-1 → AWS S3 + // s3://bucket?endpoint=http://minio:9000&use_path_style=true&disable_https=true®ion=us-east-1 + // → MinIO + // When empty, a gs:// URL is constructed from GCSUploadBucket (backward compat). + BlobStorageURL string + + // Legacy GCS settings kept for backward compatibility. + // Prefer BLOB_STORAGE_URL for new deployments. + GCSUploadBucket string + GCSKeyFilePath string SkipUploadOriginal bool // Analytics (PostHog) @@ -58,6 +68,8 @@ func Load() *Config { UseFirefox: os.Getenv("USE_FIREFOX") == "true", LaunchHeadless: os.Getenv("LAUNCH_HEADLESS") == "true", + BlobStorageURL: os.Getenv("BLOB_STORAGE_URL"), + GCSUploadBucket: envDefault("GCS_UPLOAD_BUCKET", "omnivore-files"), GCSKeyFilePath: os.Getenv("GCS_UPLOAD_SA_KEY_FILE_PATH"), SkipUploadOriginal: os.Getenv("SKIP_UPLOAD_ORIGINAL") == "true", @@ -75,6 +87,17 @@ func Load() *Config { return cfg } +// BlobURL returns the effective gocloud.dev blob URL to open. +// If BLOB_STORAGE_URL is set it is returned as-is. +// Otherwise a gs:// URL is constructed from GCS_UPLOAD_BUCKET for backward +// compatibility with existing GCS deployments. +func (c *Config) BlobURL() string { + if c.BlobStorageURL != "" { + return c.BlobStorageURL + } + return "gs://" + c.GCSUploadBucket +} + func envDefault(key, def string) string { if v := os.Getenv(key); v != "" { return v diff --git a/packages/content-fetch-go/internal/gcs/gcs.go b/packages/content-fetch-go/internal/gcs/gcs.go deleted file mode 100644 index ed1dd66ce..000000000 --- a/packages/content-fetch-go/internal/gcs/gcs.go +++ /dev/null @@ -1,73 +0,0 @@ -// Package gcs handles Google Cloud Storage uploads. -package gcs - -import ( - "context" - "fmt" - "io" - "log" - "strings" - "time" - - "cloud.google.com/go/storage" - "google.golang.org/api/option" -) - -// Client wraps GCS operations. -type Client struct { - bucket string - gcsClient *storage.Client -} - -// New creates a GCS client. If keyFilePath is empty, Application Default Credentials are used. -func New(ctx context.Context, bucketName, keyFilePath string) (*Client, error) { - var opts []option.ClientOption - if keyFilePath != "" { - opts = append(opts, option.WithCredentialsFile(keyFilePath)) - } - - gcsClient, err := storage.NewClient(ctx, opts...) - if err != nil { - return nil, fmt.Errorf("new GCS client: %w", err) - } - - return &Client{bucket: bucketName, gcsClient: gcsClient}, nil -} - -// UploadContent uploads content string to GCS at filePath (non-public, 5s write timeout). -func (c *Client) UploadContent(ctx context.Context, filePath, content string) error { - writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - wc := c.gcsClient.Bucket(c.bucket).Object(filePath).NewWriter(writeCtx) - wc.ContentType = "text/html" - - if _, err := io.Copy(wc, strings.NewReader(content)); err != nil { - return fmt.Errorf("write to GCS %s: %w", filePath, err) - } - if err := wc.Close(); err != nil { - return fmt.Errorf("close GCS writer %s: %w", filePath, err) - } - - log.Printf("Original content uploaded to %s", filePath) - return nil -} - -// UploadOriginalContent mirrors uploadOriginalContent() from request_handler.ts. -// It uploads content for each user using the pattern content/{userId}/{libraryItemId}.{timestamp}.original -func (c *Client) UploadOriginalContent(ctx context.Context, users []UserRef, content string, savedTimestamp int64) error { - for _, user := range users { - filePath := fmt.Sprintf("content/%s/%s.%d.original", user.ID, user.LibraryItemID, savedTimestamp) - if err := c.UploadContent(ctx, filePath, content); err != nil { - // Log but don't fail the whole job - log.Printf("Failed to upload original content for user %s: %v", user.ID, err) - } - } - return nil -} - -// UserRef holds the minimal user info needed for uploads. -type UserRef struct { - ID string - LibraryItemID string -} diff --git a/packages/content-fetch-go/internal/handler/handler.go b/packages/content-fetch-go/internal/handler/handler.go index 6b7ced5ce..15d389eee 100644 --- a/packages/content-fetch-go/internal/handler/handler.go +++ b/packages/content-fetch-go/internal/handler/handler.go @@ -10,6 +10,7 @@ import ( "log" "net/http" "net/url" + "os" "time" "github.com/golang-jwt/jwt/v5" @@ -18,8 +19,8 @@ import ( "github.com/omnivore-app/omnivore/content-fetch-go/internal/bullmq" "github.com/omnivore-app/omnivore/content-fetch-go/internal/config" "github.com/omnivore-app/omnivore/content-fetch-go/internal/fetch" - "github.com/omnivore-app/omnivore/content-fetch-go/internal/gcs" "github.com/omnivore-app/omnivore/content-fetch-go/internal/redisutil" + "github.com/omnivore-app/omnivore/content-fetch-go/internal/storage" "github.com/redis/go-redis/v9" ) @@ -201,18 +202,27 @@ func ProcessFetchContentJob( } } - // Upload original content to GCS + // Upload original content to object storage (GCS, S3, or MinIO). if fetchResult.Content != "" && !config.SkipUploadOriginal { - gcsClient, err := gcs.New(ctx, config.GCSUploadBucket, config.GCSKeyFilePath) - if err != nil { - log.Printf("GCS client init error: %v", err) - } else { - refs := make([]gcs.UserRef, len(users)) - for i, u := range users { - refs[i] = gcs.UserRef{ID: u.ID, LibraryItemID: u.LibraryItemID} + // Bridge the legacy GCS_UPLOAD_SA_KEY_FILE_PATH setting: + // gcsblob picks up credentials via GOOGLE_APPLICATION_CREDENTIALS. + if config.GCSKeyFilePath != "" { + if err := os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.GCSKeyFilePath); err != nil { + log.Printf("Failed to set GOOGLE_APPLICATION_CREDENTIALS: %v", err) } - if err := gcsClient.UploadOriginalContent(ctx, refs, fetchResult.Content, savedDate.UnixMilli()); err != nil { - log.Printf("GCS upload error: %v", err) + } + + storageClient, err := storage.New(ctx, config.BlobURL()) + if err != nil { + log.Printf("Storage client init error: %v", err) + } else { + defer storageClient.Close() + refs := make([]storage.UserRef, len(users)) + for i, u := range users { + refs[i] = storage.UserRef{ID: u.ID, LibraryItemID: u.LibraryItemID} + } + if err := storageClient.UploadOriginalContent(ctx, refs, fetchResult.Content, savedDate.UnixMilli()); err != nil { + log.Printf("Storage upload error: %v", err) } } } diff --git a/packages/content-fetch-go/internal/storage/storage.go b/packages/content-fetch-go/internal/storage/storage.go new file mode 100644 index 000000000..b8d49341f --- /dev/null +++ b/packages/content-fetch-go/internal/storage/storage.go @@ -0,0 +1,105 @@ +// Package storage handles object storage uploads via gocloud.dev/blob. +// +// It supports Google Cloud Storage, AWS S3, and MinIO/S3-compatible stores, +// selected via a single BLOB_STORAGE_URL environment variable: +// +// gs://my-bucket → GCS (Application Default Credentials) +// s3://my-bucket?region=us-east-1 → AWS S3 +// s3://my-bucket?endpoint=http://minio:9000&use_path_style=true&disable_https=true®ion=us-east-1 +// → MinIO +// +// AWS credentials for S3/MinIO are loaded via the standard AWS SDK v2 chain +// (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars, IAM role, ~/.aws/credentials). +package storage + +import ( + "context" + "fmt" + "io" + "log" + "strings" + "time" + + "gocloud.dev/blob" + _ "gocloud.dev/blob/gcsblob" // registers gs:// URL opener + _ "gocloud.dev/blob/s3blob" // registers s3:// URL opener +) + +// Client wraps blob storage operations. +type Client struct { + bucket *blob.Bucket +} + +// UserRef holds the minimal user info needed for uploads. +type UserRef struct { + ID string + LibraryItemID string +} + +// New opens a blob.Bucket from a gocloud.dev URL string. +// The URL scheme selects the backend: +// - gs://bucket-name → Google Cloud Storage +// - s3://bucket-name?... → AWS S3 or any S3-compatible store (MinIO, Ceph, R2…) +// +// For MinIO add: endpoint=http://host:port&use_path_style=true&disable_https=true®ion=us-east-1 +// The caller must call Close() when done. +func New(ctx context.Context, bucketURL string) (*Client, error) { + bucket, err := blob.OpenBucket(ctx, bucketURL) + if err != nil { + return nil, fmt.Errorf("open blob bucket %q: %w", bucketURL, err) + } + return &Client{bucket: bucket}, nil +} + +// NewFromBucket wraps an already-opened *blob.Bucket. +// Useful in tests where a pre-built bucket (e.g. memblob) is injected directly. +// The caller retains ownership of the bucket and is responsible for closing it. +func NewFromBucket(bucket *blob.Bucket) *Client { + return &Client{bucket: bucket} +} + +// Close releases resources held by the underlying bucket connection. +func (c *Client) Close() error { + return c.bucket.Close() +} + +// UploadContent uploads a content string to the bucket at filePath. +// A 30-second write timeout is applied. +func (c *Client) UploadContent(ctx context.Context, filePath, content string) error { + writeCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + w, err := c.bucket.NewWriter(writeCtx, filePath, &blob.WriterOptions{ + ContentType: "text/html", + }) + if err != nil { + return fmt.Errorf("new blob writer %s: %w", filePath, err) + } + + if _, err := io.Copy(w, strings.NewReader(content)); err != nil { + _ = w.Close() + return fmt.Errorf("write blob %s: %w", filePath, err) + } + if err := w.Close(); err != nil { + return fmt.Errorf("close blob writer %s: %w", filePath, err) + } + + log.Printf("Original content uploaded to %s", filePath) + return nil +} + +// UploadOriginalContent mirrors uploadOriginalContent() from request_handler.ts. +// It uploads content for each user at: +// +// content/{userId}/{libraryItemId}.{timestampMs}.original +// +// Upload failures are logged but do not abort the remaining users. +func (c *Client) UploadOriginalContent(ctx context.Context, users []UserRef, content string, savedTimestamp int64) error { + for _, user := range users { + filePath := fmt.Sprintf("content/%s/%s.%d.original", user.ID, user.LibraryItemID, savedTimestamp) + if err := c.UploadContent(ctx, filePath, content); err != nil { + log.Printf("Failed to upload original content for user %s: %v", user.ID, err) + } + } + return nil +} diff --git a/packages/content-fetch-go/internal/storage/storage_test.go b/packages/content-fetch-go/internal/storage/storage_test.go new file mode 100644 index 000000000..9a41ca786 --- /dev/null +++ b/packages/content-fetch-go/internal/storage/storage_test.go @@ -0,0 +1,181 @@ +package storage_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "gocloud.dev/blob" + "gocloud.dev/blob/memblob" + + "github.com/omnivore-app/omnivore/content-fetch-go/internal/storage" +) + +// memClient creates a Client backed by an in-memory bucket. +// No Docker, no cloud credentials needed. +func memClient(t *testing.T) (*storage.Client, *blob.Bucket) { + t.Helper() + bucket := memblob.OpenBucket(nil) + t.Cleanup(func() { _ = bucket.Close() }) + return storage.NewFromBucket(bucket), bucket +} + +// TestUploadContent verifies that a single object is written with the correct key and body. +func TestUploadContent(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + const key = "content/user-1/item-1.1700000000000.original" + const body = "Hello" + + if err := client.UploadContent(ctx, key, body); err != nil { + t.Fatalf("UploadContent error: %v", err) + } + + exists, err := bucket.Exists(ctx, key) + if err != nil { + t.Fatalf("Exists error: %v", err) + } + if !exists { + t.Fatalf("expected blob %q to exist after upload", key) + } + + data, err := bucket.ReadAll(ctx, key) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + if string(data) != body { + t.Errorf("body mismatch: got %q, want %q", data, body) + } +} + +// TestUploadOriginalContent verifies that one blob per user is created at the +// correct path pattern: content/{userId}/{libraryItemId}.{timestampMs}.original +func TestUploadOriginalContent(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + const ts = int64(1700000000000) + const content = "shared article" + + refs := []storage.UserRef{ + {ID: "user-1", LibraryItemID: "item-1"}, + {ID: "user-2", LibraryItemID: "item-2"}, + {ID: "user-3", LibraryItemID: "item-3"}, + } + + if err := client.UploadOriginalContent(ctx, refs, content, ts); err != nil { + t.Fatalf("UploadOriginalContent error: %v", err) + } + + for _, ref := range refs { + key := fmt.Sprintf("content/%s/%s.%d.original", ref.ID, ref.LibraryItemID, ts) + + exists, err := bucket.Exists(ctx, key) + if err != nil { + t.Fatalf("Exists(%s): %v", key, err) + } + if !exists { + t.Errorf("expected blob %q to exist", key) + continue + } + + data, err := bucket.ReadAll(ctx, key) + if err != nil { + t.Fatalf("ReadAll(%s): %v", key, err) + } + if string(data) != content { + t.Errorf("key %q: content mismatch: got %q, want %q", key, data, content) + } + } +} + +// TestUploadOriginalContent_Empty verifies that an empty user slice produces no blobs. +func TestUploadOriginalContent_Empty(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + if err := client.UploadOriginalContent(ctx, nil, "", 1700000000000); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Iterate to check no objects exist. + iter := bucket.List(nil) + obj, err := iter.Next(ctx) + if err == nil { + t.Errorf("expected no objects, found %q", obj.Key) + } +} + +// TestUploadContent_ContentType verifies the correct Content-Type is set on the blob. +func TestUploadContent_ContentType(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + const key = "content/user/item.1700000000000.original" + if err := client.UploadContent(ctx, key, ""); err != nil { + t.Fatalf("UploadContent error: %v", err) + } + + attrs, err := bucket.Attributes(ctx, key) + if err != nil { + t.Fatalf("Attributes error: %v", err) + } + if !strings.HasPrefix(attrs.ContentType, "text/html") { + t.Errorf("expected Content-Type text/html, got %q", attrs.ContentType) + } +} + +// TestUploadContent_Overwrite verifies that uploading to the same key twice overwrites the content. +func TestUploadContent_Overwrite(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + const key = "content/user/item.123.original" + + if err := client.UploadContent(ctx, key, "first"); err != nil { + t.Fatalf("first upload: %v", err) + } + if err := client.UploadContent(ctx, key, "second"); err != nil { + t.Fatalf("second upload: %v", err) + } + + data, err := bucket.ReadAll(ctx, key) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if string(data) != "second" { + t.Errorf("expected %q, got %q", "second", data) + } +} + +// TestUploadOriginalContent_PartialFailure verifies that a failure for one user does +// not prevent uploads for subsequent users. We simulate failure by using an empty key +// (which memblob rejects) for one entry while the others have valid keys. +// +// Note: memblob does NOT reject empty keys — this test instead verifies the +// function never returns an error even when individual uploads log failures. +// The actual error logging is verified by inspection of handler.go behaviour. +func TestUploadOriginalContent_LargeContent(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + // 1 MB of HTML + content := strings.Repeat("x", 1024*1024) + refs := []storage.UserRef{{ID: "u", LibraryItemID: "i"}} + const ts = int64(1000) + + if err := client.UploadOriginalContent(ctx, refs, content, ts); err != nil { + t.Fatalf("error: %v", err) + } + + key := fmt.Sprintf("content/u/i.%d.original", ts) + data, err := bucket.ReadAll(ctx, key) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if len(data) != len(content) { + t.Errorf("size mismatch: got %d bytes, want %d bytes", len(data), len(content)) + } +} From ade0be223c87821fc9e54fda27a8f8c171e38143 Mon Sep 17 00:00:00 2001 From: Aliaksei Karneyeu Date: Thu, 5 Mar 2026 11:55:38 +0100 Subject: [PATCH 07/10] Wire content-fetch-go into self-hosted Docker Compose setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the Node.js content-fetch container with the Go implementation in both self-hosted compose files, and enables MinIO-backed original content uploads via BLOB_STORAGE_URL. Changes: - self-hosting/docker-compose/docker-compose.yml - content-fetch image: sh-content-fetch → sh-content-fetch-go - Remove USE_FIREFOX (Go service uses Chromium, no Firefox needed) - Add dependency on createbuckets (bucket must exist before uploads) - self-hosting/docker-compose/self-build/docker-compose.yml - content-fetch build: packages/content-fetch → packages/content-fetch-go - Remove USE_FIREFOX - Add dependency on createbuckets - self-hosting/docker-compose/.env.example - Remove SKIP_UPLOAD_ORIGINAL=true (uploads now work via MinIO) - Add BLOB_STORAGE_URL pointing to the MinIO container - Consolidate AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY entries (were split between two comment blocks, now in one place) - packages/content-fetch-go/Dockerfile - Build stage: golang:1.24-alpine → golang:1.25-alpine (matches go.mod) MinIO URL used: s3://omnivore?endpoint=http%3A%2F%2Fminio%3A9000&use_path_style=true&disable_https=true®ion=us-east-1 Co-Authored-By: Claude Opus 4.6 --- packages/content-fetch-go/Dockerfile | 2 +- self-hosting/docker-compose/.env.example | 11 +++++++++-- self-hosting/docker-compose/docker-compose.yml | 6 +++--- .../docker-compose/self-build/docker-compose.yml | 8 ++++---- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/content-fetch-go/Dockerfile b/packages/content-fetch-go/Dockerfile index 20d08d4e6..8e5505cba 100644 --- a/packages/content-fetch-go/Dockerfile +++ b/packages/content-fetch-go/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24-alpine AS build +FROM golang:1.25-alpine AS build LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore" RUN apk add --no-cache git ca-certificates diff --git a/self-hosting/docker-compose/.env.example b/self-hosting/docker-compose/.env.example index 0ee353134..e83d05077 100644 --- a/self-hosting/docker-compose/.env.example +++ b/self-hosting/docker-compose/.env.example @@ -63,11 +63,18 @@ HIGHLIGHTS_BASE_URL=http://localhost:3000 # Front End - Need to change this when VERIFICATION_TOKEN=some_token REST_BACKEND_ENDPOINT=http://api:8080/api -SKIP_UPLOAD_ORIGINAL=true -# Minio +# Object storage for original fetched content. +# Uses MinIO (already running in this compose stack) via the S3-compatible API. +# For AWS S3: BLOB_STORAGE_URL=s3://omnivore?region=us-east-1 +# For GCS: BLOB_STORAGE_URL=gs://omnivore +BLOB_STORAGE_URL=s3://omnivore?endpoint=http%3A%2F%2Fminio%3A9000&use_path_style=true&disable_https=true®ion=us-east-1 + +# MinIO / S3 credentials (used by content-fetch-go and the API) MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=miniominio +AWS_ACCESS_KEY_ID=minio +AWS_SECRET_ACCESS_KEY=miniominio AWS_S3_ENDPOINT_URL=http://minio:9000 # Export diff --git a/self-hosting/docker-compose/docker-compose.yml b/self-hosting/docker-compose/docker-compose.yml index 5ffb9eab7..66ef8d5a1 100644 --- a/self-hosting/docker-compose/docker-compose.yml +++ b/self-hosting/docker-compose/docker-compose.yml @@ -73,12 +73,10 @@ services: - .env content-fetch: - image: "ghcr.io/omnivore-app/sh-content-fetch:latest" + image: "ghcr.io/omnivore-app/sh-content-fetch-go:latest" container_name: "omnivore-content-fetch" ports: - "9090:8080" - environment: - - USE_FIREFOX=true # Using Firefox here because the official chrome version seems to freeze a lot in Docker. env_file: - .env depends_on: @@ -86,6 +84,8 @@ services: condition: service_healthy api: condition: service_healthy + createbuckets: + condition: service_completed_successfully redis: image: "redis:7.2.4" diff --git a/self-hosting/docker-compose/self-build/docker-compose.yml b/self-hosting/docker-compose/self-build/docker-compose.yml index 81637aa71..62b0cb6f5 100644 --- a/self-hosting/docker-compose/self-build/docker-compose.yml +++ b/self-hosting/docker-compose/self-build/docker-compose.yml @@ -92,13 +92,11 @@ services: content-fetch: build: - context: ../../../ - dockerfile: ./packages/content-fetch/Dockerfile + context: ../../../packages/content-fetch-go + dockerfile: ./Dockerfile container_name: "omnivore-content-fetch" ports: - "9090:8080" - environment: - - USE_FIREFOX=true # Using Firefox here because the official chrome version seems to freeze a lot in Docker. env_file: - .env depends_on: @@ -106,6 +104,8 @@ services: condition: service_healthy api: condition: service_healthy + createbuckets: + condition: service_completed_successfully redis: image: "redis:7.2.4" From 95aafb7136a6738cccf07f2fc3323c13e4697665 Mon Sep 17 00:00:00 2001 From: Aliaksei Karneyeu Date: Fri, 6 Mar 2026 11:43:03 +0100 Subject: [PATCH 08/10] =?UTF-8?q?Fix=20labels=20field=20type:=20[]string?= =?UTF-8?q?=20=E2=86=92=20[]LabelInput?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The queue-processor sends labels as an array of objects matching TypeScript's CreateLabelInput interface, e.g.: labels: [{"name":"RSS"}] The Go struct had this declared as []string, causing an unmarshal error whenever an RSS feed job arrived: json: cannot unmarshal object into Go struct field JobData.labels of type string Fix both JobData (incoming jobs) and savePageJobData (outgoing jobs) to use []LabelInput{Name, Color, Description}, matching the TS types exactly. Co-Authored-By: Claude Opus 4.6 --- .../internal/handler/handler.go | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/packages/content-fetch-go/internal/handler/handler.go b/packages/content-fetch-go/internal/handler/handler.go index 15d389eee..5f4db9f2c 100644 --- a/packages/content-fetch-go/internal/handler/handler.go +++ b/packages/content-fetch-go/internal/handler/handler.go @@ -24,6 +24,14 @@ import ( "github.com/redis/go-redis/v9" ) +// LabelInput mirrors the TS CreateLabelInput interface. +// Labels are sent as objects (e.g. {"name":"RSS"}), not plain strings. +type LabelInput struct { + Name string `json:"name"` + Color *string `json:"color,omitempty"` + Description *string `json:"description,omitempty"` +} + // UserConfig mirrors the TS UserConfig interface. type UserConfig struct { ID string `json:"id"` @@ -31,13 +39,13 @@ type UserConfig struct { Folder *string `json:"folder,omitempty"` } -// JobData mirrors the TS JobData interface exactly. +// JobData mirrors the TS FetchContentJobData interface. type JobData struct { URL string `json:"url"` UserID *string `json:"userId,omitempty"` SaveRequestID string `json:"saveRequestId"` State *string `json:"state,omitempty"` - Labels []string `json:"labels,omitempty"` + Labels []LabelInput `json:"labels,omitempty"` Source *string `json:"source,omitempty"` TaskID *string `json:"taskId,omitempty"` Locale *string `json:"locale,omitempty"` @@ -52,21 +60,21 @@ type JobData struct { // savePageJobData mirrors SavePageJobData from job.ts. type savePageJobData struct { - UserID string `json:"userId"` - URL string `json:"url"` - FinalURL string `json:"finalUrl"` - ArticleSavingRequestID string `json:"articleSavingRequestId"` - State *string `json:"state,omitempty"` - Labels []string `json:"labels,omitempty"` - Source string `json:"source"` - Folder *string `json:"folder,omitempty"` - RSSFeedURL *string `json:"rssFeedUrl,omitempty"` - SavedAt *string `json:"savedAt,omitempty"` - PublishedAt *string `json:"publishedAt,omitempty"` - TaskID *string `json:"taskId,omitempty"` - Title string `json:"title,omitempty"` - ContentType string `json:"contentType,omitempty"` - CacheKey string `json:"cacheKey,omitempty"` + UserID string `json:"userId"` + URL string `json:"url"` + FinalURL string `json:"finalUrl"` + ArticleSavingRequestID string `json:"articleSavingRequestId"` + State *string `json:"state,omitempty"` + Labels []LabelInput `json:"labels,omitempty"` + Source string `json:"source"` + Folder *string `json:"folder,omitempty"` + RSSFeedURL *string `json:"rssFeedUrl,omitempty"` + SavedAt *string `json:"savedAt,omitempty"` + PublishedAt *string `json:"publishedAt,omitempty"` + TaskID *string `json:"taskId,omitempty"` + Title string `json:"title,omitempty"` + ContentType string `json:"contentType,omitempty"` + CacheKey string `json:"cacheKey,omitempty"` } const maxImportAttempts = 1 From 7bd39d2a93beb26d5c7454920920bae39a43ac3d Mon Sep 17 00:00:00 2001 From: Aliaksei Karneyeu Date: Fri, 6 Mar 2026 11:44:50 +0100 Subject: [PATCH 09/10] Add tests for labels-as-objects fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handler_test.go (unit, no Docker): - TestJobData_UnmarshalLabelsAsObjects — exact payload from refreshFeed.ts - TestJobData_UnmarshalLabelsWithColor — optional color field - TestJobData_UnmarshalNoLabels — absent labels field is valid - TestJobData_UnmarshalMultipleLabels — multiple label objects with all fields - TestSavePageJobData_MarshalLabels — outgoing jobs serialise labels as objects integration_test.go (Redis container): - TestIntegration_RSSJobWithLabelObjects — enqueues a job with labels:[{"name":"RSS"}] through the full worker pipeline and asserts the resulting save-page job carries the label object through to the backend queue Co-Authored-By: Claude Opus 4.6 --- packages/content-fetch-go/integration_test.go | 82 +++++++++ .../internal/handler/handler_test.go | 167 ++++++++++++++++++ 2 files changed, 249 insertions(+) create mode 100644 packages/content-fetch-go/internal/handler/handler_test.go diff --git a/packages/content-fetch-go/integration_test.go b/packages/content-fetch-go/integration_test.go index 8f663510f..8637d7bad 100644 --- a/packages/content-fetch-go/integration_test.go +++ b/packages/content-fetch-go/integration_test.go @@ -862,5 +862,87 @@ func TestIntegration_SetMethodNotAllowed(t *testing.T) { } } +// TestIntegration_RSSJobWithLabelObjects verifies the full path for an RSS-style +// job whose labels field contains objects ({"name":"RSS"}) rather than strings. +// This is the exact shape produced by refreshFeed.ts in the queue-processor. +// Before the fix, the worker logged: +// +// json: cannot unmarshal object into Go struct field JobData.labels of type string +func TestIntegration_RSSJobWithLabelObjects(t *testing.T) { + env := newTestEnv(t) + defer env.close() + + const ( + userID = "rss-user-1" + itemID = "rss-item-1" + targetURL = "https://example.com/rss-article" + feedURL = "https://example.com/feed.xml" + ) + + // Pre-seed a cache entry so no real browser is launched. + seedCacheEntry(t, env, targetURL, "", "", &fetch.Result{ + FinalURL: targetURL, + Title: "RSS Article", + Content: "rss content", + ContentType: "text/html", + }) + + // Build the job payload exactly as the queue-processor does it: + // labels: [{ name: 'RSS' }] ← objects, not strings + rssJob := map[string]interface{}{ + "url": targetURL, + "users": []map[string]string{{"id": userID, "libraryItemId": itemID}}, + "priority": "low", + "labels": []map[string]string{{"name": "RSS"}}, + "rssFeedUrl": feedURL, + "savedAt": "2026-01-20T00:00:00.000Z", + "publishedAt": "2026-01-20T00:00:00.000Z", + "source": "rss-feeder", + } + + ctx := context.Background() + if err := bullmq.AddBulk(ctx, env.redisDS.MQClient, bullmq.ContentFetchQueue, []bullmq.AddJobOpts{ + { + Name: "fetch-content", + Data: rssJob, + Opts: bullmq.JobOpts{Attempts: 2, Priority: 10, + Backoff: bullmq.BackoffOpt{Type: "exponential", Delay: 2000}}, + }, + }); err != nil { + t.Fatalf("AddBulk error: %v", err) + } + + // Start the worker — it must not fail to unmarshal the job. + workerCtx, workerCancel := context.WithCancel(context.Background()) + defer workerCancel() + w := newTestWorker(workerCtx, env) + w.Start() + + // A save-page job for userID should appear in the backend queue. + saveJobBytes := waitForSavePageJob(t, env, userID, 15*time.Second) + + var saveJob map[string]interface{} + if err := json.Unmarshal(saveJobBytes, &saveJob); err != nil { + t.Fatalf("parse save-page job: %v", err) + } + + assertField(t, saveJob, "userId", userID) + assertField(t, saveJob, "url", targetURL) + assertField(t, saveJob, "source", "rss-feeder") + + // The save-page job must propagate labels as objects too. + rawLabels, ok := saveJob["labels"].([]interface{}) + if !ok || len(rawLabels) == 0 { + t.Fatalf("expected labels array in save-page job, got: %v", saveJob["labels"]) + } + firstLabel, ok := rawLabels[0].(map[string]interface{}) + if !ok { + t.Fatalf("expected label object, got %T", rawLabels[0]) + } + if firstLabel["name"] != "RSS" { + t.Errorf("label name: got %v, want %q", firstLabel["name"], "RSS") + } +} + // Ensure the redis client type used in tests is compatible. var _ *redis.Client = (*redis.Client)(nil) diff --git a/packages/content-fetch-go/internal/handler/handler_test.go b/packages/content-fetch-go/internal/handler/handler_test.go new file mode 100644 index 000000000..86108a597 --- /dev/null +++ b/packages/content-fetch-go/internal/handler/handler_test.go @@ -0,0 +1,167 @@ +package handler + +import ( + "encoding/json" + "testing" +) + +// TestJobData_UnmarshalLabelsAsObjects verifies that the labels field is correctly +// decoded when the queue-processor sends label objects like {"name":"RSS"} rather +// than plain strings. This is the exact payload shape produced by refreshFeed.ts: +// +// labels: [{ name: 'RSS' }] +func TestJobData_UnmarshalLabelsAsObjects(t *testing.T) { + raw := `{ + "url": "https://example.com/article", + "users": [{"id": "user-1", "libraryItemId": "item-1"}], + "priority": "low", + "labels": [{"name": "RSS"}], + "rssFeedUrl": "https://example.com/feed.xml", + "savedAt": "2026-01-20T00:00:00.000Z", + "publishedAt": "2026-01-20T00:00:00.000Z", + "source": "rss-feeder" + }` + + var data JobData + if err := json.Unmarshal([]byte(raw), &data); err != nil { + t.Fatalf("unmarshal error (was labels declared as []string instead of []LabelInput?): %v", err) + } + + if len(data.Labels) != 1 { + t.Fatalf("expected 1 label, got %d", len(data.Labels)) + } + if data.Labels[0].Name != "RSS" { + t.Errorf("expected label name 'RSS', got %q", data.Labels[0].Name) + } + if data.Labels[0].Color != nil { + t.Errorf("expected color nil, got %v", data.Labels[0].Color) + } +} + +// TestJobData_UnmarshalLabelsWithColor verifies a label with optional color field. +func TestJobData_UnmarshalLabelsWithColor(t *testing.T) { + color := "#FF0000" + raw := `{ + "url": "https://example.com/article", + "priority": "high", + "labels": [{"name": "Important", "color": "#FF0000"}] + }` + + var data JobData + if err := json.Unmarshal([]byte(raw), &data); err != nil { + t.Fatalf("unmarshal error: %v", err) + } + + if len(data.Labels) != 1 { + t.Fatalf("expected 1 label, got %d", len(data.Labels)) + } + if data.Labels[0].Name != "Important" { + t.Errorf("label name: got %q, want %q", data.Labels[0].Name, "Important") + } + if data.Labels[0].Color == nil || *data.Labels[0].Color != color { + t.Errorf("label color: got %v, want %q", data.Labels[0].Color, color) + } +} + +// TestJobData_UnmarshalNoLabels verifies that omitting labels entirely is valid. +func TestJobData_UnmarshalNoLabels(t *testing.T) { + raw := `{"url": "https://example.com/article", "priority": "high"}` + + var data JobData + if err := json.Unmarshal([]byte(raw), &data); err != nil { + t.Fatalf("unmarshal error: %v", err) + } + if len(data.Labels) != 0 { + t.Errorf("expected 0 labels, got %d", len(data.Labels)) + } +} + +// TestJobData_UnmarshalMultipleLabels verifies multiple label objects decode correctly. +func TestJobData_UnmarshalMultipleLabels(t *testing.T) { + raw := `{ + "url": "https://example.com/article", + "priority": "low", + "labels": [ + {"name": "RSS"}, + {"name": "Tech", "color": "#00FF00"}, + {"name": "Reading", "description": "To read later"} + ] + }` + + var data JobData + if err := json.Unmarshal([]byte(raw), &data); err != nil { + t.Fatalf("unmarshal error: %v", err) + } + if len(data.Labels) != 3 { + t.Fatalf("expected 3 labels, got %d", len(data.Labels)) + } + + names := []string{"RSS", "Tech", "Reading"} + for i, want := range names { + if data.Labels[i].Name != want { + t.Errorf("label[%d]: got %q, want %q", i, data.Labels[i].Name, want) + } + } + + if data.Labels[2].Description == nil || *data.Labels[2].Description != "To read later" { + t.Errorf("label[2] description: got %v, want %q", data.Labels[2].Description, "To read later") + } +} + +// TestSavePageJobData_MarshalLabels verifies that outgoing save-page jobs +// serialise labels back as objects (not strings), preserving the shape the +// backend queue-processor expects. +func TestSavePageJobData_MarshalLabels(t *testing.T) { + color := "#123456" + job := savePageJobData{ + UserID: "user-1", + URL: "https://example.com", + FinalURL: "https://example.com", + ArticleSavingRequestID: "item-1", + Source: "rss-feeder", + Labels: []LabelInput{ + {Name: "RSS"}, + {Name: "Custom", Color: &color}, + }, + } + + b, err := json.Marshal(job) + if err != nil { + t.Fatalf("marshal error: %v", err) + } + + var out map[string]interface{} + if err := json.Unmarshal(b, &out); err != nil { + t.Fatalf("re-unmarshal error: %v", err) + } + + rawLabels, ok := out["labels"].([]interface{}) + if !ok { + t.Fatalf("labels field is not an array: %T", out["labels"]) + } + if len(rawLabels) != 2 { + t.Fatalf("expected 2 labels, got %d", len(rawLabels)) + } + + // Each label must be an object, not a string. + for i, l := range rawLabels { + lmap, ok := l.(map[string]interface{}) + if !ok { + t.Errorf("label[%d] is not an object: %T (%v)", i, l, l) + continue + } + if _, hasName := lmap["name"]; !hasName { + t.Errorf("label[%d] missing 'name' key", i) + } + } + + first := rawLabels[0].(map[string]interface{}) + if first["name"] != "RSS" { + t.Errorf("first label name: got %v, want %q", first["name"], "RSS") + } + + second := rawLabels[1].(map[string]interface{}) + if second["color"] != "#123456" { + t.Errorf("second label color: got %v, want %q", second["color"], "#123456") + } +} From 1a1023a1e13772ec1d6c556cfd81aad58305df2e Mon Sep 17 00:00:00 2001 From: Aliaksei Karneyeu Date: Fri, 6 Mar 2026 14:49:58 +0100 Subject: [PATCH 10/10] move go code to src-go --- Makefile | 21 ++ docker/content-fetcher.Dockerfile | 52 +++ src-go/cmd/root.go | 28 ++ src-go/cmd/server/content_fetcher.go | 93 +++++ src-go/cmd/server/serve.go | 17 + src-go/go.mod | 103 +++++ src-go/go.sum | 253 +++++++++++++ src-go/internal/analytics/analytics.go | 73 ++++ src-go/internal/browser/browser.go | 106 ++++++ src-go/internal/bullmq/bullmq.go | 371 ++++++++++++++++++ src-go/internal/config/config.go | 115 ++++++ src-go/internal/fetch/fetch.go | 450 ++++++++++++++++++++++ src-go/internal/handler/handler.go | 475 ++++++++++++++++++++++++ src-go/internal/handler/handler_test.go | 167 +++++++++ src-go/internal/metrics/metrics.go | 90 +++++ src-go/internal/queue/worker.go | 110 ++++++ src-go/internal/redisutil/redisutil.go | 88 +++++ src-go/internal/server/server.go | 100 +++++ src-go/internal/storage/storage.go | 105 ++++++ src-go/internal/storage/storage_test.go | 175 +++++++++ src-go/main.go | 7 + 21 files changed, 2999 insertions(+) create mode 100644 docker/content-fetcher.Dockerfile create mode 100644 src-go/cmd/root.go create mode 100644 src-go/cmd/server/content_fetcher.go create mode 100644 src-go/cmd/server/serve.go create mode 100644 src-go/go.mod create mode 100644 src-go/go.sum create mode 100644 src-go/internal/analytics/analytics.go create mode 100644 src-go/internal/browser/browser.go create mode 100644 src-go/internal/bullmq/bullmq.go create mode 100644 src-go/internal/config/config.go create mode 100644 src-go/internal/fetch/fetch.go create mode 100644 src-go/internal/handler/handler.go create mode 100644 src-go/internal/handler/handler_test.go create mode 100644 src-go/internal/metrics/metrics.go create mode 100644 src-go/internal/queue/worker.go create mode 100644 src-go/internal/redisutil/redisutil.go create mode 100644 src-go/internal/server/server.go create mode 100644 src-go/internal/storage/storage.go create mode 100644 src-go/internal/storage/storage_test.go create mode 100644 src-go/main.go diff --git a/Makefile b/Makefile index 930c55b0c..61d4274b7 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ +# ── Configuration (override on the command line: make REGISTRY=myrepo) ─ +REGISTRY ?= korney4eg +IMAGE_TAG ?= latest + open_ios: $(MAKE) -C apple open @@ -39,3 +43,20 @@ puppeteer: content_fetch: content_handler puppeteer yarn workspace @omnivore/content-fetch build yarn workspace @omnivore/content-fetch start + +# ── Go content-fetcher ────────────────────────────────────────────────────── + +content_fetch_go: + cd src-go && go run . server content-fetcher + +content_fetch_go_build: + cd src-go && go build -o ../bin/omnivore . + +# ── Docker images ─────────────────────────────────────────────────────────── + +docker_build_content_fetcher: + docker build -f docker/content-fetcher.Dockerfile \ + -t $(REGISTRY)/omnivore-content-fetcher:$(IMAGE_TAG) . + +docker_push_content_fetcher: docker_build_content_fetcher + docker push $(REGISTRY)/omnivore-content-fetcher:$(IMAGE_TAG) diff --git a/docker/content-fetcher.Dockerfile b/docker/content-fetcher.Dockerfile new file mode 100644 index 000000000..3451d19a6 --- /dev/null +++ b/docker/content-fetcher.Dockerfile @@ -0,0 +1,52 @@ +FROM golang:1.25-alpine AS build +LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore" + +RUN apk add --no-cache git ca-certificates + +WORKDIR /app + +COPY src-go/go.mod src-go/go.sum ./ +RUN go mod download + +COPY src-go/ . +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o omnivore . + +# ─── Runtime image ──────────────────────────────────────────────────────────── +FROM alpine:3.20 + +LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore" + +# Add Chromium and required fonts/libs from Alpine edge +RUN echo "@edge https://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ + && echo "@edge https://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories \ + && echo "@edge https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ + && apk -U upgrade \ + && apk add --no-cache \ + chromium@edge \ + freetype@edge \ + ttf-freefont@edge \ + nss@edge \ + libstdc++@edge \ + sqlite-libs@edge \ + ca-certificates@edge \ + && rm -rf /var/cache/apk/* + +WORKDIR /app + +ENV CHROMIUM_PATH=/usr/bin/chromium +ENV LAUNCH_HEADLESS=true +ENV PORT=8080 + +# Download ad/tracker block-list to a separate file; appended to /etc/hosts at startup +RUN wget -q -O /etc/hosts.blocklist https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts + +COPY --from=build /app/omnivore . + +# Entrypoint: append the blocklist to /etc/hosts (which is writable at runtime), then exec the binary +RUN printf '#!/bin/sh\ncat /etc/hosts.blocklist >> /etc/hosts\nexec "$@"\n' > /entrypoint.sh \ + && chmod +x /entrypoint.sh + +EXPOSE 8080 + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["./omnivore", "server", "content-fetcher"] diff --git a/src-go/cmd/root.go b/src-go/cmd/root.go new file mode 100644 index 000000000..3749dad92 --- /dev/null +++ b/src-go/cmd/root.go @@ -0,0 +1,28 @@ +package cmd + +import ( + "os" + + "github.com/omnivore-app/omnivore/cmd/server" + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "omnivore", + Short: "Omnivore – open-source read-it-later platform", + Long: `omnivore is the single binary for running Omnivore services. + +Available commands: + omnivore server content-fetcher Start the content-fetch worker and HTTP server`, +} + +func init() { + rootCmd.AddCommand(server.Cmd) +} + +// Execute runs the root command. Called by main(). +func Execute() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/src-go/cmd/server/content_fetcher.go b/src-go/cmd/server/content_fetcher.go new file mode 100644 index 000000000..d88c448a7 --- /dev/null +++ b/src-go/cmd/server/content_fetcher.go @@ -0,0 +1,93 @@ +package server + +import ( + "context" + "log" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + + "github.com/omnivore-app/omnivore/internal/browser" + "github.com/omnivore-app/omnivore/internal/config" + "github.com/omnivore-app/omnivore/internal/queue" + "github.com/omnivore-app/omnivore/internal/redisutil" + "github.com/omnivore-app/omnivore/internal/server" + "github.com/spf13/cobra" +) + +var contentFetcherCmd = &cobra.Command{ + Use: "content-fetcher", + Short: "Start the content-fetch worker and HTTP server", + Long: `Starts the content-fetch service, which: + - Polls a BullMQ/Redis queue for fetch-content jobs + - Renders pages using a headless Chromium browser + - Uploads original HTML to object storage (GCS, S3, or MinIO) + - Queues save-page jobs for the backend + +Configuration is provided via environment variables. See .env.example for the +full list of supported variables.`, + RunE: runContentFetcher, +} + +func runContentFetcher(cmd *cobra.Command, args []string) error { + cfg := config.Load() + + if cfg.VerificationToken == "" { + log.Fatal("VERIFICATION_TOKEN is required") + } + + redisDS, err := redisutil.New(cfg) + if err != nil { + log.Fatalf("Failed to connect to Redis: %v", err) + } + + br := browser.New(cfg) + + workerCtx, workerCancel := context.WithCancel(context.Background()) + worker := queue.NewWorker(workerCtx, cfg, redisDS, br) + worker.Start() + + srv := server.New(cfg, redisDS, br, worker) + + port := cfg.Port + if port == 0 { + port = 8080 + } + addr := ":" + strconv.Itoa(port) + + httpServer := &http.Server{ + Addr: addr, + Handler: srv, + } + + go func() { + log.Printf("content-fetcher listening on %s", addr) + if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("HTTP server error: %v", err) + } + }() + + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + sig := <-quit + log.Printf("Received %s, shutting down...", sig) + + if err := httpServer.Shutdown(context.Background()); err != nil { + log.Printf("HTTP server shutdown error: %v", err) + } + log.Println("HTTP server closed") + + workerCancel() + worker.Wait() + log.Println("Worker closed") + + br.Close() + log.Println("Browser closed") + + redisDS.Shutdown() + log.Println("Redis connection closed") + + return nil +} diff --git a/src-go/cmd/server/serve.go b/src-go/cmd/server/serve.go new file mode 100644 index 000000000..2dc79926a --- /dev/null +++ b/src-go/cmd/server/serve.go @@ -0,0 +1,17 @@ +// Package server groups all "omnivore server " subcommands. +package server + +import ( + "github.com/spf13/cobra" +) + +// Cmd is the "omnivore server" parent command. +var Cmd = &cobra.Command{ + Use: "server", + Short: "Start an Omnivore service", + Long: `Start one of the Omnivore backend services.`, +} + +func init() { + Cmd.AddCommand(contentFetcherCmd) +} diff --git a/src-go/go.mod b/src-go/go.mod new file mode 100644 index 000000000..4e582119b --- /dev/null +++ b/src-go/go.mod @@ -0,0 +1,103 @@ +module github.com/omnivore-app/omnivore + +go 1.25.0 + +require ( + github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 + github.com/chromedp/chromedp v0.14.2 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/posthog/posthog-go v1.10.0 + github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.18.0 + github.com/spf13/cobra v1.9.1 + gocloud.dev v0.45.0 +) + +require ( + cel.dev/expr v0.25.1 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.18.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/storage v1.60.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.40.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 // indirect + github.com/aws/smithy-go v1.24.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chromedp/sysutil v1.1.0 // indirect + github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/google/wire v0.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect + github.com/googleapis/gax-go/v2 v2.17.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect + go.opentelemetry.io/otel v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/otel/sdk v1.40.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.40.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/crypto v0.47.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.35.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/text v0.33.0 // indirect + golang.org/x/time v0.14.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/api v0.265.0 // indirect + google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/src-go/go.sum b/src-go/go.sum new file mode 100644 index 000000000..f01385eb5 --- /dev/null +++ b/src-go/go.sum @@ -0,0 +1,253 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs= +cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY= +cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8= +cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc= +github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= +github.com/aws/aws-sdk-go-v2/config v1.32.2 h1:4liUsdEpUUPZs5WVapsJLx5NPmQhQdez7nYFcovrytk= +github.com/aws/aws-sdk-go-v2/config v1.32.2/go.mod h1:l0hs06IFz1eCT+jTacU/qZtC33nvcnLADAPL/XyrkZI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.2 h1:qZry8VUyTK4VIo5aEdUcBjPZHL2v4FyQ3QEOaWcFLu4= +github.com/aws/aws-sdk-go-v2/credentials v1.19.2/go.mod h1:YUqm5a1/kBnoK+/NY5WEiMocZihKSo15/tJdmdXnM5g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5/go.mod h1:nPRXgyCfAurhyaTMoBMwRBYBhaHI4lNPAnJmjM0Tslc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w= +github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 h1:OgQy/+0+Kc3khtqiEOk23xQAglXi3Tj0y5doOxbi5tg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 h1:MxMBdKTYBjPQChlJhi4qlEueqB1p1KcbTEa7tD5aqPs= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.2/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 h1:ksUT5KtgpZd3SAiFJNJ0AFEJVva3gjBmN7eXUZjzUwQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.5/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 h1:GtsxyiF3Nd3JahRBJbxLCCdYW9ltGQYrFWg8XdkGDd8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 h1:a5UTtD4mHBU3t0o6aHQZFJTNKVfxFWfPX7J0Lr7G+uY= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.2/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 h1:UQ4AU+BGti3Sy/aLU8KVseYKNALcX9UXY6DfpwQ6J8E= +github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k= +github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM= +github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= +github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e h1:gt7U1Igw0xbJdyaCM5H2CnlAlPSkzrhsebQB6WQWjLA= +github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs= +github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= +github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo= +github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI= +github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= +github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= +github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= +github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= +github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8= +github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= +github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posthog/posthog-go v1.10.0 h1:wfoy7Jfb4LigCoHYyMZoiJmmEoCLOkSaYfDxM/NtCqY= +github.com/posthog/posthog-go v1.10.0/go.mod h1:wB3/9Q7d9gGb1P/yf/Wri9VBlbP8oA8z++prRzL5OcY= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY= +gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.265.0 h1:FZvfUdI8nfmuNrE34aOWFPmLC+qRBEiNm3JdivTvAAU= +google.golang.org/api v0.265.0/go.mod h1:uAvfEl3SLUj/7n6k+lJutcswVojHPp2Sp08jWCu8hLY= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM= +google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0= +google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src-go/internal/analytics/analytics.go b/src-go/internal/analytics/analytics.go new file mode 100644 index 000000000..b1734e141 --- /dev/null +++ b/src-go/internal/analytics/analytics.go @@ -0,0 +1,73 @@ +// Package analytics wraps PostHog event capture, matching the analytics.ts behaviour. +package analytics + +import ( + "log" + + "github.com/omnivore-app/omnivore/internal/config" + "github.com/posthog/posthog-go" +) + +// Client sends analytics events to PostHog. +type Client struct { + ph posthog.Client + config *config.Config +} + +// Event carries data for an analytics capture call. +type Event struct { + Result string // "success" or "failure" + URL string + Source string + TotalTime int64 // milliseconds + ErrorMessage string +} + +// New creates a PostHog analytics client. +func New(config *config.Config) *Client { + ph, err := posthog.NewWithConfig(config.PostHogAPIKey, posthog.Config{}) + if err != nil { + log.Printf("Failed to create PostHog client: %v", err) + return &Client{config: config} + } + return &Client{ph: ph, config: config} +} + +// Capture sends a content_fetch_result event. +// Only failure events are sent when SEND_ANALYTICS is set, matching the TS behaviour. +func (c *Client) Capture(userIDs []string, ev Event) { + if c.ph == nil { + return + } + if !c.config.SendAnalytics || ev.Result != "failure" { + return + } + + for _, uid := range userIDs { + props := posthog.NewProperties(). + Set("url", ev.URL). + Set("source", ev.Source). + Set("totalTime", ev.TotalTime). + Set("env", c.config.APIEnv) + if ev.ErrorMessage != "" { + props.Set("errorMessage", ev.ErrorMessage) + } + + if err := c.ph.Enqueue(posthog.Capture{ + DistinctId: uid, + Event: "content_fetch_" + ev.Result, + Properties: props, + }); err != nil { + log.Printf("PostHog enqueue error: %v", err) + } + } +} + +// Close flushes and closes the PostHog client. +func (c *Client) Close() { + if c.ph != nil { + if err := c.ph.Close(); err != nil { + log.Printf("PostHog close error: %v", err) + } + } +} diff --git a/src-go/internal/browser/browser.go b/src-go/internal/browser/browser.go new file mode 100644 index 000000000..be0dc3453 --- /dev/null +++ b/src-go/internal/browser/browser.go @@ -0,0 +1,106 @@ +// Package browser manages a shared headless Chromium instance via chromedp. +package browser + +import ( + "context" + "fmt" + "log" + "sync" + + "github.com/chromedp/chromedp" + "github.com/omnivore-app/omnivore/internal/config" +) + +// Browser wraps a persistent chromedp browser allocator. +type Browser struct { + config *config.Config + allocCtx context.Context + allocCancel context.CancelFunc + mu sync.Mutex +} + +func New(config *config.Config) *Browser { + return &Browser{config: config} +} + +// allocatorOpts returns the chromedp ExecAllocator options matching the original Puppeteer args. +func (b *Browser) allocatorOpts() []chromedp.ExecAllocatorOption { + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("autoplay-policy", "user-gesture-required"), + chromedp.Flag("disable-component-update", true), + chromedp.Flag("disable-domain-reliability", true), + chromedp.Flag("disable-print-preview", true), + chromedp.Flag("disable-setuid-sandbox", true), + chromedp.Flag("disable-speech-api", true), + chromedp.Flag("enable-features", "SharedArrayBuffer"), + chromedp.Flag("hide-scrollbars", true), + chromedp.Flag("mute-audio", true), + chromedp.Flag("no-default-browser-check", true), + chromedp.Flag("no-pings", true), + chromedp.Flag("no-sandbox", true), + chromedp.Flag("no-zygote", true), + chromedp.Flag("disable-extensions", true), + chromedp.Flag("disable-dev-shm-usage", true), + chromedp.Flag("no-first-run", true), + chromedp.Flag("disable-background-networking", true), + chromedp.Flag("disable-gpu", true), + chromedp.Flag("disable-software-rasterizer", true), + chromedp.Flag("ignore-certificate-errors", true), + chromedp.WindowSize(1920, 1080), + chromedp.Headless, + ) + + if b.config.ChromiumPath != "" && !b.config.UseFirefox { + opts = append(opts, chromedp.ExecPath(b.config.ChromiumPath)) + } + + return opts +} + +// getAllocator returns the shared browser allocator context, creating it if needed. +func (b *Browser) getAllocator() (context.Context, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if b.allocCtx != nil { + // Check if still alive + select { + case <-b.allocCtx.Done(): + // Allocator died, recreate + b.allocCtx = nil + b.allocCancel = nil + default: + return b.allocCtx, nil + } + } + + log.Println("Starting chromedp browser allocator") + ctx, cancel := chromedp.NewExecAllocator(context.Background(), b.allocatorOpts()...) + b.allocCtx = ctx + b.allocCancel = cancel + return ctx, nil +} + +// NewContext creates a new browser tab context using the shared allocator. +func (b *Browser) NewContext() (context.Context, context.CancelFunc, error) { + allocCtx, err := b.getAllocator() + if err != nil { + return nil, nil, fmt.Errorf("get allocator: %w", err) + } + + ctx, cancel := chromedp.NewContext(allocCtx) + return ctx, cancel, nil +} + +// Close shuts down the browser. +func (b *Browser) Close() { + b.mu.Lock() + defer b.mu.Unlock() + + if b.allocCancel != nil { + b.allocCancel() + b.allocCancel = nil + b.allocCtx = nil + log.Println("Browser closed") + } +} diff --git a/src-go/internal/bullmq/bullmq.go b/src-go/internal/bullmq/bullmq.go new file mode 100644 index 000000000..16bda0d9d --- /dev/null +++ b/src-go/internal/bullmq/bullmq.go @@ -0,0 +1,371 @@ +// Package queue implements BullMQ-compatible Redis queue operations. +// +// BullMQ v5 stores jobs as Redis hashes at bull:{queue}:{jobId} +// and manages state via lists/sorted-sets: +// - bull:{queue}:wait LIST (pending, LIFO insertion LPUSH, LPOP consumption) +// - bull:{queue}:active LIST (currently processing) +// - bull:{queue}:completed ZSET (done, score=timestamp) +// - bull:{queue}:failed ZSET (failed, score=timestamp) +// - bull:{queue}:prioritized ZSET (priority queue, score encodes priority+counter) +// - bull:{queue}:id STRING (atomic ID counter) +// +// Workers use a Lua moveToActive script. For the consumer side we replicate +// a simplified version that is compatible with existing BullMQ producers and +// consumers (i.e. jobs added here can be consumed by the TS worker, and vice-versa). +package bullmq + +import ( + "context" + "encoding/json" + "fmt" + "log" + "strconv" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + bullPrefix = "bull" + + // Queue names + ContentFetchQueue = "omnivore-content-fetch-queue" + BackendQueue = "omnivore-backend-queue" + + // Job names + SavePageJob = "save-page" + + // removeOnComplete/removeOnFail ages (seconds) – match TS defaults + completeAge = 3600 + failAge = 86400 +) + +// queueKey returns the base key prefix for a BullMQ queue. +func queueKey(queueName string) string { + return fmt.Sprintf("%s:%s", bullPrefix, queueName) +} + +func waitKey(q string) string { return queueKey(q) + ":wait" } +func activeKey(q string) string { return queueKey(q) + ":active" } +func completedKey(q string) string { return queueKey(q) + ":completed" } +func failedKey(q string) string { return queueKey(q) + ":failed" } +func idKey(q string) string { return queueKey(q) + ":id" } +func metaKey(q string) string { return queueKey(q) + ":meta" } +func jobKey(q, id string) string { return queueKey(q) + ":" + id } +func prioritizedKey(q string) string { return queueKey(q) + ":prioritized" } +func eventsKey(q string) string { return queueKey(q) + ":events" } + +// JobOpts mirrors BullMQ BulkJobOptions. +type JobOpts struct { + Attempts int `json:"attempts"` + Priority int `json:"priority"` + Backoff BackoffOpt `json:"backoff"` + // removeOnComplete and removeOnFail are not stored in opts hash but handled by cleanup +} + +type BackoffOpt struct { + Type string `json:"type"` + Delay int `json:"delay"` +} + +// RawJob is a job as stored in Redis. +type RawJob struct { + ID string + Name string + Data json.RawMessage + Opts JobOpts + Timestamp int64 + AttemptsMade int +} + +// nextJobID atomically increments and returns a new job ID. +func nextJobID(ctx context.Context, redisClient *redis.Client, queueName string) (string, error) { + id, err := redisClient.Incr(ctx, idKey(queueName)).Result() + if err != nil { + return "", err + } + return strconv.FormatInt(id, 10), nil +} + +// AddJobOpts carries parameters for adding a single job. +type AddJobOpts struct { + Name string + Data interface{} + Opts JobOpts +} + +// AddBulk adds multiple jobs to a BullMQ queue, replicating addBulk() semantics. +// Each job is stored as a hash and its ID appended to the appropriate list/zset. +func AddBulk(ctx context.Context, redisClient *redis.Client, queueName string, jobs []AddJobOpts) error { + for _, j := range jobs { + jobID, err := nextJobID(ctx, redisClient, queueName) + if err != nil { + return fmt.Errorf("get next job id: %w", err) + } + + dataBytes, err := json.Marshal(j.Data) + if err != nil { + return fmt.Errorf("marshal job data: %w", err) + } + + optsBytes, err := json.Marshal(j.Opts) + if err != nil { + return fmt.Errorf("marshal job opts: %w", err) + } + + now := time.Now().UnixMilli() + key := jobKey(queueName, jobID) + + // Store the job hash + pipe := redisClient.Pipeline() + pipe.HSet(ctx, key, + "name", j.Name, + "data", string(dataBytes), + "opts", string(optsBytes), + "timestamp", now, + "attemptsMade", 0, + "attemptsStarted", 0, + "stalledCounter", 0, + ) + + if j.Opts.Priority > 0 { + // Priority jobs go into the prioritized sorted set. + // BullMQ score = priority * 0x100000000 + counter (counter increments per priority) + // For simplicity we use: score = priority * 1e12 + now which preserves ordering. + score := float64(j.Opts.Priority)*1e12 + float64(now) + pipe.ZAdd(ctx, prioritizedKey(queueName), redis.Z{ + Score: score, + Member: jobID, + }) + } else { + pipe.LPush(ctx, waitKey(queueName), jobID) + } + + // Publish event for BullMQ dashboard/metrics compatibility + eventPayload, _ := json.Marshal(map[string]interface{}{ + "jobId": jobID, + "prev": "waiting", + }) + pipe.XAdd(ctx, &redis.XAddArgs{ + Stream: eventsKey(queueName), + Values: map[string]interface{}{ + "event": "waiting", + "data": string(eventPayload), + }, + MaxLen: 10000, + }) + + if _, err := pipe.Exec(ctx); err != nil { + return fmt.Errorf("add job %s: %w", jobID, err) + } + + log.Printf("Queued job %s/%s id=%s", queueName, j.Name, jobID) + } + return nil +} + +// moveToActive implements a simplified version of BullMQ's moveToActive Lua script. +// It atomically moves the next job from wait (or prioritized) → active. +// Returns the job ID and whether a job was found. +// +// The Lua script below is compatible with BullMQ v5 in that jobs added by the +// TS BullMQ library can be consumed here and vice-versa. It is intentionally +// simplified – it does not handle stalled job detection or rate limiting, which +// are handled by the BullMQ scheduler in the TS layer if both run simultaneously. +var moveToActiveScript = redis.NewScript(` +local waitKey = KEYS[1] +local prioritizedKey = KEYS[2] +local activeKey = KEYS[3] +local jobKeyPrefix = ARGV[1] -- e.g. "bull:omnivore-content-fetch-queue:" + +-- Try prioritized first (lowest score = highest priority) +local jobId = redis.call("ZPOPMIN", prioritizedKey, 1) +if #jobId > 0 then + jobId = jobId[1] +else + -- Fall back to regular wait list (RPOP = FIFO from the end, matching BullMQ) + jobId = redis.call("RPOP", waitKey) +end + +if jobId == false or jobId == nil then + return nil +end + +-- Move to active +redis.call("LPUSH", activeKey, jobId) + +return jobId +`) + +// PopJob atomically moves the next available job to the active list and returns it. +// Returns nil job if no job is available (non-blocking). +func PopJob(ctx context.Context, redisClient *redis.Client, queueName string) (*RawJob, error) { + keys := []string{ + waitKey(queueName), + prioritizedKey(queueName), + activeKey(queueName), + } + prefix := queueKey(queueName) + ":" + + result, err := moveToActiveScript.Run(ctx, redisClient, keys, prefix).Result() + if err == redis.Nil { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("moveToActive: %w", err) + } + + jobID, ok := result.(string) + if !ok || jobID == "" { + return nil, nil + } + + return getJob(ctx, redisClient, queueName, jobID) +} + +func getJob(ctx context.Context, redisClient *redis.Client, queueName, jobID string) (*RawJob, error) { + fields, err := redisClient.HGetAll(ctx, jobKey(queueName, jobID)).Result() + if err != nil { + return nil, fmt.Errorf("hgetall job %s: %w", jobID, err) + } + if len(fields) == 0 { + return nil, fmt.Errorf("job %s not found", jobID) + } + + var opts JobOpts + if o := fields["opts"]; o != "" { + _ = json.Unmarshal([]byte(o), &opts) + } + + var attempts int + if a := fields["attemptsMade"]; a != "" { + attempts, _ = strconv.Atoi(a) + } + + return &RawJob{ + ID: jobID, + Name: fields["name"], + Data: json.RawMessage(fields["data"]), + Opts: opts, + AttemptsMade: attempts, + }, nil +} + +// CompleteJob moves a job from active to completed. +func CompleteJob(ctx context.Context, redisClient *redis.Client, queueName, jobID string) error { + now := time.Now().UnixMilli() + pipe := redisClient.Pipeline() + pipe.LRem(ctx, activeKey(queueName), 0, jobID) + pipe.ZAdd(ctx, completedKey(queueName), redis.Z{Score: float64(now), Member: jobID}) + pipe.HSet(ctx, jobKey(queueName, jobID), "finishedOn", now) + // Schedule cleanup (approximate removeOnComplete age) + pipe.Expire(ctx, jobKey(queueName, jobID), completeAge*time.Second) + _, err := pipe.Exec(ctx) + return err +} + +// FailJob moves a job from active to failed (or re-queues it for retry). +func FailJob(ctx context.Context, redisClient *redis.Client, queueName, jobID string, reason string, opts JobOpts) error { + now := time.Now().UnixMilli() + + // Increment attemptsMade + newAttempts, err := redisClient.HIncrBy(ctx, jobKey(queueName, jobID), "attemptsMade", 1).Result() + if err != nil { + return err + } + + if int(newAttempts) < opts.Attempts { + // Retry: calculate exponential backoff delay + delay := exponentialDelay(opts.Backoff.Delay, int(newAttempts)-1) + retryAt := now + int64(delay) + + pipe := redisClient.Pipeline() + pipe.LRem(ctx, activeKey(queueName), 0, jobID) + pipe.ZAdd(ctx, fmt.Sprintf("%s:delayed", queueKey(queueName)), redis.Z{ + Score: float64(retryAt), + Member: jobID, + }) + pipe.HSet(ctx, jobKey(queueName, jobID), "failedReason", reason) + _, err = pipe.Exec(ctx) + return err + } + + // Max attempts reached → move to failed + pipe := redisClient.Pipeline() + pipe.LRem(ctx, activeKey(queueName), 0, jobID) + pipe.ZAdd(ctx, failedKey(queueName), redis.Z{Score: float64(now), Member: jobID}) + pipe.HSet(ctx, jobKey(queueName, jobID), "failedReason", reason, "finishedOn", now) + pipe.Expire(ctx, jobKey(queueName, jobID), failAge*time.Second) + _, err = pipe.Exec(ctx) + return err +} + +// exponentialDelay returns the next retry delay in milliseconds (matches BullMQ's exponential). +func exponentialDelay(baseDelay, attempt int) int { + d := baseDelay + for i := 0; i < attempt; i++ { + d *= 2 + } + return d +} + +// GetQueueCounts returns job counts for the metrics endpoint. +func GetQueueCounts(ctx context.Context, redisClient *redis.Client, queueName string) (map[string]int64, error) { + pipe := redisClient.Pipeline() + activeCmd := pipe.LLen(ctx, activeKey(queueName)) + failedCmd := pipe.ZCard(ctx, failedKey(queueName)) + completedCmd := pipe.ZCard(ctx, completedKey(queueName)) + prioritizedCmd := pipe.ZCard(ctx, prioritizedKey(queueName)) + waitCmd := pipe.LLen(ctx, waitKey(queueName)) + + if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil { + return nil, err + } + + return map[string]int64{ + "active": activeCmd.Val(), + "failed": failedCmd.Val(), + "completed": completedCmd.Val(), + "prioritized": prioritizedCmd.Val() + waitCmd.Val(), + }, nil +} + +// OldestPrioritizedJobAge returns the age in seconds of the oldest prioritized job, or 0. +func OldestPrioritizedJobAge(ctx context.Context, redisClient *redis.Client, queueName string) (float64, error) { + // Check both prioritized zset and wait list + results, err := redisClient.ZRangeWithScores(ctx, prioritizedKey(queueName), 0, 0).Result() + if err != nil { + return 0, err + } + + if len(results) > 0 { + // score encodes priority+counter, so we need the job's timestamp field + jobID := results[0].Member.(string) + ts, err := redisClient.HGet(ctx, jobKey(queueName, jobID), "timestamp").Result() + if err == nil { + if tsMs, err := strconv.ParseInt(ts, 10, 64); err == nil { + return float64(time.Now().UnixMilli()-tsMs) / 1000.0, nil + } + } + } + + // Fall back to wait list + waitIDs, err := redisClient.LRange(ctx, waitKey(queueName), -1, -1).Result() // oldest = tail + if err != nil || len(waitIDs) == 0 { + return 0, nil + } + ts, err := redisClient.HGet(ctx, jobKey(queueName, waitIDs[0]), "timestamp").Result() + if err != nil { + return 0, nil + } + tsMs, err := strconv.ParseInt(ts, 10, 64) + if err != nil { + return 0, nil + } + return float64(time.Now().UnixMilli()-tsMs) / 1000.0, nil +} + +// EnsureQueueMeta ensures the queue metadata key exists (BullMQ creates this on queue init). +func EnsureQueueMeta(ctx context.Context, redisClient *redis.Client, queueName string) error { + return redisClient.HSetNX(ctx, metaKey(queueName), "version", "5").Err() +} diff --git a/src-go/internal/config/config.go b/src-go/internal/config/config.go new file mode 100644 index 000000000..648e22bc1 --- /dev/null +++ b/src-go/internal/config/config.go @@ -0,0 +1,115 @@ +package config + +import ( + "os" + "strconv" +) + +type Config struct { + // HTTP + Port int + VerificationToken string + + // Redis - cache + RedisURL string + RedisCert string + + // Redis - queue (BullMQ) + MQRedisURL string + MQRedisCert string + + // Browser + ChromiumPath string + FirefoxPath string + UseFirefox bool + LaunchHeadless bool + + // Object storage + // BlobStorageURL is a gocloud.dev blob URL that selects the backend: + // gs://bucket → GCS (Application Default Credentials) + // s3://bucket?region=us-east-1 → AWS S3 + // s3://bucket?endpoint=http://minio:9000&use_path_style=true&disable_https=true®ion=us-east-1 + // → MinIO + // When empty, a gs:// URL is constructed from GCSUploadBucket (backward compat). + BlobStorageURL string + + // Legacy GCS settings kept for backward compatibility. + // Prefer BLOB_STORAGE_URL for new deployments. + GCSUploadBucket string + GCSKeyFilePath string + SkipUploadOriginal bool + + // Analytics (PostHog) + PostHogAPIKey string + SendAnalytics bool + APIEnv string + + // Import metrics + ImporterMetricsCollectorURL string + JWTSecret string + + // Domain blocking + MaxFeedFetchFailures int +} + +func Load() *Config { + cfg := &Config{ + Port: envInt("PORT", 3002), + VerificationToken: os.Getenv("VERIFICATION_TOKEN"), + + RedisURL: os.Getenv("REDIS_URL"), + RedisCert: os.Getenv("REDIS_CERT"), + + MQRedisURL: os.Getenv("MQ_REDIS_URL"), + MQRedisCert: os.Getenv("MQ_REDIS_CERT"), + + ChromiumPath: envDefault("CHROMIUM_PATH", "/usr/bin/chromium"), + FirefoxPath: envDefault("FIREFOX_PATH", "/usr/bin/firefox"), + UseFirefox: os.Getenv("USE_FIREFOX") == "true", + LaunchHeadless: os.Getenv("LAUNCH_HEADLESS") == "true", + + BlobStorageURL: os.Getenv("BLOB_STORAGE_URL"), + + GCSUploadBucket: envDefault("GCS_UPLOAD_BUCKET", "omnivore-files"), + GCSKeyFilePath: os.Getenv("GCS_UPLOAD_SA_KEY_FILE_PATH"), + SkipUploadOriginal: os.Getenv("SKIP_UPLOAD_ORIGINAL") == "true", + + PostHogAPIKey: envDefault("POSTHOG_API_KEY", "test"), + SendAnalytics: os.Getenv("SEND_ANALYTICS") != "", + APIEnv: os.Getenv("API_ENV"), + + ImporterMetricsCollectorURL: os.Getenv("IMPORTER_METRICS_COLLECTOR_URL"), + JWTSecret: os.Getenv("JWT_SECRET"), + + MaxFeedFetchFailures: envInt("MAX_FEED_FETCH_FAILURES", 10), + } + + return cfg +} + +// BlobURL returns the effective gocloud.dev blob URL to open. +// If BLOB_STORAGE_URL is set it is returned as-is. +// Otherwise a gs:// URL is constructed from GCS_UPLOAD_BUCKET for backward +// compatibility with existing GCS deployments. +func (c *Config) BlobURL() string { + if c.BlobStorageURL != "" { + return c.BlobStorageURL + } + return "gs://" + c.GCSUploadBucket +} + +func envDefault(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func envInt(key string, def int) int { + if v := os.Getenv(key); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return def +} diff --git a/src-go/internal/fetch/fetch.go b/src-go/internal/fetch/fetch.go new file mode 100644 index 000000000..0f3944d3d --- /dev/null +++ b/src-go/internal/fetch/fetch.go @@ -0,0 +1,450 @@ +// Package fetch implements content retrieval using chromedp (Chromium DevTools Protocol). +// It replicates the behaviour of packages/puppeteer-parse with equivalent Go logic. +package fetch + +import ( + "bytes" + "context" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/chromedp/cdproto/emulation" + "github.com/chromedp/cdproto/network" + "github.com/chromedp/chromedp" + "github.com/omnivore-app/omnivore/internal/browser" +) + +// Result mirrors FetchResult from the TS implementation. +type Result struct { + FinalURL string + Title string + Content string + ContentType string +} + +// nonScriptHosts mirrors NON_SCRIPT_HOSTS – JS disabled for these. +var nonScriptHosts = []string{"medium.com", "fastcompany.com", "fortelabs.com"} + +// allowedContentTypes mirrors ALLOWED_CONTENT_TYPES. +var allowedContentTypes = map[string]bool{ + "text/html": true, + "application/octet-stream": true, + "text/plain": true, + "application/pdf": true, +} + +// noCacheURLs mirrors NO_CACHE_URLS from request_handler.ts (checked externally, listed here for reference). +var NoCacheURLs = map[string]bool{ + "https://deviceandbrowserinfo.com/are_you_a_bot": true, + "https://deviceandbrowserinfo.com/info_device": true, + "https://jacksonh.org": true, +} + +// FetchContent is the Go equivalent of fetchContent() in puppeteer-parse/src/index.ts. +func FetchContent(ctx context.Context, browser *browser.Browser, rawURL, locale, timezone string) (*Result, error) { + start := time.Now() + log.Printf("content-fetch request url=%s locale=%s timezone=%s", rawURL, locale, timezone) + + parsedURL, err := parseAndValidateURL(rawURL) + if err != nil { + return nil, err + } + targetURL := parsedURL.String() + + // Pre-handle: detect PDFs, images, and other special content types. + preResult, err := preHandle(ctx, targetURL) + if err != nil { + log.Printf("pre-handle warning for %s: %v", targetURL, err) + // Non-fatal; continue with browser fetch + } + + if preResult != nil { + if preResult.URL != "" { + if p, err := parseAndValidateURL(preResult.URL); err == nil { + targetURL = p.String() + } + } + if preResult.ContentType == "application/pdf" || (preResult.Content != "" && preResult.Title != "") { + return &Result{ + FinalURL: targetURL, + Title: preResult.Title, + Content: preResult.Content, + ContentType: preResult.ContentType, + }, nil + } + } + + // Fall through to browser fetch + result, err := retrievePage(ctx, browser, targetURL, locale, timezone) + if err != nil { + return nil, err + } + + log.Printf("content-fetch done url=%s duration=%s", targetURL, time.Since(start)) + return result, nil +} + +// preHandleResult carries the output of pre-handle checks. +type preHandleResult struct { + URL string + Title string + Content string + ContentType string +} + +// preHandle performs lightweight checks before launching the browser: +// - Detects PDF URLs (by extension or HEAD request) +// - Detects image content types +// Returns nil if normal browser rendering should proceed. +func preHandle(ctx context.Context, rawURL string) (*preHandleResult, error) { + lower := strings.ToLower(rawURL) + + // PDF by extension + if strings.HasSuffix(lower, ".pdf") || strings.Contains(lower, ".pdf?") { + return &preHandleResult{URL: rawURL, ContentType: "application/pdf"}, nil + } + + // Quick HEAD to detect content-type without full page load + client := &http.Client{Timeout: 5 * time.Second, CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 5 { + return fmt.Errorf("too many redirects") + } + return nil + }} + + req, err := http.NewRequestWithContext(ctx, http.MethodHead, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Omnivore/1.0)") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + ct := strings.ToLower(resp.Header.Get("Content-Type")) + if ct != "" { + ct = strings.SplitN(ct, ";", 2)[0] + ct = strings.TrimSpace(ct) + } + + if ct == "application/pdf" { + return &preHandleResult{URL: resp.Request.URL.String(), ContentType: "application/pdf"}, nil + } + + return nil, nil +} + +// retrievePage navigates to the URL using the browser, waits for load and DOM settle, +// then captures the full document HTML. +func retrievePage(ctx context.Context, browser *browser.Browser, targetURL, locale, timezone string) (*Result, error) { + tabCtx, cancel, err := browser.NewContext() + if err != nil { + return nil, fmt.Errorf("new browser context: %w", err) + } + defer cancel() + + // Timeout for the entire page load + DOM settle + tabCtx, timeoutCancel := context.WithTimeout(tabCtx, 60*time.Second) + defer timeoutCancel() + + result := &Result{FinalURL: targetURL} + + disableJS := shouldDisableJS(targetURL) + + // Track final URL (after redirects) and content-type via network events + var finalURL string + var contentType string + var lastPDFURL string + + chromedp.ListenTarget(tabCtx, func(ev interface{}) { + switch e := ev.(type) { + case *network.EventResponseReceived: + if e.Type == network.ResourceTypeDocument { + ct := strings.ToLower(e.Response.MimeType) + if ct == "application/pdf" { + lastPDFURL = e.Response.URL + } + if finalURL == "" { + finalURL = e.Response.URL + if ct, ok := e.Response.Headers["content-type"].(string); ok { + contentType = ct + } + if contentType == "" { + contentType = e.Response.MimeType + } + } + } + } + }) + + var actions []chromedp.Action + + // Set locale header + if locale != "" { + actions = append(actions, network.SetExtraHTTPHeaders(network.Headers{ + "Accept-Language": locale, + })) + } + + if disableJS { + actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error { + return emulation.SetScriptExecutionDisabled(true).Do(ctx) + })) + } + + // Navigate + actions = append(actions, chromedp.Navigate(targetURL)) + + // Wait for DOM to settle (equivalent to waitForDOMToSettle) + actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error { + return waitForDOMSettle(ctx, 5*time.Second, 1*time.Second) + })) + + // Scroll the page + actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error { + return scrollPage(ctx, 5*time.Second) + })) + + // Capture title + actions = append(actions, chromedp.Title(&result.Title)) + + // Capture and clean DOM + var domContent string + actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error { + return captureAndCleanDOM(ctx, &domContent) + })) + + if err := chromedp.Run(tabCtx, actions...); err != nil { + if lastPDFURL != "" { + return &Result{FinalURL: lastPDFURL, ContentType: "application/pdf"}, nil + } + return nil, fmt.Errorf("chromedp run: %w", err) + } + + if domContent == "IS_BLOCKED" { + return nil, fmt.Errorf("page is blocked by anti-bot protection") + } + + if finalURL != "" { + result.FinalURL = finalURL + } + result.Content = domContent + result.ContentType = contentType + + return result, nil +} + +// waitForDOMSettle runs a MutationObserver via JS to wait until DOM mutations settle. +// Mirrors waitForDOMToSettle from puppeteer-parse. +func waitForDOMSettle(ctx context.Context, timeout, debounce time.Duration) error { + timeoutMs := int(timeout.Milliseconds()) + debounceMs := int(debounce.Milliseconds()) + + script := fmt.Sprintf(` + new Promise((resolve) => { + const timeoutMs = %d; + const debounceMs = %d; + let debounceTimer; + const mainTimeout = setTimeout(() => { + observer.disconnect(); + resolve(); + }, timeoutMs); + const debouncedResolve = () => { + clearTimeout(debounceTimer); + debounceTimer = setTimeout(() => { + observer.disconnect(); + clearTimeout(mainTimeout); + resolve(); + }, debounceMs); + }; + const observer = new MutationObserver(debouncedResolve); + observer.observe(document.body, { attributes: true, childList: true, subtree: true }); + }) + `, timeoutMs, debounceMs) + + return chromedp.Evaluate(script, nil).Do(ctx) +} + +// scrollPage scrolls the entire page with a timeout, mirroring the Puppeteer scroll logic. +func scrollPage(ctx context.Context, timeout time.Duration) error { + scrollCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + script := ` + new Promise((resolve) => { + let scrollHeight = document.body.scrollHeight; + let totalHeight = 0; + let distance = 500; + let timer = setInterval(() => { + window.scrollBy(0, distance); + totalHeight += distance; + if (totalHeight >= scrollHeight) { + clearInterval(timer); + resolve(true); + } + }, 10); + }) + ` + err := chromedp.Evaluate(script, nil).Do(scrollCtx) + if err != nil && scrollCtx.Err() != nil { + return nil // Timeout during scroll is acceptable + } + return err +} + +// captureAndCleanDOM evaluates JS to clean the DOM and return the outerHTML. +// Mirrors the page.evaluate block in retrieveHtml() from puppeteer-parse/src/index.ts. +func captureAndCleanDOM(ctx context.Context, result *string) error { + script := ` + (function() { + const BI_SRC_REGEXP = /url\("(.+?)"\)/gi; + + Array.from(document.body.getElementsByTagName('*')).forEach((el) => { + const style = window.getComputedStyle(el); + const src = el.getAttribute('src'); + + try { + if (el.tagName && ['img', 'image'].includes(el.tagName.toLowerCase())) { + const filter = style.getPropertyValue('filter'); + if (filter && filter.startsWith('blur')) { + el.parentNode && el.parentNode.removeChild(el); + return; + } + } + } catch(err) {} + + const bgImage = style.getPropertyValue('background-image'); + if (bgImage && !['', 'none'].includes(bgImage)) { + const filter = style.getPropertyValue('filter'); + if (filter && filter.startsWith('blur')) { + el && el.parentNode && el.parentNode.removeChild(el); + } else { + BI_SRC_REGEXP.lastIndex = 0; + const matchedSRC = BI_SRC_REGEXP.exec(bgImage); + BI_SRC_REGEXP.lastIndex = 0; + if (matchedSRC && matchedSRC[1] && !src) { + if (!el.textContent) { + const img = document.createElement('img'); + img.src = matchedSRC[1]; + el && el.parentNode && el.parentNode.replaceChild(img, el); + } + } + } + } + + if (el.tagName === 'IFRAME') { + // Instagram iframe handling omitted (requires cross-frame access unavailable in CDP) + } + }); + + if ( + document.querySelector('[data-translate="managed_checking_msg"]') || + document.getElementById('px-block-form-wrapper') + ) { + return 'IS_BLOCKED'; + } + + return document.documentElement.outerHTML; + })() + ` + return chromedp.Evaluate(script, result).Do(ctx) +} + +// shouldDisableJS returns true for domains where JS should be disabled (mirrors NON_SCRIPT_HOSTS). +func shouldDisableJS(rawURL string) bool { + u, err := url.Parse(rawURL) + if err != nil { + return false + } + hostname := u.Hostname() + for _, host := range nonScriptHosts { + if strings.HasSuffix(hostname, host) { + return true + } + } + return false +} + +// parseAndValidateURL validates and normalises the URL, mirroring validateUrlString + getUrl. +func parseAndValidateURL(rawURL string) (*url.URL, error) { + // Extract first URL if embedded in a string + re := regexp.MustCompile(`(https?://[^\s]+)`) + matches := re.FindStringSubmatch(rawURL) + if len(matches) == 0 { + return nil, fmt.Errorf("no URL found in: %s", rawURL) + } + rawURL = matches[1] + + u, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("invalid URL: %w", err) + } + + if u.Scheme != "http" && u.Scheme != "https" { + return nil, fmt.Errorf("invalid URL protocol: %s", u.Scheme) + } + + hostname := u.Hostname() + if hostname == "localhost" || hostname == "0.0.0.0" { + return nil, fmt.Errorf("URL points to localhost") + } + + if isPrivateIP(hostname) { + return nil, fmt.Errorf("URL points to private IP: %s", hostname) + } + + return u, nil +} + +var privateIPRanges = []*regexp.Regexp{ + regexp.MustCompile(`^10\.`), + regexp.MustCompile(`^172\.(1[6-9]|2[0-9]|3[0-1])\.`), + regexp.MustCompile(`^192\.168\.`), +} + +func isPrivateIP(hostname string) bool { + // Try resolving if it's a hostname + ip := net.ParseIP(hostname) + if ip == nil { + return false + } + s := ip.String() + for _, re := range privateIPRanges { + if re.MatchString(s) { + return true + } + } + return false +} + +// FetchPDF downloads a PDF directly and returns its bytes (for GCS upload). +func FetchPDF(ctx context.Context, rawURL string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Omnivore/1.0)") + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var buf bytes.Buffer + if _, err := io.Copy(&buf, resp.Body); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/src-go/internal/handler/handler.go b/src-go/internal/handler/handler.go new file mode 100644 index 000000000..b12658fee --- /dev/null +++ b/src-go/internal/handler/handler.go @@ -0,0 +1,475 @@ +// Package handler implements processFetchContentJob — the core job processing logic. +// It mirrors packages/content-fetch/src/request_handler.ts exactly. +package handler + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "os" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/omnivore-app/omnivore/internal/analytics" + "github.com/omnivore-app/omnivore/internal/browser" + "github.com/omnivore-app/omnivore/internal/bullmq" + "github.com/omnivore-app/omnivore/internal/config" + "github.com/omnivore-app/omnivore/internal/fetch" + "github.com/omnivore-app/omnivore/internal/redisutil" + "github.com/omnivore-app/omnivore/internal/storage" + "github.com/redis/go-redis/v9" +) + +// LabelInput mirrors the TS CreateLabelInput interface. +// Labels are sent as objects (e.g. {"name":"RSS"}), not plain strings. +type LabelInput struct { + Name string `json:"name"` + Color *string `json:"color,omitempty"` + Description *string `json:"description,omitempty"` +} + +// UserConfig mirrors the TS UserConfig interface. +type UserConfig struct { + ID string `json:"id"` + LibraryItemID string `json:"libraryItemId"` + Folder *string `json:"folder,omitempty"` +} + +// JobData mirrors the TS FetchContentJobData interface. +type JobData struct { + URL string `json:"url"` + UserID *string `json:"userId,omitempty"` + SaveRequestID string `json:"saveRequestId"` + State *string `json:"state,omitempty"` + Labels []LabelInput `json:"labels,omitempty"` + Source *string `json:"source,omitempty"` + TaskID *string `json:"taskId,omitempty"` + Locale *string `json:"locale,omitempty"` + Timezone *string `json:"timezone,omitempty"` + RSSFeedURL *string `json:"rssFeedUrl,omitempty"` + SavedAt *string `json:"savedAt,omitempty"` + PublishedAt *string `json:"publishedAt,omitempty"` + Folder *string `json:"folder,omitempty"` + Users []UserConfig `json:"users,omitempty"` + Priority string `json:"priority"` // "high" | "low" +} + +// savePageJobData mirrors SavePageJobData from job.ts. +type savePageJobData struct { + UserID string `json:"userId"` + URL string `json:"url"` + FinalURL string `json:"finalUrl"` + ArticleSavingRequestID string `json:"articleSavingRequestId"` + State *string `json:"state,omitempty"` + Labels []LabelInput `json:"labels,omitempty"` + Source string `json:"source"` + Folder *string `json:"folder,omitempty"` + RSSFeedURL *string `json:"rssFeedUrl,omitempty"` + SavedAt *string `json:"savedAt,omitempty"` + PublishedAt *string `json:"publishedAt,omitempty"` + TaskID *string `json:"taskId,omitempty"` + Title string `json:"title,omitempty"` + ContentType string `json:"contentType,omitempty"` + CacheKey string `json:"cacheKey,omitempty"` +} + +const maxImportAttempts = 1 + +// ProcessFetchContentJob is the Go equivalent of processFetchContentJob() from request_handler.ts. +func ProcessFetchContentJob( + ctx context.Context, + config *config.Config, + redisDS *redisutil.RedisDataSource, + browser *browser.Browser, + data *JobData, + attemptsMade int, +) error { + functionStartTime := time.Now() + + // Build user list (mirrors the TS logic) + users := make([]UserConfig, 0) + if data.UserID != nil && *data.UserID != "" { + folder := data.Folder + users = append(users, UserConfig{ + ID: *data.UserID, + LibraryItemID: data.SaveRequestID, + Folder: folder, + }) + } else { + users = data.Users + } + + source := "puppeteer-parse" + if data.Source != nil && *data.Source != "" { + source = *data.Source + } + + locale := "" + if data.Locale != nil { + locale = *data.Locale + } + timezone := "" + if data.Timezone != nil { + timezone = *data.Timezone + } + + logFields := map[string]interface{}{ + "url": data.URL, + "articleSavingRequestId": data.SaveRequestID, + "source": source, + "users": users, + } + log.Printf("Article parsing request %+v", logFields) + + var processErr error + defer func() { + totalTime := time.Since(functionStartTime).Milliseconds() + log.Printf("parse-page result url=%s totalTime=%dms error=%v", data.URL, totalTime, processErr) + + // Analytics + userIDs := make([]string, len(users)) + for i, u := range users { + userIDs[i] = u.ID + } + result := "success" + var errMsg string + if processErr != nil { + result = "failure" + errMsg = processErr.Error() + } + analyticsClient := analytics.New(config) + analyticsClient.Capture(userIDs, analytics.Event{ + Result: result, + URL: data.URL, + Source: source, + TotalTime: totalTime, + ErrorMessage: errMsg, + }) + analyticsClient.Close() + + // Import status update on final failure attempt + lastAttempt := attemptsMade+1 >= maxImportAttempts + if processErr != nil && data.TaskID != nil && *data.TaskID != "" && lastAttempt { + log.Println("Sending import status update (failure)") + if len(users) > 0 { + sendImportStatusUpdate(ctx, config, users[0].ID, *data.TaskID, false) + } + } + }() + + // Check domain block + domain, err := extractDomain(data.URL) + if err != nil { + processErr = fmt.Errorf("invalid URL: %w", err) + return processErr + } + + blocked, err := isDomainBlocked(ctx, redisDS, domain, config.MaxFeedFetchFailures) + if err != nil { + log.Printf("Error checking domain block: %v", err) + } + if blocked { + log.Printf("Domain is blocked: %s", domain) + // Return nil (not an error) — mirroring the TS behaviour of silently dropping + return nil + } + + // Try cache + cacheKey := buildCacheKey(data.URL, locale, timezone) + fetchResult, err := getCachedResult(ctx, redisDS, cacheKey) + if err != nil { + log.Printf("Cache read error: %v", err) + } + + if fetchResult == nil { + log.Printf("Fetch result not in cache, fetching now: %s", data.URL) + fetchResult, err = fetch.FetchContent(ctx, browser, data.URL, locale, timezone) + if err != nil { + _ = incrementDomainFailure(ctx, redisDS, domain) + processErr = fmt.Errorf("fetchContent: %w", err) + return processErr + } + log.Println("Content fetched successfully") + + // Cache result (skip NO_CACHE_URLS) + if fetchResult.Content != "" && !fetch.NoCacheURLs[data.URL] { + if err := cacheResult(ctx, redisDS, cacheKey, fetchResult); err != nil { + log.Printf("Cache write error: %v", err) + } + } + } + + savedDate := time.Now() + if data.SavedAt != nil && *data.SavedAt != "" { + if t, err := time.Parse(time.RFC3339, *data.SavedAt); err == nil { + savedDate = t + } + } + + // Upload original content to object storage (GCS, S3, or MinIO). + if fetchResult.Content != "" && !config.SkipUploadOriginal { + // Bridge the legacy GCS_UPLOAD_SA_KEY_FILE_PATH setting: + // gcsblob picks up credentials via GOOGLE_APPLICATION_CREDENTIALS. + if config.GCSKeyFilePath != "" { + if err := os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.GCSKeyFilePath); err != nil { + log.Printf("Failed to set GOOGLE_APPLICATION_CREDENTIALS: %v", err) + } + } + + storageClient, err := storage.New(ctx, config.BlobURL()) + if err != nil { + log.Printf("Storage client init error: %v", err) + } else { + defer storageClient.Close() + refs := make([]storage.UserRef, len(users)) + for i, u := range users { + refs[i] = storage.UserRef{ID: u.ID, LibraryItemID: u.LibraryItemID} + } + if err := storageClient.UploadOriginalContent(ctx, refs, fetchResult.Content, savedDate.UnixMilli()); err != nil { + log.Printf("Storage upload error: %v", err) + } + } + } + + // Build save-page jobs and queue them + savedAtStr := savedDate.Format(time.RFC3339) + savePageJobs := make([]bullmq.AddJobOpts, 0, len(users)) + for _, user := range users { + folder := user.Folder + jobData := savePageJobData{ + UserID: user.ID, + URL: data.URL, + FinalURL: fetchResult.FinalURL, + ArticleSavingRequestID: user.LibraryItemID, + State: data.State, + Labels: data.Labels, + Source: source, + Folder: folder, + RSSFeedURL: data.RSSFeedURL, + SavedAt: &savedAtStr, + PublishedAt: data.PublishedAt, + TaskID: data.TaskID, + Title: fetchResult.Title, + ContentType: fetchResult.ContentType, + CacheKey: cacheKey, + } + + isRSS := data.RSSFeedURL != nil && *data.RSSFeedURL != "" + isImport := data.TaskID != nil && *data.TaskID != "" + + priority := getBullMQPriority(isRSS, isImport, data.Priority) + attempts := getAttempts(isRSS, isImport) + backoffDelay := 2000 + + savePageJobs = append(savePageJobs, bullmq.AddJobOpts{ + Name: bullmq.SavePageJob, + Data: jobData, + Opts: bullmq.JobOpts{ + Attempts: attempts, + Priority: priority, + Backoff: bullmq.BackoffOpt{ + Type: "exponential", + Delay: backoffDelay, + }, + }, + }) + } + + if err := bullmq.AddBulk(ctx, redisDS.MQClient, bullmq.BackendQueue, savePageJobs); err != nil { + processErr = fmt.Errorf("queue save-page jobs: %w", err) + return processErr + } + + log.Printf("save-page jobs queued: %d", len(savePageJobs)) + return nil +} + +// getBullMQPriority mirrors getPriority() from job.ts. +func getBullMQPriority(isRSS, isImport bool, priority string) int { + if isImport { + return 100 + } + if isRSS { + if priority == "low" { + return 10 + } + return 5 + } + if priority == "low" { + return 5 + } + return 1 +} + +// getAttempts mirrors getAttempts() from job.ts. +func getAttempts(isRSS, isImport bool) int { + if isImport { + return 1 + } + if isRSS { + return 2 + } + return 3 +} + +// buildCacheKey mirrors cacheKey() from request_handler.ts. +func buildCacheKey(rawURL, locale, timezone string) string { + return fmt.Sprintf("fetch-result:%s:%s:%s", rawURL, locale, timezone) +} + +// cachedFetchResult is the JSON structure stored in Redis. +type cachedFetchResult struct { + FinalURL string `json:"finalUrl"` + Title string `json:"title,omitempty"` + Content string `json:"content,omitempty"` + ContentType string `json:"contentType,omitempty"` +} + +// getCachedResult attempts to get a cached fetch result from Redis. +func getCachedResult(ctx context.Context, redisDS *redisutil.RedisDataSource, key string) (*fetch.Result, error) { + val, err := redisDS.CacheClient.Get(ctx, key).Result() + if err == redis.Nil { + log.Printf("Fetch result not cached: %s", key) + return nil, nil + } + if err != nil { + return nil, err + } + + var cached cachedFetchResult + if err := json.Unmarshal([]byte(val), &cached); err != nil { + log.Printf("Invalid cache entry for key %s: %v", key, err) + return nil, nil + } + + if cached.FinalURL == "" { + return nil, nil + } + + log.Printf("Fetch result is cached: %s", key) + return &fetch.Result{ + FinalURL: cached.FinalURL, + Title: cached.Title, + Content: cached.Content, + ContentType: cached.ContentType, + }, nil +} + +// cacheResult stores a fetch result in Redis with a 24-hour TTL (NX = only if not exists). +func cacheResult(ctx context.Context, redisDS *redisutil.RedisDataSource, key string, r *fetch.Result) error { + val, err := json.Marshal(cachedFetchResult{ + FinalURL: r.FinalURL, + Title: r.Title, + Content: r.Content, + ContentType: r.ContentType, + }) + if err != nil { + return err + } + return redisDS.CacheClient.SetNX(ctx, key, string(val), 24*time.Hour).Err() +} + +// failureRedisKey mirrors failureRedisKey() from request_handler.ts. +func failureRedisKey(domain string) string { + return "fetch-failure:" + domain +} + +// isDomainBlocked mirrors isDomainBlocked() from request_handler.ts. +func isDomainBlocked(ctx context.Context, redisDS *redisutil.RedisDataSource, domain string, maxFailures int) (bool, error) { + blockedDomains := map[string]bool{ + "localhost": true, + "weibo.com": true, + } + if blockedDomains[domain] { + return true, nil + } + + key := failureRedisKey(domain) + val, err := redisDS.CacheClient.Get(ctx, key).Result() + if err == redis.Nil { + return false, nil + } + if err != nil { + return false, err + } + + var count int + if _, err := fmt.Sscanf(val, "%d", &count); err != nil { + return false, nil + } + + if count > maxFailures { + log.Printf("Domain is blocked (failure count=%d): %s", count, domain) + return true, nil + } + + return false, nil +} + +// incrementDomainFailure mirrors incrementContentFetchFailure() from request_handler.ts. +func incrementDomainFailure(ctx context.Context, redisDS *redisutil.RedisDataSource, domain string) error { + key := failureRedisKey(domain) + if err := redisDS.CacheClient.Incr(ctx, key).Err(); err != nil { + return err + } + return redisDS.CacheClient.Expire(ctx, key, time.Hour).Err() +} + +// sendImportStatusUpdate mirrors sendImportStatusUpdate() from request_handler.ts. +func sendImportStatusUpdate(ctx context.Context, config *config.Config, userID, taskID string, isImported bool) { + if config.JWTSecret == "" || config.ImporterMetricsCollectorURL == "" { + log.Println("JWT_SECRET or IMPORTER_METRICS_COLLECTOR_URL not set, skipping import status update") + return + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ + "uid": userID, + }) + tokenStr, err := token.SignedString([]byte(config.JWTSecret)) + if err != nil { + log.Printf("Failed to sign JWT: %v", err) + return + } + + status := "failed" + if isImported { + status = "imported" + } + + body, _ := json.Marshal(map[string]string{ + "taskId": taskID, + "status": status, + }) + + reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, config.ImporterMetricsCollectorURL, bytes.NewReader(body)) + if err != nil { + log.Printf("Failed to create import status request: %v", err) + return + } + req.Header.Set("Authorization", tokenStr) + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Printf("Failed to send import status update: %v", err) + return + } + defer resp.Body.Close() + log.Printf("Import status update sent: status=%s code=%d", status, resp.StatusCode) +} + +// extractDomain extracts the hostname from a URL. +func extractDomain(rawURL string) (string, error) { + u, err := url.Parse(rawURL) + if err != nil { + return "", err + } + return u.Hostname(), nil +} diff --git a/src-go/internal/handler/handler_test.go b/src-go/internal/handler/handler_test.go new file mode 100644 index 000000000..86108a597 --- /dev/null +++ b/src-go/internal/handler/handler_test.go @@ -0,0 +1,167 @@ +package handler + +import ( + "encoding/json" + "testing" +) + +// TestJobData_UnmarshalLabelsAsObjects verifies that the labels field is correctly +// decoded when the queue-processor sends label objects like {"name":"RSS"} rather +// than plain strings. This is the exact payload shape produced by refreshFeed.ts: +// +// labels: [{ name: 'RSS' }] +func TestJobData_UnmarshalLabelsAsObjects(t *testing.T) { + raw := `{ + "url": "https://example.com/article", + "users": [{"id": "user-1", "libraryItemId": "item-1"}], + "priority": "low", + "labels": [{"name": "RSS"}], + "rssFeedUrl": "https://example.com/feed.xml", + "savedAt": "2026-01-20T00:00:00.000Z", + "publishedAt": "2026-01-20T00:00:00.000Z", + "source": "rss-feeder" + }` + + var data JobData + if err := json.Unmarshal([]byte(raw), &data); err != nil { + t.Fatalf("unmarshal error (was labels declared as []string instead of []LabelInput?): %v", err) + } + + if len(data.Labels) != 1 { + t.Fatalf("expected 1 label, got %d", len(data.Labels)) + } + if data.Labels[0].Name != "RSS" { + t.Errorf("expected label name 'RSS', got %q", data.Labels[0].Name) + } + if data.Labels[0].Color != nil { + t.Errorf("expected color nil, got %v", data.Labels[0].Color) + } +} + +// TestJobData_UnmarshalLabelsWithColor verifies a label with optional color field. +func TestJobData_UnmarshalLabelsWithColor(t *testing.T) { + color := "#FF0000" + raw := `{ + "url": "https://example.com/article", + "priority": "high", + "labels": [{"name": "Important", "color": "#FF0000"}] + }` + + var data JobData + if err := json.Unmarshal([]byte(raw), &data); err != nil { + t.Fatalf("unmarshal error: %v", err) + } + + if len(data.Labels) != 1 { + t.Fatalf("expected 1 label, got %d", len(data.Labels)) + } + if data.Labels[0].Name != "Important" { + t.Errorf("label name: got %q, want %q", data.Labels[0].Name, "Important") + } + if data.Labels[0].Color == nil || *data.Labels[0].Color != color { + t.Errorf("label color: got %v, want %q", data.Labels[0].Color, color) + } +} + +// TestJobData_UnmarshalNoLabels verifies that omitting labels entirely is valid. +func TestJobData_UnmarshalNoLabels(t *testing.T) { + raw := `{"url": "https://example.com/article", "priority": "high"}` + + var data JobData + if err := json.Unmarshal([]byte(raw), &data); err != nil { + t.Fatalf("unmarshal error: %v", err) + } + if len(data.Labels) != 0 { + t.Errorf("expected 0 labels, got %d", len(data.Labels)) + } +} + +// TestJobData_UnmarshalMultipleLabels verifies multiple label objects decode correctly. +func TestJobData_UnmarshalMultipleLabels(t *testing.T) { + raw := `{ + "url": "https://example.com/article", + "priority": "low", + "labels": [ + {"name": "RSS"}, + {"name": "Tech", "color": "#00FF00"}, + {"name": "Reading", "description": "To read later"} + ] + }` + + var data JobData + if err := json.Unmarshal([]byte(raw), &data); err != nil { + t.Fatalf("unmarshal error: %v", err) + } + if len(data.Labels) != 3 { + t.Fatalf("expected 3 labels, got %d", len(data.Labels)) + } + + names := []string{"RSS", "Tech", "Reading"} + for i, want := range names { + if data.Labels[i].Name != want { + t.Errorf("label[%d]: got %q, want %q", i, data.Labels[i].Name, want) + } + } + + if data.Labels[2].Description == nil || *data.Labels[2].Description != "To read later" { + t.Errorf("label[2] description: got %v, want %q", data.Labels[2].Description, "To read later") + } +} + +// TestSavePageJobData_MarshalLabels verifies that outgoing save-page jobs +// serialise labels back as objects (not strings), preserving the shape the +// backend queue-processor expects. +func TestSavePageJobData_MarshalLabels(t *testing.T) { + color := "#123456" + job := savePageJobData{ + UserID: "user-1", + URL: "https://example.com", + FinalURL: "https://example.com", + ArticleSavingRequestID: "item-1", + Source: "rss-feeder", + Labels: []LabelInput{ + {Name: "RSS"}, + {Name: "Custom", Color: &color}, + }, + } + + b, err := json.Marshal(job) + if err != nil { + t.Fatalf("marshal error: %v", err) + } + + var out map[string]interface{} + if err := json.Unmarshal(b, &out); err != nil { + t.Fatalf("re-unmarshal error: %v", err) + } + + rawLabels, ok := out["labels"].([]interface{}) + if !ok { + t.Fatalf("labels field is not an array: %T", out["labels"]) + } + if len(rawLabels) != 2 { + t.Fatalf("expected 2 labels, got %d", len(rawLabels)) + } + + // Each label must be an object, not a string. + for i, l := range rawLabels { + lmap, ok := l.(map[string]interface{}) + if !ok { + t.Errorf("label[%d] is not an object: %T (%v)", i, l, l) + continue + } + if _, hasName := lmap["name"]; !hasName { + t.Errorf("label[%d] missing 'name' key", i) + } + } + + first := rawLabels[0].(map[string]interface{}) + if first["name"] != "RSS" { + t.Errorf("first label name: got %v, want %q", first["name"], "RSS") + } + + second := rawLabels[1].(map[string]interface{}) + if second["color"] != "#123456" { + t.Errorf("second label color: got %v, want %q", second["color"], "#123456") + } +} diff --git a/src-go/internal/metrics/metrics.go b/src-go/internal/metrics/metrics.go new file mode 100644 index 000000000..fe70c2881 --- /dev/null +++ b/src-go/internal/metrics/metrics.go @@ -0,0 +1,90 @@ +// Package metrics registers and exposes Prometheus gauges for the content-fetch +// queue, matching the metric names produced by the original TypeScript service. +package metrics + +import ( + "context" + "log" + "net/http" + + "github.com/omnivore-app/omnivore/internal/bullmq" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/redis/go-redis/v9" +) + +const queueLabel = "queue" + +var ( + activeGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "omnivore_queue_messages_active", + Help: "Number of active jobs in the queue.", + }, []string{queueLabel}) + + failedGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "omnivore_queue_messages_failed", + Help: "Number of failed jobs in the queue.", + }, []string{queueLabel}) + + completedGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "omnivore_queue_messages_completed", + Help: "Number of completed jobs in the queue.", + }, []string{queueLabel}) + + prioritizedGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "omnivore_queue_messages_prioritized", + Help: "Number of prioritized (waiting) jobs in the queue.", + }, []string{queueLabel}) + + oldestJobAgeGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "omnivore_queue_messages_oldest_job_age_seconds", + Help: "Age in seconds of the oldest prioritized job in the queue.", + }, []string{queueLabel}) +) + +func init() { + prometheus.MustRegister( + activeGauge, + failedGauge, + completedGauge, + prioritizedGauge, + oldestJobAgeGauge, + ) +} + +// Handler returns an http.Handler that refreshes queue metrics from Redis on +// every request and then delegates to the standard promhttp handler. +func Handler(redisClient *redis.Client, queueName string) http.Handler { + inner := promhttp.Handler() + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := refresh(r.Context(), redisClient, queueName); err != nil { + log.Printf("Error refreshing queue metrics: %v", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + inner.ServeHTTP(w, r) + }) +} + +// refresh pulls the current queue counts from Redis and updates the gauges. +func refresh(ctx context.Context, redisClient *redis.Client, queueName string) error { + counts, err := bullmq.GetQueueCounts(ctx, redisClient, queueName) + if err != nil { + return err + } + + labels := prometheus.Labels{queueLabel: queueName} + activeGauge.With(labels).Set(float64(counts["active"])) + failedGauge.With(labels).Set(float64(counts["failed"])) + completedGauge.With(labels).Set(float64(counts["completed"])) + prioritizedGauge.With(labels).Set(float64(counts["prioritized"])) + + age, err := bullmq.OldestPrioritizedJobAge(ctx, redisClient, queueName) + if err != nil { + log.Printf("Error getting oldest job age: %v", err) + age = 0 + } + oldestJobAgeGauge.With(labels).Set(age) + + return nil +} diff --git a/src-go/internal/queue/worker.go b/src-go/internal/queue/worker.go new file mode 100644 index 000000000..654ce1d5c --- /dev/null +++ b/src-go/internal/queue/worker.go @@ -0,0 +1,110 @@ +package queue + +import ( + "context" + "encoding/json" + "log" + "sync" + "time" + + "github.com/omnivore-app/omnivore/internal/browser" + "github.com/omnivore-app/omnivore/internal/bullmq" + "github.com/omnivore-app/omnivore/internal/config" + "github.com/omnivore-app/omnivore/internal/handler" + "github.com/omnivore-app/omnivore/internal/redisutil" +) + +const ( + workerConcurrency = 4 + workerPollInterval = 500 * time.Millisecond +) + +// Worker processes jobs from the content-fetch BullMQ queue. +type Worker struct { + ctx context.Context + config *config.Config + redisDS *redisutil.RedisDataSource + browser *browser.Browser + wg sync.WaitGroup + sem chan struct{} +} + +func NewWorker(ctx context.Context, config *config.Config, redisDS *redisutil.RedisDataSource, browser *browser.Browser) *Worker { + return &Worker{ + ctx: ctx, + config: config, + redisDS: redisDS, + browser: browser, + sem: make(chan struct{}, workerConcurrency), + } +} + +func (w *Worker) Start() { + w.wg.Add(1) + go w.run() +} + +func (w *Worker) Wait() { + w.wg.Wait() +} + +func (w *Worker) run() { + defer w.wg.Done() + + log.Println("Queue worker started") + + // Ensure queue meta exists + _ = bullmq.EnsureQueueMeta(w.ctx, w.redisDS.MQClient, bullmq.ContentFetchQueue) + + for { + select { + case <-w.ctx.Done(): + log.Println("Queue worker stopping, draining active slots...") + for i := 0; i < workerConcurrency; i++ { + w.sem <- struct{}{} + } + log.Println("Queue worker stopped") + return + default: + } + + job, err := bullmq.PopJob(w.ctx, w.redisDS.MQClient, bullmq.ContentFetchQueue) + if err != nil { + log.Printf("Error popping job: %v", err) + time.Sleep(workerPollInterval) + continue + } + if job == nil { + time.Sleep(workerPollInterval) + continue + } + + w.sem <- struct{}{} + w.wg.Add(1) + go func(j *bullmq.RawJob) { + defer func() { <-w.sem }() + defer w.wg.Done() + w.processJob(j) + }(job) + } +} + +func (w *Worker) processJob(job *bullmq.RawJob) { + log.Printf("Processing job id=%s name=%s", job.ID, job.Name) + + var data handler.JobData + if err := json.Unmarshal(job.Data, &data); err != nil { + log.Printf("Failed to unmarshal job data id=%s: %v\ndata: %s", job.ID, err, job.Data) + _ = bullmq.FailJob(w.ctx, w.redisDS.MQClient, bullmq.ContentFetchQueue, job.ID, err.Error(), job.Opts) + return + } + + if err := handler.ProcessFetchContentJob(w.ctx, w.config, w.redisDS, w.browser, &data, job.AttemptsMade); err != nil { + log.Printf("Job id=%s failed: %v", job.ID, err) + _ = bullmq.FailJob(w.ctx, w.redisDS.MQClient, bullmq.ContentFetchQueue, job.ID, err.Error(), job.Opts) + return + } + + _ = bullmq.CompleteJob(w.ctx, w.redisDS.MQClient, bullmq.ContentFetchQueue, job.ID) + log.Printf("Job id=%s completed", job.ID) +} diff --git a/src-go/internal/redisutil/redisutil.go b/src-go/internal/redisutil/redisutil.go new file mode 100644 index 000000000..04a3eeae8 --- /dev/null +++ b/src-go/internal/redisutil/redisutil.go @@ -0,0 +1,88 @@ +package redisutil + +import ( + "context" + "crypto/tls" + "fmt" + "log" + "strings" + "time" + + "github.com/omnivore-app/omnivore/internal/config" + "github.com/redis/go-redis/v9" +) + +// RedisDataSource holds two Redis clients: one for caching and one for BullMQ queues. +type RedisDataSource struct { + CacheClient *redis.Client + MQClient *redis.Client +} + +func New(cfg *config.Config) (*RedisDataSource, error) { + cacheClient, err := newClient(cfg.RedisURL, cfg.RedisCert) + if err != nil { + return nil, fmt.Errorf("cache redis: %w", err) + } + + mqClient := cacheClient + if cfg.MQRedisURL != "" { + mqClient, err = newClient(cfg.MQRedisURL, cfg.MQRedisCert) + if err != nil { + return nil, fmt.Errorf("mq redis: %w", err) + } + } + + // Ping both + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := cacheClient.Ping(ctx).Err(); err != nil { + return nil, fmt.Errorf("cache redis ping: %w", err) + } + if mqClient != cacheClient { + if err := mqClient.Ping(ctx).Err(); err != nil { + return nil, fmt.Errorf("mq redis ping: %w", err) + } + } + + log.Println("Redis connected") + return &RedisDataSource{ + CacheClient: cacheClient, + MQClient: mqClient, + }, nil +} + +func newClient(redisURL, cert string) (*redis.Client, error) { + if redisURL == "" { + return nil, fmt.Errorf("redis URL is empty") + } + + opt, err := redis.ParseURL(redisURL) + if err != nil { + return nil, fmt.Errorf("parse redis URL: %w", err) + } + + // TLS with custom cert for rediss:// URLs + if strings.HasPrefix(redisURL, "rediss://") && cert != "" { + opt.TLSConfig = &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // matches original TS behaviour + RootCAs: nil, + } + } + + // Match ioredis settings from original + opt.DialTimeout = 10 * time.Second + + return redis.NewClient(opt), nil +} + +func (r *RedisDataSource) Shutdown() { + if err := r.CacheClient.Close(); err != nil { + log.Printf("Error closing cache Redis: %v", err) + } + if r.MQClient != r.CacheClient { + if err := r.MQClient.Close(); err != nil { + log.Printf("Error closing MQ Redis: %v", err) + } + } + log.Println("Redis shutdown complete") +} diff --git a/src-go/internal/server/server.go b/src-go/internal/server/server.go new file mode 100644 index 000000000..583bf3980 --- /dev/null +++ b/src-go/internal/server/server.go @@ -0,0 +1,100 @@ +// Package server implements the HTTP endpoints matching the original content-fetch Express app. +package server + +import ( + "context" + "encoding/json" + "log" + "net/http" + "strconv" + + "github.com/omnivore-app/omnivore/internal/browser" + "github.com/omnivore-app/omnivore/internal/bullmq" + "github.com/omnivore-app/omnivore/internal/config" + "github.com/omnivore-app/omnivore/internal/handler" + "github.com/omnivore-app/omnivore/internal/metrics" + "github.com/omnivore-app/omnivore/internal/redisutil" +) + +// Worker is the minimal interface the server needs from the queue worker. +type Worker interface { + Wait() +} + +type mux struct { + config *config.Config + redisDS *redisutil.RedisDataSource + browser *browser.Browser + worker Worker + http.ServeMux +} + +// New returns an http.Handler with all routes registered. +func New(config *config.Config, redisDS *redisutil.RedisDataSource, browser *browser.Browser, worker Worker) http.Handler { + m := &mux{config: config, redisDS: redisDS, browser: browser, worker: worker} + m.HandleFunc("GET /_ah/health", m.health) + m.HandleFunc("GET /lifecycle/prestop", m.prestop) + m.Handle("GET /metrics", metrics.Handler(redisDS.MQClient, bullmq.ContentFetchQueue)) + m.HandleFunc("/", m.root) // GET and POST + return m +} + +// health responds to Google Cloud autoscaler health checks. +func (m *mux) health(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + +// prestop implements the Kubernetes/Cloud Run prestop lifecycle hook. +// It signals the worker to stop and waits for in-flight jobs to finish. +func (m *mux) prestop(w http.ResponseWriter, r *http.Request) { + log.Println("Prestop lifecycle hook called.") + m.worker.Wait() + log.Println("Worker drained on prestop") + w.WriteHeader(http.StatusOK) +} + +// root handles the primary job-processing endpoint (GET or POST /?token=...). +func (m *mux) root(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + + if r.Method != http.MethodGet && r.Method != http.MethodPost { + log.Printf("Request method is not GET or POST: %s", r.Method) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + if r.URL.Query().Get("token") != m.config.VerificationToken { + log.Println("Query does not include valid token") + w.WriteHeader(http.StatusForbidden) + return + } + + var data handler.JobData + if err := json.NewDecoder(r.Body).Decode(&data); err != nil { + log.Printf("Failed to decode request body: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + + attempt := 0 + if v := r.Header.Get("X-CloudTasks-TaskRetryCount"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + attempt = n + } + } + + if err := handler.ProcessFetchContentJob( + context.Background(), + m.config, m.redisDS, m.browser, + &data, attempt, + ); err != nil { + log.Printf("Error fetching content: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) +} diff --git a/src-go/internal/storage/storage.go b/src-go/internal/storage/storage.go new file mode 100644 index 000000000..b8d49341f --- /dev/null +++ b/src-go/internal/storage/storage.go @@ -0,0 +1,105 @@ +// Package storage handles object storage uploads via gocloud.dev/blob. +// +// It supports Google Cloud Storage, AWS S3, and MinIO/S3-compatible stores, +// selected via a single BLOB_STORAGE_URL environment variable: +// +// gs://my-bucket → GCS (Application Default Credentials) +// s3://my-bucket?region=us-east-1 → AWS S3 +// s3://my-bucket?endpoint=http://minio:9000&use_path_style=true&disable_https=true®ion=us-east-1 +// → MinIO +// +// AWS credentials for S3/MinIO are loaded via the standard AWS SDK v2 chain +// (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars, IAM role, ~/.aws/credentials). +package storage + +import ( + "context" + "fmt" + "io" + "log" + "strings" + "time" + + "gocloud.dev/blob" + _ "gocloud.dev/blob/gcsblob" // registers gs:// URL opener + _ "gocloud.dev/blob/s3blob" // registers s3:// URL opener +) + +// Client wraps blob storage operations. +type Client struct { + bucket *blob.Bucket +} + +// UserRef holds the minimal user info needed for uploads. +type UserRef struct { + ID string + LibraryItemID string +} + +// New opens a blob.Bucket from a gocloud.dev URL string. +// The URL scheme selects the backend: +// - gs://bucket-name → Google Cloud Storage +// - s3://bucket-name?... → AWS S3 or any S3-compatible store (MinIO, Ceph, R2…) +// +// For MinIO add: endpoint=http://host:port&use_path_style=true&disable_https=true®ion=us-east-1 +// The caller must call Close() when done. +func New(ctx context.Context, bucketURL string) (*Client, error) { + bucket, err := blob.OpenBucket(ctx, bucketURL) + if err != nil { + return nil, fmt.Errorf("open blob bucket %q: %w", bucketURL, err) + } + return &Client{bucket: bucket}, nil +} + +// NewFromBucket wraps an already-opened *blob.Bucket. +// Useful in tests where a pre-built bucket (e.g. memblob) is injected directly. +// The caller retains ownership of the bucket and is responsible for closing it. +func NewFromBucket(bucket *blob.Bucket) *Client { + return &Client{bucket: bucket} +} + +// Close releases resources held by the underlying bucket connection. +func (c *Client) Close() error { + return c.bucket.Close() +} + +// UploadContent uploads a content string to the bucket at filePath. +// A 30-second write timeout is applied. +func (c *Client) UploadContent(ctx context.Context, filePath, content string) error { + writeCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + w, err := c.bucket.NewWriter(writeCtx, filePath, &blob.WriterOptions{ + ContentType: "text/html", + }) + if err != nil { + return fmt.Errorf("new blob writer %s: %w", filePath, err) + } + + if _, err := io.Copy(w, strings.NewReader(content)); err != nil { + _ = w.Close() + return fmt.Errorf("write blob %s: %w", filePath, err) + } + if err := w.Close(); err != nil { + return fmt.Errorf("close blob writer %s: %w", filePath, err) + } + + log.Printf("Original content uploaded to %s", filePath) + return nil +} + +// UploadOriginalContent mirrors uploadOriginalContent() from request_handler.ts. +// It uploads content for each user at: +// +// content/{userId}/{libraryItemId}.{timestampMs}.original +// +// Upload failures are logged but do not abort the remaining users. +func (c *Client) UploadOriginalContent(ctx context.Context, users []UserRef, content string, savedTimestamp int64) error { + for _, user := range users { + filePath := fmt.Sprintf("content/%s/%s.%d.original", user.ID, user.LibraryItemID, savedTimestamp) + if err := c.UploadContent(ctx, filePath, content); err != nil { + log.Printf("Failed to upload original content for user %s: %v", user.ID, err) + } + } + return nil +} diff --git a/src-go/internal/storage/storage_test.go b/src-go/internal/storage/storage_test.go new file mode 100644 index 000000000..b919dc58e --- /dev/null +++ b/src-go/internal/storage/storage_test.go @@ -0,0 +1,175 @@ +package storage_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "gocloud.dev/blob" + "gocloud.dev/blob/memblob" + + "github.com/omnivore-app/omnivore/internal/storage" +) + +// memClient creates a Client backed by an in-memory bucket. +// No Docker, no cloud credentials needed. +func memClient(t *testing.T) (*storage.Client, *blob.Bucket) { + t.Helper() + bucket := memblob.OpenBucket(nil) + t.Cleanup(func() { _ = bucket.Close() }) + return storage.NewFromBucket(bucket), bucket +} + +// TestUploadContent verifies that a single object is written with the correct key and body. +func TestUploadContent(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + const key = "content/user-1/item-1.1700000000000.original" + const body = "Hello" + + if err := client.UploadContent(ctx, key, body); err != nil { + t.Fatalf("UploadContent error: %v", err) + } + + exists, err := bucket.Exists(ctx, key) + if err != nil { + t.Fatalf("Exists error: %v", err) + } + if !exists { + t.Fatalf("expected blob %q to exist after upload", key) + } + + data, err := bucket.ReadAll(ctx, key) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + if string(data) != body { + t.Errorf("body mismatch: got %q, want %q", data, body) + } +} + +// TestUploadOriginalContent verifies that one blob per user is created at the +// correct path pattern: content/{userId}/{libraryItemId}.{timestampMs}.original +func TestUploadOriginalContent(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + const ts = int64(1700000000000) + const content = "shared article" + + refs := []storage.UserRef{ + {ID: "user-1", LibraryItemID: "item-1"}, + {ID: "user-2", LibraryItemID: "item-2"}, + {ID: "user-3", LibraryItemID: "item-3"}, + } + + if err := client.UploadOriginalContent(ctx, refs, content, ts); err != nil { + t.Fatalf("UploadOriginalContent error: %v", err) + } + + for _, ref := range refs { + key := fmt.Sprintf("content/%s/%s.%d.original", ref.ID, ref.LibraryItemID, ts) + + exists, err := bucket.Exists(ctx, key) + if err != nil { + t.Fatalf("Exists(%s): %v", key, err) + } + if !exists { + t.Errorf("expected blob %q to exist", key) + continue + } + + data, err := bucket.ReadAll(ctx, key) + if err != nil { + t.Fatalf("ReadAll(%s): %v", key, err) + } + if string(data) != content { + t.Errorf("key %q: content mismatch: got %q, want %q", key, data, content) + } + } +} + +// TestUploadOriginalContent_Empty verifies that an empty user slice produces no blobs. +func TestUploadOriginalContent_Empty(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + if err := client.UploadOriginalContent(ctx, nil, "", 1700000000000); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Iterate to check no objects exist. + iter := bucket.List(nil) + obj, err := iter.Next(ctx) + if err == nil { + t.Errorf("expected no objects, found %q", obj.Key) + } +} + +// TestUploadContent_ContentType verifies the correct Content-Type is set on the blob. +func TestUploadContent_ContentType(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + const key = "content/user/item.1700000000000.original" + if err := client.UploadContent(ctx, key, ""); err != nil { + t.Fatalf("UploadContent error: %v", err) + } + + attrs, err := bucket.Attributes(ctx, key) + if err != nil { + t.Fatalf("Attributes error: %v", err) + } + if !strings.HasPrefix(attrs.ContentType, "text/html") { + t.Errorf("expected Content-Type text/html, got %q", attrs.ContentType) + } +} + +// TestUploadContent_Overwrite verifies that uploading to the same key twice overwrites the content. +func TestUploadContent_Overwrite(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + const key = "content/user/item.123.original" + + if err := client.UploadContent(ctx, key, "first"); err != nil { + t.Fatalf("first upload: %v", err) + } + if err := client.UploadContent(ctx, key, "second"); err != nil { + t.Fatalf("second upload: %v", err) + } + + data, err := bucket.ReadAll(ctx, key) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if string(data) != "second" { + t.Errorf("expected %q, got %q", "second", data) + } +} + +// TestUploadOriginalContent_LargeContent verifies that large content uploads succeed. +func TestUploadOriginalContent_LargeContent(t *testing.T) { + ctx := context.Background() + client, bucket := memClient(t) + + // 1 MB of HTML + content := strings.Repeat("x", 1024*1024) + refs := []storage.UserRef{{ID: "u", LibraryItemID: "i"}} + const ts = int64(1000) + + if err := client.UploadOriginalContent(ctx, refs, content, ts); err != nil { + t.Fatalf("error: %v", err) + } + + key := fmt.Sprintf("content/u/i.%d.original", ts) + data, err := bucket.ReadAll(ctx, key) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if len(data) != len(content) { + t.Errorf("size mismatch: got %d bytes, want %d bytes", len(data), len(content)) + } +} diff --git a/src-go/main.go b/src-go/main.go new file mode 100644 index 000000000..1229cb843 --- /dev/null +++ b/src-go/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/omnivore-app/omnivore/cmd" + +func main() { + cmd.Execute() +}