diff --git a/.dockerignore b/.dockerignore index 58157d36d..c777d7187 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,7 @@ **/Dockerfile **/.dockerignore **/*.yaml +!self-hosting/omc/pnpm-lock.yaml .secrets*.yaml apple android diff --git a/.gitignore b/.gitignore index 963660405..3712fa610 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,9 @@ package-lock.json .env.local .env.production +# OMC cron sidecar secrets +self-hosting/docker-compose/omc.env + # build & dist dirs build diff --git a/self-hosting/GUIDE.md b/self-hosting/GUIDE.md index 4d5552626..bf526c6b5 100644 --- a/self-hosting/GUIDE.md +++ b/self-hosting/GUIDE.md @@ -13,7 +13,7 @@ ## Docker Compose We recommend using Docker-compose for the simplest way to deploy Omnivore. We have provided a configuration in the `self-hosting/docker-compose` folder. - + All networking and persistent storage is handled by the docker-compose file. ### Requirements @@ -62,6 +62,26 @@ When the service is ready you can access the web-app by using localhost:3000 With the default .env file you will be able to use Omnivore, add RSS Feeds, add stories etc. +### Optional: OMC Cron Sidecar (Content Analysis + Notes) + +This repo includes an optional `omc-cron` service that runs the Omnivore Content System (OMC) on a schedule: + +- Fetch recent saves into an analysis queue +- Run `omc analyze auto` (non-interactive) using `codex exec` +- Classify analyzed items (adds topic/sentiment/type labels) +- Sync summaries + optional NOTE highlights back into Omnivore +- Run non-destructive hygiene (backup + integrity checks + daily corpus report) + +To enable: + +1. In `self-hosting/docker-compose/`, create `omc.env` from the example: + - `cp omc.env.example omc.env` +2. Fill in: + - `OMNIVORE_API_KEY` (from Omnivore Settings → API Keys) + - `OPENAI_API_KEY` (for `codex exec`) +3. Start the stack: + - `docker compose up -d` + ### Additional Services used: @@ -391,4 +411,3 @@ To learn more about setting up the OpenAI Api key, read here: https://openai.com In future releases we would like to be able to open this up to use different LLMs, such as Anthropic, Mistral, Bedrock, or any of the other myriad LLM Services. - diff --git a/self-hosting/docker-compose/docker-compose.yml b/self-hosting/docker-compose/docker-compose.yml index b155937f8..1fa82e278 100644 --- a/self-hosting/docker-compose/docker-compose.yml +++ b/self-hosting/docker-compose/docker-compose.yml @@ -153,6 +153,23 @@ services: condition: service_healthy restart: always + omc-cron: + build: + context: ../.. + dockerfile: self-hosting/docker-compose/omc/Dockerfile + container_name: "omnivore-omc-cron" + env_file: + - ./omc.env + volumes: + - omc_data:/opt/omc/data + - omc_content:/opt/omc/content + - omc_temp:/opt/omc/temp + - omc_logs:/var/log/omc + depends_on: + api: + condition: service_healthy + restart: always + print-server: profiles: ["print"] build: @@ -188,6 +205,10 @@ volumes: pgdata: redis_data: minio_data: + omc_data: + omc_content: + omc_temp: + omc_logs: print_spool: networks: diff --git a/self-hosting/docker-compose/omc.env.example b/self-hosting/docker-compose/omc.env.example new file mode 100644 index 000000000..f22d1e100 --- /dev/null +++ b/self-hosting/docker-compose/omc.env.example @@ -0,0 +1,21 @@ +# Omnivore Content System (OMC) cron sidecar +# +# Copy to `omc.env` and fill in secrets: +# cp omc.env.example omc.env + +# Omnivore GraphQL endpoint (inside compose network) +OMNIVORE_API_URL=http://api:8080/api/graphql + +# Omnivore API key (generate in Omnivore UI: Settings → API Keys) +OMNIVORE_API_KEY= + +# Codex CLI auth (used by `codex exec` during `omc analyze auto`) +OPENAI_API_KEY= + +# Optional tuning +OMC_QUEUE_HOURS=2 +OMC_BATCH_SIZE=5 +OMC_JSONL_PATH=content/analysis/analyses.jsonl +OMC_PREP_SINCE_HOURS=24 +OMC_PREP_LIMIT=50 + diff --git a/self-hosting/docker-compose/omc/Dockerfile b/self-hosting/docker-compose/omc/Dockerfile new file mode 100644 index 000000000..f1043dd72 --- /dev/null +++ b/self-hosting/docker-compose/omc/Dockerfile @@ -0,0 +1,44 @@ +FROM node:18-bookworm-slim + +ARG TARGETARCH + +ENV PNPM_HOME=/pnpm +ENV PATH=$PNPM_HOME:$PATH +ENV OMC_DIR=/opt/omc + +WORKDIR /opt/omc + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + curl \ + python3 \ + make \ + g++ \ + && rm -rf /var/lib/apt/lists/* + +# pnpm (Corepack-managed, pinned to the OMC repo's packageManager) +RUN corepack enable && corepack prepare pnpm@10.14.0 --activate + +# Codex CLI (OMC's default analyzer shells out to `codex exec`) +RUN pnpm add -g @openai/codex@0.93.0 + +# supercronic (cron runner that inherits container env) +RUN curl -fsSL -o /usr/local/bin/supercronic "https://github.com/aptible/supercronic/releases/download/v0.2.29/supercronic-linux-${TARGETARCH}" \ + && chmod +x /usr/local/bin/supercronic + +# Install/build OMC (vendored into this repo under self-hosting/omc/) +COPY self-hosting/omc/package.json self-hosting/omc/pnpm-lock.yaml self-hosting/omc/.npmrc ./ +RUN pnpm install --frozen-lockfile +COPY self-hosting/omc/ ./ +RUN pnpm run build + +RUN mkdir -p /etc/omc /var/log/omc /var/lock/omc /opt/omc/data /opt/omc/content /opt/omc/temp + +COPY self-hosting/docker-compose/omc/omc.crontab /etc/omc/omc.crontab +COPY self-hosting/docker-compose/omc/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + diff --git a/self-hosting/docker-compose/omc/docker-entrypoint.sh b/self-hosting/docker-compose/omc/docker-entrypoint.sh new file mode 100644 index 000000000..5f6fa4102 --- /dev/null +++ b/self-hosting/docker-compose/omc/docker-entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +mkdir -p /var/log/omc /var/lock/omc /opt/omc/data /opt/omc/content /opt/omc/temp /opt/omc/content/corpus-reports + +exec /usr/local/bin/supercronic -passthrough-logs /etc/omc/omc.crontab + diff --git a/self-hosting/docker-compose/omc/omc.crontab b/self-hosting/docker-compose/omc/omc.crontab new file mode 100644 index 000000000..0cf1adf51 --- /dev/null +++ b/self-hosting/docker-compose/omc/omc.crontab @@ -0,0 +1,14 @@ +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# Global lock to avoid overlapping runs. +# supercronic runs jobs with container env (so OMNIVORE_API_KEY, OMNIVORE_API_URL, OPENAI_API_KEY work). + +# Every 30 minutes: ingest recent items -> analyze -> retry failures. +*/30 * * * * flock -n /var/lock/omc/omc.lock bash -lc 'cd /opt/omc && node dist/bin/omc.js queue add --hours "${OMC_QUEUE_HOURS:-2}" && node dist/bin/omc.js analyze auto --batch-size "${OMC_BATCH_SIZE:-5}" --jsonl --jsonl-path "${OMC_JSONL_PATH:-content/analysis/analyses.jsonl}" && node dist/bin/omc.js analyze retry --failed' >> /var/log/omc/analyze.log 2>&1 + +# Hourly: content prep (classification + sync summaries/notes back to Omnivore). +10 * * * * flock -n /var/lock/omc/omc.lock bash -lc 'cd /opt/omc && node dist/bin/omc.js content classify --all --since-hours "${OMC_PREP_SINCE_HOURS:-24}" --limit "${OMC_PREP_LIMIT:-50}" && node dist/bin/omc.js content sync --all --since-hours "${OMC_PREP_SINCE_HOURS:-24}" --limit "${OMC_PREP_LIMIT:-50}" --create-notes' >> /var/log/omc/prep.log 2>&1 + +# Daily hygiene (non-destructive): backup + integrity check + summary report. +0 2 * * * flock -n /var/lock/omc/omc.lock bash -lc 'cd /opt/omc && node dist/bin/omc.js db backup && node dist/bin/omc.js db check && node dist/bin/omc.js doctor && mkdir -p content/corpus-reports && node dist/bin/omc.js report corpus > "content/corpus-reports/$(date -I)-daily.md"' >> /var/log/omc/hygiene.log 2>&1 diff --git a/self-hosting/omc/.gitignore b/self-hosting/omc/.gitignore new file mode 100644 index 000000000..d9141944b --- /dev/null +++ b/self-hosting/omc/.gitignore @@ -0,0 +1,67 @@ +# Environment variables +.env +.env.local +.env.*.local + +# Dependencies +node_modules/ +.pnpm-store/ +pnpm-lock.yaml +package-lock.json +yarn.lock + +# Runtime data +data/ +*.sqlite +*.sqlite-journal +*.db + +# Test outputs +test-scripts/test-output/ + +# Cache +cache/ +.cache/ + +# Generated content +generated/ +output/ + +# Logs +logs/ +*.log +npm-debug.log* +pnpm-debug.log* +yarn-debug.log* +yarn-error.log* + +# OS files +.DS_Store +Thumbs.db + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Test coverage +coverage/ + +# Build output +dist/ +build/ + +# GraphQL codegen output (currently not used by runtime) +src/types/generated/ + +# Temporary files +tmp/ +temp/ +*.tmp + +# API keys and secrets (extra safety) +**/secrets/ +**/*secret* +**/*key*.json diff --git a/self-hosting/omc/.npmrc b/self-hosting/omc/.npmrc new file mode 100644 index 000000000..6c59086d8 --- /dev/null +++ b/self-hosting/omc/.npmrc @@ -0,0 +1 @@ +enable-pre-post-scripts=true diff --git a/self-hosting/omc/AGENTS.md b/self-hosting/omc/AGENTS.md new file mode 100644 index 000000000..df7a4af98 --- /dev/null +++ b/self-hosting/omc/AGENTS.md @@ -0,0 +1,40 @@ +# Agent Instructions + +This project uses **bd** (beads) for issue tracking. Run `bd onboard` to get started. + +## Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --status in_progress # Claim work +bd close # Complete work +bd sync # Sync with git +``` + +## Landing the Plane (Session Completion) + +**When ending a work session**, you MUST complete ALL steps below. Work is NOT complete until `git push` succeeds. + +**MANDATORY WORKFLOW:** + +1. **File issues for remaining work** - Create issues for anything that needs follow-up +2. **Run quality gates** (if code changed) - Tests, linters, builds +3. **Update issue status** - Close finished work, update in-progress items +4. **PUSH TO REMOTE** - This is MANDATORY: + ```bash + git pull --rebase + bd sync + git push + git status # MUST show "up to date with origin" + ``` +5. **Clean up** - Clear stashes, prune remote branches +6. **Verify** - All changes committed AND pushed +7. **Hand off** - Provide context for next session + +**CRITICAL RULES:** +- Work is NOT complete until `git push` succeeds +- NEVER stop before pushing - that leaves work stranded locally +- NEVER say "ready to push when you are" - YOU must push +- If push fails, resolve and retry until it succeeds + diff --git a/self-hosting/omc/CLAUDE.md b/self-hosting/omc/CLAUDE.md new file mode 100644 index 000000000..416a6cfb8 --- /dev/null +++ b/self-hosting/omc/CLAUDE.md @@ -0,0 +1,524 @@ +# Omnivore Content System - Agent Context + +## Project Purpose +Transform voracious reading habits into monetizable content through AI-powered analysis and generation. + +## Content Strategy + +### Primary Topics +- **AI & Machine Learning**: LLMs, agents, training, deployment +- **Tech Infrastructure**: Cloud, DevOps, databases, system design +- **Software Engineering**: Best practices, tools, frameworks +- **Startup/Business**: Product, growth, monetization strategies + +### Content Goals +1. **Weekly Blog Roundup**: Top 5-10 AI/Tech stories with analysis +2. **Deep Dive Posts**: Multi-article synthesis into original analysis +3. **Newsletter**: Curated links with personal commentary +4. **SEO Optimization**: Drive organic traffic through strategic keywords +5. **Monetization**: Build audience → affiliate links → sponsorships + +## Writing Style + +### Voice & Tone +- **Authoritative yet accessible**: Technical depth without jargon overload +- **Opinionated**: Strong takes backed by evidence from reading +- **Practical**: Focus on actionable insights and implications +- **Conversational**: Write like talking to a smart colleague + +### Content Structure +- **Hook**: Start with surprising insight or provocative question +- **Context**: Brief background from articles +- **Analysis**: Your unique take connecting multiple sources +- **Implications**: What this means for readers +- **CTA**: Subscribe, share, engage + +### Writing Guidelines +- Use active voice +- Short paragraphs (2-3 sentences max) +- Subheadings for scannability +- Bullet points for lists +- Examples and analogies +- Link to sources (Omnivore articles) + +## Content Types + +### 1. Weekly Roundup (Fridays) +**Format**: "AI/Tech This Week: Top 5 Stories You Need to Know" +- 5-10 articles from the week +- 100-150 words per story +- Overall theme/trend analysis +- 800-1200 words total + +### 2. Deep Dive (Monthly) +**Format**: "The Complete Guide to [Topic]" +- Synthesize 10-20 articles on single topic +- Original analysis and insights +- Code examples, diagrams +- 2000-3000 words + +### 3. Newsletter (Sundays) +**Format**: "Weekend Reading: AI/Tech Digest" +- 7-10 curated links +- Personal commentary (50-75 words each) +- Quick hits section (5-10 line items) +- 600-800 words total + +### 4. Social Media +**Twitter Threads**: Key insights from deep dives (8-10 tweets) +**LinkedIn Posts**: Professional takeaways (300-500 words) + +## SEO Strategy + +### Primary Keywords +- "AI trends 2025" +- "LLM development" +- "Claude agents tutorial" +- "Tech infrastructure best practices" +- "Startup engineering advice" + +### Content Optimization +- Title: 60 chars, keyword-rich, curiosity-driven +- Meta description: 155 chars, compelling summary +- H2 headers: Question-based, keyword-targeted +- Internal links: Cross-reference related posts +- External links: High-authority sources (Omnivore articles) + +## Monetization Path + +### Phase 1: Audience Building (Months 1-3) +- Publish 2x/week (roundup + deep dive) +- Email list growth (newsletter signups) +- SEO optimization for organic traffic +- Social media distribution + +### Phase 2: Engagement (Months 4-6) +- Reader surveys and topic requests +- Comments and discussions +- Guest posts and collaborations +- Community building + +### Phase 3: Monetization (Months 7+) +- Affiliate links (tools, books, courses) +- Sponsored posts (relevant products) +- Premium newsletter tier +- Consulting/advisory services + +## Quality Standards + +### Before Publishing +- [ ] Fact-check all claims against source articles +- [ ] Verify all links work +- [ ] Run through Grammarly or similar +- [ ] Check SEO optimization (Yoast, etc.) +- [ ] Preview formatting on target platform +- [ ] Add cover image (if applicable) + +### Performance Tracking +- Google Analytics: Traffic, engagement, conversions +- Email metrics: Open rate, click rate, growth +- Social metrics: Shares, comments, saves +- Database: Track which articles → which posts → performance + +## Article Analysis Workflow + +### How to Process Queued Articles + +When user asks to analyze articles, use the unified CLI: + +**Automated (cron/launchd):** +```bash +# End-to-end, non-interactive (uses codex exec in read-only mode) +omc queue add --hours 24 +omc analyze auto --batch-size 5 + +# Retry failures (optional) +omc analyze retry --failed +``` + +**Basic Analysis:** +```bash +# Prepare batch (creates stub files, outputs agent params) +omc analyze run --batch-size 5 + +# Prepare specific article +omc analyze run --article-id + +# After agents complete: save results to database + markdown +omc analyze complete + +# Save with keeping temp files for debugging +omc analyze complete --keep-temp +``` + +**Queue Management:** +```bash +# Add articles to queue +omc queue add --hours 24 # Last 24 hours +omc queue add --label "ai-ml" # By label +omc queue add --url # Single article +omc queue add --slug # By slug + +# Check queue status +omc queue list # All queued articles +omc queue list --status pending # Filter by status +omc queue stats # Statistics + +# Retry failed analyses +omc analyze retry --failed # All failed +omc analyze retry --article-id # Specific article +``` + +**Automation notes:** +- `omc analyze auto` shells out to `codex exec -s read-only` and expects a single JSON object response. +- For schedulers, we set `CODEX_HOME=temp/codex-home` so runs don’t depend on `~/.codex` permissions. +- Prefer running installs/builds via Corepack (`corepack pnpm ...`) to avoid native-module ABI mismatches. + +**View Results:** +```bash +# Show analysis for specific article +omc content show + +# List all analyzed content +omc content list + +# Search analyses +omc content search "keyword" + +# Generate reports +omc report corpus # Full corpus analysis +omc report topics # Topic distribution +omc report trends # Trending topics +``` + +**Sync to Omnivore (Create Notebook Notes):** +```bash +# Sync single article with note creation +omc content sync --create-notes + +# Sync all analyzed articles with notes +omc content sync --all --create-notes + +# Sync without creating notes (metadata only) +omc content sync +``` + +### Complete Workflow (Agent-Assisted Analysis) + +**From Reading to Published Notes:** + +1. **Add to Queue**: `omc queue add --hours 168` (or by label, URL, slug) +2. **Prepare Batch**: `omc analyze run --batch-size 5` + - Creates stub files in `temp/*.jsonl` + - Marks jobs as "in_progress" + - Outputs agent parameters (copy these) +3. **Invoke Agents**: Copy agent parameters and invoke article-content-analyzer agents via Task tool + - Agents fetch content via `omc omnivore get` + - Agents analyze and enrich stub files + - Run 5 agents in parallel for performance +4. **Save Results**: `omc analyze complete` + - Reads enriched JSONL from temp/ + - Writes markdown to `content/analysis/*.md` + - Saves to database + - Marks jobs "completed" +5. **Review**: `omc content show ` or `omc content list` +6. **Sync to Omnivore**: `omc content sync --all --create-notes` + - Updates article descriptions + - Creates notebook notes with full analysis +7. **Repeat**: Go back to step 2 until queue empty + +**Key Commands:** +- `omc analyze run --batch-size 5` - Prepare next batch +- `omc analyze complete` - Save completed analyses +- `omc content sync --all --create-notes` - Push to Omnivore + +### Legacy Scripts (Archived) +Old workflow scripts have been archived in `cli/archived-scripts/`. Use the CLI commands above instead. + +## Code Quality Tools + +### OACC (Omniarcs Code Checker) +The project has access to `oacc analyze` for automated code quality analysis. **Use this instead of manual analysis.** + +**Available via Bash tool (auto-approved):** +```bash +oacc analyze +oacc analyze --functions +``` + +**What oacc provides:** +- **Function length analysis**: Automatic counting with thresholds + - GREEN: ≤20 lines (target) + - YELLOW: 21-25 lines (warning) + - RED: >25 lines (must fix) +- **Function-level reports**: Shows each function with line count +- **Exit codes**: 0 = GREEN, 1 = YELLOW/RED +- **FTA scores**: Code quality metrics + +**DO NOT manually:** +- Write Python/shell scripts to count lines +- Parse files with regex to find functions +- Manually count function lengths +- Create custom analysis tools + +**DO use oacc:** +```bash +# Check single file +oacc analyze src/commands/queue/export.ts + +# Get function-level details +oacc analyze --functions src/commands/queue/export.ts + +# Analyze multiple files in quality-guard agent +for file in src/commands/**/*.ts; do + oacc analyze "$file" +done +``` + +**Quality-guard agent**: Always use `oacc analyze` for function length validation. It's pre-approved and faster than manual analysis. + +## CLI Development Ground Truths + +### Architecture Patterns (Established 2025-01-05) + +**1. All Commands MUST Extend BaseCommand** +- Location: `src/lib/cli/base-command.ts:16` +- Pattern: `export default class MyCommand extends BaseCommand` +- Why: Provides standard error handling, run() method, and OCLIF integration +- Evidence: All 63 commands use this pattern (verified 2025-01-05) + +**2. Database Operations MUST Use withDatabase()** +- Location: `src/lib/cli/database.ts:17` +- Pattern: `await withDatabase(async (db, repo) => { ... })` +- Why: Ensures proper connection cleanup and error handling +- Evidence: 45/46 database commands use this (db/backup.ts fixed 2025-01-05) + +**3. JSON Parsing MUST Use parseJsonSafely()** +- Location: `src/lib/cli/command-utils.ts:45` +- Pattern: `const data = parseJsonSafely(job.analysisJson)` +- Why: Prevents crashes on malformed JSON, provides type safety +- Evidence: 15+ commands updated to use this (2025-01-05) + +**4. Shared Flags MUST Use Utility Functions** +- `jsonFlag()`: `src/lib/cli/shared-flags.ts:14` - Standard JSON output flag +- `statusFlag()`: `src/lib/cli/shared-flags.ts:8` - Queue status filtering +- Why: DRY principle, consistent flag behavior +- Evidence: 45/63 commands use jsonFlag() + +**5. Output Formatting MUST Use Shared Formatters** +- `formatHeader()`: `src/lib/cli/formatters.ts:18` - Section headers +- `formatSuccess()`: `src/lib/cli/formatters.ts:37` - Success messages +- `formatError()`: `src/lib/cli/formatters.ts:48` - Error messages +- Why: Consistent user experience, single source of truth +- Evidence: All 63 commands use these formatters + +**6. GraphQL Operations MUST Use checkGraphQLResult()** +- Location: `src/lib/cli/graphql.ts:21` +- Pattern: `checkGraphQLResult(result)` after every GraphQL call +- Why: Checks both result.errors AND domain errorCodes +- Evidence: All 9 omnivore commands use this + +**7. Environment Variables MUST Use loadEnvFile()** +- Location: `src/lib/cli/command-utils.ts:58` +- Pattern: `const env = loadEnvFile('.env')` +- Why: Centralized .env parsing, no duplicate implementations +- Evidence: 2 duplicate implementations removed (2025-01-05) + +### Shared Utilities Reference + +**Database Utilities:** +```typescript +// src/lib/cli/database.ts +withDatabase(callback: (db, repo) => Promise): Promise +``` + +**Data Utilities:** +```typescript +// src/lib/cli/command-utils.ts +parseJsonSafely(jsonString: string, fallback?: T): T | undefined +loadEnvFile(envPath: string = '.env'): Record +handleCommandError(command: Command, error: unknown): void +outputResult(command: Command, data: any, successMessage: string, jsonMode: boolean): void +``` + +**Flag Utilities:** +```typescript +// src/lib/cli/shared-flags.ts +jsonFlag(): Flags.Boolean +statusFlag(): Flags.String +``` + +**Formatting Utilities:** +```typescript +// src/lib/cli/formatters.ts +formatHeader(title: string): string +formatSuccess(message: string): string +formatError(message: string): string +formatDivider(): string +``` + +**GraphQL Utilities:** +```typescript +// src/lib/cli/graphql.ts +checkGraphQLResult(result: any): void +fetchUsername(): Promise +``` + +**Display Utilities:** +```typescript +// src/lib/cli/queue-display.ts +displayQueueStats(stats: QueueStats): void +displayJobs(jobs: AnalysisJob[]): void +``` + +### Command Structure Pattern + +**Every command follows this structure:** +```typescript +import { Args, Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { formatSuccess } from '@lib/cli/formatters.js'; + +export default class MyCommand extends BaseCommand { + static override description = 'Clear description'; + + static override examples = [ + '$ omc command arg --flag' + ]; + + static override args = { + myArg: Args.string({ description: 'Arg description', required: true }) + }; + + static override flags = { + json: jsonFlag(), + myFlag: Flags.string({ description: 'Flag description' }) + }; + + protected async execute(flags: any): Promise { + await withDatabase(async (db, repo) => { + // 1. Validate input + // 2. Execute business logic + // 3. Format output + if (flags.json) { + this.log(JSON.stringify(result, null, 2)); + } else { + this.log(formatSuccess('Success message')); + } + }); + } +} +``` + +### Complete Command Inventory (63 total) + +**Queue Management (9 commands):** +- `queue:add` - Add articles to queue +- `queue:list` - List queued articles +- `queue:stats` - Show queue statistics (with --detailed) +- `queue:reset` - Reset article status +- `queue:remove` - Remove from queue +- `queue:clear` - Bulk clear operations +- `queue:export` - Export queue state +- `queue:import` - Import queue state +- `queue:retry` - Retry failed items + +**Analysis Operations (4 commands):** +- `analyze:run` - Process articles +- `analyze:retry` - Retry failed analyses +- `analyze:status` - Show analysis status +- `analyze:watch` - Real-time monitoring + +**Omnivore Integration (9 commands):** +- `omnivore:get` - Fetch article by slug +- `omnivore:search` - Search articles +- `omnivore:list` - List recent articles +- `omnivore:update` - Update article metadata +- `omnivore:note:add` - Add note +- `omnivore:note:get` - Get notes +- `omnivore:note:update` - Update note +- `omnivore:highlight:add` - Add highlight +- `omnivore:highlight:list` - List highlights + +**Content Operations (5 commands):** +- `content:show` - Display analysis +- `content:list` - List analyzed content +- `content:search` - Full-text search +- `content:sync` - Sync to Omnivore +- `content:export` - Export for blog + +**Database Management (9 commands):** +- `db:migrate` - Run migrations +- `db:schema` - Show schema +- `db:seed` - Load sample data +- `db:vacuum` - Optimize database +- `db:backup` - Create backup +- `db:restore` - Restore from backup +- `db:reset` - Drop and recreate +- `db:check` - Integrity check +- `db:stats` - Database statistics + +**Reporting (7 commands):** +- `report:corpus` - Full corpus analysis +- `report:topics` - Topic distribution +- `report:trends` - Trending topics +- `report:monetization` - Opportunities +- `report:sentiment` - Sentiment analysis +- `report:custom` - Custom queries +- `report:export` - Export reports + +**Configuration (7 commands):** +- `config:show` - Display config +- `config:get` - Get value +- `config:set` - Set value +- `config:test` - Test API connection +- `config:validate` - Validate config +- `config:env:list` - List environments +- `config:env:use` - Switch environment + +**System (3 commands):** +- `init` - Initialize project +- `doctor` - System health check +- `version` - Version info + +### DRY Principles (Zero Tolerance) + +**Violations to Avoid:** +1. ❌ Direct `JSON.parse()` without error handling → Use `parseJsonSafely()` +2. ❌ Manual database init/close → Use `withDatabase()` +3. ❌ Duplicate flag definitions → Use `jsonFlag()`, `statusFlag()` +4. ❌ Custom error handling → Extend `BaseCommand` +5. ❌ Duplicate .env parsing → Use `loadEnvFile()` +6. ❌ Direct Command extension → Extend `BaseCommand` +7. ❌ Custom formatters → Use formatHeader/Success/Error +8. ❌ GraphQL without error check → Use `checkGraphQLResult()` + +**Evidence of Compliance:** +- All 63 commands extend BaseCommand ✓ +- All 45 database commands use withDatabase() ✓ +- All 15+ JSON parsing uses parseJsonSafely() ✓ +- Zero duplicate helper functions ✓ +- (Last verified: 2025-01-05) + +### Quality Metrics + +**Function Length Distribution (2025-01-05):** +- GREEN (≤20 lines): 52 commands (83%) +- YELLOW (21-25 lines): 11 commands (17%) +- RED (>25 lines): 0 commands (0%) + +**Build Status:** +- TypeScript: Strict mode enabled +- Compilation: 0 errors, 0 warnings +- Bundle size: Optimized with ESBuild +- Commands registered: 63/63 (100%) + +## Notes for Agents +- **Always cite sources**: Link back to Omnivore articles +- **Maintain authenticity**: Sound like the human, not generic AI +- **Be selective**: Quality content over quantity +- **Stay current**: Focus on recent articles (last 7-30 days) +- **Think monetization**: Every piece should serve the business goal +- **Use oacc for code analysis**: Don't reinvent code quality tools diff --git a/self-hosting/omc/EXTRACTION_CHECKLIST.md b/self-hosting/omc/EXTRACTION_CHECKLIST.md new file mode 100644 index 000000000..b2104fe07 --- /dev/null +++ b/self-hosting/omc/EXTRACTION_CHECKLIST.md @@ -0,0 +1,140 @@ +# Extraction Checklist + +## ✅ Pre-Extraction Verification + +Run these commands to verify everything is ready: + +```bash +cd scripts/omnivore-content-system + +# 1. Dependencies resolved +pnpm install +# Should complete without errors + +# 2. TypeScript compiles +pnpm run build +# Should create dist/ directory + +# 3. Type checking passes +pnpm run typecheck +# Should show no errors + +# 4. Omnivore client works +node lib/omnivore/client.js --test +# Should connect and show user info +``` + +## 📦 Extraction Commands + +```bash +# From omnivore repo root +cd /Volumes/devel/personal/keybase/edgerouter/mac-mini/home/omnivore/omnivore + +# Copy to new location +cp -r scripts/omnivore-content-system /path/to/new-location/ + +# OR move (if extracting permanently) +mv scripts/omnivore-content-system /path/to/new-location/ +``` + +## 🔧 Post-Extraction Setup + +```bash +cd /path/to/new-location/omnivore-content-system + +# 1. Install dependencies +pnpm install + +# 2. Configure environment +cp .env.example .env +# Edit .env with your API keys + +# 3. Test connection +node lib/omnivore/client.js --test + +# 4. Verify build +pnpm run build +pnpm run typecheck + +# 5. Initialize Git (if new repo) +git init +git add . +git commit -m "Initial commit: Omnivore content monetization system" +``` + +## 🔍 Verification After Extraction + +### Check 1: No External Dependencies +```bash +# Should find no references to parent repo paths +grep -r "\.\./\.\./self-hosting" . +grep -r "omnivore/scripts" . +# Both should return no results (or only in IMPLEMENTATION_PLAN.md for documentation) +``` + +### Check 2: All Files Present +```bash +# Required files +ls -la .env.example # ✅ Should exist +ls -la package.json # ✅ Should exist +ls -la tsconfig.json # ✅ Should exist +ls -la .gitignore # ✅ Should exist +ls -la README.md # ✅ Should exist +ls -la IMPLEMENTATION_PLAN.md # ✅ Should exist +ls -la CLAUDE.md # ✅ Should exist + +# Required directories +ls -d lib/ # ✅ Should exist +ls -d src/ # ✅ Should exist +ls -d content/ # ✅ Should exist +``` + +### Check 3: Build Works +```bash +pnpm run build +# Should succeed and create dist/types/ +ls -la dist/ +``` + +### Check 4: Client Works +```bash +# After configuring .env with real API keys +node lib/omnivore/client.js --test +# Should output: +# ✅ Connected to Omnivore API +# User: Your Name (your@email.com) +# Username: yourusername +``` + +## ❌ What Should NOT Be Included + +These remain in the omnivore repo: +- ❌ `/scripts/migrate-omnivore.js` +- ❌ `/scripts/import-pocket.js` +- ❌ `/self-hosting/` directory +- ❌ Parent repo's package.json + +## 📝 Final Checklist + +Before considering extraction complete: + +- [ ] All dependencies install without errors +- [ ] TypeScript compiles successfully +- [ ] Type checking passes with no errors +- [ ] Omnivore client connects to API +- [ ] .env.example is present with correct structure +- [ ] README.md reflects current status +- [ ] IMPLEMENTATION_PLAN.md is up to date +- [ ] .gitignore properly configured +- [ ] No symlinks to parent repo +- [ ] No hardcoded paths to parent repo structure + +## 🚀 Ready When + +All checkboxes above are checked ✅ + +## 📚 Documentation References + +- [README.md](./README.md) - Overview and quick start +- [IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md) - Detailed roadmap +- [CLAUDE.md](./CLAUDE.md) - Agent context and strategy diff --git a/self-hosting/omc/IMPLEMENTATION_PLAN.md b/self-hosting/omc/IMPLEMENTATION_PLAN.md new file mode 100644 index 000000000..f92e94785 --- /dev/null +++ b/self-hosting/omc/IMPLEMENTATION_PLAN.md @@ -0,0 +1,1612 @@ +# Omnivore Content System - Implementation Plan (Revised) + +## 🚀 NEXT SESSION START HERE + +**Current Status**: ✅ Parallel analysis workflow COMPLETE - 65 articles analyzed, 0 pending, 2 failed + +**Next Steps**: +1. **Investigate 2 failed analyses**: + - Both failed with "Cannot read properties of undefined (reading 'topicScores')" + - Likely incomplete agent analysis missing required fields + - May need to re-queue or analyze manually + +2. **Generate first content** from analyzed corpus: + - Weekly roundup from top 5-10 AI/Tech articles + - Use `omc report corpus` to identify trending topics + - Focus on AI/ML, Developer Tools, Security topics (most common in corpus) + +3. **Optional: Update groundtruth documentation** if needed: + - Current groundtruth already documents parallel workflow + - No significant architectural changes in this session + +--- + +## Overview + +AI-powered content monetization system using TypeScript, Claude Agent SDK, and modern ESM conventions. Transforms Omnivore reading into blog posts, newsletters, and social media content. + +**Key Decision**: Build mechanics-first approach - prove each building block works independently before integration. + +## Groundtruth: What Already Exists + +**Before implementing any phase**, read the groundtruth documentation to understand what's available to build upon: + +📚 **[Foundation & Type System](docs/_meta/foundation-and-types.md)** - TypeScript setup, Omnivore client library (lib/omnivore/), type definitions (src/types/), environment configuration, dependencies, and usage examples. + +📚 **[GraphQL Organization](docs/_meta/graphql-organization.md)** - GraphQL fragment system for preventing query drift, reusable fragments, composed queries, and anti-drift architecture. + +📚 **[System Architecture](docs/_meta/architecture.md)** - Three-layer architecture, storage boundaries, database schema, and type definitions. + +📚 **[CLI Reference](docs/_meta/cli-reference.md)** - Complete command-line interface documentation for queue management, analysis, and reporting. + +📚 **[Workflow Internals](docs/_meta/workflow-internals.md)** - Zero-context-pollution design, parallel analysis workflow, agent invocation patterns, and file handling. + +**Quick Reference**: +- **Omnivore Client** (lib/omnivore/client.js): 11 functions for fetching articles, labels, highlights +- **Query Builders** (lib/omnivore/queries.js): Pre-built GraphQL queries and pattern builders +- **Type System** (src/types/*.ts): Complete TypeScript types for API, storage, and analysis +- **Tracking DB** (data/omnivore-content.db): SQLite for immutable analysis snapshots + job coordination (gitignored but permanent) +- **CLI** (`omc`): queue/analyze/content/report/db/config/omnivore commands (oclif) +- **Storage** (content/analysis/): Markdown analysis results (permanent, git-tracked) +- **Workflow**: `omc queue add --hours 24` → `omc analyze auto --batch-size 5` → `omc report corpus` +- **LLM**: analysis uses `codex exec` (read-only) + Codex CLI auth (no API key in `.env`) +- **Package manager**: Prefer Corepack-managed pnpm (`corepack pnpm ...`) to avoid native-module ABI mismatches + +## Maintaining Groundtruth Documentation + +### When to Update Groundtruth + +**Update `docs/_meta/.md` immediately after**: +- Implementing a new phase or building block +- Adding new modules, functions, or classes that will be reused +- Creating new utilities, helpers, or shared code +- Establishing new patterns or conventions +- Fixing implementation drift from original design + +**Do NOT wait** until "the end" - update groundtruth as you go. + +### How to Update Groundtruth + +**Use the @documentation-writer agent**: +``` +@documentation-writer Update docs/_meta/foundation-and-types.md to include +the new ArticleWriter class in src/storage/ArticleWriter.ts with: +- What it does +- How to import and use it +- Key methods and their signatures +- Usage examples +``` + +**Documentation Standards**: +1. **Hard-to-vary facts only** - document what exists, not plans or status +2. **No status markers** - never write "complete", "done", "implemented" +3. **Focus on usage** - show how to use what's built, with examples +4. **Technical accuracy** - document actual code, not idealized version +5. **Reusability first** - emphasize how to reuse existing code + +### Anti-Drift Protocol + +**Before implementing new code**: +1. Read relevant groundtruth docs to understand what exists +2. Check if similar functionality already exists +3. Reuse existing patterns and utilities when possible + +**After implementing new code**: +1. Compare implementation against groundtruth and original design +2. Identify any drift from established patterns +3. Self-correct if drift is found: + - **Minor drift** (naming, structure): Update implementation to match patterns + - **Major drift** (architecture, approach): Document why drift occurred, update groundtruth if justified + +**When adding new groundtruth docs**: +1. Review existing groundtruth files for similar content +2. Check for inconsistencies with established patterns +3. Self-correct documentation if conflicts found +4. Maintain consistent voice and structure across all groundtruth files + +### Groundtruth File Organization + +**File naming**: `docs/_meta/.md` + +**Logical groups** (examples): +- `foundation-and-types.md` - TypeScript, dependencies, type system, client library +- `storage-and-files.md` - Markdown storage, front-matter, file operations (future) +- `analysis-and-generation.md` - Claude integration, prompts, content generation (future) +- `workflows-and-cli.md` - CLI tools, automated workflows (future) + +**When to create a new file**: +- Group is distinct from existing groundtruth +- Contains 3+ related components +- Will be referenced frequently by agents + +**When to update existing file**: +- New component fits existing logical group +- Extends or enhances documented functionality + +### Groundtruth Update Checklist + +When updating groundtruth documentation: + +- [ ] Use @documentation-writer agent (not manual edits) +- [ ] Document actual code (verify with file reads) +- [ ] Include import patterns and usage examples +- [ ] Remove any status/completion language +- [ ] Cross-reference related groundtruth sections +- [ ] Verify no conflicts with existing patterns +- [ ] Self-correct any identified drift +- [ ] Update "What Does NOT Exist Yet" section if needed + +### Example: Good vs Bad Groundtruth + +**❌ Bad (status-oriented)**: +```markdown +## ArticleWriter + +Status: Complete (2025-09-30) + +We implemented the ArticleWriter class to write articles to Markdown. +It works great and is ready to use. +``` + +**✅ Good (fact-oriented)**: +```markdown +## ArticleWriter + +**Location**: `src/storage/ArticleWriter.ts` + +**Purpose**: Writes Omnivore articles to Markdown files with YAML front-matter. + +**Import**: +```typescript +import { ArticleWriter } from '@storage/ArticleWriter'; +``` + +**Usage**: +```typescript +const writer = new ArticleWriter('content/articles'); +await writer.write(omnivoreArticle); +// Creates: content/articles/2025-09-30-article-slug.md +``` + +**Methods**: +- `write(article: OmnivoreArticle): Promise` - Write article, returns file path +- `generateSlug(title: string): string` - Create URL-friendly slug +``` + +## Architecture Decisions + +### 1. TypeScript-First (New Code Only) +- Strict TypeScript with full type safety +- Compile to `dist/` using latest ESM conventions +- **Use existing JS client as library** (import from `lib/omnivore/`) +- Source maps for debugging +- Type definitions for all public APIs + +### 2. Storage Strategy: Tracking + Permanent Storage (Three-Layer Architecture) + +**Layer 1: Omnivore (Source of Truth)** +- **Source articles**: Query Omnivore GraphQL API (NEVER store locally) +- **Labels, highlights**: Always fetch via API +- **CRITICAL**: Omnivore IS the storage for source articles + +**Layer 2: SQLite (Tracking + Immutable Snapshots)** +- **Purpose**: Coordinate parallel analysis + store original AI output +- **Location**: `data/omnivore-content.db` (gitignored) +- **Contains**: + - `analysis_queue` table: job status + immutable analysis JSON + markdown references + - Article metadata: publishedAt, updatedAt from Omnivore + - READ-ONLY access to existing Omnivore cache tables (if present) +- **Lifecycle**: Permanent storage - contains original AI analysis (queryable); Markdown has user-edited versions +- **Boundary**: Stores ORIGINAL AI analysis (queryable catalog); Markdown files are editable + +**Layer 3: Git-Tracked Files (Human-Editable Storage)** +- **Analysis results**: Markdown files with YAML front-matter (user-editable) +- **Generated content**: Markdown files for blog posts, newsletters +- **Location**: `content/analysis/`, `content/generated/` +- **User workflow**: Edit Markdown files to refine, improve, add context +- **Git versioned**: Full history of user edits and content evolution +- **Relationship to Layer 2**: SQLite stores original AI output; Markdown stores current version after edits + +**Critical Boundaries** (enforced with AIDEV-NOTE annotations): +1. **SQLite = Original + Tracking**: Immutable AI snapshots + job coordination +2. **Markdown = Editable**: User can refine AI analysis in git-tracked files +3. **Omnivore tables = READ-ONLY**: Never modify, always use GraphQL API + +### 3. Test Promotion Path +Clear progression for code maturity: +1. **test-scripts/** - Quick experiments, proof of concept +2. **cli/** - Working CLI tools for manual use +3. **tests/** - Automated test scenarios +4. **src/** - Production-ready TypeScript code + +### 4. Agent Development +- Write prompts first, test in chat +- Develop mechanics before wrapping in Agent SDK +- Use prompts directly with Anthropic API initially +- Wrap in Agent SDK once mechanics proven + +### 5. Publishing Priority +1. **Markdown files** - Initial output format +2. **Ghost** - If self-hostable +3. Other platforms later + +### 6. DRY (Don't Repeat Yourself) +- Single source of truth for API interactions (existing client) +- Shared base classes for common functionality +- Composable template system +- Unified error handling and logging +- Reusable utilities + +### 7. AIDEV Boundary Documentation +All code dealing with architectural boundaries must be annotated with `AIDEV-NOTE` comments: + +**Tracking Code** (permanent storage): +```typescript +// AIDEV-NOTE: tracking + immutable snapshots - coordination and original AI output +// AIDEV-NOTE: tracking-db - stores original analysis JSON for querying +// AIDEV-NOTE: tracking-lock - prevents duplicate analysis by concurrent runs +``` + +**Analysis Output Code** (permanent): +```typescript +// AIDEV-NOTE: analysis-output-boundary - results written to Markdown/JSONL, NOT database +// AIDEV-NOTE: git-tracked-output - analysis result stored permanently +``` + +**Omnivore Boundary Code** (read-only): +```typescript +// AIDEV-NOTE: omnivore-boundary - always use GraphQL API, never local cache +// AIDEV-NOTE: boundary-check - ensure Omnivore tables never modified by our code +``` + +### 8. Parallel Analysis Architecture +- **Concurrency**: Process 5 articles simultaneously +- **Coordination**: SQLite queue prevents duplicate work +- **Execution**: Single message with 5 Task tool calls to `@article-content-analyzer` +- **Error handling**: Failed jobs tracked, retryable up to 3 times +- **Progress tracking**: Real-time status via `analysis_queue` table +- **Output**: All results written to git-tracked Markdown/JSONL + +## Extraction Strategy: Moving to Standalone Repo + +### Timeline +The omnivore-content-system will be extracted to its own repository soon. This section describes how to handle the lib/omnivore/ dependency. + +### Recommended Approach: Copy lib/ Directory + +When extracting to standalone repo, **copy lib/omnivore/** with the content-system: + +```bash +# Extraction command +cp -r omnivore/scripts/omnivore-content-system /path/to/new-repo + +# Result: New repo includes lib/ +omnivore-content-system/ # New standalone repo +├── lib/ # ✅ Copied from old repo +│ └── omnivore/ +│ ├── client.js # Working GraphQL client +│ └── queries.js # Query builders +├── src/ # TypeScript code +├── test-scripts/ +├── content/ +└── package.json +``` + +### Why Copy Instead of NPM Package? + +**Advantages:** +- ✅ Zero setup - works immediately +- ✅ No external dependencies to manage +- ✅ Can modify if needed for content-system use case +- ✅ lib/ is stable (GraphQL client rarely changes) +- ✅ TypeScript can import JavaScript files directly + +**Trade-offs:** +- Code duplication (acceptable - lib is ~700 lines total) +- No automatic updates from omnivore repo (not needed - stable API) + +### How TypeScript Imports JavaScript + +TypeScript can import JavaScript modules directly: + +```typescript +// src/storage/ArticleWriter.ts +import { searchArticles } from '../../lib/omnivore/client.js'; +import { buildTopicQuery } from '../../lib/omnivore/queries.js'; + +// Works because: +// 1. TypeScript allows .js imports +// 2. lib/ is in tsconfig "include" paths +// 3. ESM modules work across JS/TS boundary +``` + +### Type Safety for JavaScript Client + +Create TypeScript definitions for the JavaScript client: + +```typescript +// src/types/omnivore.ts +export interface OmnivoreArticle { + id: string; + title: string; + url: string; + content?: string; + // ... matches client.js response structure +} + +// Wrapper with types (optional) +// src/lib/omnivore-typed.ts +import * as client from '../../lib/omnivore/client.js'; +import type { OmnivoreArticle, SearchResult } from '@types/omnivore'; + +export const searchArticles = client.searchArticles as ( + params: SearchParams +) => Promise; +``` + +### Future Option: NPM Package + +If later you need to: +- Use omnivore-client in multiple projects +- Share updates between projects +- Publish for community use + +Then convert lib/omnivore/ to `@yourorg/omnivore-client` npm package. + +**For now:** Copy is simpler and sufficient. + +### Extraction Checklist + +When moving to standalone repo: +- [ ] Copy entire omnivore-content-system/ directory +- [ ] Verify lib/omnivore/ is included +- [ ] Update .env with API credentials +- [ ] Run `pnpm install` +- [ ] Test: `node lib/omnivore/client.js --test` +- [ ] Verify TypeScript can import from lib/ + +--- + +## Extraction Readiness Status + +### ✅ READY TO EXTRACT - 100% Self-Contained + +The omnivore-content-system is **ready to be moved** to a standalone repository. All dependencies on the parent omnivore repo have been eliminated. + +### What's Already Copied and Working + +**Omnivore Client Library (WORKING):** +- ✅ `lib/omnivore/client.js` (8.0k) - Complete GraphQL client + - 11 functions: getMe, searchArticles, getArticle, getArticlesByDate, getArticlesByLabel, getRecentArticles, searchByTopic, getUnreadArticles, getLabels, getHighlights, testConnection + - Features: Pagination, label filtering, topic queries, content inclusion + - Built-in CLI test: `node lib/omnivore/client.js --test` +- ✅ `lib/omnivore/queries.js` (8.8k) - Query builders and patterns + - Pre-built queries: SEARCH_ARTICLES_FULL, GET_ARTICLE_FULL, SEARCH_WITH_CONTENT + - Topic queries: AI_ML, DEVOPS, PROGRAMMING, DATABASES, CLOUD, STARTUP, SECURITY + - Query builders: buildComplexQuery, buildTopicQuery, buildDateRangeQuery, buildLabelQuery + +**Legacy Migration Scripts (PRESERVED - All Copied from Parent):** +- ✅ `legacy-scripts/` contains copies of all migration scripts from parent `/scripts/`: + - `apply-labels-with-mapping.js` (11k) - Bulk label application with mapping + - `apply-labels.js` (11k) - Apply labels to articles + - `apply-single-label.js` (2.9k) - Single label application + - `compare-urls.js` (9.5k) - Compare URLs between databases + - `download-items-mapping.js` (5.9k) - Download and map items + - `import-pocket.js` (42k) - Import from Pocket with progress tracking + - `migrate-omnivore.js` (14k) - Migrate from SQLite to Omnivore + - `test-auth.js` (984b) - Test API authentication + - `test-create-label.js` (1.3k) - Test label creation + - `check-missing-labeled.sql` (1.0k) - SQL query for missing labels + - `find-redirected-urls.sql` (490b) - SQL query for redirected URLs + +**TypeScript Foundation (CONFIGURED AND TESTED):** +- ✅ `tsconfig.json` - Development config with path aliases (@lib, @storage, etc.) +- ✅ `tsconfig.build.json` - Production build config (no source maps) +- ✅ Build system: `pnpm run build` successfully creates `dist/types/` +- ✅ Type checking: `pnpm run typecheck` passes with no errors +- ✅ Dev mode: `pnpm run dev` for watch mode + +**Dependencies (COMPLETE - All in package.json):** + +*Runtime Dependencies (10 packages):* +- ✅ `@anthropic-ai/sdk` (^0.20.0) - NEW: For content analysis/generation +- ✅ `@anthropic-ai/claude-agent-sdk` (^1.0.0) - For future agent integration +- ✅ `better-sqlite3` (^12.4.1) - Legacy scripts use SQLite +- ✅ `chalk` (^5.3.0) - Console output +- ✅ `csv-parse` (^6.1.0) - Legacy import scripts +- ✅ `dotenv` (^16.4.0) - Environment config +- ✅ `gray-matter` (^4.0.3) - Front-matter parsing +- ✅ `markdown-it` (^14.0.0) - Markdown processing +- ✅ `node-fetch` (^3.3.2) - HTTP (lib/omnivore/client.js uses this) +- ✅ `p-limit` (^5.0.0) - Rate limiting in legacy scripts + +*Dev Dependencies (5 packages):* +- ✅ `@types/node` (^20.0.0) - Node.js types +- ✅ `typescript` (^5.4.0) - TypeScript compiler +- ✅ `tsx` (^4.0.0) - TypeScript execution +- ✅ `vitest` (^1.0.0) - Testing framework +- ✅ `nodemon` (^3.0.0) - Dev file watcher + +**Configuration Files:** +- ✅ `.env.example` - Environment template (Omnivore + Anthropic config) +- ✅ `.gitignore` - Excludes node_modules, dist, .env, test-output +- ✅ `package.json` - Updated with all dependencies and build scripts +- ✅ `CLAUDE.md` - Agent context and content strategy + +**Documentation (UPDATED):** +- ✅ `README.md` - Clear "NOT IMPLEMENTED" markers for all planned features +- ✅ `IMPLEMENTATION_PLAN.md` - This file with complete roadmap +- ✅ `EXTRACTION_CHECKLIST.md` - Step-by-step extraction guide + +**Directory Structure (CREATED):** +- ✅ `src/` - TypeScript source directories (types, storage, analysis, generation, publishing, workflows, utils) +- ✅ `content/` - Storage directories (articles, analysis, generated/blog-posts, generated/newsletters, .metadata) +- ✅ `test-scripts/` - Test script directory with test-output/ +- ✅ `cli/` - CLI tools directory +- ✅ `tests/` - Test directory + +### What's NOT Included (Intentionally) + +These remain in the parent omnivore repo and are NOT needed: +- ❌ `/scripts/*.js` - Original scripts location (COPIES in legacy-scripts/) +- ❌ `/self-hosting/` - Omnivore server setup (separate concern) +- ❌ Parent repo's package.json (omnivore-content-system has its own) +- ❌ Any SQLite databases (legacy scripts included but DBs not needed) + +### Zero External Dependencies Verified + +**No file system references outside this directory:** +- ✅ Verified: No `../../` paths to parent repo files +- ✅ Verified: No symlinks to parent repo +- ✅ Verified: No hardcoded absolute paths to parent repo +- ✅ Verified: All imports are relative within omnivore-content-system/ +- ✅ Verified: Legacy scripts don't depend on parent repo (self-contained) + +**Only configuration dependencies (set in .env file):** +- API endpoint: `OMNIVORE_API_URL=https://omnivore-api.caladan.haus/api/graphql` +- API key: `OMNIVORE_API_KEY=your-api-key` +- Anthropic key: `ANTHROPIC_API_KEY=your-anthropic-key` + +### Extraction Commands + +```bash +# From omnivore repo root +cd /Volumes/devel/personal/keybase/edgerouter/mac-mini/home/omnivore/omnivore + +# Option 1: Copy to new location +cp -r scripts/omnivore-content-system /path/to/new-repo/ + +# Option 2: Move (if ready to extract) +mv scripts/omnivore-content-system /path/to/new-repo/ + +# Initialize new git repo +cd /path/to/new-repo/omnivore-content-system +git init +git add . +git commit -m "Initial commit: Omnivore content monetization system" +``` + +### Post-Extraction Setup + +```bash +# In new standalone repo +cd omnivore-content-system + +# 1. Install dependencies +pnpm install + +# 2. Configure environment +cp .env.example .env +# Edit .env with your credentials: +# OMNIVORE_API_KEY=your-api-key +# OMNIVORE_API_URL=https://omnivore-api.caladan.haus/api/graphql +# ANTHROPIC_API_KEY=your-anthropic-key + +# 3. Test Omnivore client +node lib/omnivore/client.js --test + +# 4. Test TypeScript build +pnpm run build + +# 5. Verify no errors +pnpm run typecheck +``` + +### Environment Variables Required + +Create `.env` file with: + +```bash +# Omnivore API Configuration +OMNIVORE_API_KEY=your_omnivore_api_key_here +OMNIVORE_API_URL=https://omnivore-api.caladan.haus/api/graphql + +# Anthropic API Configuration +ANTHROPIC_API_KEY=your_anthropic_api_key_here + +# Content Output (optional) +CONTENT_OUTPUT_DIR=./content +``` + +### What's NOT Included (Intentionally) + +These remain in the parent omnivore repo and are NOT needed: +- ❌ `/scripts/migrate-omnivore.js` - Migration script (stays in omnivore repo) +- ❌ `/scripts/import-pocket.js` - Import script (stays in omnivore repo) +- ❌ `/self-hosting/` - Omnivore self-hosting setup (separate concern) +- ❌ Parent omnivore repo's package.json + +The `legacy-scripts/` folder contains COPIES of migration scripts for reference, but content-system doesn't depend on them. + +### Validation After Extraction + +Run these commands to verify standalone operation: + +```bash +# Should all pass +pnpm run typecheck # ✅ No errors +pnpm run build # ✅ Creates dist/ +node lib/omnivore/client.js --test # ✅ Connects to API +``` + +### GitHub Repository Setup (Optional) + +```bash +# After extraction, if creating new GitHub repo +git remote add origin https://github.com/yourusername/omnivore-content-system.git +git branch -M main +git push -u origin main +``` + +### .gitignore Already Configured + +The directory already has proper .gitignore (if not, create): + +``` +# Dependencies +node_modules/ +pnpm-lock.yaml + +# Build output +dist/ + +# Environment +.env +.env.local + +# Test output +test-scripts/test-output/ + +# Runtime data +data/ +*.sqlite +*.sqlite-journal + +# IDE +.vscode/ +.idea/ + +# OS +.DS_Store +Thumbs.db +``` + +### Dependencies on Parent Repo: NONE + +✅ **The omnivore-content-system has ZERO dependencies on the parent omnivore repo structure.** + +All references to self-hosted Omnivore are via: +- API endpoint (configured in .env) +- API key (configured in .env) + +No file system dependencies outside the omnivore-content-system/ directory. + +--- + +## Project Structure (Updated) + +``` +omnivore-content-system/ +├── lib/ # ✅ Existing JavaScript client library +│ └── omnivore/ +│ ├── client.js # ✅ Working GraphQL client +│ └── queries.js # ✅ Query builders +│ +├── src/ # 🆕 TypeScript source (new code) +│ ├── storage/ # Storage layer (tracking + permanent) +│ │ ├── schema/ +│ │ │ └── tracking-schema.sql # AIDEV: tracking tables only +│ │ ├── database.ts # AIDEV: DB init with boundary checks +│ │ ├── AnalysisQueueRepository.ts # AIDEV: tracking CRUD +│ │ ├── AnalysisWriter.ts # AIDEV: write to Markdown/JSONL +│ │ ├── ContentReader.ts # Read Markdown files +│ │ └── SearchIndex.ts # JSON-based search index (future) +│ │ +│ ├── analysis/ # Content analysis +│ │ ├── ContentAnalyzer.ts # Analyze article content +│ │ ├── TopicExtractor.ts # Extract topics +│ │ ├── SummaryGenerator.ts # Generate summaries +│ │ └── prompts/ # Agent prompts +│ │ ├── analyze.md +│ │ └── summarize.md +│ │ +│ ├── generation/ # Content generation +│ │ ├── BlogPostGenerator.ts # Generate blog posts +│ │ ├── NewsletterGenerator.ts # Generate newsletters +│ │ └── prompts/ +│ │ ├── blog-post.md +│ │ └── newsletter.md +│ │ +│ ├── publishing/ # Publishing to platforms +│ │ ├── MarkdownPublisher.ts # Output to Markdown files +│ │ └── GhostPublisher.ts # Publish to Ghost (later) +│ │ +│ ├── workflows/ # Automated workflows +│ │ ├── DailyAnalysis.ts +│ │ ├── WeeklyRoundup.ts +│ │ └── NewsletterCreation.ts +│ │ +│ ├── utils/ # Shared utilities +│ │ ├── frontmatter.ts # Front-matter parsing +│ │ ├── markdown.ts # Markdown utilities +│ │ ├── logger.ts # Logging +│ │ └── anthropic.ts # Anthropic API wrapper +│ │ +│ └── types/ # TypeScript types +│ ├── omnivore.ts # Omnivore types +│ ├── content.ts # Content types +│ └── index.ts +│ +├── data/ # ✅ Database storage (gitignored but permanent) +│ ├── omnivore-content.db # SQLite: immutable AI snapshots + tracking +│ │ # AIDEV: stores original analysis JSON + metadata +│ └── batches/ # DEPRECATED: replaced by SQLite queue +│ └── current-batch.json +│ +├── content/ # ✅ Content storage (git-tracked) +│ ├── analysis/ # Analysis results (Markdown, user-editable) +│ │ └── YYYY-MM-DD-{slug}-analysis.md +│ ├── corpus-reports/ # Corpus statistics +│ │ └── YYYY-MM-DD-report.md +│ └── generated/ # Generated content (future) +│ ├── blog-posts/ +│ │ └── YYYY-MM-DD-{title}.md +│ └── newsletters/ +│ └── YYYY-WW-roundup.md +│ +├── test-scripts/ # 🆕 Legacy test scripts +│ ├── 01-fetch-articles.js # Reference: original fetch test +│ ├── 03-test-fixed-queries.js # Reference: query validation +│ └── test-output/ # Test outputs (gitignored) +│ +├── cli/ # ✅ Working CLI tools +│ ├── fetch-articles.ts # AIDEV: Omnivore API → tracking queue +│ ├── parallel-analyze.ts # Archived (see cli/archived-scripts/) +│ ├── analysis-status.ts # AIDEV: Show tracking queue status +│ ├── retry-failed.ts # AIDEV: Retry failed analyses +│ ├── cleanup-completed.ts # AIDEV: Remove completed from queue +│ ├── corpus-report.ts # Generate statistics from Markdown +│ └── (future) # generate.ts, publish.ts +│ +├── templates/ # Template files +│ ├── blog-posts/ +│ │ ├── single-article.md +│ │ └── weekly-roundup.md +│ └── newsletters/ +│ └── weekly.md +│ +├── legacy-scripts/ # ✅ Old migration scripts (preserved) +├── dist/ # Compiled TypeScript (gitignored) +├── tests/ # Automated tests +├── docs/ # Documentation +│ +├── .env.example +├── CLAUDE.md # Agent context +├── package.json +├── tsconfig.json +└── tsconfig.build.json +``` + +--- + +## Database Boundaries & Tables + +### SQLite Database Structure + +**Location**: `data/omnivore-content.db` (gitignored) + +**Tables**: + +1. **Existing Omnivore Tables** (READ-ONLY - if present) + - Created by legacy import/migration scripts + - NEVER modify from our code + - All Omnivore updates via GraphQL API only + - AIDEV-NOTE: `omnivore-boundary` - GraphQL only + +2. **Tracking Tables** (READ-WRITE - our code) + ```sql + -- AIDEV-NOTE: tracking + immutable snapshots for querying + CREATE TABLE analysis_queue ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + article_id TEXT NOT NULL UNIQUE, + article_url TEXT NOT NULL, + article_title TEXT NOT NULL, + status TEXT CHECK(status IN ('pending', 'in_progress', 'completed', 'failed')), + error_message TEXT, + retry_count INTEGER DEFAULT 0, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + ``` + +### Boundary Enforcement + +**Code Annotations** (mandatory): +```typescript +// AIDEV-NOTE: tracking + immutable snapshots - permanent storage +// AIDEV-NOTE: analysis-output-boundary - Markdown/JSONL, not DB +// AIDEV-NOTE: omnivore-boundary - GraphQL API only, never local cache +// AIDEV-NOTE: boundary-check - verify Omnivore tables unchanged +``` + +**Validation Checks**: +- `database.ts` validates no modification to Omnivore tables +- `AnalysisQueueRepository` only touches `analysis_queue` table +- All analysis output goes to Markdown/JSONL, never database + +**Data Flow**: +``` +Omnivore API (GraphQL) + ↓ fetch article +SQLite analysis_queue (tracking: pending → in_progress) + ↓ coordinate +@article-content-analyzer agent (5 concurrent) + ↓ analyze +Markdown + JSONL (permanent, git-tracked) + ↓ save +SQLite analysis_queue (tracking: in_progress → completed) +``` + +--- + +## Implementation Phases (Mechanics-First) + +### Phase 0: Foundation Setup +**Goal**: Set up TypeScript tooling and project structure + +**Note**: This phase is complete. See [Foundation & Type System](docs/_meta/foundation-and-types.md) for details on what exists. + +**What This Phase Provides**: +```json +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "declaration": true, + "sourceMap": true, + "paths": { + "@lib/*": ["./lib/*"], + "@storage/*": ["./src/storage/*"], + "@analysis/*": ["./src/analysis/*"], + "@generation/*": ["./src/generation/*"], + "@utils/*": ["./src/utils/*"], + "@types/*": ["./src/types/*"] + } + }, + "include": ["src/**/*", "lib/**/*"], + "exclude": ["node_modules", "dist", "test-scripts", "legacy-scripts"] +} +``` + +**Deliverable**: `pnpm run build` compiles successfully +**Exit Criteria**: TypeScript compiles, path aliases resolve + +--- + +### Phase 1: Type Definitions +**Goal**: Define TypeScript types for Omnivore API and content system + +**Note**: This phase is complete. See [Foundation & Type System](docs/_meta/foundation-and-types.md) for all type definitions and usage examples. + +**What This Phase Provides**: +- Omnivore API types (src/types/omnivore.ts): OmnivoreArticle, Label, Highlight, SearchResult, PageInfo +- Content storage types (src/types/content.ts): ArticleFrontMatter, AnalysisFrontMatter, StoredArticle, etc. +- Analysis types (src/types/analysis.ts): ContentAnalysis, AnalysisRequest, TopicScore, BatchAnalysisResult +- Central exports (src/types/index.ts): All types plus utility types and config interfaces + +**Omnivore Article Type** (based on existing client.js): +```typescript +interface OmnivoreArticle { + id: string; + title: string; + url: string; + originalArticleUrl?: string; + slug?: string; + content?: string; + description?: string; + author?: string; + image?: string; + siteName?: string; + pageType?: string; + wordCount?: number; + createdAt: string; + savedAt: string; + publishedAt?: string; + updatedAt: string; + readingProgressTopPercent?: number; + isArchived: boolean; + folder?: string; + labels: Label[]; + highlights: Highlight[]; +} + +interface Label { + id: string; + name: string; + color: string; + description?: string; +} + +interface Highlight { + id: string; + quote: string; + annotation?: string; + createdAt: string; +} +``` + +**Analysis Type**: +```typescript +interface ContentAnalysis { + articleId: string; + topics: string[]; // ["AI", "Machine Learning", "LLMs"] + topicScores: Record; // { "AI": 0.95, "ML": 0.88 } + summary: string; // 2-3 sentence summary + keyPoints: string[]; // 3-5 key takeaways + sentiment: "positive" | "neutral" | "negative"; + monetizationAngle: string; // How to turn into valuable content + analyzedAt: string; // ISO timestamp +} +``` + +**Article Front-matter**: +```typescript +interface ArticleFrontMatter { + id: string; // Omnivore article ID + url: string; + title: string; + author?: string; + savedAt: string; + publishedAt?: string; + labels: string[]; + highlights: number; // Count + wordCount: number; + topics?: string[]; // Added by analysis + sentiment?: string; // Added by analysis + analyzed?: boolean; +} +``` + +**Deliverable**: All types compile, can be imported throughout codebase +**Exit Criteria**: `pnpm run typecheck` passes with no errors + +--- + +### Phase 2: Building Block 1 - Content Fetching +**Goal**: Validate existing client can fetch articles with content + +**Prerequisites** (see groundtruth): +- lib/omnivore/client.js - Full GraphQL client +- lib/omnivore/queries.js - Query builders +- See [Foundation & Type System](docs/_meta/foundation-and-types.md) + +**Additions in Phase 2**: +- GraphQL fragment system (src/graphql/) - see [GraphQL Organization](docs/_meta/graphql-organization.md) + +**Test Script**: `test-scripts/01-fetch-articles.js` validates fetching AI/ML articles with content + +**Deliverable**: Can reliably fetch 10 AI articles with full content +**Exit Criteria**: Test script runs successfully, outputs valid JSON + +**Anti-Drift Guardrails**: +- Use ONLY existing client.js (no rewrites) +- Test with real self-hosted API +- Save test outputs for inspection + +--- + +### Phase 3: Building Block 2 - Analysis Storage & Tracking ✅ COMPLETE +**Goal**: Store AI-generated analysis results as Markdown files with YAML front-matter + SQLite tracking for parallel processing + +**Status**: ✅ Complete (2025-10-01) +**Update**: ✅ Enhanced with slug support for article(slug, username) GQL query (2025-10-01) + +**CRITICAL**: Omnivore IS the storage for source articles. Do NOT store source articles locally. + +**Implemented Architecture**: Three-layer system (Omnivore → SQLite Tracking → Git-tracked Storage) + +**What Was Built**: + +1. **SQLite Storage Layer** (permanent, gitignored) + - Database: `data/omnivore-content.db` + - Table: `analysis_queue` (pending/in_progress/completed/failed status) + - Purpose: Coordinate parallel analysis, prevent duplicate work + - AIDEV-NOTE: tracking-only, not for analysis storage + +2. **Analysis Storage** (permanent, git-tracked) + - Location: `content/analysis/` (Markdown + JSONL) + - Format: YAML front-matter + structured analysis + - AIDEV-NOTE: analysis-output-boundary - results to Markdown/JSONL, NOT database + +3. **CLI Tools** (see CLAUDE.md for complete command list) + - Queue management: add, list, stats, reset, remove, clear, export, import, retry + - Analysis operations: run, retry, status, watch + - Reporting: corpus, topics, trends, monetization, sentiment + +4. **Repository Classes** (TypeScript) + - `AnalysisQueueRepository.ts` - CRUD for tracking queue (AIDEV: tracking-db) + - `AnalysisWriter.ts` - Write to Markdown/JSONL (AIDEV: git-tracked-output) + - Complete boundary enforcement with AIDEV annotations + +**Analysis File Format** (as implemented): +```markdown +--- +articleId: omnivore-abc123 +articleUrl: https://example.com/article +articleTitle: Anthropic's New Claude Model +analyzedAt: 2025-09-30T10:00:00Z +topics: [ai, machine-learning, anthropic] +sentiment: positive +topicScores: + ai: 0.95 + machine-learning: 0.88 + anthropic: 0.92 +--- + +# Analysis: Anthropic's New Claude Model + +## Summary +2-3 sentence summary capturing the main points... + +## Key Points +- First key takeaway +- Second key takeaway +- Third key takeaway + +## Monetization Angle +How this article could be turned into valuable content for your audience... +``` + +**Deliverables**: +- ✅ SQLite database with immutable analysis snapshots (permanent, gitignored) +- ✅ Analysis results in Markdown + JSONL (permanent, git-tracked) +- ✅ Parallel analysis workflow (5 concurrent agents via @article-content-analyzer) +- ✅ Complete job tracking and error recovery +- ✅ All AIDEV boundary annotations in place +- ✅ TypeScript builds successfully +- ✅ Package scripts updated (fetch, analyze:auto, analyze:retry, report:corpus) + +**Exit Criteria Met**: +- ✅ Can track 100+ articles through analysis pipeline +- ✅ Parallel processing prevents duplicate work +- ✅ Analysis results saved to git-tracked files +- ✅ Boundary enforcement via AIDEV annotations +- ✅ Queue management (status, retry, cleanup) working +- ✅ Does NOT store source articles (Omnivore IS the storage) +- ✅ References source articles by ID/URL only + +**Architectural Enhancement**: +Original design called for "Analysis Storage (ONLY)" with simple Markdown files. Implementation enhanced this with: +- SQLite layer for immutable AI snapshots + parallel coordination (permanent) +- JSONL format for machine-readable batch processing +- Retry mechanism with attempt tracking +- Status monitoring and queue management tools +- **Slug field in queue** for Omnivore article(slug, username) GQL query support + +This maintains the original "git-tracked permanent storage" principle while adding robust parallel processing capabilities. + +**GQL Query Enhancement** (2025-10-01): +- Added `article_slug` column to `analysis_queue` table +- Updated AnalysisQueueRepository to handle slug field +- Agents can fetch articles via getArticle(slug, username) per schema + +**Workflow Documentation**: +- See CLAUDE.md "Article Analysis Workflow" section for CLI usage +- See `docs/_meta/architecture.md` for system design and storage boundaries +- See `docs/_meta/cli-reference.md` for command-line interface documentation +- See `docs/_meta/workflow-internals.md` for parallel analysis workflow details +- All boundary annotations documented in code + +**Anti-Drift Guardrails Enforced**: +- ✅ **DO**: Store analysis results, generated content +- ❌ **DON'T**: Store source articles (Omnivore IS the storage) +- ✅ **DO**: Reference articles by ID/URL +- ❌ **DON'T**: Duplicate article content locally +- ✅ Git-track content/analysis/ and content/generated/ ONLY +- ✅ SQLite for tracking ONLY, not permanent storage +- ✅ All boundaries marked with AIDEV-NOTE annotations + +--- + +### Phase 4: Building Block 3 - Content Analysis +**Goal**: Extract topics, summary, key points from article using Claude + +**Agent Prompt** (src/analysis/prompts/analyze.md): +```markdown +# Content Analysis Prompt + +You are analyzing an article for content monetization. Your goal is to extract structured insights that can be used to create valuable blog posts and newsletters. + +## Input Article + +**Title**: {{ article.title }} +**Author**: {{ article.author }} +**URL**: {{ article.url }} +**Published**: {{ article.publishedAt }} +**Word Count**: {{ article.wordCount }} + +**Content**: +{{ article.content }} + +{% if article.highlights.length > 0 %} +**Highlights**: +{% for highlight in article.highlights %} +- {{ highlight.quote }} + {% if highlight.annotation %}Note: {{ highlight.annotation }}{% endif %} +{% endfor %} +{% endif %} + +## Analysis Task + +Extract the following information in JSON format: + +```json +{ + "topics": ["topic1", "topic2", "topic3"], + "topicScores": { + "topic1": 0.95, + "topic2": 0.88, + "topic3": 0.75 + }, + "summary": "2-3 sentence summary capturing the main points", + "keyPoints": [ + "First key takeaway", + "Second key takeaway", + "Third key takeaway" + ], + "sentiment": "positive|neutral|negative", + "monetizationAngle": "How this article could be turned into valuable content for your audience" +} +``` + +## Guidelines + +1. **Topics**: 2-5 main topics, prioritizing AI, machine learning, software engineering, DevOps, databases, cloud, startups +2. **Topic Scores**: 0-1 confidence score for each topic +3. **Summary**: Focus on "so what?" - why does this matter? +4. **Key Points**: Actionable insights or surprising facts +5. **Sentiment**: Overall tone of the article +6. **Monetization Angle**: How to package this for your audience (e.g., "Compare with 2 other LLM papers for comparison post", "Tutorial on applying this technique") + +Return ONLY valid JSON, no markdown code blocks. +``` + +**Tasks**: +- [ ] Implement ContentAnalyzer.ts (src/analysis/ContentAnalyzer.ts) + - Load prompt template + - Call Anthropic API with article content + - Parse JSON response + - Handle API errors/retries + - Rate limiting (respect API limits) +- [ ] Implement AnalysisWriter.ts (src/storage/AnalysisWriter.ts) + - Save analysis as Markdown with front-matter + - Update article's front-matter with topics/sentiment +- [ ] Implement anthropic.ts utility (src/utils/anthropic.ts) + - Wrapper for Anthropic API + - Retry logic with exponential backoff + - Token counting + - Cost tracking + +**Test Script**: `test-scripts/03-analyze-content.ts` +```typescript +#!/usr/bin/env node + +/** + * Test: Analyze article content with Claude + * Validates: API call, prompt rendering, JSON parsing + */ + +import { ContentReader } from '../src/storage/ContentReader'; +import { ContentAnalyzer } from '../src/analysis/ContentAnalyzer'; +import { AnalysisWriter } from '../src/storage/AnalysisWriter'; + +// Read stored article +const reader = new ContentReader('content/articles'); +const articles = await reader.list(); +const article = await reader.read(articles[0]); + +console.log(`Analyzing: ${article.frontMatter.title}`); + +// Analyze (calls Claude API) +const analyzer = new ContentAnalyzer(); +const analysis = await analyzer.analyze({ + title: article.frontMatter.title, + author: article.frontMatter.author, + url: article.frontMatter.url, + content: article.content, + wordCount: article.frontMatter.wordCount, + highlights: article.highlights || [] +}); + +console.log('\nAnalysis Result:'); +console.log(JSON.stringify(analysis, null, 2)); + +// Save analysis +const writer = new AnalysisWriter('content/analysis'); +const analysisPath = await writer.write(article.frontMatter.id, analysis); + +console.log(`\n✓ Saved analysis to: ${analysisPath}`); +``` + +**Deliverable**: Can analyze article and extract structured insights +**Exit Criteria**: +- Claude API call succeeds +- Returns valid JSON +- Topics extracted correctly +- Analysis saved as Markdown + +**Post-Phase: Update Groundtruth** +- Create `docs/_meta/analysis-and-generation.md` or update existing groundtruth +- Document ContentAnalyzer, AnalysisWriter classes +- Document prompt templates and how to use them +- Include Anthropic API usage patterns and error handling +- Use @documentation-writer agent + +**Anti-Drift Guardrails**: +- Use Anthropic API directly (not Agent SDK yet) +- Test prompt in this chat first +- Track API costs +- Handle rate limiting + +--- + +### Phase 5: Building Block 4 - Content Generation +**Goal**: Generate blog post from analyzed articles + +**Agent Prompt** (src/generation/prompts/blog-post.md): +```markdown +# Blog Post Generation Prompt + +Generate a blog post based on the analyzed articles below. Follow the writing style and content strategy from the user's guidelines. + +## Writing Style (from CLAUDE.md) +- Authoritative yet accessible +- Opinionated with evidence +- Practical and actionable +- Conversational tone +- Short paragraphs (2-3 sentences) +- Use subheadings, bullet points, examples + +## Analyzed Articles + +{% for article in articles %} +### {{ article.title }} +**URL**: {{ article.url }} +**Author**: {{ article.author }} +**Topics**: {{ article.analysis.topics.join(', ') }} + +**Summary**: {{ article.analysis.summary }} + +**Key Points**: +{% for point in article.analysis.keyPoints %} +- {{ point }} +{% endfor %} + +**Monetization Angle**: {{ article.analysis.monetizationAngle }} + +{% endfor %} + +## Generation Task + +Create a blog post with: + +1. **SEO-Optimized Title**: 60 chars max, keyword-rich, curiosity-driven +2. **Meta Description**: 155 chars, compelling summary +3. **Hook**: Start with surprising insight or provocative question +4. **Content Structure**: + - Introduction with context + - Analysis of each article (with links) + - Synthesis connecting multiple sources + - Practical implications + - Call-to-action +5. **Length**: 800-1200 words +6. **Links**: Include article URLs as sources + +Return in this format: + +```yaml +--- +title: "SEO Title Here" +metaDescription: "Meta description here" +topics: [topic1, topic2] +publishedAt: {{ now }} +sources: + - {{ article1.url }} + - {{ article2.url }} +--- + +# Actual Title (Can Be Different from SEO Title) + +Hook paragraph... + +## First Section + +Content... + +[Link to article](url) + +## Conclusion + +CTA... +``` +``` + +**Tasks**: +- [ ] Implement BlogPostGenerator.ts (src/generation/BlogPostGenerator.ts) + - Load analyzed articles + - Render prompt with article data + - Call Claude API + - Parse Markdown response + - Save to content/generated/blog-posts/ +- [ ] Implement template utilities (src/utils/markdown.ts) + - Markdown parsing and manipulation + - Slug generation + - Link validation + +**Test Script**: `test-scripts/04-generate-post.ts` +```typescript +#!/usr/bin/env node + +/** + * Test: Generate blog post from analyzed articles + * Validates: Prompt rendering, content generation, post structure + */ + +import { ContentReader } from '../src/storage/ContentReader'; +import { BlogPostGenerator } from '../src/generation/BlogPostGenerator'; + +// Read analyzed articles (last 5) +const articleReader = new ContentReader('content/articles'); +const analysisReader = new ContentReader('content/analysis'); + +const articles = (await articleReader.list()).slice(0, 5); + +// Load with analysis +const articlesWithAnalysis = await Promise.all( + articles.map(async (filePath) => { + const article = await articleReader.read(filePath); + const analysisPath = filePath.replace('/articles/', '/analysis/'); + const analysis = await analysisReader.read(analysisPath); + return { ...article, analysis }; + }) +); + +console.log(`Generating blog post from ${articlesWithAnalysis.length} articles...`); + +// Generate +const generator = new BlogPostGenerator(); +const blogPost = await generator.generate(articlesWithAnalysis, { + type: 'weekly-roundup', + title: 'AI This Week: Top Stories You Need to Know' +}); + +console.log('\nGenerated Blog Post:'); +console.log('Title:', blogPost.frontMatter.title); +console.log('Meta:', blogPost.frontMatter.metaDescription); +console.log('Word count:', blogPost.content.split(/\s+/).length); +console.log('Sources:', blogPost.frontMatter.sources.length); + +// Save +const outputPath = `content/generated/blog-posts/${blogPost.slug}.md`; +await generator.save(blogPost, outputPath); + +console.log(`\n✓ Saved to: ${outputPath}`); +``` + +**Deliverable**: Can generate blog post from analyzed articles +**Exit Criteria**: +- Post has proper structure +- Links to source articles +- Follows writing style +- 800-1200 words +- SEO-optimized title/meta + +**Post-Phase: Update Groundtruth** +- Update `docs/_meta/analysis-and-generation.md` with generation classes +- Document BlogPostGenerator, template system +- Include generation prompt patterns and configuration +- Show examples of different content types (roundup, deep-dive, etc.) +- Use @documentation-writer agent + +**Anti-Drift Guardrails**: +- Test prompt in chat first +- Follow CLAUDE.md guidelines +- Generate one format (weekly roundup) first +- Validate output structure + +--- + +### Phase 6: Building Block 5 - Publishing +**Goal**: Output generated content to Markdown (Ghost later) + +**Tasks**: +- [ ] Implement MarkdownPublisher.ts (src/publishing/MarkdownPublisher.ts) + - Copy from content/generated/ to public output directory + - Add publication metadata to front-matter + - Generate index of published posts +- [ ] (Later) Implement GhostPublisher.ts + - Authenticate with Ghost Admin API + - Create draft post + - Upload as draft (not published) + - Return post URL + +**Test Script**: `test-scripts/05-publish.ts` +```typescript +#!/usr/bin/env node + +/** + * Test: Publish generated blog post + * Validates: File operations, metadata updates + */ + +import { MarkdownPublisher } from '../src/publishing/MarkdownPublisher'; + +const publisher = new MarkdownPublisher('public/blog'); + +// Publish latest generated post +const generated = await publisher.getUnpublished('content/generated/blog-posts'); +const latestPost = generated[0]; + +console.log(`Publishing: ${latestPost.frontMatter.title}`); + +const result = await publisher.publish(latestPost); + +console.log(`\n✓ Published to: ${result.publicPath}`); +console.log(` Added publishedAt: ${result.publishedAt}`); +``` + +**Deliverable**: Can output Markdown files for publishing +**Exit Criteria**: +- Files copied to public directory +- Metadata updated +- Index generated + +**Post-Phase: Update Groundtruth** +- Create `docs/_meta/publishing-and-workflows.md` or update existing groundtruth +- Document MarkdownPublisher and any other publishing modules +- Include publishing patterns and metadata handling +- Use @documentation-writer agent + +--- + +## Integrated Workflows (After Building Blocks Proven) + +### Workflow 1: Daily Analysis +**Trigger**: Manual command +**Steps**: +1. Fetch articles saved in last 24 hours (AI/Tech topics) +2. Store as Markdown files +3. Analyze each article +4. Update search index + +**CLI**: `pnpm run fetch-daily` + +### Workflow 2: Weekly Roundup +**Trigger**: Manual command (Friday) +**Steps**: +1. List analyzed articles from last 7 days +2. Filter by topic (AI/Tech) +3. Generate weekly roundup blog post +4. Output to Markdown + +**CLI**: `pnpm run generate-roundup` + +### Workflow 3: Newsletter Creation +**Trigger**: Manual command (Sunday) +**Steps**: +1. List analyzed articles from last week +2. Generate newsletter with curated links +3. Output to Markdown + +**CLI**: `pnpm run generate-newsletter` + +--- + +## Success Criteria + +### Building Block Completion +Each building block is complete when: +- [ ] Test script runs successfully +- [ ] Output validated manually +- [ ] No errors in console +- [ ] Follows DRY principles +- [ ] Anti-drift guardrails documented + +### Promotion to Production +Code moves from test-scripts → cli → src when: +- [ ] Proven to work reliably (5+ successful runs) +- [ ] Error handling added +- [ ] Logging added +- [ ] Configuration externalized +- [ ] Code reviewed for DRY violations + +### System Complete +System is complete when: +- [ ] Can fetch articles from Omnivore ✅ (proven) +- [ ] Can store articles as Markdown +- [ ] Can analyze article content +- [ ] Can generate blog posts +- [ ] Can generate newsletters +- [ ] All workflows run end-to-end +- [ ] Documentation complete + +--- + +## Anti-Drift Guardrails + +### Phase 0 (Foundation) +- ✅ **DO**: Use existing lib/omnivore/ client as library +- ✅ **DO**: Set up TypeScript for new code only +- ✅ **DO**: Create clear directory structure +- ❌ **DON'T**: Rewrite existing working client +- ❌ **DON'T**: Add database (use front-matter + Git) +- ❌ **DON'T**: Over-engineer build system + +### Phase 2 (Fetching) +- ✅ **DO**: Use existing client.js and queries.js +- ✅ **DO**: Test with real self-hosted API +- ✅ **DO**: Use topic queries from queries.js +- ❌ **DON'T**: Build new GraphQL client +- ❌ **DON'T**: Cache responses (fetch fresh each time) +- ❌ **DON'T**: Add rate limiting yet (API handles it) + +### Phase 3 (Storage & Tracking) +- ✅ **DO**: Use gray-matter for front-matter (permanent storage) +- ✅ **DO**: Git-track content/ directory (analysis results) +- ✅ **DO**: Use SQLite for immutable AI snapshots + tracking (permanent storage) +- ✅ **DO**: Add AIDEV-NOTE annotations for all boundaries +- ✅ **DO**: Keep Omnivore tables READ-ONLY (never modify) +- ✅ **DO**: Query Omnivore via GraphQL API, not local cache +- ✅ **DO**: Write analysis results to Markdown/JSONL, not database +- ❌ **DON'T**: Store analysis results in SQLite +- ❌ **DON'T**: Modify Omnivore cache tables +- ❌ **DON'T**: Use SQLite for anything permanent +- ❌ **DON'T**: Skip boundary annotations + +### Phase 4 (Analysis) +- ✅ **DO**: Test prompt in this chat first +- ✅ **DO**: Use Anthropic API directly initially +- ✅ **DO**: Track API costs +- ❌ **DON'T**: Use Agent SDK yet +- ❌ **DON'T**: Over-complicate prompt +- ❌ **DON'T**: Add ML/embeddings + +### Phase 5 (Generation) +- ✅ **DO**: Follow CLAUDE.md writing guidelines +- ✅ **DO**: Test prompt in chat first +- ✅ **DO**: Generate one format first (weekly roundup) +- ❌ **DON'T**: Build complex template engine +- ❌ **DON'T**: Add WYSIWYG editor +- ❌ **DON'T**: Implement multiple formats at once + +### General +- ✅ **DO**: Build one mechanic at a time +- ✅ **DO**: Test manually before automating +- ✅ **DO**: Keep it simple +- ❌ **DON'T**: Skip test-scripts phase +- ❌ **DON'T**: Optimize prematurely +- ❌ **DON'T**: Add features not in plan + +### Groundtruth Documentation (CRITICAL) +- ✅ **DO**: Read groundtruth docs BEFORE starting any phase +- ✅ **DO**: Update groundtruth IMMEDIATELY after implementing reusable code +- ✅ **DO**: Use @documentation-writer agent for groundtruth updates +- ✅ **DO**: Check for drift by comparing implementation to groundtruth +- ✅ **DO**: Self-correct if drift is found +- ✅ **DO**: Document with hard-to-vary facts and usage examples +- ❌ **DON'T**: Add status markers ("complete", "done", "implemented") to groundtruth +- ❌ **DON'T**: Wait until "the end" to update groundtruth +- ❌ **DON'T**: Manually edit groundtruth (use documentation-writer agent) +- ❌ **DON'T**: Skip drift checking when adding new features +- ❌ **DON'T**: Duplicate functionality without checking groundtruth first + +--- + +## Dependencies Summary + +**Current State** (omnivore-content-system/package.json): +```json +{ + "dependencies": { + // EXISTING - Keep (used by legacy-scripts/ and lib/omnivore/) + "@anthropic-ai/claude-agent-sdk": "^1.0.0", // For future agent integration + "better-sqlite3": "^12.4.1", // Legacy scripts use SQLite + "chalk": "^5.3.0", // Console output + "csv-parse": "^6.1.0", // Legacy import scripts + "dotenv": "^16.4.0", // Environment config + "gray-matter": "^4.0.3", // Front-matter parsing + "markdown-it": "^14.0.0", // Markdown processing + "node-fetch": "^3.3.2", // HTTP (lib/omnivore/client.js) + "p-limit": "^5.0.0", // Rate limiting in legacy scripts + + // NEW - Add for content system + "@anthropic-ai/sdk": "^0.20.0" // Claude API for analysis/generation + }, + "devDependencies": { + "nodemon": "^3.0.0", // EXISTING - Dev watcher + + // NEW - Add for TypeScript + "@types/node": "^20.0.0", // Node types + "typescript": "^5.4.0", // TypeScript compiler + "tsx": "^4.0.0", // TypeScript execution + "vitest": "^1.0.0" // Testing framework + } +} +``` + +**Important**: +- **ONLY ADD new dependencies** - never remove existing ones +- `better-sqlite3`, `csv-parse`, `p-limit` are used by legacy-scripts/ (migrate, import) +- `node-fetch` is used by existing lib/omnivore/client.js +- New content system uses front-matter + Git (not SQLite), but legacy scripts still need it +- When content system moves to its own repo, dependencies can be cleaned up then + +--- + +## Test Promotion Path + +``` +┌─────────────────┐ +│ test-scripts/ │ Quick proof-of-concept +│ 01-fetch.js │ No error handling +│ 02-store.ts │ Console output only +│ 03-analyze.ts │ Test real API +└────────┬────────┘ + │ PROVEN (5+ successful runs) + ↓ +┌─────────────────┐ +│ cli/ │ Working CLI tools +│ fetch.ts │ + Error handling +│ analyze.ts │ + Logging +│ generate.ts │ + Config +└────────┬────────┘ + │ RELIABLE (daily use) + ↓ +┌─────────────────┐ +│ src/ │ Production TypeScript +│ storage/ │ + Tests +│ analysis/ │ + Documentation +│ generation/ │ + Type safety +└─────────────────┘ +``` + +--- + +## Notes + +- **Existing client library** (lib/omnivore/) should NOT be modified or rewritten +- **Self-hosted Omnivore** at omnivore-api.caladan.haus is the source of truth +- **Front-matter + Git** replaces database for simplicity +- **Agent prompts** should be tested in chat before implementation +- **Test scripts** are throwaway code - fast iteration, no perfectionism +- **CLI tools** are daily-use scripts - reliable but not production-grade +- **TypeScript src/** is production code - tested, documented, type-safe +- **One building block at a time** - no parallel development until mechanics proven diff --git a/self-hosting/omc/README.md b/self-hosting/omc/README.md new file mode 100644 index 000000000..49e6a171b --- /dev/null +++ b/self-hosting/omc/README.md @@ -0,0 +1,278 @@ +# Omnivore Content Monetization System + +AI-powered content generation system that transforms your Omnivore reading into monetizable content: blog posts, newsletters, and social media posts. + +## 🔎 Current Reality (Audit: 2026-01-30) + +- The CLI command set is largely implemented (`omc queue/*`, `omc analyze/*`, `omc content/*`, `omc report/*`, `omc omnivore/*`, `omc db/*`, `omc config/*`). +- There are still critical fixups needed before this repo is “clean build + reliable run”; see `docs/_meta/current-state.md`. + +## 🚀 Features + +### ✅ Currently Working +- **Omnivore API Client**: Fetch/search/update articles, highlights, and notes +- **Queue + Tracking DB**: SQLite-backed analysis queue (`data/omnivore-content.db`) +- **Analysis Workflow**: Prepare stub files → run external analyzer → persist results to Markdown + DB +- **Reporting**: Topic/sentiment/trend aggregation across saved analyses +- **Sync Back to Omnivore**: Push summaries (and optional NOTE highlights) back to Omnivore + +### 🚧 Planned Features (NOT YET IMPLEMENTED) +- **Blog Post Generation** *(Phase 5)*: Create weekly roundups, deep dives, tutorials from your saved articles +- **Newsletter Creation** *(Phase 5)*: Generate curated newsletters with your commentary +- **SEO Optimization** *(Phase 5)*: Automatic title, description, and tag generation +- **Trend Tracking** *(Phase 7)*: Identify emerging topics before they peak +- **Multi-Agent System** *(Phase 6-7)*: Specialized Claude agents for each content type +- **Publishing Integration** *(Phase 6)*: Ghost, WordPress, Medium support +- **First-Class Content Generation Pipeline**: Turn analyses into publishable posts (beyond reporting/export) + +## 📋 Prerequisites + +- Node.js 18+ +- Omnivore account with API key +- Anthropic API key (Claude) +- Self-hosted Omnivore instance OR Omnivore cloud account + +## 📦 Current Status + +**Phase 0: Foundation ✅ COMPLETE** +- TypeScript toolchain configured +- Omnivore GraphQL client working +- Directory structure created +- Ready for standalone extraction + +**Phases 1-6: In Progress** +- See [IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md) for detailed roadmap + +## 🛠️ Installation + +1. **Extract to standalone directory** (if from omnivore repo) + ```bash + cp -r omnivore/scripts/omnivore-content-system /path/to/new-repo/ + cd /path/to/new-repo/omnivore-content-system + ``` + +2. **Install dependencies** + ```bash + pnpm install + ``` + +3. **Configure environment** + ```bash + cp .env.example .env + # Edit .env with your API keys: + # - OMNIVORE_API_KEY + # - OMNIVORE_API_URL + # - ANTHROPIC_API_KEY + ``` + +4. **Test setup** + ```bash + # Test Omnivore connection + node lib/omnivore/client.js --test + + # Test TypeScript build + pnpm run build + + # Note: `pnpm run typecheck` is currently failing due to known alias/import issues. + # See: docs/_meta/current-state.md + ``` + +## ⚙️ Configuration + +### Required Environment Variables + +```bash +OMNIVORE_API_KEY=your_omnivore_api_key +OMNIVORE_API_URL=https://your-omnivore-instance.com/api/graphql +ANTHROPIC_API_KEY=your_claude_api_key +``` + +### Optional Configuration (NOT IMPLEMENTED YET) + +- **Publishing**: Configure Ghost, WordPress, or Medium credentials *(Phase 6)* +- **Topics**: Edit `config/topics.json` to customize AI/Tech categories *(Phase 1)* +- **Templates**: Modify templates in `templates/` for your style *(Phase 5)* +- **Workflows**: Adjust workflow schedules in `config/workflows.json` *(Phase 9)* + +## 🎯 Usage (Current) + +### Currently Working +```bash +# Setup checks +omc doctor + +# Add recent articles to queue +omc queue add --hours 24 + +# Prepare stub files for external analysis +omc analyze run --batch-size 5 + +# After your external analyzer writes `analysis` into temp/*.jsonl: +omc analyze complete + +# View/search +omc content list +omc content search "kubernetes" + +# Categorized reports +omc report topics +omc report trends +``` + +For a deeper breakdown (what works, what’s broken, and what to fix next), see `docs/_meta/current-state.md`. + +## 📁 Project Structure + +``` +omnivore-content-system/ +├── lib/ # ✅ Omnivore GraphQL client (WORKING) +│ └── omnivore/ +│ ├── client.js # ✅ Full-featured GraphQL client +│ └── queries.js # ✅ Query builders +├── src/ # ✅ TypeScript source (READY) +│ ├── types/ # (Phase 1 - not started) +│ ├── storage/ # (Phase 3 - not started) +│ ├── analysis/ # (Phase 4 - not started) +│ ├── generation/ # (Phase 5 - not started) +│ ├── publishing/ # (Phase 6 - not started) +│ ├── workflows/ # (Phase 9 - not started) +│ └── utils/ # (Phase 2 - not started) +├── content/ # ✅ Storage directories (READY) +│ ├── articles/ # (empty - Phase 3) +│ ├── analysis/ # (empty - Phase 4) +│ └── generated/ # (empty - Phase 5) +├── test-scripts/ # ✅ Test scripts (READY) +├── cli/ # (Phase 2-6 - not started) +├── tests/ # ✅ Test directory (READY) +├── legacy-scripts/ # ✅ Original migration scripts (preserved) +├── templates/ # (empty - Phase 5 needed) +├── .claude/ # (Agent SDK config - Phase 6+) +├── tsconfig.json # ✅ TypeScript config (WORKING) +├── package.json # ✅ Dependencies (WORKING) +└── .env.example # ✅ Config template (WORKING) +``` + +**Legend:** +- ✅ = Working and ready to use +- (empty) = Directory created but no files yet +- (Phase X) = Implementation planned, not started +- (WORKING) = Fully functional + +## 🤖 Agent Architecture (NOT IMPLEMENTED - Phase 6-8) + +### Orchestrator Agent *(NOT IMPLEMENTED)* +Main controller that coordinates all subagents and workflows. + +### Specialized Agents *(NOT IMPLEMENTED)* +- **Content Analyzer** *(Phase 7)*: Analyzes reading patterns, extracts themes +- **Blog Writer** *(Phase 7)*: Generates blog posts from analyzed content +- **Newsletter Creator** *(Phase 7)*: Creates newsletters with commentary +- **SEO Optimizer** *(Phase 7)*: Optimizes titles, descriptions, tags +- **Trend Tracker** *(Phase 7)*: Identifies emerging topics and opportunities + +### MCP Servers *(NOT IMPLEMENTED - Phase 8)* +- **Omnivore MCP**: Tools for searching, fetching articles, highlights +- **Content DB MCP**: Tools for saving and tracking generated content + +## 📝 Content Templates (NOT IMPLEMENTED - Phase 5) + +### Blog Post Types *(NOT IMPLEMENTED)* +- **Weekly Roundup**: Top 5-10 articles from the week +- **Deep Dive**: Multi-article synthesis into long-form analysis +- **Tutorial**: How-to content extracted from technical articles +- **Comparison**: "X vs Y" posts from related articles + +### Newsletter Formats *(NOT IMPLEMENTED)* +- **Weekly Digest**: Curated links with your commentary +- **Themed Edition**: Deep dive into single topic + +### Social Media *(NOT IMPLEMENTED)* +- **Twitter Threads**: Key insights from articles +- **LinkedIn Posts**: Professional summaries + +## 🔧 Legacy Scripts + +All original Omnivore scripts are preserved in `legacy-scripts/`: +- `import-pocket.js` - Import from Pocket +- `apply-labels.js` - Label management +- `migrate-omnivore.js` - Data migration +- And more... + +These can be called from agents as needed or run independently. + +## 💾 Content Storage (Phase 3 - NOT IMPLEMENTED) + +Will use **Markdown + front-matter + Git** (no database): +- **content/articles/** *(empty)*: Omnivore articles saved as Markdown +- **content/analysis/** *(empty)*: AI analysis results +- **content/generated/** *(empty)*: Generated blog posts and newsletters +- **Front-matter**: YAML metadata in each file +- **Git**: Version control and history + +Directories are created but no storage implementation yet. + +## 🚀 Publishing (Phase 6 - NOT IMPLEMENTED) + +### Ghost CMS *(NOT IMPLEMENTED)* +```bash +# Configuration ready in .env.example, but no publisher code yet +GHOST_API_URL=https://your-blog.ghost.io +GHOST_API_KEY=your_admin_api_key +``` + +### WordPress *(NOT IMPLEMENTED)* +```bash +WORDPRESS_URL=https://your-site.com +WORDPRESS_API_KEY=your_api_key +``` + +### Medium *(NOT IMPLEMENTED)* +```bash +MEDIUM_TOKEN=your_integration_token +``` + +## 🐛 Troubleshooting + +### API Connection Issues (Currently Applicable) +- Verify `OMNIVORE_API_KEY` is correct +- Check `OMNIVORE_API_URL` is accessible +- Test with: `node lib/omnivore/client.js --test` + +### TypeScript Build Issues (Currently Applicable) +- Run `pnpm install` to ensure dependencies are installed +- Run `pnpm run typecheck` to verify no type errors +- Check that Node.js version is ≥18.0.0 + +### Agent Issues *(NOT APPLICABLE YET - Phase 6+)* +- Check `ANTHROPIC_API_KEY` is set +- Verify Claude Agent SDK is installed +- Review logs in `data/logs/` + +### Storage Issues *(NOT APPLICABLE YET - Phase 3+)* +- Check permissions on `content/` directory +- Verify Git is initialized if using version control + +## 📚 Documentation + +- **[IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md)** - ✅ Detailed implementation roadmap (current) +- **[EXTRACTION_CHECKLIST.md](./EXTRACTION_CHECKLIST.md)** - ✅ Steps to move to standalone repo (current) +- **[CLAUDE.md](./CLAUDE.md)** - ✅ Agent context and content strategy (current) +- ~~Setup Guide~~ *(NOT CREATED - covered in this README)* +- ~~Workflows~~ *(NOT CREATED - Phase 9)* +- ~~Agent Architecture~~ *(NOT CREATED - Phase 6-8)* +- ~~API Documentation~~ *(NOT CREATED - Phase 1-7)* + +## 🤝 Contributing + +This is a personal content system, but suggestions welcome! + +## 📄 License + +MIT + +## 🙏 Credits + +Built with: +- [Claude Agent SDK](https://github.com/anthropics/claude-agent-sdk-typescript) by Anthropic +- [Omnivore](https://omnivore.app/) for reading management +- Your voracious reading habit! diff --git a/self-hosting/omc/bin/omc.ts b/self-hosting/omc/bin/omc.ts new file mode 100755 index 000000000..8270e207a --- /dev/null +++ b/self-hosting/omc/bin/omc.ts @@ -0,0 +1,8 @@ +#!/usr/bin/env node --import tsx +/** + * OMC (Omnivore Content System) CLI Executable + * AIDEV-NOTE: Main executable entry point for the OMC CLI system + */ + +// Import and run the main CLI +import '../index.js'; diff --git a/self-hosting/omc/cli/archived-scripts/README.md b/self-hosting/omc/cli/archived-scripts/README.md new file mode 100644 index 000000000..61236a283 --- /dev/null +++ b/self-hosting/omc/cli/archived-scripts/README.md @@ -0,0 +1,23 @@ +# Archived Scripts + +These scripts have been replaced by the new OCLIF-based CLI (`omc`). They are preserved here for reference but should not be used. + +## Migration Map + +| Old Script | New CLI Command | +|-----------|-----------------| +| `parallel-analyze.ts` | `omc analyze run` | +| `save-analysis-results.ts` | Internal (called by analyze run) | +| `retry-failed.ts` | `omc analyze retry` | +| `corpus-report.ts` | `omc report corpus` | +| `get-article-content.ts` | `omc omnivore get ` | +| `get-article-notes.ts` | `omc omnivore note get ` | +| `test-update-article-notes.ts` | `omc omnivore note update ` | +| `update-note-test.ts` | `omc omnivore note update ` | +| `migrate-database.ts` | `omc db migrate` | + +## Archived Date +2025-01-05 + +## Reason +All functionality has been migrated to the unified CLI system with better error handling, help text, and consistent interfaces. diff --git a/self-hosting/omc/cli/archived-scripts/corpus-report.ts b/self-hosting/omc/cli/archived-scripts/corpus-report.ts new file mode 100644 index 000000000..6a1aff66d --- /dev/null +++ b/self-hosting/omc/cli/archived-scripts/corpus-report.ts @@ -0,0 +1,127 @@ +#!/usr/bin/env tsx + +/** + * Generate corpus analysis report + * AIDEV-NOTE: tracking-db - reads from SQLite analysis_queue table + */ + +import Database from 'better-sqlite3'; + +console.log('\n' + '═'.repeat(80)); +console.log('CORPUS ANALYSIS REPORT'); +console.log('═'.repeat(80)); + +// AIDEV-NOTE: tracking-db - read from SQLite, not deprecated batch file +const db = new Database('data/omnivore-content.db', { readonly: true }); + +// Queue status from tracking DB +const statusCounts = db.prepare(` + SELECT status, COUNT(*) as count + FROM analysis_queue + GROUP BY status +`).all() as { status: string; count: number }[]; + +const total = statusCounts.reduce((sum, row) => sum + row.count, 0); + +console.log(`\n📊 Queue Status:`); +console.log(` Total articles: ${total}`); +for (const { status, count } of statusCounts) { + console.log(` ${status}: ${count}`); +} + +// Load completed analyses from SQLite +const rows = db.prepare(` + SELECT article_id, article_title, analysis_json + FROM analysis_queue + WHERE status = 'completed' AND analysis_json IS NOT NULL +`).all() as { article_id: string; article_title: string; analysis_json: string }[]; + +console.log(`\n📚 Analyzed Articles: ${rows.length}\n`); + +// Parse analysis JSON from each row +const analyses = rows.map(row => ({ + articleId: row.article_id, + articleTitle: row.article_title, + ...JSON.parse(row.analysis_json) +})).filter(a => a.topics && a.topics[0] !== 'N/A'); // Filter out failed analyses + +// Topic distribution +const topicCounts: Record = {}; +const topicScoreTotals: Record = {}; + +for (const analysis of analyses) { + for (const topic of analysis.topics) { + topicCounts[topic] = (topicCounts[topic] || 0) + 1; + const score = analysis.topicScores[topic] || 0; + topicScoreTotals[topic] = (topicScoreTotals[topic] || 0) + score; + } +} + +console.log('═'.repeat(80)); +console.log('TOPIC DISTRIBUTION'); +console.log('═'.repeat(80)); + +const sortedTopics = Object.entries(topicCounts) + .sort((a, b) => b[1] - a[1]); + +console.log(`\n📈 Topics by frequency:\n`); +for (const [topic, count] of sortedTopics) { + const avgScore = (topicScoreTotals[topic] / count).toFixed(2); + console.log(` ${topic.padEnd(30)} ${count} articles (avg score: ${avgScore})`); +} + +// Sentiment distribution +const sentiments: Record = {}; +for (const analysis of analyses) { + const sentiment = analysis.sentiment; + sentiments[sentiment] = (sentiments[sentiment] || 0) + 1; +} + +console.log(`\n😊 Sentiment distribution:\n`); +for (const [sentiment, count] of Object.entries(sentiments)) { + console.log(` ${sentiment}: ${count}`); +} + +// Identify clusters (articles with shared topics) +console.log('\n' + '═'.repeat(80)); +console.log('TOPIC CLUSTERS (Content Opportunities)'); +console.log('═'.repeat(80)); + +for (const [topic, count] of sortedTopics) { + if (count < 2) continue; // Only show topics with 2+ articles + + console.log(`\n📂 ${topic} (${count} articles):`); + + const articlesWithTopic = analyses.filter(a => + a.topics.includes(topic) + ); + + for (const analysis of articlesWithTopic) { + const score = analysis.topicScores[topic].toFixed(2); + console.log(` [${score}] ${analysis.articleTitle.substring(0, 70)}`); + } + + // Content opportunity + if (count >= 3) { + console.log(` 💡 OPPORTUNITY: Weekly roundup or comparison post possible`); + } else if (count === 2) { + console.log(` 💡 OPPORTUNITY: Comparison or "X vs Y" post possible`); + } +} + +// Monetization angles summary +console.log('\n' + '═'.repeat(80)); +console.log('MONETIZATION ANGLES'); +console.log('═'.repeat(80) + '\n'); + +for (const analysis of analyses) { + console.log(`📝 ${analysis.articleTitle.substring(0, 70)}`); + console.log(` ${analysis.monetizationAngle}\n`); +} + +console.log('═'.repeat(80)); +console.log('END OF REPORT'); +console.log('═'.repeat(80) + '\n'); + +// AIDEV-NOTE: tracking-db - close connection after report +db.close(); diff --git a/self-hosting/omc/cli/archived-scripts/get-article-content.ts b/self-hosting/omc/cli/archived-scripts/get-article-content.ts new file mode 100644 index 000000000..ffa162869 --- /dev/null +++ b/self-hosting/omc/cli/archived-scripts/get-article-content.ts @@ -0,0 +1,33 @@ +#!/usr/bin/env tsx +// AIDEV-NOTE: agent-helper - fetches article content for agent analysis +// AIDEV-NOTE: omnivore-boundary - GraphQL API only + +import { getArticle } from '../lib/omnivore/client.js'; +import 'dotenv/config'; + +async function main() { + const articleSlug = process.argv[2]; + const username = process.argv[3]; + + if (!articleSlug || !username) { + console.error('Usage: tsx cli/get-article-content.ts '); + process.exit(1); + } + + try { + const result = await getArticle(articleSlug, username); + + if (!result?.article?.content) { + console.error('No content found'); + process.exit(1); + } + + // Output ONLY content to stdout (no JSON wrapper) + console.log(result.article.content); + } catch (err: any) { + console.error(`Error fetching article: ${err.message}`); + process.exit(1); + } +} + +main(); diff --git a/self-hosting/omc/cli/archived-scripts/get-article-notes.ts b/self-hosting/omc/cli/archived-scripts/get-article-notes.ts new file mode 100644 index 000000000..8bfdaf3ee --- /dev/null +++ b/self-hosting/omc/cli/archived-scripts/get-article-notes.ts @@ -0,0 +1,84 @@ +#!/usr/bin/env tsx + +/** + * Fetch and display notes for an article + */ + +import fetch from 'node-fetch'; +import { config } from 'dotenv'; + +config(); + +const API_URL = process.env.OMNIVORE_API_URL || 'https://api-prod.omnivore.app/api/graphql'; +const API_KEY = process.env.OMNIVORE_API_KEY; + +const articleId = process.argv[2] || '5977ff9f-01ea-4977-aa0e-1dbe43cd2a20'; + +console.log(`\n📝 Fetching notes for article: ${articleId}\n`); + +const query = ` + query GetArticle($id: ID!) { + article(id: $id) { + ... on ArticleSuccess { + article { + id + title + highlights { + id + type + quote + annotation + createdAt + updatedAt + } + } + } + ... on ArticleError { + errorCodes + } + } + } +`; + +const response = await fetch(API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': API_KEY, + }, + body: JSON.stringify({ query, variables: { id: articleId } }), +}); + +const result = await response.json(); + +if (result.errors) { + console.error('GraphQL errors:', JSON.stringify(result.errors, null, 2)); + process.exit(1); +} + +const article = result.data?.article?.article; + +if (!article) { + console.error('Article not found or error:', result.data?.article?.errorCodes); + process.exit(1); +} + +console.log(`Article: ${article.title}\n`); + +const highlights = article.highlights || []; +const notes = highlights.filter((h: any) => h.type === 'NOTE'); + +console.log(`Found ${notes.length} notes:\n`); + +for (const note of notes) { + console.log('═'.repeat(80)); + console.log(`Note ID: ${note.id}`); + console.log(`Type: ${note.type}`); + console.log(`Created: ${note.createdAt}`); + console.log(`Updated: ${note.updatedAt}`); + console.log('─'.repeat(80)); + console.log('Annotation (raw Markdown):'); + console.log(note.annotation); + console.log('═'.repeat(80)); + console.log(); +} diff --git a/self-hosting/omc/cli/archived-scripts/migrate-database.ts b/self-hosting/omc/cli/archived-scripts/migrate-database.ts new file mode 100644 index 000000000..8bf642f0f --- /dev/null +++ b/self-hosting/omc/cli/archived-scripts/migrate-database.ts @@ -0,0 +1,55 @@ +#!/usr/bin/env tsx +// AIDEV-NOTE: database-migration - adds new columns to existing analysis_queue table + +import { initDatabase } from '../src/storage/database'; + +async function main() { + console.log('Starting database migration...\n'); + + const db = initDatabase(); + + try { + // Check current schema + const columns = db.prepare(`PRAGMA table_info(analysis_queue)`).all() as Array<{ name: string }>; + const columnNames = columns.map(c => c.name); + + console.log('Current columns:', columnNames.join(', ')); + + // Add missing columns + const migrations = [ + { name: 'saved_at', sql: 'ALTER TABLE analysis_queue ADD COLUMN saved_at TEXT NOT NULL DEFAULT ""' }, + { name: 'published_at', sql: 'ALTER TABLE analysis_queue ADD COLUMN published_at TEXT' }, + { name: 'updated_at_article', sql: 'ALTER TABLE analysis_queue ADD COLUMN updated_at_article TEXT' }, + { name: 'analysis_json', sql: 'ALTER TABLE analysis_queue ADD COLUMN analysis_json TEXT' }, + { name: 'markdown_path', sql: 'ALTER TABLE analysis_queue ADD COLUMN markdown_path TEXT' }, + ]; + + let added = 0; + for (const migration of migrations) { + if (!columnNames.includes(migration.name)) { + console.log(`Adding column: ${migration.name}`); + db.exec(migration.sql); + added++; + } else { + console.log(`Column ${migration.name} already exists, skipping`); + } + } + + console.log(`\n✓ Migration complete: ${added} columns added`); + + // Show final schema + const finalColumns = db.prepare(`PRAGMA table_info(analysis_queue)`).all() as Array<{ name: string, type: string }>; + console.log('\nFinal schema:'); + finalColumns.forEach(col => { + console.log(` - ${col.name} (${col.type})`); + }); + + } catch (err: any) { + console.error('Migration failed:', err.message); + process.exit(1); + } finally { + db.close(); + } +} + +main().catch(console.error); diff --git a/self-hosting/omc/cli/archived-scripts/parallel-analyze.ts b/self-hosting/omc/cli/archived-scripts/parallel-analyze.ts new file mode 100644 index 000000000..29944981b --- /dev/null +++ b/self-hosting/omc/cli/archived-scripts/parallel-analyze.ts @@ -0,0 +1,98 @@ +#!/usr/bin/env tsx +// AIDEV-NOTE: tracking-coordination - uses SQLite queue for parallel execution +// AIDEV-NOTE: analysis-output-boundary - results written to Markdown/JSONL, NOT database +// AIDEV-NOTE: omnivore-boundary - fetches articles via GraphQL API, never local cache + +import { getArticle, getMe } from '../lib/omnivore/client.js'; +import { initDatabase } from '../src/storage/database'; +import { AnalysisQueueRepository } from '../src/storage/AnalysisQueueRepository'; +import { AnalysisWriter } from '../src/storage/AnalysisWriter'; +import type { ContentAnalysis } from '../src/types/analysis'; +import { writeFileSync, mkdirSync } from 'fs'; +import 'dotenv/config'; + +const BATCH_SIZE = 5; // Process 5 articles in parallel + +async function main() { + // AIDEV-NOTE: tracking-db - coordination only, not analysis storage + const db = initDatabase(); + const queueRepo = new AnalysisQueueRepository(db); + + const stats = queueRepo.getStats(); + console.log('═'.repeat(80)); + console.log('Analysis Queue Status'); + console.log('═'.repeat(80)); + console.log(` Total: ${stats.total}`); + console.log(` Pending: ${stats.pending}`); + console.log(` In Progress: ${stats.inProgress}`); + console.log(` Completed: ${stats.completed}`); + console.log(` Failed: ${stats.failed}`); + + if (stats.pending === 0) { + console.log('\n✓ No pending analyses'); + db.close(); + return; + } + + // Get next batch + // AIDEV-NOTE: tracking-batch - fetch jobs for parallel processing + const jobs = queueRepo.getPending(BATCH_SIZE); + console.log(`\n${'─'.repeat(80)}`); + console.log(`Created ${jobs.length} stub files for parallel analysis...`); + console.log('─'.repeat(80)); + + // Mark as in_progress (coordination lock) + // AIDEV-NOTE: tracking-lock - prevents duplicate analysis by concurrent runs + for (const job of jobs) { + queueRepo.markInProgress(job.articleId); + } + + // AIDEV-NOTE: gql-article-query - get username from authenticated user + const me = await getMe(); + const username = me.profile.username; + + // Create temp directory for stub files + mkdirSync('temp', { recursive: true }); + + // AIDEV-NOTE: stub-file-creation - fetch articles and populate complete metadata + const agentParams = []; + + for (let index = 0; index < jobs.length; index++) { + const job = jobs[index]; + const filename = `temp/${job.articleSlug}.jsonl`; + + console.log(`Fetching article ${index + 1}/${jobs.length}: ${job.articleTitle.substring(0, 60)}...`); + + // Fetch article to get publishedAt/updatedAt + const result = await getArticle(job.articleSlug, username); + + // Write complete stub JSONL with all metadata + const stub = { + articleId: job.articleId, + articleSlug: job.articleSlug, + username: username, + articleUrl: result.article.url, + articleTitle: result.article.title, + savedAt: result.article.savedAt, + publishedAt: result.article.publishedAt || null, + updatedAt: result.article.updatedAt || null + }; + + writeFileSync(filename, JSON.stringify(stub) + '\n', 'utf-8'); + + // Return metadata for agent invocation + agentParams.push({ + filename: filename, + articleId: job.articleId, + articleSlug: job.articleSlug, + username: username, + articleTitle: result.article.title.substring(0, 60) + '...' // truncated for display + }); + } + + console.log(JSON.stringify(agentParams, null, 2)); + + db.close(); +} + +main().catch(console.error); diff --git a/self-hosting/omc/cli/archived-scripts/retry-failed.ts b/self-hosting/omc/cli/archived-scripts/retry-failed.ts new file mode 100644 index 000000000..3ece3a6bc --- /dev/null +++ b/self-hosting/omc/cli/archived-scripts/retry-failed.ts @@ -0,0 +1,52 @@ +#!/usr/bin/env tsx +// AIDEV-NOTE: tracking-retry - resets failed jobs to pending for another attempt + +import { initDatabase } from '../src/storage/database'; +import { AnalysisQueueRepository } from '../src/storage/AnalysisQueueRepository'; + +const MAX_RETRIES = 3; + +async function main() { + const db = initDatabase(); + const queueRepo = new AnalysisQueueRepository(db); + + const failed = queueRepo.getFailed(); + + if (failed.length === 0) { + console.log('✓ No failed jobs to retry'); + db.close(); + return; + } + + console.log('═'.repeat(80)); + console.log(`Found ${failed.length} failed jobs`); + console.log('═'.repeat(80)); + + let reset = 0; + let skipped = 0; + + for (const job of failed) { + if (job.retryCount < MAX_RETRIES) { + queueRepo.resetToPending(job.articleId); + console.log(`✓ Reset: ${job.articleTitle.substring(0, 50)}... (attempt ${job.retryCount + 2})`); + reset++; + } else { + console.log(`✗ Skip: ${job.articleTitle.substring(0, 50)}... (max retries exceeded)`); + skipped++; + } + } + + console.log('\n' + '═'.repeat(80)); + console.log('Summary:'); + console.log(` Reset for retry: ${reset}`); + console.log(` Skipped (max retries): ${skipped}`); + console.log('═'.repeat(80)); + + if (reset > 0) { + console.log('\nRun: pnpm analyze:parallel'); + } + + db.close(); +} + +main().catch(console.error); diff --git a/self-hosting/omc/cli/archived-scripts/save-analysis-results.ts b/self-hosting/omc/cli/archived-scripts/save-analysis-results.ts new file mode 100644 index 000000000..7036e8f3c --- /dev/null +++ b/self-hosting/omc/cli/archived-scripts/save-analysis-results.ts @@ -0,0 +1,106 @@ +#!/usr/bin/env tsx +// AIDEV-NOTE: analysis-output-boundary - saves results to git-tracked Markdown/JSONL +// AIDEV-NOTE: tracking-update - updates SQLite queue status after save + +import { readFileSync, unlinkSync } from 'fs'; +import { initDatabase } from '../src/storage/database'; +import { AnalysisQueueRepository } from '../src/storage/AnalysisQueueRepository'; +import { AnalysisWriter } from '../src/storage/AnalysisWriter'; +import type { ContentAnalysis } from '../src/types/analysis'; + +interface EnrichedResult { + articleId: string; + articleSlug: string; + username: string; + articleUrl: string; + articleTitle: string; + savedAt: string; + publishedAt: string | null; + updatedAt: string | null; + analysis: ContentAnalysis; +} + +async function main() { + const patterns = process.argv.slice(2); + + if (patterns.length === 0) { + patterns.push('temp/*.jsonl'); // Default: all JSONL files in temp/ + } + + // Read all matching JSONL files from all patterns + const { globSync } = await import('glob'); + const files = patterns.flatMap(pattern => globSync(pattern)); + + if (files.length === 0) { + console.error(`No files found matching patterns: ${patterns.join(', ')}`); + process.exit(1); + } + + const resultsWithFiles: Array<{ result: EnrichedResult; file: string }> = files.map(file => { + const content = readFileSync(file, 'utf-8'); + return { result: JSON.parse(content), file }; + }); + + console.log(`Loaded ${resultsWithFiles.length} enriched analysis results from ${files.length} files`); + + // AIDEV-NOTE: tracking-db - coordination only + const db = initDatabase(); + const queueRepo = new AnalysisQueueRepository(db); + + // AIDEV-NOTE: analysis-output - permanent git-tracked storage + const writer = new AnalysisWriter({ outputDir: 'content/analysis' }); + + let saved = 0; + let failed = 0; + + for (const { result, file } of resultsWithFiles) { + const { articleId, articleUrl, articleTitle, savedAt, publishedAt, updatedAt, analysis } = result; + + try { + // Write to Markdown (git-tracked, human-editable) + const mdPath = await writer.write( + articleId, + articleUrl, + articleTitle, + savedAt, + analysis + ); + + // Store in database (immutable AI snapshot) + queueRepo.storeAnalysis( + articleId, + publishedAt, + updatedAt, + JSON.stringify(analysis), + mdPath + ); + + // Remove temp file after successful save + unlinkSync(file); + + console.log(`✓ Saved: ${articleTitle.substring(0, 60)}...`); + saved++; + } catch (err: any) { + // Mark as failed in tracking DB + queueRepo.markFailed(articleId, `Save failed: ${err.message}`); + console.error(`✗ Failed: ${articleTitle.substring(0, 60)}... - ${err.message}`); + failed++; + } + } + + // Show updated stats + const stats = queueRepo.getStats(); + console.log(`\n${'═'.repeat(80)}`); + console.log('Results:'); + console.log(` Saved: ${saved}`); + console.log(` Failed: ${failed}`); + console.log('\nUpdated Queue Status:'); + console.log(` Pending: ${stats.pending}`); + console.log(` Completed: ${stats.completed}`); + console.log(` Failed: ${stats.failed}`); + console.log('═'.repeat(80)); + + db.close(); +} + +main().catch(console.error); diff --git a/self-hosting/omc/cli/archived-scripts/test-update-article-notes.ts b/self-hosting/omc/cli/archived-scripts/test-update-article-notes.ts new file mode 100644 index 000000000..de7858df6 --- /dev/null +++ b/self-hosting/omc/cli/archived-scripts/test-update-article-notes.ts @@ -0,0 +1,48 @@ +#!/usr/bin/env tsx + +/** + * Test updating article description AND creating notebook notes + */ + +import { updatePage, createHighlight } from '../lib/omnivore/client.js'; +import { randomUUID } from 'crypto'; + +// Use the LLM Observability article +const articleId = '5977ff9f-01ea-4977-aa0e-1dbe43cd2a20'; +const articleTitle = 'LLM Observability in the Wild - Why OpenTelemetry should be the Standard | SigNoz'; + +console.log(`\n🧪 Testing article notes update for: ${articleTitle}\n`); + +// Test 1: Update article description (the "info" section in Omnivore UI) +console.log('1️⃣ Updating article description (Info section)...'); + +const updateResult = await updatePage({ + pageId: articleId, + description: '🔍 WHY I SAVED THIS: Excellent deep dive into OpenTelemetry patterns for LLM observability. Potential for comparison post: OpenTelemetry vs proprietary solutions (Langfuse, Weights & Biases). Key insight: standardization matters more in LLM ops than traditional observability.' +}); + +console.log(' ✅ Description updated:', updateResult.updatedPage.description); + +// Test 2: Create a NOTE highlight (the "Notebook" section in Omnivore UI) +console.log('\n2️⃣ Creating notebook note (NOTE highlight)...'); + +const noteId = randomUUID(); +const shortId = Math.random().toString(36).substring(2, 10); // 8-char random string +const noteResult = await createHighlight({ + id: noteId, + shortId: shortId, + articleId: articleId, + type: 'NOTE', + annotation: '💡 CONTENT STRATEGY NOTE:\n\nThis article fits perfectly into our AI Infrastructure theme for the weekly roundup. Key angles:\n\n1. Comparison piece: OpenTelemetry vs Langfuse vs W&B\n2. Tutorial: Setting up OTel for LLM apps\n3. Opinion piece: Why standardization matters in AI ops\n\nTarget audience: ML engineers building production LLM apps\nEstimated value: High - trending topic, practical advice', + quote: '', // Empty quote = standalone note + patch: '', + prefix: '', + suffix: '' +}); + +console.log(' Result:', JSON.stringify(noteResult, null, 2)); + +console.log('\n📌 Check Omnivore UI to verify:'); +console.log(' 1. Description field shows "WHY I SAVED THIS" note'); +console.log(' 2. Notebook section shows the content strategy note'); +console.log(` Article ID: ${articleId}\n`); diff --git a/self-hosting/omc/cli/archived-scripts/update-note-test.ts b/self-hosting/omc/cli/archived-scripts/update-note-test.ts new file mode 100644 index 000000000..2b1995b5f --- /dev/null +++ b/self-hosting/omc/cli/archived-scripts/update-note-test.ts @@ -0,0 +1,42 @@ +#!/usr/bin/env tsx + +/** + * Update a note with Markdown to test format + */ + +import { updateHighlight } from '../lib/omnivore/client.js'; + +const noteId = '922361d2-f09c-4b81-8826-396de3690c5d'; + +const markdown = `# Analysis Report + +**Article**: LLM Observability +**Status**: High Priority + +## Key Insights + +1. **OpenTelemetry vs OpenInference** - competing standards +2. **Ruby SDK gap** - No OpenInference support +3. **Production debugging** - Visibility challenges + +## Content Strategy + +- [ ] Write comparison post +- [ ] Tutorial on OTel setup +- [ ] Interview Pranav + +### Target Audience +ML engineers building production LLM apps + +*Last updated: 2025-10-02* +`; + +console.log('Updating note with Markdown...\n'); + +const result = await updateHighlight({ + highlightId: noteId, + annotation: markdown +}); + +console.log('Result:', JSON.stringify(result, null, 2)); +console.log('\nCheck Omnivore UI to see how the Markdown renders!'); diff --git a/self-hosting/omc/codegen.yml b/self-hosting/omc/codegen.yml new file mode 100644 index 000000000..da91ecaac --- /dev/null +++ b/self-hosting/omc/codegen.yml @@ -0,0 +1,58 @@ +# GraphQL Code Generator Configuration +# Generates TypeScript types from GraphQL schema and operations + +schema: './docs/graphql-schema/schema.graphql' + +# Watch mode for development +watch: false + +# The CLI runtime currently uses `lib/omnivore/client.js` (string-based queries). +# Typed GraphQL operations are not maintained right now; reintroduce documents as part of a future migration. +documents: [] + +# Output configuration +generates: + # Generate TypeScript types for the schema + src/types/generated/graphql.ts: + plugins: + - typescript + config: + # Type-safe configuration + strictScalars: true + scalars: + Date: string + JSON: 'Record' + # Naming conventions + enumsAsTypes: true + # Add helpful comments + addUnderscoreToArgsType: true + skipTypename: false + + # Generate TypeScript types for operations (queries/mutations) + src/types/generated/operations.ts: + plugins: + - typescript + - typescript-operations + - typed-document-node + config: + # Type-safe configuration + strictScalars: true + scalars: + Date: string + JSON: 'Record' + # Avoid duplicating types from schema + avoidOptionals: false + # Better type names + namingConvention: + typeNames: pascal-case#pascalCase + enumValues: upper-case#upperCase + # Fragment configuration + fragmentVariablePrefix: '' + fragmentVariableSuffix: 'Fragment' + # Deduplication + dedupeFragments: true + +# Hooks to run after generation +# hooks: +# afterAllFileWrite: +# - prettier --write diff --git a/self-hosting/omc/docs/CLI_DESIGN.md b/self-hosting/omc/docs/CLI_DESIGN.md new file mode 100644 index 000000000..f068c2596 --- /dev/null +++ b/self-hosting/omc/docs/CLI_DESIGN.md @@ -0,0 +1,1042 @@ +# Omnivore Content System CLI Design + +## Overview + +A unified CLI (`omnivore-content` or `omc`) that abstracts workflow operations, uses GraphQL fragments for extensibility, and provides a clean interface for both humans and AI agents. + +## Audit Note (2026-01-30) + +This document contains a **historical design snapshot** and includes many “NOT IMPLEMENTED” markers that are now outdated. The CLI command surface area is largely present in `src/commands/**`, but there are still critical fixes required before the repo is “clean build + reliable run”. + +For ground-truth implementation status and a prioritized fix list, see `docs/_meta/current-state.md`. + +## Historical Status Snapshot (2025-01-05) + +**Overall Progress:** 100% complete (52 of 53 commands) - All command structures implemented + +| Command Group | Progress | Implemented | Total | Status | +|---------------|----------|-------------|-------|--------| +| queue | 89% | 8 | 9 | 🚧 Missing --label/--url/--slug in add | +| analyze | 100% | 4 | 4 | 🚧 Missing --article-id/--all in run | +| content | 100% | 5 | 5 | 🚧 Missing --topic filter in list | +| report | 100% | 7 | 7 | ✅ Fully implemented | +| omnivore | 100% | 9 | 9 | ✅ Fully implemented | +| db | 100% | 9 | 9 | 🚧 migrate/seed are placeholders | +| config | 100% | 7 | 7 | ✅ Fully implemented | +| init | 100% | 3 | 3 | ✅ Fully implemented | + +**Total Commands:** 63 registered (52 implemented + 10 original + 1 help) + +**Status Markers:** +- ✅ Fully implemented with all features +- 🚧 Implemented but missing some flags/features +- ❌ Placeholder only (no real implementation) + +**Implementation Notes (2025-01-05 snapshot):** +- ✅ All 43 new commands created and registered +- ✅ All commands extend BaseCommand +- ✅ All use shared utilities (withDatabase, parseJsonSafely, etc.) +- ✅ TypeScript strict mode, 0 build errors +- ✅ Quality: 83% GREEN (≤20 lines), 17% YELLOW (21-25 lines), 0% RED +- ✅ All old scripts migrated to CLI commands +- ✅ DRY violations identified and fixed + +**Remaining Gaps (Updated Summary, 2026-01-30):** +- TypeScript `typecheck` is currently failing due to alias/import issues (see `docs/_meta/current-state.md`) +- The `dist/` build likely cannot locate the DB schema file due to schema path drift +- Some commands still diverge from the `BaseCommand` execution contract (args/flags plumbing) + +**Files Modified/Created:** +- Modified: `src/storage/AnalysisQueueRepository.ts` (added 3 delete methods + PENDING_QUERY constant) +- Created: `src/commands/queue/remove.ts` (52 lines) +- Created: `src/commands/queue/clear.ts` (68 lines) +- Created: `src/commands/queue/reset.ts` (74 lines) +- Created: `src/commands/analyze/retry.ts` (75 lines) +- Created: `src/commands/content/show.ts` (97 lines) +- Created: `src/commands/content/list.ts` (78 lines) + +**Phase 1 Infrastructure:** +- ✅ OCLIF v3 framework +- ✅ ESBuild compilation +- ✅ CLI utilities (database, formatters, graphql, queue-display) +- ✅ TypeScript ESM modules +- ✅ Repository pattern (AnalysisQueueRepository with delete operations) +- ⚠️ Vitest (configured but no tests yet) + +## Command Structure + +``` +omc [options] +``` + +### Command Groups + +#### 1. `omc queue` - Analysis Queue Management + +Manages the article analysis queue (abstracts SQLite operations). + +```bash +# Add articles to queue +✅ omc queue add --hours 24 # Add articles from last 24 hours +🚧 omc queue add --label "ai-ml" # (NOT IMPLEMENTED) Add articles with specific label +🚧 omc queue add --url # (NOT IMPLEMENTED) Add single article by URL + omc queue add --slug # (NOT IMPLEMENTED) Add single article by slug + +# List queue status +✅ omc queue list # Show all queued articles +✅ omc queue list --status pending # Filter by status + omc queue list --status completed + omc queue list --status failed + +# Show queue statistics +🚧 omc queue stats # Overall queue stats + omc queue stats --detailed # (NOT IMPLEMENTED) Per-status breakdown + +# Manage queue items +✅ omc queue reset # Reset article to pending +✅ omc queue remove # Remove from queue +✅ omc queue clear --status failed # Clear all failed items +✅ omc queue clear --all # Clear entire queue (requires confirm) + +# Export/import queue (NOT IMPLEMENTED) + omc queue export > queue-backup.jsonl # Export queue state + omc queue import queue-backup.jsonl # Import queue state +``` + +#### 2. `omc analyze` - Analysis Operations + +Runs content analysis on queued articles. + +```bash +# Run analysis +🚧 omc analyze run # Process next batch (5 articles) +🚧 omc analyze run --batch-size 10 # Custom batch size + omc analyze run --article-id # (NOT IMPLEMENTED) Analyze specific article + omc analyze run --all # (NOT IMPLEMENTED) Process entire queue + +# Resume/retry +✅ omc analyze retry --failed # Retry all failed analyses +✅ omc analyze retry --article-id # Retry specific article + +# Monitor analysis (NOT IMPLEMENTED) + omc analyze status # Show current batch progress + omc analyze watch # Watch analysis in real-time +``` + +#### 3. `omc content` - Content Operations + +Manages analyzed content and synchronization with Omnivore. + +```bash +# View content +✅ omc content show # Show analysis for article +🚧 omc content show --raw # (NOT IMPLEMENTED) Show raw JSONL +✅ omc content list # List all analyzed content + omc content list --topic "ai-ml" # (NOT IMPLEMENTED) Filter by topic + omc content search "opentelemetry" # (NOT IMPLEMENTED) Full-text search in analyses + +# Sync to Omnivore (NOT IMPLEMENTED) + omc content sync # Sync specific article + omc content sync --all # Sync all analyzed articles + omc content sync --since "2025-10-01" # Sync articles analyzed since date + omc content sync --dry-run # Preview what would be synced + +# Export content (NOT IMPLEMENTED) + omc content export --format markdown # Export all as Markdown + omc content export --format json # Export all as JSON + omc content export --topic "ai-ml" # Export filtered content +``` + +#### 4. `omc report` - Reporting & Analytics (NOT IMPLEMENTED) + +Generates reports from analyzed content. + +```bash +# Generate reports (NOT IMPLEMENTED) + omc report corpus # Full corpus analysis report + omc report topics # Topic distribution + omc report trends # Trending topics over time + omc report monetization # Monetization opportunities + omc report sentiment # Sentiment analysis + +# Custom reports (NOT IMPLEMENTED) + omc report custom --query "..." # SQL-based custom report + omc report export --format csv # Export report data +``` + +#### 5. `omc omnivore` - Omnivore API Operations (NOT IMPLEMENTED) + +Direct Omnivore API operations (abstracted GraphQL). + +```bash +# Article operations (NOT IMPLEMENTED) + omc omnivore get # Fetch article by slug + omc omnivore get --format json # Output as JSON + omc omnivore search "opentelemetry" # Search articles + omc omnivore list --hours 24 # List recent articles + +# Note operations (NOT IMPLEMENTED) + omc omnivore note add "content" # Add note to article + omc omnivore note get # Get article notes + omc omnivore note update "content" # Update note + +# Metadata operations (NOT IMPLEMENTED) + omc omnivore update --description "..." # Update description + omc omnivore update --labels "ai,ml" # Update labels + +# Highlight operations (NOT IMPLEMENTED) + omc omnivore highlight add --quote "..." --annotation "..." + omc omnivore highlight list +``` + +#### 6. `omc db` - Database Management (NOT IMPLEMENTED) + +Database operations (migrations, seeding, maintenance). + +```bash +# Schema management (NOT IMPLEMENTED) + omc db migrate # Run pending migrations + omc db migrate --down # Rollback last migration + omc db migrate status # Show migration status + omc db schema # Show current schema + +# Seeding (NOT IMPLEMENTED) + omc db seed # Seed with sample data + omc db seed --fixture test-articles # Seed specific fixture + +# Maintenance (NOT IMPLEMENTED) + omc db vacuum # Optimize database + omc db backup # Create backup + omc db restore backup.db # Restore from backup + omc db reset # Drop and recreate (requires confirm) + +# Diagnostics (NOT IMPLEMENTED) + omc db check # Verify data integrity + omc db stats # Show database statistics +``` + +#### 7. `omc config` - Configuration Management (NOT IMPLEMENTED) + +Manage configuration and credentials. + +```bash +# View config (NOT IMPLEMENTED) + omc config show # Show all config + omc config get # Get specific config value + +# Set config (NOT IMPLEMENTED) + omc config set api.url # Set API URL + omc config set api.key # Set API key (stored securely) + omc config set analysis.batch-size 10 # Set batch size + +# Test configuration (NOT IMPLEMENTED) + omc config test # Test API connection + omc config validate # Validate all config values + +# Environment management (NOT IMPLEMENTED) + omc config env list # List available environments + omc config env use production # Switch to production env + omc config env use development # Switch to development env +``` + +#### 8. `omc init` - Project Setup (NOT IMPLEMENTED) + +Initialize or reset the system. + +```bash +# Initialize new installation (NOT IMPLEMENTED) + omc init # Interactive setup wizard + omc init --api-key # Non-interactive setup + omc init --force # Reinitialize (drops existing data) + +# Verify installation (NOT IMPLEMENTED) + omc doctor # Check system health + omc version # Show version info +``` + +## Configuration File + +**`.omnivore-content.toml`** or **`omnivore-content.config.json`** + +```toml +[api] +url = "https://api-prod.omnivore.app/api/graphql" +key = "encrypted:..." # Encrypted API key + +[analysis] +batch_size = 5 +concurrent_agents = 5 +retry_limit = 3 + +[storage] +data_dir = "data" +content_dir = "content/analysis" + +[sync] +auto_sync_to_omnivore = true +sync_description = true +sync_notebook = true + +[reporting] +default_format = "text" +``` + +## GraphQL Fragment System + +**Design for extensibility:** + +``` +lib/omnivore/ +├── fragments/ +│ ├── article.fragments.ts # Article-related fragments +│ ├── highlight.fragments.ts # Highlight/note fragments +│ ├── label.fragments.ts # Label fragments +│ └── index.ts # Export all fragments +├── queries/ +│ ├── article.queries.ts # Composed from fragments +│ ├── search.queries.ts +│ └── index.ts +├── mutations/ +│ ├── article.mutations.ts +│ ├── highlight.mutations.ts +│ └── index.ts +└── client.ts # GraphQL client +``` + +**Example Fragment:** + +```typescript +// lib/omnivore/fragments/article.fragments.ts +export const ARTICLE_BASIC = gql` + fragment ArticleBasic on Article { + id + slug + title + url + author + description + } +`; + +export const ARTICLE_WITH_METADATA = gql` + fragment ArticleWithMetadata on Article { + ...ArticleBasic + publishedAt + savedAt + updatedAt + wordsCount + readingProgressPercent + } + ${ARTICLE_BASIC} +`; + +export const ARTICLE_FULL = gql` + fragment ArticleFull on Article { + ...ArticleWithMetadata + content + highlights { + ...HighlightBasic + } + labels { + ...LabelBasic + } + } + ${ARTICLE_WITH_METADATA} + ${HIGHLIGHT_BASIC} + ${LABEL_BASIC} +`; +``` + +**Usage in Queries:** + +```typescript +// lib/omnivore/queries/article.queries.ts +import { ARTICLE_FULL } from '../fragments'; + +export const GET_ARTICLE = gql` + query GetArticle($slug: String!, $username: String!) { + article(slug: $slug, username: $username) { + ... on ArticleSuccess { + article { + ...ArticleFull + } + } + ... on ArticleError { + errorCodes + } + } + } + ${ARTICLE_FULL} +`; +``` + +## Repository Pattern (No Direct SQLite) + +**Abstracts all database operations:** + +``` +src/storage/ +├── repositories/ +│ ├── AnalysisQueueRepository.ts # Queue operations +│ ├── AnalysisRepository.ts # Analysis CRUD +│ ├── ReportRepository.ts # Report queries +│ └── index.ts +├── migrations/ +│ ├── 001_initial_schema.sql +│ ├── 002_add_tracking_fields.sql +│ └── index.ts # Migration runner +├── seeds/ +│ ├── test-articles.seed.ts +│ └── index.ts +└── database.ts # Database connection +``` + +**Example Repository:** + +```typescript +// src/storage/repositories/AnalysisQueueRepository.ts +export class AnalysisQueueRepository { + constructor(private db: Database) {} + + // Add article to queue + add(article: QueueArticle): string { + const stmt = this.db.prepare(` + INSERT INTO analysis_queue (article_id, article_url, article_title, ...) + VALUES (?, ?, ?, ...) + `); + stmt.run(article.id, article.url, article.title, ...); + return article.id; + } + + // List with filters + list(filters: QueueFilters = {}): QueueArticle[] { + let sql = 'SELECT * FROM analysis_queue WHERE 1=1'; + const params: any[] = []; + + if (filters.status) { + sql += ' AND status = ?'; + params.push(filters.status); + } + + return this.db.prepare(sql).all(...params); + } + + // No raw SQL exposed to CLI layer +} +``` + +## CLI Implementation Structure + +``` +cli/ +├── commands/ +│ ├── queue/ +│ │ ├── add.ts +│ │ ├── list.ts +│ │ ├── stats.ts +│ │ └── index.ts +│ ├── analyze/ +│ │ ├── run.ts +│ │ ├── retry.ts +│ │ └── index.ts +│ ├── content/ +│ │ ├── show.ts +│ │ ├── sync.ts +│ │ └── index.ts +│ ├── report/ +│ ├── omnivore/ +│ ├── db/ +│ ├── config/ +│ └── init/ +├── index.ts # CLI entry point (commander.js) +└── utils/ + ├── formatters.ts # Output formatting + ├── validators.ts # Input validation + └── logger.ts # Structured logging +``` + +## Output Formats + +Support multiple output formats for agent consumption: + +```bash +# Human-readable (default) +omc queue list +┌──────────────────────────────────────┬──────────────────┬───────────┐ +│ Article ID │ Title │ Status │ +├──────────────────────────────────────┼──────────────────┼───────────┤ +│ 5977ff9f-01ea-4977-aa0e-1dbe43cd2a20 │ LLM Observabi... │ completed │ +└──────────────────────────────────────┴──────────────────┴───────────┘ + +# JSON (for agents) +omc queue list --format json +[{"articleId":"5977ff9f...","title":"...","status":"completed"}] + +# CSV (for export) +omc queue list --format csv +articleId,title,status +5977ff9f-01ea-4977-aa0e-1dbe43cd2a20,LLM Observability,completed + +# JSONL (for streaming) +omc queue list --format jsonl +{"articleId":"5977ff9f...","title":"...","status":"completed"} +{"articleId":"369f6a08...","title":"...","status":"completed"} +``` + +## Error Handling + +Consistent error codes for agent parsing: + +```bash +# Success +$ omc queue add --hours 24 +echo $? # 0 + +# User error (bad input) +$ omc queue add --hours invalid +Error: Invalid value for --hours: must be a number +echo $? # 1 + +# System error (API failure) +$ omc analyze run +Error: Failed to connect to Omnivore API +echo $? # 2 + +# Not found +$ omc content show invalid-id +Error: Article not found +echo $? # 3 +``` + +## Agent-Friendly Features + +1. **Machine-readable output**: `--format json|jsonl|csv` +2. **Quiet mode**: `--quiet` (only errors to stderr) +3. **Progress indicators**: Optional `--progress` flag +4. **Exit codes**: Consistent status codes +5. **Dry-run**: `--dry-run` for preview +6. **Idempotent operations**: Safe to retry +7. **Batch operations**: Support for piping IDs + +## Migration Path + +Current scripts → CLI commands: + +| Current Script | New CLI Command | +|-----------------------------|-------------------------------------| +| `parallel-analyze.ts` | `omc analyze run` | +| `save-analysis-results.ts` | `omc analyze complete` | +| `corpus-report.ts` | `omc report corpus` | +| `get-article-content.ts` | `omc omnivore get --json` | +| `test-update-article-notes` | `omc omnivore note update` | +| Direct SQLite queries | `omc queue list`, `omc content list`| + +## Implementation Priority + +**Phase 1: Core Commands** +- [ ] `omc init` - Setup wizard +- [ ] `omc config` - Configuration management +- [x] `omc queue add/list/stats/reset/remove/clear` - Queue management (PARTIAL: stats needs --detailed) +- [x] `omc analyze run/retry` - Core analysis workflow (PARTIAL: run needs --article-id, --all flags) +- [x] `omc content show/list` - View analyzed content (PARTIAL: show needs --raw, list needs filters) +- [ ] `omc content sync` - Omnivore synchronization + +**Phase 2: Enhanced Operations** +- [ ] `omc omnivore` - GraphQL operations +- [ ] `omc report` - Reporting commands +- [ ] `omc db migrate/seed` - Database management + +**Phase 3: Advanced Features** +- [ ] `omc content search` - Full-text search +- [ ] `omc analyze watch` - Real-time monitoring +- [ ] `omc report custom` - Custom queries + +## Benefits + +1. **For Agents**: Clean, predictable interface with machine-readable output +2. **For Humans**: Intuitive commands with good help text +3. **For Maintenance**: Centralized logic, no scattered scripts +4. **For Extension**: Fragment system makes adding new fields easy +5. **For Safety**: Repository pattern prevents SQL injection +6. **For Testing**: Each command is independently testable + +## Technical Stack (Based on OACC Patterns) + +### Framework & Build +- **CLI Framework**: OCLIF v3 (@oclif/core + @oclif/plugin-help) +- **Language**: TypeScript with ESM modules +- **Build Tool**: ESBuild (fast compilation, path alias support) +- **Testing**: Vitest with command mocking patterns +- **Module System**: Native ESM ("type": "module") + +### Project Structure (OCLIF Convention) +``` +omnivore-content-system/ +├── bin/ +│ └── omc.js # CLI entry point +├── src/ +│ ├── commands/ +│ │ ├── queue/ +│ │ │ ├── add.ts +│ │ │ ├── list.ts +│ │ │ └── stats.ts +│ │ ├── analyze/ +│ │ │ ├── run.ts +│ │ │ └── retry.ts +│ │ ├── content/ +│ │ │ ├── show.ts +│ │ │ └── sync.ts +│ │ ├── omnivore/ +│ │ │ ├── get.ts +│ │ │ └── note/ +│ │ │ ├── add.ts +│ │ │ └── update.ts +│ │ └── init.ts +│ ├── lib/ +│ │ ├── omnivore/ +│ │ │ ├── fragments/ # GraphQL fragments +│ │ │ ├── queries/ # GraphQL queries +│ │ │ ├── mutations/ # GraphQL mutations +│ │ │ └── client.ts # API client +│ │ ├── storage/ +│ │ │ ├── repositories/ # Data access layer +│ │ │ ├── migrations/ # DB migrations +│ │ │ └── database.ts +│ │ ├── formatters/ # Output formatting +│ │ ├── validators/ # Input validation +│ │ ├── constants.ts # EXIT_CODES, etc. +│ │ └── path-resolver.ts # Project root detection +│ └── types/ +│ └── index.ts +├── test/ +│ ├── unit/ +│ │ └── commands/ +│ └── fixtures/ +├── dist/ # ESBuild output +├── esbuild.config.mjs +├── tsconfig.json +└── package.json +``` + +### Command Class Pattern (from OACC) +```typescript +import { Command, Args, Flags } from '@oclif/core'; +import { EXIT_CODES } from '@omc/lib/constants'; + +export default class QueueAdd extends Command { + static override description = 'Add articles to analysis queue'; + + static override examples = [ + '$ omc queue add --hours 24', + '$ omc queue add --label "ai-ml"', + '$ omc queue add --url https://...' + ]; + + static override flags = { + hours: Flags.integer({ + char: 'h', + description: 'Add articles from last N hours', + exclusive: ['label', 'url'] + }), + label: Flags.string({ + char: 'l', + description: 'Add articles with specific label', + exclusive: ['hours', 'url'] + }), + url: Flags.string({ + char: 'u', + description: 'Add single article by URL', + exclusive: ['hours', 'label'] + }), + json: Flags.boolean({ + description: 'Output as JSON', + default: false + }) + }; + + async run(): Promise { + const { flags } = await this.parse(QueueAdd); + + try { + const result = await this.addToQueue(flags); + + if (flags.json) { + this.outputJson(result); + } else { + this.outputHuman(result); + } + + process.exit(EXIT_CODES.SUCCESS); + } catch (error) { + this.error(error.message, { exit: false }); + process.exit(EXIT_CODES.ERROR); + } + } + + private async addToQueue(flags: any): Promise { + // Implementation + } + + private outputJson(result: QueueResult): void { + this.log(JSON.stringify(result)); + } + + private outputHuman(result: QueueResult): void { + this.log(`✅ Added ${result.count} articles to queue`); + } +} +``` + +### Configuration Management Pattern +```typescript +// lib/path-resolver.ts +export function getProjectRoot(): string { + const findPackageJsonUp = (dir: string): string | null => { + const pkgPath = join(dir, 'package.json'); + if (existsSync(pkgPath)) return dir; + const parent = dirname(dir); + if (parent === dir) return null; + return findPackageJsonUp(parent); + }; + + const root = findPackageJsonUp(process.cwd()); + if (!root) { + throw new Error('Not in omnivore-content-system project'); + } + return root; +} + +export function resolveFromRoot(...paths: string[]): string { + return join(getProjectRoot(), ...paths); +} + +// lib/config-loader.ts +export function loadConfig(): Config { + const configPath = resolveFromRoot('.omnivore-content.json'); + + if (!existsSync(configPath)) { + throw new Error('Config not found. Run: omc init'); + } + + const content = readFileSync(configPath, 'utf-8'); + return JSON.parse(content); +} +``` + +### Exit Codes (from OACC Pattern) +```typescript +// lib/constants.ts +export const EXIT_CODES = { + SUCCESS: 0, + ERROR: 1, // General errors + VALIDATION: 2, // Validation failures + NOT_FOUND: 3, // Resource not found + API_ERROR: 4, // API failures +} as const; +``` + +### Output Formatting Pattern +```typescript +// lib/formatters/table.ts +export function formatTable(data: any[], columns: string[]): string { + // Use cli-table3 or similar +} + +// lib/formatters/json.ts +export function formatJson(data: any): string { + return JSON.stringify(data, null, 2); +} + +// In commands: +private output(data: any, flags: OutputFlags): void { + if (flags.json) { + this.log(formatJson(data)); + } else if (flags.csv) { + this.log(formatCsv(data)); + } else { + this.log(formatTable(data, flags.columns)); + } +} +``` + +### Testing Pattern (from OACC) +```typescript +// test/unit/commands/queue-add.test.ts +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import QueueAdd from '@omc/commands/queue/add'; + +describe('QueueAdd Command', () => { + let mockProcessExit: any; + let mockLog: any; + let errorSpy: any; + + beforeEach(() => { + mockProcessExit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + errorSpy = vi.spyOn(QueueAdd.prototype, 'error').mockImplementation(() => undefined as never); + mockLog = vi.spyOn(QueueAdd.prototype, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('should add articles from last 24 hours', async () => { + await QueueAdd.run(['--hours', '24']); + expect(mockLog).toHaveBeenCalledWith(expect.stringContaining('Added')); + expect(mockProcessExit).toHaveBeenCalledWith(0); + }); + + it('should output JSON when --json flag is used', async () => { + await QueueAdd.run(['--hours', '24', '--json']); + expect(mockLog).toHaveBeenCalledWith(expect.stringMatching(/^\{.*\}$/)); + }); +}); +``` + +### Build Configuration +```javascript +// esbuild.config.mjs +import { build } from 'esbuild'; +import { glob } from 'glob'; + +const entryPoints = await glob('src/**/*.ts', { + ignore: ['src/**/*.test.ts'] +}); + +await build({ + entryPoints, + bundle: true, + platform: 'node', + target: 'node18', + format: 'esm', + outdir: 'dist', + packages: 'external', + alias: { + '@omc': './src' + }, + splitting: true, + sourcemap: true, + loader: { '.ts': 'ts' }, +}); +``` + +### TypeScript Configuration +```json +{ + "compilerOptions": { + "target": "es2022", + "module": "esnext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "outDir": "./dist", + "rootDir": "./src", + "paths": { + "@omc/*": ["./src/*"] + } + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "test"] +} +``` + +### Package.json Setup +```json +{ + "name": "omnivore-content-system", + "version": "1.0.0", + "type": "module", + "bin": { + "omc": "./dist/bin/omc.js" + }, + "scripts": { + "build": "node esbuild.config.mjs", + "dev": "node --loader tsx/esm index.ts", + "test": "vitest" + }, + "dependencies": { + "@oclif/core": "^3.26.6", + "@oclif/plugin-help": "^6.0.21" + }, + "oclif": { + "commands": "./dist/src/commands", + "bin": "omc", + "dirname": "omc", + "plugins": ["@oclif/plugin-help"], + "topicSeparator": " " + } +} +``` + +## Next Steps + +1. ✅ CLI framework chosen: OCLIF v3 +2. ✅ Setup project structure with OCLIF conventions +3. ✅ Implement Phase 1 commands using command class pattern +4. ✅ Create GraphQL fragment system +5. ✅ Build repository pattern for database abstraction +6. ⏳ Write tests using Vitest (in progress) +7. ✅ Configure ESBuild for compilation + +## Implementation Gaps & Next Steps + +### Critical Gaps (Must Fix) + +**1. Queue Add Command - Missing Flag Implementations** +- **Location:** `src/commands/queue/add.ts:37,57` +- **Issue:** `--label`, `--url`, `--slug` flags defined but throw "not implemented" errors +- **Current:** Only `--hours` flag works +- **Required:** + - Implement Omnivore label-based search for `--label` + - Implement single article fetch by URL for `--url` + - Implement single article fetch by slug for `--slug` +- **Priority:** HIGH - Core functionality gap + +**2. Analyze Run Command - Missing Targeting Flags** +- **Location:** `src/commands/analyze/run.ts:30-36` +- **Issue:** Missing `--article-id` and `--all` flags from design +- **Current:** Only batch processing supported +- **Required:** + - Add `--article-id ` flag to analyze specific article + - Add `--all` flag to process entire queue +- **Priority:** MEDIUM - Useful for debugging/targeted operations + +**3. Database Migration System** +- **Location:** `src/commands/db/migrate.ts:7` +- **Issue:** Placeholder only, no actual migration system +- **Current:** Returns "No migrations pending" +- **Required:** + - Implement migration file scanning (src/storage/migrations/) + - Track migration versions in database + - Execute pending migrations in order +- **Priority:** LOW - Can use manual schema updates for now + +**4. Database Seeding System** +- **Location:** `src/commands/db/seed.ts:7` +- **Issue:** Placeholder only, no actual seeding system +- **Current:** Returns "No seed data configured" +- **Required:** + - Create seed data files (src/storage/seeds/) + - Implement seed execution + - Support different environments (dev, test) +- **Priority:** LOW - Mainly for testing/development + +### Minor Gaps (Nice to Have) + +**5. Content List Topic Filter** +- **Location:** Design doc line 133 +- **Issue:** `--topic` filter not implemented in `content list` +- **Current:** Lists all completed analyses +- **Required:** Filter analyses by topic field +- **Priority:** LOW - Can use grep or other tools + +**6. Content Search Full-Text Capability** +- **Location:** Design doc line 134 +- **Issue:** Full-text search implementation not verified +- **Current:** Basic search exists but capability unclear +- **Required:** Verify SQLite FTS5 integration or grep-based search +- **Priority:** LOW - Current search may be sufficient + +### Script Migration Status + +All old scripts successfully migrated: + +| Old Script | New Command | Status | +|-----------|-------------|--------| +| `cli/parallel-analyze.ts` | `omc analyze run` | ✅ Migrated | +| `cli/save-analysis-results.ts` | `omc analyze complete` | ✅ Migrated | +| `cli/corpus-report.ts` | `omc report corpus` | ✅ Migrated | +| `cli/get-article-content.ts` | `omc omnivore get --json` | ✅ Migrated | +| `cli/test-update-article-notes.ts` | `omc omnivore note update` | ✅ Migrated | +| `cli/retry-failed.ts` | `omc analyze retry` | ✅ Migrated | +| `cli/migrate-database.ts` | `omc db migrate` | 🚧 Partial (placeholder) | +| `cli/get-article-notes.ts` | `omc omnivore note get` | ✅ Migrated | +| `cli/update-note-test.ts` | `omc omnivore note update` | ✅ Migrated | + +### Recommended Implementation Order + +**Note (2026-01-30)**: Much of the feature work listed below is now implemented; remaining work is largely around build/runtime correctness and documentation drift. See `docs/_meta/current-state.md`. + +**Phase 1: Critical Functionality (1-2 days)** +1. Implement `queue add --label` - Most commonly needed +2. Implement `queue add --url` - Single article workflow +3. Implement `analyze run --article-id` - Debugging support + +**Phase 2: Enhanced Features (2-3 days)** +4. Implement `queue add --slug` - API consistency +5. Implement `analyze run --all` - Batch processing +6. Add `content list --topic` filter - Better UX + +**Phase 3: Infrastructure (3-5 days)** +7. Build migration system for `db migrate` +8. Build seeding system for `db seed` +9. Enhance `content search` with FTS5 + +### Testing Gaps + +**Current State:** +- All commands have help text ✅ +- All commands build successfully ✅ +- No unit tests exist ❌ +- No integration tests exist ❌ +- No E2E tests exist ❌ + +**Required Tests:** +``` +test/ +├── unit/ +│ ├── commands/ +│ │ ├── queue/add.test.ts # Test all flag combinations +│ │ ├── analyze/run.test.ts # Test batch processing +│ │ └── db/migrate.test.ts # Test migration logic +│ └── lib/ +│ ├── command-utils.test.ts # Test parseJsonSafely, loadEnvFile +│ └── database.test.ts # Test withDatabase wrapper +├── integration/ +│ ├── queue-workflow.test.ts # Add → analyze → report +│ └── omnivore-sync.test.ts # API integration +└── e2e/ + └── full-workflow.test.ts # Complete user journey +``` + +### Documentation Gaps + +**Current State:** +- CLI_DESIGN.md exists ✅ +- CLAUDE.md has ground truths ✅ +- Command help text complete ✅ +- No user guides exist ❌ +- No API docs exist ❌ + +**Required Documentation:** +``` +docs/ +├── guides/ +│ ├── getting-started.md # First-time setup +│ ├── common-workflows.md # Task patterns +│ ├── omnivore-integration.md # API usage +│ └── troubleshooting.md # FAQ +├── api/ +│ ├── queue-commands.md # Queue API reference +│ ├── analyze-commands.md # Analysis API +│ └── report-commands.md # Reporting API +└── architecture/ + ├── command-structure.md # How commands work + └── utilities.md # Shared utilities guide +``` + +### Summary + +**Implementation Status:** +- ✅ **Structure:** 100% complete (all 52 commands exist) +- 🚧 **Features:** 90% complete (5 missing features) +- ❌ **Tests:** 0% complete (no tests written) +- 🚧 **Docs:** 40% complete (design + ground truths only) + +**Next Actions:** +1. Fix queue add flags (highest impact) +2. Add analyze run targeting (debugging support) +3. Write unit tests (quality assurance) +4. Create user guides (adoption) diff --git a/self-hosting/omc/docs/_meta/architecture.md b/self-hosting/omc/docs/_meta/architecture.md new file mode 100644 index 000000000..8d731f5a0 --- /dev/null +++ b/self-hosting/omc/docs/_meta/architecture.md @@ -0,0 +1,395 @@ +# System Architecture + +**Purpose**: Architectural design decisions, storage boundaries, and structural patterns for the Omnivore Content System. + +**Last Updated**: 2026-01-30 + +**Status**: Architectural intent remains valid, but some implementation details have drifted (notably JSONL output and the legacy workflow scripts). See `docs/_meta/current-state.md` for the full audit. + +## Overview + +The analysis engine processes articles from Omnivore through the `@article-content-analyzer` agent, extracting structured insights for content monetization. The system uses a three-layer architecture designed to maintain clear boundaries and prevent data corruption. + +## Three-Layer Architecture + +The system enforces strict separation between source data, coordination, and permanent storage: + +**Layer 1: Omnivore (Source of Truth)** +- Always fetch articles via GraphQL API +- Never duplicate article content locally +- Omnivore labels used as intent hints for analysis + +**Layer 2: SQLite (Immutable Snapshots + Tracking)** +- Database: `data/omnivore-content.db` (gitignored) +- Purpose: Store immutable AI analysis snapshots + coordinate parallel execution +- Tracks: pending/in_progress/completed/failed status +- Tables: `analysis_queue`, `analysis_sessions` +- Existing Omnivore tables (if present): READ-ONLY, never modify + +**Layer 3: Markdown (Permanent Storage)** +- Analysis results: `content/analysis/*.md` (human-readable) +- Git-tracked for version history +- Only value-added content stored + +### What Gets Stored + +- ✅ **Analysis results** (`content/analysis/*.md`) - Topics, summaries, key points, monetization angles +- ✅ **JSONL records (optional)** (`content/analysis/analyses.jsonl`) - Produced when enabled via CLI flags (`omc analyze complete --jsonl` / `omc analyze auto --jsonl`) +- ✅ **Tracking queue** (SQLite) - Job status, retry counts, error messages +- ✅ **Generated content** (`content/generated/**/*.md`) - Blog posts, newsletters (future) +- ❌ **Source articles** - Query Omnivore API; do NOT duplicate locally + +### Storage Formats + +1. **Markdown with YAML front-matter** - Human-readable, git-trackable +2. **JSONL (JSON Lines)** - Append-only, machine-readable (planned/optional; not currently produced by default) +3. **SQLite** - Immutable AI snapshots + tracking (gitignored but permanent) + +## Database Schema + +**Location**: `src/storage/schema/tracking-schema.sql` + +**Purpose**: Define tracking tables for job queue coordination. + +### Tables + +**1. `analysis_queue`** - Job tracking for parallel execution +- `id` (INTEGER PRIMARY KEY) - Auto-increment job ID +- `article_id` (TEXT UNIQUE) - Omnivore article ID +- `article_url` (TEXT) - Source article URL +- `article_title` (TEXT) - Source article title +- `status` (TEXT) - Job status: `pending` | `in_progress` | `completed` | `failed` +- `assigned_at` (TEXT) - When marked in_progress (ISO 8601) +- `completed_at` (TEXT) - When marked completed (ISO 8601) +- `error_message` (TEXT) - Error details if failed +- `retry_count` (INTEGER) - Number of retry attempts (default: 0) +- `created_at` (TEXT) - When added to queue (ISO 8601) +- `updated_at` (TEXT) - Last status change (ISO 8601) + +**2. `analysis_sessions`** - Optional batch metadata +- `id` (INTEGER PRIMARY KEY) - Auto-increment session ID +- `started_at` (TEXT) - Session start time +- `completed_at` (TEXT) - Session completion time +- `total_articles` (INTEGER) - Total articles in session +- `completed_articles` (INTEGER) - Completed count +- `failed_articles` (INTEGER) - Failed count +- `notes` (TEXT) - Session notes + +### Indexes + +- `idx_analysis_queue_status` - Fast status filtering +- `idx_analysis_queue_article_id` - Fast article lookup +- `idx_analysis_queue_created_at` - Chronological ordering + +### AIDEV Annotations + +- `tracking + immutable snapshots` - Stores original AI output + coordination +- `analysis-output-boundary` - Analysis results stored in git-tracked Markdown/JSONL, NOT here + +## Database Initialization + +**Location**: `src/storage/database.ts` + +**Purpose**: Initialize SQLite database with boundary enforcement for tracking coordination. + +**Import**: +```typescript +import { initDatabase } from '@storage/database'; +``` + +### Key Functions + +**1. `initDatabase(dbPath?: string): Database`** - Initialize database connection +- Default path: `data/omnivore-content.db` +- Enables WAL mode for concurrent access +- Creates tracking tables via `tracking-schema.sql` +- Returns `better-sqlite3` Database instance + +**2. `listTables(db: Database): string[]`** - List all tables +- Returns array of table names +- Used to identify Omnivore vs tracking tables + +**3. `isOmnivoreTable(db: Database, tableName: string): boolean`** - Identify Omnivore tables +- Detects Core Data tables by Z-prefixed column names +- Returns `true` if table belongs to Omnivore cache + +**4. `validateOmnivoreTablesReadOnly(db: Database): void`** - Boundary check +- Logs Omnivore tables (READ-ONLY) vs tracking tables (READ-WRITE) +- Safety check to ensure no modification of Omnivore data + +**5. `getTableCounts(db: Database): Record`** - Get row counts +- Returns object mapping table names to row counts +- Useful for debugging and monitoring + +### AIDEV Annotations + +- `tracking-db-boundary` - SQLite stores immutable AI snapshots + job tracking +- `omnivore-boundary` - Existing Omnivore tables are READ-ONLY, never modify + +### Example Usage + +```typescript +import { initDatabase, validateOmnivoreTablesReadOnly, listTables } from '@storage/database'; + +// Initialize database +const db = initDatabase('data/omnivore-content.db'); + +// Verify boundaries +validateOmnivoreTablesReadOnly(db); + +// List all tables +const tables = listTables(db); +console.log('Tables:', tables); +// Output: ['analysis_queue', 'analysis_sessions', 'ZMEDIA', 'ZUSER', ...] + +// Close when done +db.close(); +``` + +## Storage Formats + +### Markdown File Structure + +**Location**: `content/analysis/YYYY-MM-DD-{slug}-analysis.md` + +**Format**: +```markdown +--- +articleId: 1039961b-8de3-4ccf-b3e8-df888d6174b8 +articleSlug: cchistory-tracking-claude-code-system-prompt +articleUrl: https://example.com/article +articleTitle: "Article Title" +savedAt: 2025-09-30T02:27:46.000Z +analyzedAt: 2025-10-01T04:25:16.676Z +topics: [developer-tools, ai-tooling, reverse-engineering] +topicScores: + developer-tools: 0.95 + ai-tooling: 0.9 + reverse-engineering: 0.85 +sentiment: positive +--- + +## Summary + +2-3 sentence summary capturing the main points and why this matters... + +## Key Points + +- First key takeaway or insight +- Second key takeaway or insight +- Third key takeaway or insight + +## Monetization Angle + +Specific content opportunity description... +``` + +**Front-Matter Fields**: +- `articleId` (string) - Omnivore article ID (references source) +- `articleSlug` (string, optional) - Omnivore slug (preferred filename key) +- `articleUrl` (string) - Source article URL +- `articleTitle` (string) - Source article title (escaped quotes) +- `savedAt` (ISO 8601) - When article was saved to Omnivore +- `analyzedAt` (ISO 8601) - When analysis was performed +- `topics` (array) - 2-5 topic labels +- `topicScores` (object) - Topic → confidence score (0-1) +- `sentiment` (enum) - `positive` | `neutral` | `negative` + +**Markdown Sections**: +- `## Summary` - Strategic summary (2-3 sentences) +- `## Key Points` - Bullet list of insights (3-5 items) +- `## Monetization Angle` - Content opportunity description + +### JSONL File Structure + +**Location**: `content/analysis/analyses.jsonl` + +**Status**: Optional. `AnalysisWriter.appendToJsonl()` is invoked when JSONL is enabled via CLI flags. + +**Format**: One JSON object per line (JSON Lines / newline-delimited JSON) + +**Purpose**: Append-only machine-readable format for batch processing + +**Record Structure**: +```json +{"articleId":"abc-123","articleUrl":"https://example.com/article","articleTitle":"Article Title","savedAt":"2025-09-30T14:12:18.000Z","analyzedAt":"2025-10-01T04:30:53.761Z","topics":["developer-tools","api-design"],"topicScores":{"developer-tools":0.95,"api-design":0.9},"sentiment":"positive","summary":"Article summary...","keyPoints":["First point","Second point"],"monetizationAngle":"Content opportunity..."} +``` + +**Fields**: Same as front-matter plus `summary`, `keyPoints`, `monetizationAngle` (flattened structure) + +**Advantages**: +- Append-only (no file rewriting) +- Easy parsing line-by-line +- Works with streaming processing +- Standard format for data pipelines + +**Usage Example**: +```javascript +import { readFileSync } from 'fs'; + +const lines = readFileSync('content/analysis/analyses.jsonl', 'utf-8') + .split('\n') + .filter(line => line.trim()); + +const analyses = lines.map(line => JSON.parse(line)); +console.log(`Loaded ${analyses.length} analyses`); +``` + +## Boundary Enforcement + +The system enforces strict boundaries between three layers to prevent data corruption and maintain clarity. + +### AIDEV Annotation Patterns + +All boundary-critical code uses AIDEV annotations for searchability: + +**Tracking Annotations** (SQLite coordination): +- `tracking + immutable snapshots` - Stores original AI output + coordination +- `tracking-db-boundary` - SQLite stores immutable AI snapshots + job tracking +- `tracking-initialization` - Sets up job queue +- `tracking-coordination` - Fetches jobs for parallel processing +- `tracking-lock` - Prevents duplicate analysis by concurrent runs +- `tracking-completion` - Job done, analysis in git-tracked files +- `tracking-error` - Increments retry counter +- `tracking-retry` - Resets failed job for another attempt +- `tracking-stats` - Shows progress, not analysis content +- `tracking-deduplication` - Prevents duplicate queue entries +- `tracking-cleanup` - Removes completed jobs +- `tracking-inspection` - Shows queue status +- `tracking-update` - Updates queue status after save + +**Output Annotations** (Git-tracked storage): +- `analysis-output-boundary` - Results written to Markdown (and optionally JSONL), NOT stored as editable content in the database +- `git-tracked-output` - Permanent storage for analysis results + +**Omnivore Annotations** (Source of truth): +- `omnivore-boundary` - Always fetch via GraphQL, never from local cache + +### Search Annotations + +Find all boundary-critical code: +```bash +# SQLite tracking code +rg "AIDEV-NOTE:.*tracking" + +# Analysis output code +rg "AIDEV-NOTE:.*analysis-output" + +# Omnivore API usage +rg "AIDEV-NOTE:.*omnivore-boundary" + +# Git-tracked output +rg "AIDEV-NOTE:.*git-tracked" +``` + +### Boundary Rules + +**Rule 1: Omnivore is Source of Truth** +- Always fetch articles via GraphQL API +- Never duplicate article content locally +- Use Omnivore labels as intent hints + +**Rule 2: SQLite Stores Immutable Analysis Snapshots** +- Stores original AI analysis output (immutable snapshots) + coordination for parallel execution +- Database file (`data/omnivore-content.db`) is gitignored but permanent +- Contains valuable AI output that should NOT be deleted +- Existing Omnivore tables (if present) are READ-ONLY + +**Rule 3: Markdown/JSONL is Permanent Storage** +- Analysis results are git-tracked +- Only value-added content stored +- Never store source articles + +**Rule 4: No Cross-Layer Leakage** +- Analysis results never stored in SQLite +- Article content never stored in git +- Queue status never persisted in Markdown + +### Validation Functions + +Use database boundary checks to verify compliance: + +```typescript +import { initDatabase, validateOmnivoreTablesReadOnly, isOmnivoreTable } from '@storage/database'; + +const db = initDatabase(); + +// Check for Omnivore tables +validateOmnivoreTablesReadOnly(db); +// Output: +// [BOUNDARY CHECK] Omnivore table detected: ZMEDIA (READ-ONLY) +// [BOUNDARY CHECK] Tracking table: analysis_queue (READ-WRITE) + +// Verify table type before operations +if (isOmnivoreTable(db, 'ZMEDIA')) { + throw new Error('Cannot modify Omnivore table'); +} +``` + +## Type Definitions + +### ContentAnalysis + +**Location**: `src/types/analysis.ts` + +**Purpose**: Analysis result from `@article-content-analyzer` agent. + +```typescript +export interface ContentAnalysis { + articleId: string; // Omnivore article ID + topics: string[]; // 2-5 main topics + topicScores: Record; // Topic → confidence (0-1) + summary: string; // 2-3 sentence summary + keyPoints: string[]; // 3-5 key takeaways + sentiment: 'positive' | 'neutral' | 'negative'; + monetizationAngle: string; // Content opportunity + analyzedAt: string; // ISO 8601 timestamp +} +``` + +**Import**: +```typescript +import type { ContentAnalysis } from '@omc-types/analysis.js'; +// OR +import type { ContentAnalysis } from '../types/analysis'; +``` + +### AnalysisFrontMatter + +**Location**: `src/types/content.ts` + +**Purpose**: Front-matter schema for analysis Markdown files. + +```typescript +export interface AnalysisFrontMatter { + articleId: string; // Omnivore article ID + articleUrl: string; // Source article URL + articleTitle: string; // Source article title + savedAt: string; // ISO 8601 + analyzedAt: string; // ISO 8601 + topics: string[]; // Topic labels + topicScores: Record; // Topic → score + sentiment: 'positive' | 'neutral' | 'negative'; +} +``` + +### StoredAnalysis + +**Location**: `src/types/content.ts` + +**Purpose**: Complete stored analysis (after reading Markdown file). + +```typescript +export interface StoredAnalysis { + frontMatter: AnalysisFrontMatter; // Parsed YAML + summary: string; // From ## Summary section + keyPoints: string[]; // From ## Key Points section + monetizationAngle: string; // From ## Monetization Angle section +} +``` + +## Related Documentation + +- [Workflow Internals](workflow-internals.md) - How the parallel analysis workflow operates +- [CLI Reference](cli-reference.md) - Command-line interface documentation +- [Foundation & Type System](foundation-and-types.md) - TypeScript setup, type definitions, Omnivore client diff --git a/self-hosting/omc/docs/_meta/automation-patterns.md b/self-hosting/omc/docs/_meta/automation-patterns.md new file mode 100644 index 000000000..fae39f575 --- /dev/null +++ b/self-hosting/omc/docs/_meta/automation-patterns.md @@ -0,0 +1,47 @@ +# Automation Patterns (Codex CLI) + +**Purpose**: Document the non-interactive automation path for daily analysis, and how it integrates with `codex exec`. + +## Goal + +Enable a fully schedulable workflow: + +```bash +omc queue add --hours 24 +omc analyze auto --batch-size 5 +``` + +This replaces the interactive “Task tool / parallel agents” middle step. + +## Key Design Constraints + +- The analysis step must be **non-interactive** and runnable from cron/launchd. +- The LLM caller must **not mutate the repository** (safe-by-default). +- The CLI must remain usable as a “tool” inside future Codex sessions (content generation phase). + +## Codex CLI Invocation + +The automated analysis pipeline uses the local Codex CLI in non-interactive mode: + +- Command: `codex exec` +- Sandbox: `-s read-only` +- Prompt input: stdin (`-`) + +Important runtime behavior: + +- Codex writes session files under `CODEX_HOME`. For scheduled runs, `CODEX_HOME` is set to a repo-local directory (`temp/codex-home`) so we don’t depend on `~/.codex/*` permissions. + +## Implementation Notes + +- LLM wrapper: `src/lib/ai/codex-cli-client.ts` + - Builds a strict “JSON only” prompt. + - Runs `codex exec` and parses the first JSON object from stdout. +- Analyzer: `src/analysis/ContentAnalyzer.ts` + - Loads `src/analysis/prompts/analyze-article.md` + - Calls the Codex CLI wrapper and normalizes required fields. +- Orchestration: `src/commands/analyze/auto.ts` + - Selects jobs from SQLite queue + - Fetches article content from Omnivore + - Calls `ContentAnalyzer` + - Persists results to Markdown + SQLite snapshot + diff --git a/self-hosting/omc/docs/_meta/cli-reference.md b/self-hosting/omc/docs/_meta/cli-reference.md new file mode 100644 index 000000000..ce31956e0 --- /dev/null +++ b/self-hosting/omc/docs/_meta/cli-reference.md @@ -0,0 +1,832 @@ +# CLI Reference + +**Purpose**: Comprehensive command-line interface documentation for the Omnivore Content System. + +**Last Updated**: 2026-01-30 + +**Status**: This file historically mixed design intent with implementation notes. For a full ground-truth audit (including known breakages and fixes), see `docs/_meta/current-state.md`. + +## Overview + +The CLI provides commands for managing the article analysis queue, fetching content from Omnivore, running analysis, and generating reports. All commands use the `omc` binary. + +## Queue Management Commands + +### queue add + +**Command**: `omc queue add` + +**Purpose**: Fetch articles from Omnivore and populate tracking queue. + +**Usage**: +```bash +omc queue add --hours 24 # Last 24 hours +omc queue add --label "ai-ml" # By label +omc queue add --url # Single article +omc queue add --slug # By slug +``` + +**Parameters**: +- `--hours=N` - Fetch articles from last N hours +- `--label=X` - Fetch articles with specific label +- `--url=X` - Add single article by URL +- `--slug=X` - Add single article by slug + +**What It Does (Current)**: +1. Fetches articles from Omnivore (by hours / label / url / slug) +2. Converts results into `{id, slug, url, title, savedAt}` queue rows +3. Inserts into `analysis_queue` via `INSERT OR IGNORE` (deduplication by `article_id`) + +**Notes**: +- Content-length filtering and label distribution output are not currently implemented in `omc queue add`. + +**Output Example**: +``` +Added 38 articles to queue (45 total found) +``` + +**AIDEV Annotations**: +- `omnivore-boundary` - Fetches via GraphQL API, populates tracking queue +- `tracking-initialization` - Adds articles to SQLite queue for parallel analysis + +## Omnivore Integration Commands + +### omnivore get + +**Command**: `omc omnivore get` + +**Purpose**: Fetch an article by slug (metadata by default; raw content with `--content`; full JSON including `content` with `--json`). + +**Usage**: +```bash +omc omnivore get +omc omnivore get --content +omc omnivore get --json +``` + +**Parameters**: +- `articleSlug` - Article slug from Omnivore URL or metadata +- Username is loaded from .env file automatically + +**What It Does (Current)**: +1. Queries Omnivore GraphQL API using `article(slug, username)` +2. Default output is a human-readable summary (title/url/author/description) +3. With `--json`, prints the full Article payload (including `content`) +4. With `--content`, prints the raw article content to stdout (agent-friendly) + +**Agent usage**: +- Prefer `omc omnivore get --content` for stable, parse-free access to the article body. +- If you also need metadata, use `--json` and extract `.content`. + +**Why Needed**: +- Agents execute via Bash tool and cannot import TypeScript modules +- Provides clean separation between content fetching and analysis +- Enables zero-context-pollution agent workflow (each agent fetches independently) + +**Usage in Agent Workflow**: +```typescript +// Agent reads stub file for slug +const stub = JSON.parse(fs.readFileSync(`temp/${filename}`, 'utf-8')); + +// Agent calls CLI tool to get content +const contentResult = await bash(`omc omnivore get ${stub.articleSlug}`); +const articleContent = contentResult.stdout; + +// Agent analyzes content... +``` + +**AIDEV Annotations**: +- `omnivore-boundary` - Fetches via GraphQL API, outputs to stdout +- `agent-workflow` - Enables agents to fetch content without TypeScript imports + +## Analysis Commands + +### analyze auto + +**Command**: `omc analyze auto` + +**Purpose**: Run the full analysis pipeline non-interactively (cron/launchd friendly). + +**Usage**: +```bash +omc analyze auto --batch-size 5 +omc analyze auto --article-id +omc analyze auto --all --batch-size 5 +omc analyze auto --json +omc analyze auto --batch-size 5 --jsonl +``` + +**What It Does (Current)**: +1. Selects jobs from the SQLite queue (`pending` by default; `pending+failed` with `--all`) +2. Fetches article content from Omnivore +3. Calls `codex exec` in **read-only** mode to produce a `ContentAnalysis` JSON object +4. Writes Markdown under `content/analysis/` and stores an immutable analysis snapshot in SQLite +5. Deletes `temp/{slug}.jsonl` unless `--keep-temp` + +**Notes**: +- Codex session files are written under `CODEX_HOME`. For automation, the CLI sets `CODEX_HOME=temp/codex-home` so runs don’t depend on `~/.codex` permissions. +- JSONL output is optional: enable with `--jsonl` (defaults to `content/analysis/analyses.jsonl`). + +### analyze run + +**Command**: `omc analyze run` + +**Purpose**: Run parallel analysis on queued articles with zero-context-pollution design. + +**Usage**: +```bash +omc analyze run # Process next 5 articles (default) +omc analyze run --batch-size 10 # Custom batch size +omc analyze run --article-id # Analyze specific article +omc analyze run --all # Process entire queue +``` + +**What It Does**: +1. Fetches next 5 pending jobs from tracking queue +2. Marks jobs as `in_progress` (coordination lock) +3. Fetches article metadata from Omnivore API using `getArticle(slug, username)` +4. Creates slug-based stub files: `temp/{articleSlug}.jsonl` (prevents overwrites when running multiple batches) +5. Each stub file contains: `articleId`, `articleSlug`, `username`, `articleUrl`, `articleTitle`, `savedAt`, `publishedAt`, `updatedAt` +6. Outputs JSON array with agent parameters for parallel invocation +7. Closes database connection (agents get fresh connections with no context pollution) + +**File Naming Strategy**: +- Stub files use article slug: `temp/{articleSlug}.jsonl` (e.g., `temp/understanding-llm-fine-tuning.jsonl`) +- Prevents overwrites when running multiple batches before agents complete +- Enables agents to work independently without coordination + +**Output Example**: +``` +Analysis Queue Status + Total: 75 + Pending: 42 + In Progress: 0 + Completed: 30 + Failed: 3 + +Processing batch of 5 articles... + +Fetching article metadata from Omnivore API... +✓ Fetched 5 articles +✓ Created stub files: temp/*.jsonl + +READY FOR PARALLEL ANALYSIS + +[ + { + "filename": "temp/understanding-llm-fine-tuning.jsonl", + "articleId": "abc-123", + "articleSlug": "understanding-llm-fine-tuning", + "username": "myusername", + "articleTitle": "Understanding LLM Fine-Tuning Techniques" + }, + ... +] + +Instructions: +1. Invoke 5 @article-content-analyzer agents in parallel (single message) +2. Each agent receives: filename, articleId, articleSlug, username, articleTitle +3. Wait for all agents to complete +4. Run: `omc analyze complete` +``` + +**AIDEV Annotations**: +- `tracking-coordination` - Uses SQLite queue for parallel execution +- `omnivore-boundary` - Fetches articles via GraphQL API, never local cache +- `tracking-lock` - Marks jobs in_progress to prevent duplicate analysis +- `zero-context-pollution` - Creates stub files, closes DB before agent invocation + +### analyze complete + +**Command**: `omc analyze complete` + +**Purpose**: Persist enriched temp files by writing Markdown + storing an immutable `analysis_json` snapshot in SQLite, then marking jobs completed. + +**Input Format** (enriched temp files): +```json +{ + "articleId": "abc-123", + "articleSlug": "understanding-llm-fine-tuning", + "username": "myusername", + "articleUrl": "https://example.com/article", + "articleTitle": "Article Title", + "savedAt": "2025-09-30T10:00:00Z", + "publishedAt": "2025-09-29T14:00:00Z", + "updatedAt": "2025-09-30T08:00:00Z", + "analysis": { + "topics": ["ai", "developer-tools"], + "topicScores": { "ai": 0.95, "developer-tools": 0.88 }, + "summary": "Article summary...", + "keyPoints": ["point1", "point2"], + "sentiment": "positive", + "monetizationAngle": "Content opportunity...", + "analyzedAt": "2025-10-01T04:30:00Z" + } +} +``` + +**What It Does (Current)**: +1. Scans `temp/*.jsonl` for files that contain an `analysis` field +2. For each enriched temp file: + - Writes Markdown file under `content/analysis/` (date from `savedAt`; slug derived from `articleSlug` when available, otherwise title) + - Stores `analysis_json` + `markdown_path` into `analysis_queue` + - Updates tracking status to `completed` + - Deletes the temp file unless `--keep-temp` +3. On error: marks job as `failed` in tracking queue and preserves the temp file + +**JSONL Output (Optional)**: +- Use `--jsonl` to append a machine-readable record to `content/analysis/analyses.jsonl` for each completed analysis. + +**File Naming Strategy**: +- Markdown: `content/analysis/YYYY-MM-DD-{slug}-analysis.md` (date from `savedAt`) +- JSONL: written when `--jsonl` is set (`content/analysis/analyses.jsonl` by default) +- Temp files: Automatically deleted after successful save using `unlinkSync(file)` + +**Output Example**: +``` +Loaded 5 analysis results from temp files +✓ Saved: Understanding LLM Fine-Tuning Techniques... + → content/analysis/2025-09-30-understanding-llm-fine-tuning-analysis.md + → Deleted: temp/understanding-llm-fine-tuning.jsonl +✓ Saved: Building Scalable Microservices with Kubernetes... + → content/analysis/2025-09-29-building-scalable-microservices.md + → Deleted: temp/building-scalable-microservices.jsonl +✗ Failed: Another Article... - Parse error + → Preserved: temp/another-article.jsonl + +Results: + Saved: 4 + Failed: 1 + +Updated Queue Status: + Pending: 37 + Completed: 34 + Failed: 4 +``` + +**AIDEV Annotations**: +- `analysis-output-boundary` - Saves results to git-tracked Markdown (plus immutable DB snapshot) +- `tracking-update` - Updates SQLite queue status after save +- `git-tracked-output` - Analysis result stored permanently +- `temp-file-cleanup` - Deletes temp files after successful DB save + +### analyze status + +**Command**: `omc analyze status` + +**Purpose**: Show queue statistics and failed/in-progress jobs. + +**Usage**: +```bash +omc analyze status +``` + +**What It Does**: +- Shows queue statistics (total, pending, in_progress, completed, failed) +- Lists failed jobs with error messages and retry counts +- Lists in-progress jobs with assignment timestamps + +**Output Example**: +``` +Analysis Queue Status + Total: 75 + Pending: 37 (49%) + In Progress: 0 + Completed: 33 (44%) + Failed: 5 + +Failed Jobs: + + Article: Article That Failed Analysis Due to Timeout... + ID: xyz-789 + Error: Agent timeout after 120s + Retries: 2 + + Article: Another Failed Article with Parse Error... + ID: abc-321 + Error: JSON parse error + Retries: 1 +``` + +**AIDEV Annotations**: +- `tracking-inspection` - Shows queue status, not analysis content + +### analyze retry + +**Command**: `omc analyze retry` + +**Purpose**: Reset failed jobs to pending for retry (max 3 attempts). +**Purpose**: Reset failed jobs to `pending` for retry. + +**Usage**: +```bash +omc analyze retry --failed # Retry all failed +omc analyze retry --article-id # Retry specific article +``` + +**What It Does (Current)**: +1. Fetches failed jobs (or a single `--article-id`) +2. Resets status to `pending` and clears error/assigned fields +3. Does not currently enforce a max retry limit (the DB keeps `retry_count` for visibility) + +**Output Example**: +``` +Found 5 failed jobs + +✓ Reset: Article That Failed Analysis... (attempt 3) +✓ Reset: Another Failed Article... (attempt 2) +✗ Skip: Permanently Failed Article... (max retries exceeded) + +Summary: + Reset for retry: 4 + Skipped (max retries): 1 + +Next step: `omc analyze run` +``` + +**AIDEV Annotations**: +- `tracking-retry` - Resets failed jobs to pending for another attempt + +## Reporting Commands + +### report corpus + +**Command**: `omc report corpus` + +**Purpose**: Generate corpus statistics and topic distribution report. + +**Usage**: +```bash +omc report corpus +``` + +**What It Does**: +- Loads completed analyses from SQLite (`analysis_queue.analysis_json`) via `AnalysisQueueRepository.getCompletedWithAnalysis()` +- Generates statistics: total analyses, topic distribution, sentiment distribution, content-type distribution +- Prints the report to stdout (use `--json` for machine-readable output) + +**Note**: +- `omc report corpus` reads from SQLite (`analysis_queue.analysis_json`). JSONL is an optional *write-only* output for downstream tooling. + +## Storage Layer Commands + +**Note**: The most reliable source of truth for the storage-layer API is `src/storage/AnalysisQueueRepository.ts` and `src/storage/AnalysisWriter.ts`. Some type snippets below may drift as the code evolves. + +### AnalysisQueueRepository + +**Location**: `src/storage/AnalysisQueueRepository.ts` + +**Purpose**: Repository for analysis job queue management. + +**Import**: +```typescript +import { AnalysisQueueRepository } from '@storage/AnalysisQueueRepository'; +import type { AnalysisJob, QueueStats } from '@storage/AnalysisQueueRepository'; +``` + +**Types**: + +```typescript +interface AnalysisJob { + id: number; + articleId: string; + articleUrl: string; + articleTitle: string; + status: 'pending' | 'in_progress' | 'completed' | 'failed'; + assignedAt?: string; + completedAt?: string; + errorMessage?: string; + retryCount: number; + createdAt: string; + updatedAt: string; +} + +interface QueueStats { + total: number; + pending: number; + inProgress: number; + completed: number; + failed: number; +} +``` + +**Constructor**: +```typescript +const queueRepo = new AnalysisQueueRepository(db); +``` + +**Methods**: + +1. **`initializeQueue(articles): number`** - Initialize queue from article metadata + - Parameter: `articles: Array<{ id: string; url: string; title: string }>` + - Returns: Number of articles inserted (duplicates ignored by UNIQUE constraint) + - Sets status to `pending` for all new articles + - AIDEV: `tracking-initialization` + +2. **`getPending(limit?: number): AnalysisJob[]`** - Get next batch of pending jobs + - Default limit: 5 + - Returns jobs in chronological order (oldest first) + - AIDEV: `tracking-coordination` + +3. **`markInProgress(articleId: string): void`** - Lock job for processing + - Sets status to `in_progress` + - Records `assigned_at` timestamp + - Prevents duplicate analysis by parallel workers + - AIDEV: `tracking-lock` + +4. **`markCompleted(articleId: string): void`** - Mark job as completed + - Sets status to `completed` + - Records `completed_at` timestamp + - Called AFTER analysis saved to Markdown/JSONL + - AIDEV: `tracking-completion` + +5. **`markFailed(articleId: string, errorMessage: string): void`** - Mark job as failed + - Sets status to `failed` + - Stores error message + - Increments `retry_count` + - AIDEV: `tracking-error` + +6. **`resetToPending(articleId: string): void`** - Reset failed job for retry + - Sets status back to `pending` + - Clears error message and `assigned_at` + - Does NOT reset `retry_count` (used to enforce max retries) + - AIDEV: `tracking-retry` + +7. **`getStats(): QueueStats`** - Get queue statistics + - Returns counts for all statuses + - Used for progress monitoring + - AIDEV: `tracking-stats` + +8. **`getFailed(): AnalysisJob[]`** - Get all failed jobs + - Sorted by retry count (most retries first), then updated time + - Used for error investigation + - AIDEV: `tracking-failures` + +9. **`getByStatus(status: string): AnalysisJob[]`** - Get jobs by status + - Returns jobs matching specified status + - Sorted by creation time (newest first) + +10. **`getByArticleId(articleId: string): AnalysisJob | null`** - Get specific job + - Returns job or `null` if not found + +11. **`hasArticle(articleId: string): boolean`** - Check if article in queue + - Returns `true` if article exists (any status) + - AIDEV: `tracking-deduplication` + +12. **`clearCompleted(): number`** - Remove completed jobs + - Deletes all jobs with `completed` status + - Returns number of deleted jobs + - AIDEV: `tracking-cleanup` + +**Example Usage**: +```typescript +import { initDatabase } from '@storage/database'; +import { AnalysisQueueRepository } from '@storage/AnalysisQueueRepository'; + +const db = initDatabase(); +const queueRepo = new AnalysisQueueRepository(db); + +// Add articles to queue +const articles = [ + { id: 'abc-123', url: 'https://example.com/1', title: 'Article 1' }, + { id: 'def-456', url: 'https://example.com/2', title: 'Article 2' } +]; +const inserted = queueRepo.initializeQueue(articles); +console.log(`Added ${inserted} articles`); + +// Get pending jobs +const jobs = queueRepo.getPending(5); +console.log(`Processing ${jobs.length} jobs`); + +// Mark as in_progress +for (const job of jobs) { + queueRepo.markInProgress(job.articleId); +} + +// After successful analysis → mark completed +queueRepo.markCompleted('abc-123'); + +// If analysis fails → mark failed +queueRepo.markFailed('def-456', 'Agent timeout'); + +// Check statistics +const stats = queueRepo.getStats(); +console.log(`Pending: ${stats.pending}, Completed: ${stats.completed}`); + +// Get failed jobs for retry +const failed = queueRepo.getFailed(); +for (const job of failed) { + if (job.retryCount < 3) { + queueRepo.resetToPending(job.articleId); + } +} + +db.close(); +``` + +### AnalysisWriter + +**Location**: `src/storage/AnalysisWriter.ts` + +**Purpose**: Write `ContentAnalysis` results to Markdown files with YAML front-matter. + +**Import**: +```typescript +import { AnalysisWriter } from '@storage/AnalysisWriter'; +``` + +**Configuration**: +```typescript +const writer = new AnalysisWriter({ + outputDir: 'content/analysis' +}); +``` + +**Methods**: + +1. **`write(articleId, articleUrl, articleTitle, savedAt, analysis): Promise`** - Write analysis to Markdown + ```typescript + const filePath = await writer.write( + articleId, // Omnivore article ID + articleUrl, // Source article URL + articleTitle, // Source article title + savedAt, // When saved to Omnivore (ISO 8601) + analysis, // ContentAnalysis from agent + articleSlug // Optional Omnivore slug (preferred for filename stability) + ); + // Returns: 'content/analysis/2025-09-30-omnivore-slug-analysis.md' + ``` + - Creates Markdown file with YAML front-matter + - File naming: `YYYY-MM-DD-{slug}-analysis.md` + - Date from `savedAt` timestamp + - Slug from `articleSlug` when provided; otherwise derived from article title (lowercase, alphanumeric, max 50 chars) + - AIDEV: `analysis-output-boundary`, `git-tracked-output` + + 2. **`appendToJsonl(jsonlPath, data): Promise`** - Append to JSONL file + ```typescript + await writer.appendToJsonl('content/analysis/analyses.jsonl', { + articleId, + articleUrl, + articleTitle, + savedAt, + analyzedAt: analysis.analyzedAt, + topics: analysis.topics, + topicScores: analysis.topicScores, + summary: analysis.summary, + keyPoints: analysis.keyPoints, + sentiment: analysis.sentiment, + monetizationAngle: analysis.monetizationAngle + }); + ``` + - Called when JSONL output is enabled (`omc analyze complete --jsonl` / `omc analyze auto --jsonl`) + - Appends one JSON line to JSONL file + - Creates file if it doesn't exist + - Machine-readable format for batch processing + - AIDEV: `git-tracked-output` + +**Private Methods**: +- `generateSlug(title: string): string` - Create URL-friendly slug +- `formatMarkdown(...)` - Build YAML front-matter and markdown body + +**Example Usage**: +```typescript +import { AnalysisWriter } from '@storage/AnalysisWriter'; + +const writer = new AnalysisWriter({ outputDir: 'content/analysis' }); + +const analysis = { + articleId: 'abc-123', + topics: ['ai', 'developer-tools'], + topicScores: { ai: 0.95, 'developer-tools': 0.88 }, + summary: 'Article explores...', + keyPoints: ['First insight', 'Second insight'], + sentiment: 'positive', + monetizationAngle: 'Tutorial series on...', + analyzedAt: new Date().toISOString() +}; + +const filePath = await writer.write( + 'abc-123', + 'https://example.com/article', + 'Example Article Title', + '2025-09-30T10:00:00Z', + analysis +); + +console.log(`Saved to: ${filePath}`); +// Saved to: content/analysis/2025-09-30-example-article-title-analysis.md +``` + +### ContentReader + +**Location**: `src/storage/ContentReader.ts` + +**Purpose**: Read and parse Markdown files with YAML front-matter. + +**Import**: +```typescript +import { ContentReader } from '@storage/ContentReader'; +``` + +**Configuration**: +```typescript +const reader = new ContentReader({ + directory: 'content/analysis' +}); +``` + +**Methods**: + +1. **`list(pattern?: string): Promise`** - List all Markdown files + - Returns array of absolute file paths + - Sorted by date (newest first, assumes `YYYY-MM-DD` prefix) + - Returns empty array if directory doesn't exist + +2. **`read(filePath: string): Promise`** - Read and parse file + - Uses `gray-matter` to parse YAML front-matter + - Extracts markdown sections (Summary, Key Points, Monetization Angle) + - Returns `StoredAnalysis` with parsed front-matter and content + +3. **`findByArticleId(articleId: string): Promise`** - Find by article ID + - Searches all files for matching `articleId` in front-matter + - Returns first match or `null` + +4. **`searchByTopic(topic: string): Promise`** - Search by topic + - Returns all analyses containing the topic + - Topic must match exactly (case-sensitive) + +**Example Usage**: +```typescript +import { ContentReader } from '@storage/ContentReader'; + +const reader = new ContentReader({ directory: 'content/analysis' }); + +// List all analyses +const files = await reader.list(); +console.log(`Found ${files.length} analyses`); + +// Read specific analysis +const analysis = await reader.read(files[0]); +console.log(analysis.frontMatter.articleTitle); +console.log(analysis.summary); +console.log(analysis.keyPoints); + +// Find by article ID +const found = await reader.findByArticleId('abc-123'); +if (found) { + console.log(`Topics: ${found.frontMatter.topics.join(', ')}`); +} + +// Search by topic +const aiArticles = await reader.searchByTopic('ai'); +console.log(`${aiArticles.length} articles about AI`); +``` + +**Private Method**: +- `parseMarkdownSections(content: string)` - Extract sections from markdown body using regex + +## Workflow Examples + +### Complete Parallel Analysis Workflow + +```bash +# 1. Fetch articles from Omnivore → tracking queue +omc queue add --hours 168 + +# 2. Prepare batch (writes temp/*.jsonl stubs + marks jobs in_progress) +omc analyze run --batch-size 5 + +# 3. User invokes 5 agents in parallel (single message) +# Each agent: +# - Reads stub file +# - Calls: omc omnivore get {slug} --json (extract `.content`) +# - Analyzes content +# - Enriches stub file with analysis field +# - Writes back to temp file + +# 4. Persist results (writes content/analysis/*.md, updates DB, cleans temp files) +omc analyze complete + +# 5. Check status / stats +omc analyze status +omc queue stats --detailed + +# 6. Retry failed jobs +omc analyze retry --failed + +# 7. Continue analysis +omc analyze run --batch-size 5 + +# 8. Generate report +omc report corpus +``` + +### Programmatic Queue Management + +```typescript +import { initDatabase } from '@storage/database'; +import { AnalysisQueueRepository } from '@storage/AnalysisQueueRepository'; +import { AnalysisWriter } from '@storage/AnalysisWriter'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; + +// Initialize database and repository +const db = initDatabase('data/omnivore-content.db'); +const queueRepo = new AnalysisQueueRepository(db); +const writer = new AnalysisWriter({ outputDir: 'content/analysis' }); + +// Add articles to queue +const articles = [ + { id: 'abc-123', url: 'https://example.com/1', title: 'Article 1' }, + { id: 'def-456', url: 'https://example.com/2', title: 'Article 2' } +]; +const inserted = queueRepo.initializeQueue(articles); +console.log(`Added ${inserted} articles to queue`); + +// Get pending jobs +const jobs = queueRepo.getPending(5); + +// Mark as in_progress +for (const job of jobs) { + queueRepo.markInProgress(job.articleId); +} + +// After agent analysis completes... +const analysis: ContentAnalysis = { + articleId: 'abc-123', + topics: ['ai', 'developer-tools'], + topicScores: { ai: 0.95, 'developer-tools': 0.88 }, + summary: 'Article explores...', + keyPoints: ['First insight', 'Second insight'], + sentiment: 'positive', + monetizationAngle: 'Tutorial series on...', + analyzedAt: new Date().toISOString() +}; + +// Save to Markdown (PERMANENT) +const mdPath = await writer.write( + 'abc-123', + 'https://example.com/1', + 'Article 1', + '2025-09-30T10:00:00Z', + analysis +); + +// Append to JSONL (PERMANENT) +await writer.appendToJsonl('content/analysis/analyses.jsonl', { + articleId: 'abc-123', + articleUrl: 'https://example.com/1', + articleTitle: 'Article 1', + savedAt: '2025-09-30T10:00:00Z', + analyzedAt: analysis.analyzedAt, + topics: analysis.topics, + topicScores: analysis.topicScores, + summary: analysis.summary, + keyPoints: analysis.keyPoints, + sentiment: analysis.sentiment, + monetizationAngle: analysis.monetizationAngle +}); + +// Update tracking status (EPHEMERAL) +queueRepo.markCompleted('abc-123'); + +// Check statistics +const stats = queueRepo.getStats(); +console.log(`Pending: ${stats.pending}, Completed: ${stats.completed}`); + +db.close(); +``` + +### Reading Analysis Results + +```typescript +import { ContentReader } from '@storage/ContentReader'; + +const reader = new ContentReader({ directory: 'content/analysis' }); + +// List all analyses (newest first) +const files = await reader.list(); +console.log(`Total analyses: ${files.length}`); + +// Read latest analysis +const latest = await reader.read(files[0]); +console.log(`Title: ${latest.frontMatter.articleTitle}`); +console.log(`Topics: ${latest.frontMatter.topics.join(', ')}`); +console.log(`Summary: ${latest.summary}`); +console.log(`Key Points:\n${latest.keyPoints.map(p => `- ${p}`).join('\n')}`); + +// Find specific article's analysis +const found = await reader.findByArticleId('abc-123'); +if (found) { + console.log(`Sentiment: ${found.frontMatter.sentiment}`); +} + +// Search by topic +const aiArticles = await reader.searchByTopic('ai'); +console.log(`AI articles: ${aiArticles.length}`); +``` + +## Related Documentation + +- [Architecture](architecture.md) - System design and storage boundaries +- [Workflow Internals](workflow-internals.md) - How the parallel analysis workflow operates +- [Foundation & Type System](foundation-and-types.md) - TypeScript setup, type definitions diff --git a/self-hosting/omc/docs/_meta/current-state.md b/self-hosting/omc/docs/_meta/current-state.md new file mode 100644 index 000000000..930af07f1 --- /dev/null +++ b/self-hosting/omc/docs/_meta/current-state.md @@ -0,0 +1,151 @@ +# Current State Audit + +**Purpose**: Ground-truth documentation of what exists *in code today*, what is broken, and what needs to be fixed to make the system reliable. + +**Last Updated**: 2026-01-31 + +## Executive Summary + +This repository has a fairly complete **command surface area** (queue/analyze/content/report/omnivore/db/config/init/doctor/version), plus an intended “3-layer” architecture: + +1. **Omnivore** as the content source-of-truth (GraphQL) +2. **SQLite** as coordination + immutable analysis snapshot storage +3. **Git-tracked Markdown** as human-editable, permanent output + +The core analysis pipeline now works end-to-end in a clean checkout: + +- `pnpm run typecheck` / `pnpm test` / `pnpm run build` pass. +- The built CLI (`dist/bin/omc.js`) can initialize the tracking DB schema. +- Commands follow the `BaseCommand.execute({ ...args, ...flags })` contract. +- Documentation reflects the current `omc analyze auto` automation path and optional JSONL output. + +## Architecture (As Implemented) + +### Layer 1: Omnivore (Source of Truth) + +- Implementation: `lib/omnivore/client.js` +- Used by CLI commands via the TypeScript re-export surface (`src/lib/omnivore/client.ts`) in most places. +- Auth is via `.env` (`OMNIVORE_API_URL`, `OMNIVORE_API_KEY`) loaded by `dotenv` at module import time. + +### Layer 2: SQLite (Coordination + Snapshots) + +- Default DB path (source): `src/storage/database.ts` uses `data/omnivore-content.db`. +- Schema: `src/storage/schema/tracking-schema.sql` +- Repository: `src/storage/AnalysisQueueRepository.ts` + +The DB is used for: +- Queue state (`pending`/`in_progress`/`completed`/`failed`) +- Retry counters and error messages +- Immutable snapshot of analysis output (`analysis_json`) and pointer to Markdown (`markdown_path`) + +### Layer 3: Markdown Output (Git-tracked) + +- Writer: `src/storage/AnalysisWriter.ts` +- The current pipeline writes **Markdown** by default. +- Optional machine-readable JSONL is available via flags (`omc analyze auto --jsonl` / `omc analyze complete --jsonl`). +- Markdown filename uses `savedAt` date and the Omnivore `articleSlug` when available (falls back to title-derived slug). + +## Command Inventory (As Implemented) + +Commands are defined under `src/commands/**` and built/run via oclif (`index.ts`). + +### `omc queue` + +- `omc queue add` (`src/commands/queue/add.ts`): adds articles to tracking queue (hours/label/url/slug). +- `omc queue list` (`src/commands/queue/list.ts`): lists queue entries (optional `--status`). +- `omc queue stats` (`src/commands/queue/stats.ts`): queue stats (optional `--detailed`). +- `omc queue reset` (`src/commands/queue/reset.ts`): resets a specific job to `pending`. +- `omc queue remove` (`src/commands/queue/remove.ts`): deletes a job. +- `omc queue clear` (`src/commands/queue/clear.ts`): bulk delete by status or all. +- `omc queue export` (`src/commands/queue/export.ts`): prints queue rows as JSONL (or JSON array with `--json`). +- `omc queue import` (`src/commands/queue/import.ts`): imports JSONL into the queue. + +### `omc analyze` + +- `omc analyze run` (`src/commands/analyze/run.ts`): + - Selects jobs (pending by default; pending+failed with `--all`; single with `--article-id`). + - Marks selected jobs `in_progress`. + - Fetches Omnivore article metadata and writes `temp/{articleSlug}.jsonl` stub files. + - Outputs agent parameter JSON describing the stub files. +- `omc analyze complete` (`src/commands/analyze/complete.ts`): + - Finds `temp/*.jsonl` files that contain an `analysis` field. + - Writes Markdown via `AnalysisWriter`. + - Stores `analysis_json` + `markdown_path` in SQLite via `AnalysisQueueRepository.storeAnalysis`. + - Deletes temp files unless `--keep-temp`. +- `omc analyze status` (`src/commands/analyze/status.ts`): shows in-progress jobs + current `temp/*.jsonl`. +- `omc analyze watch` (`src/commands/analyze/watch.ts`): polls queue stats until *queue empty* (pending=0 and in_progress=0). +- `omc analyze retry` (`src/commands/analyze/retry.ts`): resets failed jobs to pending. + +### `omc content` (Read/Export/Sync) + +- `omc content list` (`src/commands/content/list.ts`): lists completed jobs; optional filters (since/topic). +- `omc content show` (`src/commands/content/show.ts`): prints Markdown for one article ID. +- `omc content search` (`src/commands/content/search.ts`): searches completed analyses (title/analysis/content/all). +- `omc content export` (`src/commands/content/export.ts`): export data for blogging workflows (implementation varies). +- `omc content sync` (`src/commands/content/sync.ts`): syncs analysis summaries back to Omnivore (description field) and optionally creates/updates a NOTE highlight from Markdown. + +### `omc report` (Categorized Analysis / Aggregations) + +This is the “categorized analysis” surface area: reports summarize analyses by topic/sentiment/time. + +- `omc report corpus` (`src/commands/report/corpus.ts`): topic + sentiment + contentType distributions. +- `omc report topics` (`src/commands/report/topics.ts`): topic distribution. +- `omc report trends` (`src/commands/report/trends.ts`): trends over time (based on saved/completed timestamps). +- `omc report sentiment` (`src/commands/report/sentiment.ts`): sentiment distribution. +- `omc report monetization` (`src/commands/report/monetization.ts`): monetization angle aggregation. +- `omc report custom` (`src/commands/report/custom.ts`): custom SQL against the tracking DB. +- `omc report export` (`src/commands/report/export.ts`): export report output (format varies). + +### `omc omnivore` + +Direct API operations: +- `omc omnivore list`, `search`, `get`, `update` +- highlights: `omnivore highlight add/list` +- notes: `omnivore note add/get/update` + +### `omc db` + +DB maintenance: +- `migrate`, `schema`, `stats`, `check`, `vacuum`, `backup`, `restore`, `reset`, `seed` + +### `omc config` + +Config management: +- `show`, `get`, `set`, `test`, `validate`, `env list`, `env use` + +### Misc + +- `omc init` (`src/commands/init.ts`): basic setup wizard (directories, db, env). +- `omc doctor` (`src/commands/doctor.ts`): health checks. +- `omc version` (`src/commands/version.ts`): version/system info. + +## End-to-End Workflow (Current) + +1. Populate queue: `omc queue add --hours 24` (or `--label`, `--url`, `--slug`). +2. Prepare stubs: `omc analyze run --batch-size 5` (creates `temp/{slug}.jsonl`). +3. Run external agent(s): read stub, fetch content, add `analysis` field to the stub JSON. +4. Persist results: `omc analyze complete` (writes `content/analysis/*.md`, updates DB, cleans temp files). +5. View/search: `omc content list`, `omc content search`, `omc content show `. +6. Report: `omc report topics` / `omc report trends` / etc. +7. Sync back to Omnivore: `omc content sync --all --create-notes`. + +## Issues / Fix List (Prioritized) + +### Resolved (as of 2026-01-31) + +- TypeScript aliases normalized (`@omc-types/*` instead of `@types/*`); `pnpm run typecheck` passes. +- Tracking schema resolution works in `dist/` builds; built CLI can initialize the DB. +- Commands follow `BaseCommand.execute({ ...args, ...flags })`. +- Docs and scripts align with the built CLI and the current analysis workflow. +- Agent-friendly raw content output via `omc omnivore get --content`. +- Optional JSONL output via `--jsonl` flags (`omc analyze auto --jsonl` / `omc analyze complete --jsonl`). +- Markdown filename contract prefers Omnivore slug (falls back to title-derived slug). +- `omc queue add --hours` paginates until the cutoff is reached. +- Markdown front-matter emitted via `gray-matter` (valid YAML for common edge cases). +- GraphQL error handling normalized via `checkGraphQLResult(...)`. +- Typed GraphQL documents/codegen removed from the runtime path (single client source-of-truth). + +### Remaining / Next + +- Automated content generation and publishing phases (Phase 5/6). +- Decide whether to keep or retire `src/lib/ai/anthropic-client.ts` (Codex CLI is the default analysis provider today). diff --git a/self-hosting/omc/docs/_meta/foundation-and-types.md b/self-hosting/omc/docs/_meta/foundation-and-types.md new file mode 100644 index 000000000..f8e1c960a --- /dev/null +++ b/self-hosting/omc/docs/_meta/foundation-and-types.md @@ -0,0 +1,712 @@ +# Foundation and Type System + +**Groundtruth Documentation** - What exists and how to use it. + +## TypeScript Foundation + +### Project Setup + +**TypeScript Configuration** (`tsconfig.json`): +```typescript +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "declaration": true, + "sourceMap": true, + "paths": { + "@lib/*": ["./lib/*"], + "@storage/*": ["./src/storage/*"], + "@analysis/*": ["./src/analysis/*"], + "@generation/*": ["./src/generation/*"], + "@utils/*": ["./src/utils/*"], + "@omc-types/*": ["./src/types/*"] + } + }, + "include": ["src/**/*", "lib/**/*"], + "exclude": ["node_modules", "dist", "test-scripts", "legacy-scripts"] +} +``` + +**Build System**: +- Package manager: pnpm +- Build command: `pnpm run build` → compiles to `dist/` +- Type check: `pnpm run typecheck` → validates without emitting +- Dev mode: `pnpm run dev` → watch mode with tsx + +### Dependencies + +**Runtime** (package.json dependencies): +- `node-fetch` (^3.3.2) - HTTP requests for GraphQL API +- `dotenv` (^16.4.0) - Environment variable loading +- `@anthropic-ai/sdk` (^0.20.0) - Claude API for analysis +- `@anthropic-ai/claude-agent-sdk` (^0.1.1) - Agent framework +- `gray-matter` (^4.0.3) - YAML front-matter parsing +- `markdown-it` (^14.0.0) - Markdown processing +- `chalk` (^5.3.0) - Terminal colors +- `csv-parse` (^6.1.0) - CSV parsing +- `p-limit` (^5.0.0) - Concurrency control +- `better-sqlite3` (^12.4.1) - SQLite (for legacy scripts) + +**Development** (package.json devDependencies): +- `typescript` (^5.4.0) - TypeScript compiler +- `@types/node` (^20.0.0) - Node.js type definitions +- `tsx` (^4.0.0) - TypeScript execution +- `vitest` (^1.0.0) - Testing framework +- `nodemon` (^3.0.0) - File watcher + +## GraphQL Organization + +**Current**: +- Runtime GraphQL client: `lib/omnivore/client.js` (string-based queries) +- TypeScript import surface: `src/lib/omnivore/client.ts` +- In TypeScript code, prefer importing client functions from `@lib/omnivore/client.js` + +**Note**: +An older typed GraphQL layer (`src/graphql/**` + `src/types/generated/**`) was removed because it was not integrated into the CLI runtime and created drift risk. See `docs/_meta/graphql-organization.md`. + +### Query Builders (`lib/omnivore/queries.js`) + +**Purpose**: Helper functions for composing Omnivore query strings. + +**NOTE**: Query builders remain in JavaScript (`lib/omnivore/queries.js`) for compatibility with existing client code. They generate query string parameters, not GraphQL query documents. + +**Builder Functions**: + +1. **`buildComplexQuery({ keywords, labels, timeRange, status, hasHighlights, sortBy })`** + - Compose multiple filters into single query string + - Parameters: + - `keywords`: Array of search terms (OR logic) + - `labels`: Array of label names (OR logic) + - `timeRange`: 'last24hrs' | 'last7days' | 'last30days' | date expression + - `status`: 'inbox' | 'archived' | 'unread' | 'read' + - `hasHighlights`: boolean (true = has:highlights, false = no:highlights) + - `sortBy`: Sort order (default: 'saved-desc') + - Returns: Omnivore query string + - Example: + ```javascript + buildComplexQuery({ + keywords: ['ai', 'machine learning'], + labels: ['technology'], + timeRange: 'last7days', + hasHighlights: true + }) + // → 'saved:last7days ("ai" OR "machine learning") label:technology has:highlights sort:saved-desc' + ``` + +2. **`buildTopicQuery(topicKey, timeRange)`** + - Generate query for predefined topic + - Topic keys: 'ai', 'devops', 'programming', 'databases', 'web', 'cloud', 'startup', 'security' + - Each topic has predefined keywords (see TOPIC_QUERIES) + - Example: + ```javascript + buildTopicQuery('ai', 'last7days') + // → 'saved:last7days ("ai" OR "artificial intelligence" OR "machine learning" OR ...) sort:saved-desc' + ``` + +3. **`buildDateRangeQuery(startDate, endDate, sortBy)`** + - Create date range filter + - Accepts Date objects or YYYY-MM-DD strings + - Example: + ```javascript + buildDateRangeQuery('2025-09-01', '2025-09-30') + // → 'saved:>2025-09-01 saved:<2025-09-30 sort:saved-desc' + ``` + +4. **`buildLabelQuery(labels, operator)`** + - Create label filter with AND/OR logic + - Example: + ```javascript + buildLabelQuery(['ai', 'ml'], 'OR') + // → 'label:ai OR label:ml' + ``` + +5. **`buildKeywordQuery(keywords, operator)`** + - Create keyword search with AND/OR logic + - Automatically quotes multi-word terms + - Example: + ```javascript + buildKeywordQuery(['ai', 'machine learning'], 'OR') + // → '("ai" OR "machine learning")' + ``` + +**Predefined Patterns** (`QUERY_PATTERNS`): +```javascript +QUERY_PATTERNS.LAST_7_DAYS // 'saved:last7days sort:saved-desc' +QUERY_PATTERNS.INBOX // 'in:inbox sort:saved-desc' +QUERY_PATTERNS.WITH_HIGHLIGHTS // 'has:highlights sort:saved-desc' +QUERY_PATTERNS.AI_ML // '(ai OR "machine learning" OR llm ...) sort:saved-desc' +QUERY_PATTERNS.RECENT_AI // 'saved:last7days (ai OR llm OR ...) sort:saved-desc' +// ... more patterns (see lib/omnivore/queries.js:189-221) +``` + +**Topic Definitions** (`TOPIC_QUERIES`): +```javascript +TOPIC_QUERIES.ai // keywords: ['ai', 'artificial intelligence', ...] +TOPIC_QUERIES.devops // keywords: ['devops', 'kubernetes', ...] +TOPIC_QUERIES.programming // keywords: ['programming', 'coding', ...] +// ... more topics (see lib/omnivore/queries.js:305-338) +``` + +**Integration Example**: +```javascript +import { searchArticles } from '@lib/omnivore/client.js'; +import { buildComplexQuery, TOPIC_QUERIES } from '../lib/omnivore/queries.js'; + +// Build query string +const queryString = buildComplexQuery({ + keywords: TOPIC_QUERIES.ai.keywords.slice(0, 5), + labels: ['technology', 'research'], + timeRange: 'last7days', + hasHighlights: true +}); + +const result = await searchArticles({ query: queryString, first: 20, includeContent: true }); +``` + +## Omnivore Client Library + +### Location + +``` +lib/omnivore/ +├── client.js # GraphQL client using node-fetch +└── queries.js # Query builders and patterns (documented above) +``` + +### Client Module (`lib/omnivore/client.js`) + +**Implementation**: Uses `node-fetch` to make GraphQL requests. NOT using Apollo Client. + +**Core Function**: `graphqlRequest(query, variables)` +```javascript +async function graphqlRequest(query, variables = {}) { + const response = await fetch(API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': API_KEY, // From process.env.OMNIVORE_API_KEY + }, + body: JSON.stringify({ query, variables }), + }); + + const result = await response.json(); + return result.data; +} +``` + +**Exported Functions**: + +1. **`getMe()`** - Get authenticated user + - Returns: `{ id, name, email, profile: { username } }` + - Use: Verify API connection + +2. **`searchArticles({ query, first, after, includeContent })`** - Search articles + - Parameters: + - `query`: Omnivore query syntax (default: 'in:all') + - `first`: Number of results (default: 10) + - `after`: Pagination cursor (default: '') + - `includeContent`: Include full HTML content (default: false) + - Returns: `{ search: { pageInfo, edges: [{ node, cursor }] } }` + +3. **`getArticle({ slug, username })`** - Get single article with content + - Returns: Full article with content, highlights, labels + +4. **`getArticlesByDate({ startDate, endDate, first })`** - Time-range queries + - Date format: YYYY-MM-DD + +5. **`getArticlesByLabel({ label, first })`** - Filter by label name + +6. **`getRecentArticles({ period, first })`** - Recent articles + - Periods: 'last24hours', 'last7days', 'last30days' + +7. **`searchByTopic({ topic, period, first })`** - Topic + time filter + - Topics: 'ai', 'devops', 'programming', etc. + +8. **`getUnreadArticles({ first })`** - Inbox/unread items + +9. **`getLabels()`** - All available labels + - Returns: Array of `{ id, name, color, description }` + +10. **`getHighlights({ articleId })`** - Article highlights + - Returns: Array of `{ id, quote, annotation, createdAt }` + +11. **`testConnection()`** - Verify API connectivity + - Calls `getMe()` and logs result + - Returns: boolean + +**Import Pattern**: +```javascript +import { searchArticles, getArticle } from './lib/omnivore/client.js'; +``` + +**NOTE**: Query builders and patterns are documented above in "Query Builders (`lib/omnivore/queries.js`)" section. + +## Type System + +### Type Files + +``` +src/types/ +├── omnivore.ts # API response types +├── content.ts # Storage/front-matter types +├── analysis.ts # Analysis result types +└── index.ts # Central exports +``` + +### Omnivore API Types (`src/types/omnivore.ts`) + +**Core Interfaces**: + +```typescript +// Article from API +export interface OmnivoreArticle { + id: string; + title: string; + url: string; + originalArticleUrl?: string; + slug?: string; + content?: string; // Only present if includeContent: true + description?: string; + author?: string; + image?: string; + siteName?: string; + pageType?: string; + wordCount?: number; + createdAt: string; // ISO 8601 + savedAt: string; // ISO 8601 + publishedAt?: string; // ISO 8601 + updatedAt: string; // ISO 8601 + readingProgressTopPercent?: number; + isArchived: boolean; + folder?: string; + labels: Label[]; + highlights: Highlight[]; +} + +// Label metadata +export interface Label { + id: string; + name: string; + color: string; + description?: string; +} + +// User highlight/annotation +export interface Highlight { + id: string; + quote: string; + annotation?: string; + createdAt: string; // ISO 8601 +} + +// Search parameters +export interface SearchParams { + query?: string; // Omnivore query syntax + first?: number; // Results per page + after?: string; // Pagination cursor + includeContent?: boolean; // Include article HTML +} + +// Pagination metadata +export interface PageInfo { + hasNextPage: boolean; + hasPreviousPage: boolean; + startCursor?: string; + endCursor?: string; + totalCount: number; +} + +// Search result wrapper +export interface SearchResult { + edges: Array<{ + node: OmnivoreArticle; + }>; + pageInfo: PageInfo; +} + +// User profile +export interface OmnivoreUser { + id: string; + name: string; + email: string; + profile: { + username: string; + }; +} +``` + +**Import Pattern**: +```typescript +import type { OmnivoreArticle, SearchResult, Label } from '@omc-types/omnivore.js'; +// OR +import type { OmnivoreArticle, SearchResult, Label } from '../types/omnivore'; +``` + +### Content Storage Types (`src/types/content.ts`) + +**Front-matter Interfaces** for Markdown files with YAML headers: + +```typescript +// Article metadata (content/articles/*.md) +export interface ArticleFrontMatter { + id: string; // Omnivore article ID + url: string; + title: string; + author?: string; + savedAt: string; // ISO 8601 + publishedAt?: string; // ISO 8601 + labels: string[]; // Label names (not IDs) + highlights: number; // Count + wordCount: number; + siteName?: string; + topics?: string[]; // Added by analysis + sentiment?: string; // Added by analysis + analyzed?: boolean; +} + +// Analysis metadata (content/analysis/*.md) +export interface AnalysisFrontMatter { + articleId: string; + analyzedAt: string; // ISO 8601 + topics: string[]; + sentiment: 'positive' | 'neutral' | 'negative'; + topicScores: Record; +} + +// Generated content metadata (content/generated/**/*.md) +export interface GeneratedContentFrontMatter { + title: string; + metaDescription: string; // Max 155 chars + generatedAt: string; // ISO 8601 + type: 'blog-post' | 'newsletter'; + sources: string[]; // Article URLs + topics: string[]; + publishedAt?: string; // ISO 8601 + slug?: string; +} + +// Complete stored article (after reading Markdown file) +export interface StoredArticle { + frontMatter: ArticleFrontMatter; + content: string; // Markdown content + highlights?: Array<{ + quote: string; + annotation?: string; + }>; +} + +// Complete stored analysis +export interface StoredAnalysis { + frontMatter: AnalysisFrontMatter; + summary: string; + keyPoints: string[]; + monetizationAngle: string; +} + +// Complete generated content +export interface StoredGeneratedContent { + frontMatter: GeneratedContentFrontMatter; + content: string; +} + +// Search index entry (content/.metadata/index.json) +export interface SearchIndexEntry { + id: string; + slug: string; + title: string; + topics: string[]; + savedAt: string; + analyzed: boolean; +} +``` + +### Analysis Types (`src/types/analysis.ts`) + +**AI Analysis Interfaces**: + +```typescript +// Analysis result from Claude +export interface ContentAnalysis { + articleId: string; + topics: string[]; // 2-5 main topics + topicScores: Record; // Topic → confidence (0-1) + summary: string; // 2-3 sentences + keyPoints: string[]; // 3-5 takeaways + sentiment: 'positive' | 'neutral' | 'negative'; + monetizationAngle: string; // Content opportunity + analyzedAt: string; // ISO 8601 +} + +// Input to Claude for analysis +export interface AnalysisRequest { + title: string; + author?: string; + url: string; + content: string; + wordCount: number; + highlights: Array<{ + quote: string; + annotation?: string; + }>; + publishedAt?: string; +} + +// Topic with confidence +export interface TopicScore { + topic: string; + score: number; // 0-1 + keywords: string[]; +} + +// Analysis configuration +export interface AnalysisConfig { + focusTopics?: string[]; + minTopicScore?: number; + maxTopics?: number; + includeSentiment?: boolean; +} + +// Batch analysis result +export interface BatchAnalysisResult { + articles: Array<{ + articleId: string; + analysis: ContentAnalysis; + }>; + commonTopics: string[]; + trends: Array<{ + topic: string; + frequency: number; + averageScore: number; + }>; + analyzedAt: string; +} +``` + +### Central Exports (`src/types/index.ts`) + +**Re-exports all types** plus utility types: + +```typescript +// Re-exports +export * from './omnivore'; +export * from './content'; +export * from './analysis'; + +// Utility types +export type DateString = string; // ISO 8601 +export type UUID = string; +export type Slug = string; + +// Generation config +export interface BlogPostConfig { + type: 'single-article' | 'weekly-roundup' | 'deep-dive'; + title?: string; + targetWordCount?: number; + includeSources?: boolean; + seoOptimize?: boolean; +} + +export interface NewsletterConfig { + type: 'weekly' | 'monthly'; + includeSections?: string[]; + maxArticles?: number; + personalCommentary?: boolean; +} + +// Publishing +export type PublishingPlatform = 'markdown' | 'ghost' | 'wordpress' | 'medium'; + +export interface PublishResult { + platform: PublishingPlatform; + success: boolean; + url?: string; + error?: string; + publishedAt: string; +} +``` + +**Import Pattern**: +```typescript +// Import from index for convenience +import type { + OmnivoreArticle, + ContentAnalysis, + ArticleFrontMatter +} from '@omc-types'; + +// OR import from specific files +import type { OmnivoreArticle } from '@omc-types/omnivore.js'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; +``` + +## Environment Configuration + +### Required Variables + +**File**: `.env` (project root, gitignored) + +```bash +# Omnivore API (REQUIRED) +OMNIVORE_API_KEY=your_api_key_here +OMNIVORE_API_URL=https://omnivore-api.caladan.haus/api/graphql + +# Anthropic API (REQUIRED for analysis/generation phases) +ANTHROPIC_API_KEY=your_anthropic_api_key_here + +# Content directories (OPTIONAL - defaults provided) +CONTENT_OUTPUT_DIR=./content +ARTICLES_DIR=./content/articles +ANALYSIS_DIR=./content/analysis +GENERATED_DIR=./content/generated +``` + +### For Self-Hosted Omnivore + +If using self-hosted Omnivore instance: +- Set `OMNIVORE_API_URL` to your instance's GraphQL endpoint +- Get API key from your instance's settings + +### Example: `.env.example` + +Template file included in repo shows all available configuration options. + +## Usage Examples + +### Fetching Articles + +```javascript +import { searchArticles } from './lib/omnivore/client.js'; +import { QUERY_PATTERNS } from './lib/omnivore/queries.js'; + +// Fetch AI articles from last 7 days +const result = await searchArticles({ + query: QUERY_PATTERNS.AI_ML + ' ' + QUERY_PATTERNS.LAST_7_DAYS, + first: 10, + includeContent: true +}); + +// Access articles +const articles = result.search.edges.map(edge => edge.node); +articles.forEach(article => { + console.log(article.title); + console.log(article.labels.map(l => l.name)); +}); +``` + +### Type-Safe Operations + +```typescript +import type { OmnivoreArticle, SearchResult } from '@omc-types'; +import { searchArticles } from './lib/omnivore/client.js'; + +async function getAIArticles(): Promise { + const result = await searchArticles({ + query: 'label:ai', + first: 20 + }) as SearchResult; + + return result.edges.map(edge => edge.node); +} +``` + +### Testing Connection + +```bash +# Test API connection +node lib/omnivore/client.js --test + +# Expected output: +# ✅ Connected to Omnivore API +# User: Your Name (your@email.com) +# Username: yourusername +``` + +## File System Conventions + +### Content Storage + +``` +content/ +├── articles/ # Original articles as Markdown +│ └── YYYY-MM-DD-{slug}.md +├── analysis/ # Analysis results +│ └── YYYY-MM-DD-{slug}.md +├── generated/ # Generated content +│ ├── blog-posts/ +│ │ └── YYYY-MM-DD-{title}.md +│ └── newsletters/ +│ └── YYYY-WW-roundup.md +└── .metadata/ + └── index.json # Search index +``` + +### Markdown Format + +**Article File** (`content/articles/2025-09-30-example.md`): +```markdown +--- +id: omnivore-abc123 +url: https://example.com/article +title: Example Article +author: John Doe +savedAt: 2025-09-30T10:00:00Z +labels: [ai, machine-learning] +highlights: 3 +wordCount: 2500 +--- + +# Example Article + +Article content here... + +## Highlights + +> Important quote +— Note: Why this matters +``` + +## Build System Details + +### TypeScript Compilation + +```bash +# Development build (with source maps & declarations) +pnpm run build +# Output: dist/ with .js, .d.ts, .js.map files + +# Type check only (no output) +pnpm run typecheck + +# Watch mode for development +pnpm run dev +``` + +### Path Aliases + +Configured in tsconfig.json, usable in TypeScript files: +- `@lib/*` → `src/lib/*` +- `@storage/*` → `src/storage/*` +- `@analysis/*` → `src/analysis/*` +- `@generation/*` → `src/generation/*` +- `@utils/*` → `src/utils/*` +- `@omc-types/*` → `src/types/*` + +### Module System + +- **Type**: ESM (ES Modules) +- **Import syntax**: `import { x } from './file.js'` (include `.js` extension) +- **package.json**: `"type": "module"` + +## What Does NOT Exist Yet + +This groundtruth documents what IS built. These are NOT implemented: +- Automated content generation modules (blog posts/newsletters) - Phase 5 +- Publishing modules (Ghost/WordPress/etc.) - Phase 6 +- End-to-end scheduled orchestration beyond analysis (e.g., daily generation + publishing) + +See IMPLEMENTATION_PLAN.md for what needs to be built. diff --git a/self-hosting/omc/docs/_meta/graphql-organization.md b/self-hosting/omc/docs/_meta/graphql-organization.md new file mode 100644 index 000000000..9916a1b92 --- /dev/null +++ b/self-hosting/omc/docs/_meta/graphql-organization.md @@ -0,0 +1,19 @@ +# GraphQL Organization (Current) + +This repo currently uses a single runtime GraphQL client: + +- Runtime client: `lib/omnivore/client.js` (string-based GraphQL queries + `node-fetch`) +- TypeScript import surface: `src/lib/omnivore/client.ts` (re-exports runtime functions for TS/alias imports) + +## Why + +An earlier, typed GraphQL layer (`src/graphql/**` + `src/types/generated/**`) existed but was **not integrated** into the CLI runtime. Maintaining two parallel query stacks created drift risk without any runtime benefit. + +To reduce duplication and keep docs aligned with reality, the typed layer was removed. If/when we migrate the runtime client to typed operations, we can reintroduce codegen + documents as part of that migration. + +## How To Add/Change Fields + +1. Update the query string in `lib/omnivore/client.js`. +2. If needed, update the corresponding TS types in `src/types/omnivore.ts`. +3. Prefer importing client functions from `@lib/omnivore/client.js` in TypeScript code. + diff --git a/self-hosting/omc/docs/_meta/workflow-internals.md b/self-hosting/omc/docs/_meta/workflow-internals.md new file mode 100644 index 000000000..4e746d0ca --- /dev/null +++ b/self-hosting/omc/docs/_meta/workflow-internals.md @@ -0,0 +1,613 @@ +# Workflow Internals + +**Purpose**: Deep dive into how the parallel analysis workflow operates, including zero-context-pollution design, agent invocation patterns, and file handling. + +**Last Updated**: 2026-01-30 + +**Status**: This document originally described the legacy `cli/parallel-analyze.ts` + `cli/save-analysis-results.ts` flow. The current workflow is implemented via `omc analyze run` and `omc analyze complete`. See `docs/_meta/current-state.md` for the full audit and issue list. + +## Overview + +The parallel analysis workflow uses a **zero-context-pollution** design where each agent operates independently with fresh database connections and no shared state from the orchestration script. This enables true parallel execution and prevents race conditions. + +## Zero-Context-Pollution Design + +### Design Principles + +1. **Stub files as communication**: Main context creates minimal stub files containing only article metadata +2. **Fresh connections**: the orchestrator command closes its DB work before agents run; agents should operate independently +3. **Independent content fetching**: agents fetch article content via `omc omnivore get --json` and extract `.content` +4. **File-based results**: Agents enrich stub files with analysis field, no inter-agent communication +5. **Slug-based naming**: Temp files named by article slug prevents overwrites across batches + +### Why This Matters + +- **Prevents race conditions**: Each agent reads/writes different files +- **Enables true parallelism**: No database locks during agent execution +- **Supports batch overlap**: Can run multiple batches before saving results +- **Isolates failures**: Failed agent doesn't corrupt others' work +- **Scales horizontally**: Can distribute agents across machines + +## Parallel Analysis Workflow + +### Automated Mode (`omc analyze auto`) + +The automated workflow is the non-interactive, schedulable path: + +1. Populate queue: `omc queue add --hours 24` +2. Analyze end-to-end: `omc analyze auto --batch-size 5` + +Implementation notes: +- The analysis step shells out to `codex exec` in `read-only` sandbox mode and expects a single JSON object response. +- `CODEX_HOME` is set to `temp/codex-home` during execution to avoid reliance on `~/.codex` permissions in automation environments. + +### Step 1: Orchestration (`omc analyze run`) + +The orchestration script sets up the batch and prepares for agent invocation: + +```typescript +// 1. Fetch pending jobs and mark in_progress +const jobs = queueRepo.getPending(5); +jobs.forEach(job => queueRepo.markInProgress(job.articleId)); + +// 2. Fetch article metadata from Omnivore API +const result = await getArticle(job.articleSlug, username); +const article = result.article; + +// 3. Create stub file with slug-based name +const stubFile = `temp/${job.articleSlug}.jsonl`; +fs.writeFileSync(stubFile, JSON.stringify({ + articleId: job.articleId, + articleSlug: job.articleSlug, + username: username, + articleUrl: article.url, + articleTitle: article.title, + savedAt: article.savedAt, + publishedAt: article.publishedAt, + updatedAt: article.updatedAt +})); + +// 4. Output agent parameters +console.log(JSON.stringify([{ + filename: stubFile, + articleId: article.id, + articleSlug: article.slug, + username: username, + articleTitle: article.title +}])); + +// 5. Close database (agents get fresh connections) +db.close(); +``` + +**Key Points**: +- Database is closed before agents execute +- Stub files contain metadata only (no content) +- Each stub file is named by article slug +- Agent parameters output as JSON array + +### Step 2: Agent Execution (@article-content-analyzer) + +Each agent operates independently with its own workflow: + +```typescript +// Each agent receives: filename, articleId, articleSlug, username, articleTitle + +// 1. Read stub file to get metadata +const stub = JSON.parse(fs.readFileSync(filename, 'utf-8')); + +// 2. Fetch content via CLI tool (no TypeScript imports) +const contentResult = await bash(`omc omnivore get ${stub.articleSlug} --json`); +const article = JSON.parse(contentResult.stdout); +const articleContent = article.content; + +// 3. Analyze content using agent's prompt +const analysis = { + topics: [...], + topicScores: {...}, + summary: "...", + keyPoints: [...], + sentiment: "positive", + monetizationAngle: "...", + analyzedAt: new Date().toISOString() +}; + +// 4. Re-read stub file (fresh copy, no race conditions) +const enriched = JSON.parse(fs.readFileSync(filename, 'utf-8')); + +// 5. Add analysis field ONLY (preserve all original fields) +enriched.analysis = analysis; + +// 6. Write enriched data back to same file +fs.writeFileSync(filename, JSON.stringify(enriched, null, 2)); +``` + +**Key Points**: +- Agent fetches content via CLI (no direct TypeScript imports) +- Re-reads stub before writing (ensures fresh data) +- Adds `analysis` field only (preserves metadata) +- Each agent writes to unique file (slug-based naming) + +### Step 3: Save Results (`omc analyze complete`) + +The save script processes all enriched files and updates tracking: + +```typescript +// For each enriched temp file: +for (const file of files) { + const data = JSON.parse(fs.readFileSync(file, 'utf-8')); + + // 1) Write to Markdown (git-tracked, human-editable) + const markdownPath = await writer.write( + data.articleId, + data.articleUrl, + data.articleTitle, + data.savedAt, + data.analysis + ); + + // 2) Store immutable snapshot + mark completed in SQLite + queueRepo.storeAnalysis( + data.articleId, + data.publishedAt ?? null, + data.updatedAt ?? null, + JSON.stringify(data.analysis), + markdownPath + ); + + // 3) Delete temp file (cleanup) + fs.unlinkSync(file); +} +``` + +**Key Points**: +- Processes all temp files in one pass +- Updates database after successful write +- Automatically deletes temp files (cleanup) +- Preserves temp files on error for debugging + +**Note**: +- JSONL output is optional: enable with `omc analyze complete --jsonl` or `omc analyze auto --jsonl`. + +## File Lifecycle + +### Stub File Creation (`omc analyze run`) + +**File**: `temp/{articleSlug}.jsonl` + +**Content**: Article metadata only (no content, no analysis) + +**Purpose**: Minimal communication between orchestrator and agents + +**Example**: +```json +{ + "articleId": "abc-123", + "articleSlug": "understanding-llm-fine-tuning", + "username": "myusername", + "articleUrl": "https://example.com/article", + "articleTitle": "Understanding LLM Fine-Tuning", + "savedAt": "2025-09-30T10:00:00Z", + "publishedAt": "2025-09-29T14:00:00Z", + "updatedAt": "2025-09-30T08:00:00Z" +} +``` + +### Enrichment (agent) + +**Process**: +1. Reads stub file +2. Fetches content via CLI tool +3. Adds `analysis` field to JSON +4. Writes back to same file + +**Enriched Format**: +```json +{ + "articleId": "abc-123", + "articleSlug": "understanding-llm-fine-tuning", + "username": "myusername", + "articleUrl": "https://example.com/article", + "articleTitle": "Understanding LLM Fine-Tuning", + "savedAt": "2025-09-30T10:00:00Z", + "publishedAt": "2025-09-29T14:00:00Z", + "updatedAt": "2025-09-30T08:00:00Z", + "analysis": { + "topics": ["ai", "machine-learning"], + "topicScores": { "ai": 0.95, "machine-learning": 0.88 }, + "summary": "Article summary...", + "keyPoints": ["point1", "point2"], + "sentiment": "positive", + "monetizationAngle": "Content opportunity...", + "analyzedAt": "2025-10-01T04:30:00Z" + } +} +``` + +### Storage & Cleanup (`omc analyze complete`) + +**Process**: +1. Reads enriched file +2. Writes Markdown to `content/analysis/YYYY-MM-DD-{slug}-analysis.md` (slug derived from `articleSlug` when available, otherwise title) +3. Stores immutable snapshot to SQLite (`analysis_json`, `markdown_path`) and marks job `completed` +4. **Deletes temp file** with `unlinkSync(file)` + +**Output Files**: +- `content/analysis/2025-09-30-understanding-llm-fine-tuning-analysis.md` + +**Temp File**: Deleted after successful save + +## Error Handling + +### Parse Error + +**Scenario**: Enriched file contains invalid JSON + +**Handling**: +- Mark job as `failed` in tracking queue +- Preserve temp file for debugging +- Store error message in database +- Increment retry count + +**Example**: +``` +✗ Failed: Understanding LLM Fine-Tuning... - Parse error + → Preserved: temp/understanding-llm-fine-tuning.jsonl +``` + +### Agent Timeout + +**Scenario**: Agent doesn't complete within time limit + +**Handling**: +- Status remains `in_progress` in database +- Temp file preserved (partially enriched) +- Manual retry needed via `omc analyze retry` + +### Save Failure + +**Scenario**: Error writing to Markdown or JSONL + +**Handling**: +- Mark job as `failed` +- Preserve temp file +- Log error details +- Can retry after fixing issue + +## Parallel Invocation Pattern + +### Single Message, Multiple Agents + +The workflow requires invoking all agents in a single message for true parallelism: + +``` +User message: +Analyze these 5 articles in parallel: + +@article-content-analyzer { + filename: "temp/article-1.jsonl", + articleId: "abc-123", + articleSlug: "article-1", + username: "myusername", + articleTitle: "Article 1 Title" +} + +@article-content-analyzer { + filename: "temp/article-2.jsonl", + articleId: "def-456", + articleSlug: "article-2", + username: "myusername", + articleTitle: "Article 2 Title" +} + +... (3 more agents) +``` + +### Why Single Message + +- **All agents execute in parallel** (no sequential bottleneck) +- **User waits for all results** before saving +- **Failure of one agent** doesn't block others +- **Claude Code orchestrates** parallel execution + +### Agent Independence + +Each agent: +- Gets unique parameters (filename, articleId, slug, username, title) +- Reads different stub file +- Fetches content independently via CLI +- Writes to different temp file +- No coordination with other agents + +## Slug-Based Naming Benefits + +### Problem with Sequential IDs + +Using sequential names like `temp/result-1.jsonl`, `temp/result-2.jsonl`: +- Running batch 2 overwrites batch 1's files +- Must save batch 1 before running batch 2 +- Forces sequential processing + +### Solution with Slug-Based Names + +Using article slug like `temp/{articleSlug}.jsonl`: +- Each article has unique filename based on slug +- Multiple batches can run simultaneously +- Agents can complete out-of-order +- Save script processes all `temp/*.jsonl` files at once + +### Example Workflow + +```bash +# Batch 1 creates: +temp/understanding-llm-fine-tuning.jsonl +temp/kubernetes-best-practices.jsonl +temp/rust-performance-tips.jsonl + +# Batch 2 creates (while batch 1 agents still running): +temp/python-async-patterns.jsonl +temp/distributed-systems-design.jsonl + +# Save all at once: +omc analyze complete +# Processes all 5 files, deletes after successful save +``` + +## Agent Content Analyzer + +### Agent Name + +`@article-content-analyzer` + +**Location**: `.claude/agents/article-content-analyzer.md` + +**Model**: Sonnet + +### Core Responsibilities + +1. **Topic Extraction & Scoring** - Identify 2-5 topics with confidence scores (0-1) +2. **Strategic Summarization** - 2-3 sentence "so what?" summary +3. **Key Point Extraction** - 3-5 actionable insights or surprising facts +4. **Sentiment Analysis** - Classify as positive/neutral/negative +5. **Monetization Strategy** - Identify specific content opportunities + +### Topic Categories + +Aligned with content strategy: +- `developer-tools`, `ai-tooling`, `ai`, `machine-learning`, `llm` +- `software-engineering`, `architecture`, `code-quality` +- `devops`, `kubernetes`, `docker`, `cloud-native` +- `databases`, `sql`, `nosql` +- `cloud`, `aws`, `azure`, `serverless` +- `startups`, `product`, `growth`, `monetization` +- `security`, `auth`, `encryption` + +### Topic Scores + +- **0.90-1.0**: Core focus of article +- **0.70-0.89**: Significant discussion +- **0.50-0.69**: Mentioned but not central +- **Below 0.50**: Excluded + +### Output Schema + +```json +{ + "topics": ["topic1", "topic2", "topic3"], + "topicScores": { + "topic1": 0.95, + "topic2": 0.88, + "topic3": 0.75 + }, + "summary": "2-3 sentence summary capturing main points and why this matters", + "keyPoints": [ + "First key takeaway or insight", + "Second key takeaway or insight", + "Third key takeaway or insight" + ], + "sentiment": "positive|neutral|negative", + "monetizationAngle": "Specific content opportunity description" +} +``` + +### Critical Rules + +- Returns ONLY JSON object (no markdown code blocks, no explanatory text) +- All topic labels from approved category list +- Topic scores between 0-1, at least one ≥ 0.70 +- Summary is 2-3 sentences focusing on implications +- 3-5 specific, actionable key points +- Monetization angle suggests concrete content opportunity +- User highlights receive extra weight if provided + +### Usage in Workflow + +The agent is invoked with parameters from the orchestration script: + +``` +@article-content-analyzer { + filename: "temp/understanding-llm-fine-tuning.jsonl", + articleId: "abc-123", + articleSlug: "understanding-llm-fine-tuning", + username: "myusername", + articleTitle: "Understanding LLM Fine-Tuning Techniques" +} +``` + +Agent then: +1. Reads stub file for metadata +2. Calls `omc omnivore get understanding-llm-fine-tuning --json` for content (extract `.content`) +3. Analyzes content using its prompt template +4. Enriches stub file with analysis +5. Writes back to temp file + +## Analysis Prompt Template + +**Location**: `src/analysis/prompts/analyze.md` + +**Purpose**: Structured prompt template for article analysis (used by agent). + +### Template Variables + +- `{{title}}` - Article title +- `{{author}}` - Article author +- `{{url}}` - Article URL +- `{{wordCount}}` - Word count +- `{{publishedAt}}` - Publication date +- `{{content}}` - Article content (HTML or text) +- `{{#if highlights}}...{{/if}}` - Conditional highlights section +- `{{#each highlights}}...{{/each}}` - Iterate over highlights + +### Highlights Format + +If user highlights are present: + +``` +{{#each highlights}} +- "{{quote}}"{{#if annotation}} — Note: {{annotation}}{{/if}} +{{/each}} +``` + +### Output Format + +JSON matching `ContentAnalysis` schema (no markdown code blocks). + +### Example Rendered Prompt + +```markdown +# Article Content Analysis Prompt + +You are analyzing an article for content monetization opportunities... + +## Article Details + +**Title**: Understanding LLM Fine-Tuning +**Author**: Jane Smith +**URL**: https://example.com/llm-finetuning +**Word Count**: 3200 +**Published**: 2025-09-25T10:00:00Z + +**Content**: +[Full article content here...] + +**User Highlights**: +- "LoRA reduces trainable parameters by 90%" — Note: Key metric for cost savings +- "Prompt engineering vs fine-tuning trade-offs" + +## Analysis Task + +Extract the following information and return as JSON: +[JSON schema and guidelines...] +``` + +## Data Flow Diagram + +``` +┌─────────────────────────────────────────────────────────────┐ +│ 1. ORCHESTRATION (omc analyze run) │ +│ │ +│ Queue DB → Get 5 pending → Mark in_progress │ +│ → Fetch metadata from Omnivore API │ +│ → Create stub files (temp/{slug}.jsonl) │ +│ → Output agent parameters as JSON │ +│ → Close DB connection │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 2. PARALLEL AGENT EXECUTION (@article-content-analyzer × 5) │ +│ │ +│ Each agent independently: │ +│ - Read stub file for metadata │ +│ - Call `omc omnivore get {slug} --json` for content │ +│ - Analyze content using prompt template │ +│ - Enrich stub file with analysis field │ +│ - Write back to temp/{slug}.jsonl │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 3. SAVE RESULTS (omc analyze complete) │ +│ │ +│ Read all temp/*.jsonl files │ +│ For each: │ +│ - Write to content/analysis/YYYY-MM-DD-{slug}-analysis.md │ +│ - Store snapshot in DB + mark completed │ +│ - Delete temp file │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Complete Example + +### Real Analysis Output + +**File**: `content/analysis/2025-09-30-cchistory-tracking-claude-code-system-prompt-and-t-analysis.md` + +```markdown +--- +articleId: 1039961b-8de3-4ccf-b3e8-df888d6174b8 +articleUrl: https://mariozechner.at/posts/2025-08-03-cchistory/ +articleTitle: "cchistory: Tracking Claude Code System Prompt and Tool Changes" +savedAt: 2025-09-30T02:27:46.000Z +analyzedAt: 2025-10-01T04:25:16.676Z +topics: [developer-tools, ai-tooling, reverse-engineering, software-debugging] +topicScores: + developer-tools: 0.95 + ai-tooling: 0.9 + reverse-engineering: 0.85 + software-debugging: 0.75 +sentiment: positive +--- + +## Summary + +Developer creates tools to reverse-engineer and track changes in Claude Code's system prompts and tool definitions over time. The cchistory project monitors Claude Code updates by intercepting and archiving system messages, revealing how Anthropic evolves the assistant's behavior and capabilities. + +## Key Points + +- Created claude-trace to intercept and log Claude Code's internal API communications +- Built cchistory to automatically track and diff system prompt changes across versions +- Discovered Claude Code switched from Sonnet to Haiku for certain operations to reduce costs +- System prompts reveal detailed behavioral instructions and tool usage patterns +- Tracking changes helps understand how AI coding assistants evolve over time + +## Monetization Angle + +Tutorial series on 'Reverse Engineering AI Tools' - developers want to understand how Claude Code works under the hood. Could create content comparing system prompt evolution, analyzing cost optimization strategies, or teaching prompt engineering through real examples from production AI systems. +``` + +### Example JSONL Record (Optional Output) + +**File**: `content/analysis/analyses.jsonl` (written when `--jsonl` is enabled) + +```json +{"articleId":"1039961b-8de3-4ccf-b3e8-df888d6174b8","articleUrl":"https://mariozechner.at/posts/2025-08-03-cchistory/","articleTitle":"cchistory: Tracking Claude Code System Prompt and Tool Changes","savedAt":"2025-09-30T02:27:46.000Z","analyzedAt":"2025-10-01T04:25:16.676Z","topics":["developer-tools","ai-tooling","reverse-engineering","software-debugging"],"topicScores":{"developer-tools":0.95,"ai-tooling":0.9,"reverse-engineering":0.85,"software-debugging":0.75},"sentiment":"positive","summary":"Developer creates tools to reverse-engineer and track changes in Claude Code's system prompts and tool definitions over time. The cchistory project monitors Claude Code updates by intercepting and archiving system messages, revealing how Anthropic evolves the assistant's behavior and capabilities.","keyPoints":["Created claude-trace to intercept and log Claude Code's internal API communications","Built cchistory to automatically track and diff system prompt changes across versions","Discovered Claude Code switched from Sonnet to Haiku for certain operations to reduce costs","System prompts reveal detailed behavioral instructions and tool usage patterns","Tracking changes helps understand how AI coding assistants evolve over time"],"monetizationAngle":"Tutorial series on 'Reverse Engineering AI Tools' - developers want to understand how Claude Code works under the hood. Could create content comparing system prompt evolution, analyzing cost optimization strategies, or teaching prompt engineering through real examples from production AI systems."} +``` + +## What Does NOT Exist Yet + +**Implemented**: +- ✅ SQLite tracking database for parallel execution +- ✅ Job queue with status tracking (pending/in_progress/completed/failed) +- ✅ Parallel analysis workflow (batch of 5) +- ✅ Error handling with retry logic (max 3 attempts) +- ✅ CLI tools for queue management +- ✅ JSONL output format +- ✅ Boundary enforcement with AIDEV annotations +- ✅ Zero-context-pollution design +- ✅ Slug-based file naming + +**Future Enhancements**: +- **Automated agent invocation** - Currently requires manual copy/paste of prompts; could integrate Claude Agent SDK for direct invocation +- **Real-time progress tracking** - Add progress bars during fetch/analysis operations +- **Session management** - `analysis_sessions` table exists but not yet used for batch tracking +- **Queue cleanup automation** - Manual `clearCompleted()` call; could auto-cleanup after export +- **Distributed execution** - Agents across multiple machines +- **Advanced error recovery** - Automatic retry with backoff +- **Batch prioritization** - Priority queue for time-sensitive articles +- **Content pre-filtering** - Skip articles unlikely to be valuable +- **Incremental updates** - Re-analyze articles when content changes + +## Related Documentation + +- [Architecture](architecture.md) - System design and storage boundaries +- [CLI Reference](cli-reference.md) - Command-line interface documentation +- [Foundation & Type System](foundation-and-types.md) - TypeScript setup, type definitions diff --git a/self-hosting/omc/docs/graphql-schema/schema.graphql b/self-hosting/omc/docs/graphql-schema/schema.graphql new file mode 100644 index 000000000..aec51f4ba --- /dev/null +++ b/self-hosting/omc/docs/graphql-schema/schema.graphql @@ -0,0 +1,3510 @@ +schema { + query: Query + mutation: Mutation + subscription: SubscriptionRootType +} + +directive @sanitize(allowedTags: [String], maxLength: Int, minLength: Int, pattern: String) on INPUT_FIELD_DEFINITION + +type AddDiscoverFeedError { + errorCodes: [AddDiscoverFeedErrorCode!]! +} + +enum AddDiscoverFeedErrorCode { + BAD_REQUEST + CONFLICT + NOT_FOUND + UNAUTHORIZED +} + +input AddDiscoverFeedInput { + url: String! +} + +union AddDiscoverFeedResult = AddDiscoverFeedError | AddDiscoverFeedSuccess + +type AddDiscoverFeedSuccess { + feed: DiscoverFeed! +} + +type AddPopularReadError { + errorCodes: [AddPopularReadErrorCode!]! +} + +enum AddPopularReadErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union AddPopularReadResult = AddPopularReadError | AddPopularReadSuccess + +type AddPopularReadSuccess { + pageId: String! +} + +enum AllowedReply { + CONFIRM + OKAY + SUBSCRIBE + YES +} + +type ApiKey { + createdAt: Date! + expiresAt: Date! + id: ID! + key: String + name: String! + scopes: [String!] + usedAt: Date +} + +type ApiKeysError { + errorCodes: [ApiKeysErrorCode!]! +} + +enum ApiKeysErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union ApiKeysResult = ApiKeysError | ApiKeysSuccess + +type ApiKeysSuccess { + apiKeys: [ApiKey!]! +} + +type ArchiveLinkError { + errorCodes: [ArchiveLinkErrorCode!]! + message: String! +} + +enum ArchiveLinkErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input ArchiveLinkInput { + archived: Boolean! + linkId: ID! +} + +union ArchiveLinkResult = ArchiveLinkError | ArchiveLinkSuccess + +type ArchiveLinkSuccess { + linkId: String! + message: String! +} + +type Article { + author: String + content: String! + contentReader: ContentReader! + createdAt: Date! + description: String + directionality: DirectionalityType + feedContent: String + folder: String! + hasContent: Boolean + hash: String! + highlights(input: ArticleHighlightsInput): [Highlight!]! + id: ID! + image: String + isArchived: Boolean! + labels: [Label!] + language: String + linkId: ID + originalArticleUrl: String + originalHtml: String + pageType: PageType + postedByViewer: Boolean + publishedAt: Date + readAt: Date + readingProgressAnchorIndex: Int! + readingProgressPercent: Float! + readingProgressTopPercent: Float + recommendations: [Recommendation!] + savedAt: Date! + savedByViewer: Boolean + shareInfo: LinkShareInfo + sharedComment: String + siteIcon: String + siteName: String + slug: String! + state: ArticleSavingRequestStatus + subscription: String + title: String! + unsubHttpUrl: String + unsubMailTo: String + updatedAt: Date + uploadFileId: ID + url: String! + wordsCount: Int +} + +type ArticleEdge { + cursor: String! + node: Article! +} + +type ArticleError { + errorCodes: [ArticleErrorCode!]! +} + +enum ArticleErrorCode { + BAD_DATA + NOT_FOUND + UNAUTHORIZED +} + +input ArticleHighlightsInput { + includeFriends: Boolean +} + +union ArticleResult = ArticleError | ArticleSuccess + +type ArticleSavingRequest { + article: Article @deprecated(reason: "article has been replaced with slug") + createdAt: Date! + errorCode: CreateArticleErrorCode + id: ID! + slug: String! + status: ArticleSavingRequestStatus! + updatedAt: Date + url: String! + user: User! + userId: ID! @deprecated(reason: "userId has been replaced with user") +} + +type ArticleSavingRequestError { + errorCodes: [ArticleSavingRequestErrorCode!]! +} + +enum ArticleSavingRequestErrorCode { + BAD_DATA + NOT_FOUND + UNAUTHORIZED +} + +union ArticleSavingRequestResult = ArticleSavingRequestError | ArticleSavingRequestSuccess + +enum ArticleSavingRequestStatus { + ARCHIVED + CONTENT_NOT_FETCHED + DELETED + FAILED + PROCESSING + SUCCEEDED +} + +type ArticleSavingRequestSuccess { + articleSavingRequest: ArticleSavingRequest! +} + +type ArticleSuccess { + article: Article! +} + +type ArticlesError { + errorCodes: [ArticlesErrorCode!]! +} + +enum ArticlesErrorCode { + UNAUTHORIZED +} + +union ArticlesResult = ArticlesError | ArticlesSuccess + +type ArticlesSuccess { + edges: [ArticleEdge!]! + pageInfo: PageInfo! +} + +type BulkActionError { + errorCodes: [BulkActionErrorCode!]! +} + +enum BulkActionErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union BulkActionResult = BulkActionError | BulkActionSuccess + +type BulkActionSuccess { + success: Boolean! +} + +enum BulkActionType { + ADD_LABELS + ARCHIVE + DELETE + MARK_AS_READ + MARK_AS_SEEN + MOVE_TO_FOLDER +} + +enum ContentReader { + EPUB + PDF + WEB +} + +type CreateArticleError { + errorCodes: [CreateArticleErrorCode!]! +} + +enum CreateArticleErrorCode { + ELASTIC_ERROR + NOT_ALLOWED_TO_PARSE + PAYLOAD_TOO_LARGE + UNABLE_TO_FETCH + UNABLE_TO_PARSE + UNAUTHORIZED + UPLOAD_FILE_MISSING +} + +input CreateArticleInput { + articleSavingRequestId: ID + folder: String + labels: [CreateLabelInput!] + preparedDocument: PreparedDocumentInput + publishedAt: Date + rssFeedUrl: String + savedAt: Date + skipParsing: Boolean + source: String + state: ArticleSavingRequestStatus + uploadFileId: ID + url: String! +} + +union CreateArticleResult = CreateArticleError | CreateArticleSuccess + +type CreateArticleSavingRequestError { + errorCodes: [CreateArticleSavingRequestErrorCode!]! +} + +enum CreateArticleSavingRequestErrorCode { + BAD_DATA + UNAUTHORIZED +} + +input CreateArticleSavingRequestInput { + url: String! +} + +union CreateArticleSavingRequestResult = CreateArticleSavingRequestError | CreateArticleSavingRequestSuccess + +type CreateArticleSavingRequestSuccess { + articleSavingRequest: ArticleSavingRequest! +} + +type CreateArticleSuccess { + created: Boolean! + createdArticle: Article! + user: User! +} + +type CreateFolderPolicyError { + errorCodes: [CreateFolderPolicyErrorCode!]! +} + +enum CreateFolderPolicyErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input CreateFolderPolicyInput { + action: FolderPolicyAction! + afterDays: Int! + folder: String! +} + +union CreateFolderPolicyResult = CreateFolderPolicyError | CreateFolderPolicySuccess + +type CreateFolderPolicySuccess { + policy: FolderPolicy! +} + +type CreateGroupError { + errorCodes: [CreateGroupErrorCode!]! +} + +enum CreateGroupErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input CreateGroupInput { + description: String + expiresInDays: Int + maxMembers: Int + name: String! + onlyAdminCanPost: Boolean + onlyAdminCanSeeMembers: Boolean + topics: [String!] +} + +union CreateGroupResult = CreateGroupError | CreateGroupSuccess + +type CreateGroupSuccess { + group: RecommendationGroup! +} + +type CreateHighlightError { + errorCodes: [CreateHighlightErrorCode!]! +} + +enum CreateHighlightErrorCode { + ALREADY_EXISTS + BAD_DATA + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +input CreateHighlightInput { + annotation: String + articleId: ID! + color: String + highlightPositionAnchorIndex: Int + highlightPositionPercent: Float + html: String + id: ID! + patch: String + prefix: String + quote: String + representation: RepresentationType + sharedAt: Date + shortId: String! + suffix: String + type: HighlightType +} + +type CreateHighlightReplyError { + errorCodes: [CreateHighlightReplyErrorCode!]! +} + +enum CreateHighlightReplyErrorCode { + EMPTY_ANNOTATION + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +input CreateHighlightReplyInput { + highlightId: ID! + text: String! +} + +union CreateHighlightReplyResult = CreateHighlightReplyError | CreateHighlightReplySuccess + +type CreateHighlightReplySuccess { + highlightReply: HighlightReply! +} + +union CreateHighlightResult = CreateHighlightError | CreateHighlightSuccess + +type CreateHighlightSuccess { + highlight: Highlight! +} + +type CreateLabelError { + errorCodes: [CreateLabelErrorCode!]! +} + +enum CreateLabelErrorCode { + BAD_REQUEST + LABEL_ALREADY_EXISTS + NOT_FOUND + UNAUTHORIZED +} + +input CreateLabelInput { + color: String + description: String + name: String! +} + +union CreateLabelResult = CreateLabelError | CreateLabelSuccess + +type CreateLabelSuccess { + label: Label! +} + +type CreateNewsletterEmailError { + errorCodes: [CreateNewsletterEmailErrorCode!]! +} + +enum CreateNewsletterEmailErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input CreateNewsletterEmailInput { + description: String + folder: String + name: String +} + +union CreateNewsletterEmailResult = CreateNewsletterEmailError | CreateNewsletterEmailSuccess + +type CreateNewsletterEmailSuccess { + newsletterEmail: NewsletterEmail! +} + +type CreatePostError { + errorCodes: [CreatePostErrorCode!]! +} + +enum CreatePostErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input CreatePostInput { + content: String! + highlightIds: [ID!] + libraryItemIds: [ID!]! + thought: String + thumbnail: String + title: String! +} + +union CreatePostResult = CreatePostError | CreatePostSuccess + +type CreatePostSuccess { + post: Post! +} + +type CreateReactionError { + errorCodes: [CreateReactionErrorCode!]! +} + +enum CreateReactionErrorCode { + BAD_CODE + BAD_TARGET + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +input CreateReactionInput { + code: ReactionType! + highlightId: ID + userArticleId: ID +} + +union CreateReactionResult = CreateReactionError | CreateReactionSuccess + +type CreateReactionSuccess { + reaction: Reaction! +} + +type CreateReminderError { + errorCodes: [CreateReminderErrorCode!]! +} + +enum CreateReminderErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input CreateReminderInput { + archiveUntil: Boolean! + clientRequestId: ID + linkId: ID + remindAt: Date! + sendNotification: Boolean! +} + +union CreateReminderResult = CreateReminderError | CreateReminderSuccess + +type CreateReminderSuccess { + reminder: Reminder! +} + +scalar Date + +type DeleteAccountError { + errorCodes: [DeleteAccountErrorCode!]! +} + +enum DeleteAccountErrorCode { + FORBIDDEN + UNAUTHORIZED + USER_NOT_FOUND +} + +union DeleteAccountResult = DeleteAccountError | DeleteAccountSuccess + +type DeleteAccountSuccess { + userID: ID! +} + +type DeleteDiscoverArticleError { + errorCodes: [DeleteDiscoverArticleErrorCode!]! +} + +enum DeleteDiscoverArticleErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input DeleteDiscoverArticleInput { + discoverArticleId: ID! +} + +union DeleteDiscoverArticleResult = DeleteDiscoverArticleError | DeleteDiscoverArticleSuccess + +type DeleteDiscoverArticleSuccess { + id: ID! +} + +type DeleteDiscoverFeedError { + errorCodes: [DeleteDiscoverFeedErrorCode!]! +} + +enum DeleteDiscoverFeedErrorCode { + BAD_REQUEST + CONFLICT + NOT_FOUND + UNAUTHORIZED +} + +input DeleteDiscoverFeedInput { + feedId: ID! +} + +union DeleteDiscoverFeedResult = DeleteDiscoverFeedError | DeleteDiscoverFeedSuccess + +type DeleteDiscoverFeedSuccess { + id: String! +} + +type DeleteFilterError { + errorCodes: [DeleteFilterErrorCode!]! +} + +enum DeleteFilterErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union DeleteFilterResult = DeleteFilterError | DeleteFilterSuccess + +type DeleteFilterSuccess { + filter: Filter! +} + +type DeleteFolderPolicyError { + errorCodes: [DeleteFolderPolicyErrorCode!]! +} + +enum DeleteFolderPolicyErrorCode { + UNAUTHORIZED +} + +union DeleteFolderPolicyResult = DeleteFolderPolicyError | DeleteFolderPolicySuccess + +type DeleteFolderPolicySuccess { + success: Boolean! +} + +type DeleteHighlightError { + errorCodes: [DeleteHighlightErrorCode!]! +} + +enum DeleteHighlightErrorCode { + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +type DeleteHighlightReplyError { + errorCodes: [DeleteHighlightReplyErrorCode!]! +} + +enum DeleteHighlightReplyErrorCode { + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +union DeleteHighlightReplyResult = DeleteHighlightReplyError | DeleteHighlightReplySuccess + +type DeleteHighlightReplySuccess { + highlightReply: HighlightReply! +} + +union DeleteHighlightResult = DeleteHighlightError | DeleteHighlightSuccess + +type DeleteHighlightSuccess { + highlight: Highlight! +} + +type DeleteIntegrationError { + errorCodes: [DeleteIntegrationErrorCode!]! +} + +enum DeleteIntegrationErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union DeleteIntegrationResult = DeleteIntegrationError | DeleteIntegrationSuccess + +type DeleteIntegrationSuccess { + integration: Integration! +} + +type DeleteLabelError { + errorCodes: [DeleteLabelErrorCode!]! +} + +enum DeleteLabelErrorCode { + BAD_REQUEST + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +union DeleteLabelResult = DeleteLabelError | DeleteLabelSuccess + +type DeleteLabelSuccess { + label: Label! +} + +type DeleteNewsletterEmailError { + errorCodes: [DeleteNewsletterEmailErrorCode!]! +} + +enum DeleteNewsletterEmailErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union DeleteNewsletterEmailResult = DeleteNewsletterEmailError | DeleteNewsletterEmailSuccess + +type DeleteNewsletterEmailSuccess { + newsletterEmail: NewsletterEmail! +} + +type DeletePostError { + errorCodes: [DeletePostErrorCode!]! +} + +enum DeletePostErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union DeletePostResult = DeletePostError | DeletePostSuccess + +type DeletePostSuccess { + success: Boolean! +} + +type DeleteReactionError { + errorCodes: [DeleteReactionErrorCode!]! +} + +enum DeleteReactionErrorCode { + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +union DeleteReactionResult = DeleteReactionError | DeleteReactionSuccess + +type DeleteReactionSuccess { + reaction: Reaction! +} + +type DeleteReminderError { + errorCodes: [DeleteReminderErrorCode!]! +} + +enum DeleteReminderErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union DeleteReminderResult = DeleteReminderError | DeleteReminderSuccess + +type DeleteReminderSuccess { + reminder: Reminder! +} + +type DeleteRuleError { + errorCodes: [DeleteRuleErrorCode!]! +} + +enum DeleteRuleErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union DeleteRuleResult = DeleteRuleError | DeleteRuleSuccess + +type DeleteRuleSuccess { + rule: Rule! +} + +type DeleteWebhookError { + errorCodes: [DeleteWebhookErrorCode!]! +} + +enum DeleteWebhookErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union DeleteWebhookResult = DeleteWebhookError | DeleteWebhookSuccess + +type DeleteWebhookSuccess { + webhook: Webhook! +} + +type DeviceToken { + createdAt: Date! + id: ID! + token: String! +} + +type DeviceTokensError { + errorCodes: [DeviceTokensErrorCode!]! +} + +enum DeviceTokensErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union DeviceTokensResult = DeviceTokensError | DeviceTokensSuccess + +type DeviceTokensSuccess { + deviceTokens: [DeviceToken!]! +} + +type DigestConfig { + channels: [String] +} + +input DigestConfigInput { + channels: [String] +} + +enum DirectionalityType { + LTR + RTL +} + +type DiscoverFeed { + description: String + id: ID! + image: String + link: String! + title: String! + type: String! + visibleName: String +} + +type DiscoverFeedArticle { + author: String + description: String! + feed: String! + id: ID! + image: String + publishedDate: Date + savedId: String + savedLinkUrl: String + siteName: String + slug: String! + title: String! + url: String! +} + +type DiscoverFeedError { + errorCodes: [DiscoverFeedErrorCode!]! +} + +enum DiscoverFeedErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union DiscoverFeedResult = DiscoverFeedError | DiscoverFeedSuccess + +type DiscoverFeedSuccess { + feeds: [DiscoverFeed]! +} + +type DiscoverTopic { + description: String! + name: String! +} + +type EditDiscoverFeedError { + errorCodes: [EditDiscoverFeedErrorCode!]! +} + +enum EditDiscoverFeedErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input EditDiscoverFeedInput { + feedId: ID! + name: String! +} + +union EditDiscoverFeedResult = EditDiscoverFeedError | EditDiscoverFeedSuccess + +type EditDiscoverFeedSuccess { + id: ID! +} + +type EmptyTrashError { + errorCodes: [EmptyTrashErrorCode!]! +} + +enum EmptyTrashErrorCode { + UNAUTHORIZED +} + +union EmptyTrashResult = EmptyTrashError | EmptyTrashSuccess + +type EmptyTrashSuccess { + success: Boolean +} + +enum ErrorCode { + BAD_REQUEST + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +type ExportToIntegrationError { + errorCodes: [ExportToIntegrationErrorCode!]! +} + +enum ExportToIntegrationErrorCode { + FAILED_TO_CREATE_TASK + UNAUTHORIZED +} + +union ExportToIntegrationResult = ExportToIntegrationError | ExportToIntegrationSuccess + +type ExportToIntegrationSuccess { + task: Task! +} + +type Feature { + createdAt: Date! + expiresAt: Date + grantedAt: Date + id: ID! + name: String! + token: String! + updatedAt: Date +} + +type Feed { + author: String + createdAt: Date + description: String + id: ID + image: String + publishedAt: Date + title: String! + type: String + updatedAt: Date + url: String! +} + +type FeedArticle { + annotationsCount: Int + article: Article! + highlight: Highlight + highlightsCount: Int + id: ID! + reactions: [Reaction!]! + sharedAt: Date! + sharedBy: User! + sharedComment: String + sharedWithHighlights: Boolean +} + +type FeedArticleEdge { + cursor: String! + node: FeedArticle! +} + +type FeedArticlesError { + errorCodes: [FeedArticlesErrorCode!]! +} + +enum FeedArticlesErrorCode { + UNAUTHORIZED +} + +union FeedArticlesResult = FeedArticlesError | FeedArticlesSuccess + +type FeedArticlesSuccess { + edges: [FeedArticleEdge!]! + pageInfo: PageInfo! +} + +type FeedEdge { + cursor: String! + node: Feed! +} + +type FeedsError { + errorCodes: [FeedsErrorCode!]! +} + +enum FeedsErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input FeedsInput { + after: String + first: Int + query: String + sort: SortParams +} + +union FeedsResult = FeedsError | FeedsSuccess + +type FeedsSuccess { + edges: [FeedEdge!]! + pageInfo: PageInfo! +} + +type FetchContentError { + errorCodes: [FetchContentErrorCode!]! +} + +enum FetchContentErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union FetchContentResult = FetchContentError | FetchContentSuccess + +type FetchContentSuccess { + success: Boolean! +} + +enum FetchContentType { + ALWAYS + NEVER + WHEN_EMPTY +} + +type Filter { + category: String + createdAt: Date! + defaultFilter: Boolean + description: String + filter: String! + folder: String + id: ID! + name: String! + position: Int! + updatedAt: Date + visible: Boolean +} + +type FiltersError { + errorCodes: [FiltersErrorCode!]! +} + +enum FiltersErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union FiltersResult = FiltersError | FiltersSuccess + +type FiltersSuccess { + filters: [Filter!]! +} + +type FolderPoliciesError { + errorCodes: [FolderPoliciesErrorCode!]! +} + +enum FolderPoliciesErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union FolderPoliciesResult = FolderPoliciesError | FolderPoliciesSuccess + +type FolderPoliciesSuccess { + policies: [FolderPolicy!]! +} + +type FolderPolicy { + action: FolderPolicyAction! + afterDays: Int! + createdAt: Date! + folder: String! + id: ID! + updatedAt: Date! +} + +enum FolderPolicyAction { + ARCHIVE + DELETE +} + +type GenerateApiKeyError { + errorCodes: [GenerateApiKeyErrorCode!]! +} + +enum GenerateApiKeyErrorCode { + ALREADY_EXISTS + BAD_REQUEST + UNAUTHORIZED +} + +input GenerateApiKeyInput { + expiresAt: Date! + name: String! + scopes: [String!] +} + +union GenerateApiKeyResult = GenerateApiKeyError | GenerateApiKeySuccess + +type GenerateApiKeySuccess { + apiKey: ApiKey! +} + +type GetDiscoverFeedArticleError { + errorCodes: [GetDiscoverFeedArticleErrorCode!]! +} + +enum GetDiscoverFeedArticleErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union GetDiscoverFeedArticleResults = GetDiscoverFeedArticleError | GetDiscoverFeedArticleSuccess + +type GetDiscoverFeedArticleSuccess { + discoverArticles: [DiscoverFeedArticle] + pageInfo: PageInfo! +} + +type GetDiscoverTopicError { + errorCodes: [GetDiscoverTopicErrorCode!]! +} + +enum GetDiscoverTopicErrorCode { + UNAUTHORIZED +} + +union GetDiscoverTopicResults = GetDiscoverTopicError | GetDiscoverTopicSuccess + +type GetDiscoverTopicSuccess { + discoverTopics: [DiscoverTopic!] +} + +type GetFollowersError { + errorCodes: [GetFollowersErrorCode!]! +} + +enum GetFollowersErrorCode { + UNAUTHORIZED +} + +union GetFollowersResult = GetFollowersError | GetFollowersSuccess + +type GetFollowersSuccess { + followers: [User!]! +} + +type GetFollowingError { + errorCodes: [GetFollowingErrorCode!]! +} + +enum GetFollowingErrorCode { + UNAUTHORIZED +} + +union GetFollowingResult = GetFollowingError | GetFollowingSuccess + +type GetFollowingSuccess { + following: [User!]! +} + +type GetUserPersonalizationError { + errorCodes: [GetUserPersonalizationErrorCode!]! +} + +enum GetUserPersonalizationErrorCode { + UNAUTHORIZED +} + +union GetUserPersonalizationResult = GetUserPersonalizationError | GetUserPersonalizationSuccess + +type GetUserPersonalizationSuccess { + userPersonalization: UserPersonalization +} + +input GoogleLoginInput { + email: String! + secret: String! +} + +type GoogleSignupError { + errorCodes: [SignupErrorCode]! +} + +input GoogleSignupInput { + bio: String + email: String! + name: String! + pictureUrl: String! + secret: String! + sourceUserId: String! + username: String! +} + +union GoogleSignupResult = GoogleSignupError | GoogleSignupSuccess + +type GoogleSignupSuccess { + me: User! +} + +type GroupsError { + errorCodes: [GroupsErrorCode!]! +} + +enum GroupsErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union GroupsResult = GroupsError | GroupsSuccess + +type GroupsSuccess { + groups: [RecommendationGroup!]! +} + +type HiddenHomeSectionError { + errorCodes: [HiddenHomeSectionErrorCode!]! +} + +enum HiddenHomeSectionErrorCode { + BAD_REQUEST + PENDING + UNAUTHORIZED +} + +union HiddenHomeSectionResult = HiddenHomeSectionError | HiddenHomeSectionSuccess + +type HiddenHomeSectionSuccess { + section: HomeSection +} + +type Highlight { + annotation: String + color: String + createdAt: Date! + createdByMe: Boolean! + highlightPositionAnchorIndex: Int + highlightPositionPercent: Float + html: String + id: ID! + labels: [Label!] + libraryItem: Article! + patch: String + prefix: String + quote: String + reactions: [Reaction!]! + replies: [HighlightReply!]! + representation: RepresentationType! + sharedAt: Date + shortId: String! + suffix: String + type: HighlightType! + updatedAt: Date + user: User! +} + +type HighlightEdge { + cursor: String! + node: Highlight! +} + +type HighlightReply { + createdAt: Date! + highlight: Highlight! + id: ID! + text: String! + updatedAt: Date + user: User! +} + +type HighlightStats { + highlightCount: Int! +} + +enum HighlightType { + HIGHLIGHT + NOTE + REDACTION +} + +type HighlightsError { + errorCodes: [HighlightsErrorCode!]! +} + +enum HighlightsErrorCode { + BAD_REQUEST +} + +union HighlightsResult = HighlightsError | HighlightsSuccess + +type HighlightsSuccess { + edges: [HighlightEdge!]! + pageInfo: PageInfo! +} + +type HomeEdge { + cursor: String! + node: HomeSection! +} + +type HomeError { + errorCodes: [HomeErrorCode!]! +} + +enum HomeErrorCode { + BAD_REQUEST + PENDING + UNAUTHORIZED +} + +type HomeItem { + author: String + broadcastCount: Int + canArchive: Boolean + canComment: Boolean + canDelete: Boolean + canMove: Boolean + canSave: Boolean + canShare: Boolean + date: Date! + dir: String + id: ID! + likeCount: Int + previewContent: String + saveCount: Int + score: Float + seen_at: Date + slug: String + source: HomeItemSource + thumbnail: String + title: String! + url: String! + wordCount: Int +} + +type HomeItemSource { + icon: String + id: ID + name: String + type: HomeItemSourceType! + url: String +} + +enum HomeItemSourceType { + LIBRARY + NEWSLETTER + RECOMMENDATION + RSS +} + +union HomeResult = HomeError | HomeSuccess + +type HomeSection { + items: [HomeItem!]! + layout: String + thumbnail: String + title: String +} + +type HomeSuccess { + edges: [HomeEdge!]! + pageInfo: PageInfo! +} + +type ImportFromIntegrationError { + errorCodes: [ImportFromIntegrationErrorCode!]! +} + +enum ImportFromIntegrationErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union ImportFromIntegrationResult = ImportFromIntegrationError | ImportFromIntegrationSuccess + +type ImportFromIntegrationSuccess { + success: Boolean! +} + +enum ImportItemState { + ALL + ARCHIVED + UNARCHIVED + UNREAD +} + +type Integration { + createdAt: Date! + enabled: Boolean! + id: ID! + name: String! + settings: JSON + taskName: String + token: String! + type: IntegrationType! + updatedAt: Date +} + +type IntegrationError { + errorCodes: [IntegrationErrorCode!]! +} + +enum IntegrationErrorCode { + NOT_FOUND +} + +union IntegrationResult = IntegrationError | IntegrationSuccess + +type IntegrationSuccess { + integration: Integration! +} + +enum IntegrationType { + EXPORT + IMPORT +} + +type IntegrationsError { + errorCodes: [IntegrationsErrorCode!]! +} + +enum IntegrationsErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union IntegrationsResult = IntegrationsError | IntegrationsSuccess + +type IntegrationsSuccess { + integrations: [Integration!]! +} + +scalar JSON + +type JoinGroupError { + errorCodes: [JoinGroupErrorCode!]! +} + +enum JoinGroupErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union JoinGroupResult = JoinGroupError | JoinGroupSuccess + +type JoinGroupSuccess { + group: RecommendationGroup! +} + +type Label { + color: String! + createdAt: Date + description: String + id: ID! + internal: Boolean + name: String! + position: Int + source: String +} + +type LabelsError { + errorCodes: [LabelsErrorCode!]! +} + +enum LabelsErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union LabelsResult = LabelsError | LabelsSuccess + +type LabelsSuccess { + labels: [Label!]! +} + +type LeaveGroupError { + errorCodes: [LeaveGroupErrorCode!]! +} + +enum LeaveGroupErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union LeaveGroupResult = LeaveGroupError | LeaveGroupSuccess + +type LeaveGroupSuccess { + success: Boolean! +} + +type Link { + highlightStats: HighlightStats! + id: ID! + page: Page! + postedByViewer: Boolean! + readState: ReadState! + savedAt: Date! + savedBy: User! + savedByViewer: Boolean! + shareInfo: LinkShareInfo! + shareStats: ShareStats! + slug: String! + updatedAt: Date + url: String! +} + +type LinkShareInfo { + description: String! + imageUrl: String! + title: String! +} + +type LogOutError { + errorCodes: [LogOutErrorCode!]! +} + +enum LogOutErrorCode { + LOG_OUT_FAILED +} + +union LogOutResult = LogOutError | LogOutSuccess + +type LogOutSuccess { + message: String +} + +type LoginError { + errorCodes: [LoginErrorCode!]! +} + +enum LoginErrorCode { + ACCESS_DENIED + AUTH_FAILED + INVALID_CREDENTIALS + USER_ALREADY_EXISTS + USER_NOT_FOUND + WRONG_SOURCE +} + +union LoginResult = LoginError | LoginSuccess + +type LoginSuccess { + me: User! +} + +type MarkEmailAsItemError { + errorCodes: [MarkEmailAsItemErrorCode!]! +} + +enum MarkEmailAsItemErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union MarkEmailAsItemResult = MarkEmailAsItemError | MarkEmailAsItemSuccess + +type MarkEmailAsItemSuccess { + success: Boolean! +} + +type MergeHighlightError { + errorCodes: [MergeHighlightErrorCode!]! +} + +enum MergeHighlightErrorCode { + ALREADY_EXISTS + BAD_DATA + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +input MergeHighlightInput { + annotation: String + articleId: ID! + color: String + highlightPositionAnchorIndex: Int + highlightPositionPercent: Float + html: String + id: ID! + overlapHighlightIdList: [String!]! + patch: String! + prefix: String + quote: String! + representation: RepresentationType + shortId: ID! + suffix: String +} + +union MergeHighlightResult = MergeHighlightError | MergeHighlightSuccess + +type MergeHighlightSuccess { + highlight: Highlight! + overlapHighlightIdList: [String!]! +} + +type MoveFilterError { + errorCodes: [MoveFilterErrorCode!]! +} + +enum MoveFilterErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input MoveFilterInput { + afterFilterId: ID + filterId: ID! +} + +union MoveFilterResult = MoveFilterError | MoveFilterSuccess + +type MoveFilterSuccess { + filter: Filter! +} + +type MoveLabelError { + errorCodes: [MoveLabelErrorCode!]! +} + +enum MoveLabelErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input MoveLabelInput { + afterLabelId: ID + labelId: ID! +} + +union MoveLabelResult = MoveLabelError | MoveLabelSuccess + +type MoveLabelSuccess { + label: Label! +} + +type MoveToFolderError { + errorCodes: [MoveToFolderErrorCode!]! +} + +enum MoveToFolderErrorCode { + ALREADY_EXISTS + BAD_REQUEST + UNAUTHORIZED +} + +union MoveToFolderResult = MoveToFolderError | MoveToFolderSuccess + +type MoveToFolderSuccess { + success: Boolean! +} + +type Mutation { + addDiscoverFeed(input: AddDiscoverFeedInput!): AddDiscoverFeedResult! + addPopularRead(name: String!): AddPopularReadResult! + bulkAction(action: BulkActionType!, arguments: JSON, async: Boolean, expectedCount: Int, labelIds: [ID!], query: String!): BulkActionResult! + createArticle(input: CreateArticleInput!): CreateArticleResult! + createArticleSavingRequest(input: CreateArticleSavingRequestInput!): CreateArticleSavingRequestResult! + createFolderPolicy(input: CreateFolderPolicyInput!): CreateFolderPolicyResult! + createGroup(input: CreateGroupInput!): CreateGroupResult! + createHighlight(input: CreateHighlightInput!): CreateHighlightResult! + createLabel(input: CreateLabelInput!): CreateLabelResult! + createNewsletterEmail(input: CreateNewsletterEmailInput): CreateNewsletterEmailResult! + createPost(input: CreatePostInput!): CreatePostResult! + deleteAccount(userID: ID!): DeleteAccountResult! + deleteDiscoverArticle(input: DeleteDiscoverArticleInput!): DeleteDiscoverArticleResult! + deleteDiscoverFeed(input: DeleteDiscoverFeedInput!): DeleteDiscoverFeedResult! + deleteFilter(id: ID!): DeleteFilterResult! + deleteFolderPolicy(id: ID!): DeleteFolderPolicyResult! + deleteHighlight(highlightId: ID!): DeleteHighlightResult! + deleteIntegration(id: ID!): DeleteIntegrationResult! + deleteLabel(id: ID!): DeleteLabelResult! + deleteNewsletterEmail(newsletterEmailId: ID!): DeleteNewsletterEmailResult! + deletePost(id: ID!): DeletePostResult! + deleteRule(id: ID!): DeleteRuleResult! + deleteWebhook(id: ID!): DeleteWebhookResult! + editDiscoverFeed(input: EditDiscoverFeedInput!): EditDiscoverFeedResult! + emptyTrash: EmptyTrashResult! + exportToIntegration(integrationId: ID!): ExportToIntegrationResult! + fetchContent(id: ID!): FetchContentResult! + generateApiKey(input: GenerateApiKeyInput!): GenerateApiKeyResult! + googleLogin(input: GoogleLoginInput!): LoginResult! + googleSignup(input: GoogleSignupInput!): GoogleSignupResult! + importFromIntegration(integrationId: ID!): ImportFromIntegrationResult! + joinGroup(inviteCode: String!): JoinGroupResult! + leaveGroup(groupId: ID!): LeaveGroupResult! + logOut: LogOutResult! + markEmailAsItem(recentEmailId: ID!): MarkEmailAsItemResult! + mergeHighlight(input: MergeHighlightInput!): MergeHighlightResult! + moveFilter(input: MoveFilterInput!): MoveFilterResult! + moveLabel(input: MoveLabelInput!): MoveLabelResult! + moveToFolder(folder: String!, id: ID!): MoveToFolderResult! + optInFeature(input: OptInFeatureInput!): OptInFeatureResult! + recommend(input: RecommendInput!): RecommendResult! + recommendHighlights(input: RecommendHighlightsInput!): RecommendHighlightsResult! + refreshHome: RefreshHomeResult! + replyToEmail(recentEmailId: ID!, reply: AllowedReply!): ReplyToEmailResult! + reportItem(input: ReportItemInput!): ReportItemResult! + revokeApiKey(id: ID!): RevokeApiKeyResult! + saveArticleReadingProgress(input: SaveArticleReadingProgressInput!): SaveArticleReadingProgressResult! + saveDiscoverArticle(input: SaveDiscoverArticleInput!): SaveDiscoverArticleResult! + saveFile(input: SaveFileInput!): SaveResult! + saveFilter(input: SaveFilterInput!): SaveFilterResult! + savePage(input: SavePageInput!): SaveResult! + saveUrl(input: SaveUrlInput!): SaveResult! + setBookmarkArticle(input: SetBookmarkArticleInput!): SetBookmarkArticleResult! + setDeviceToken(input: SetDeviceTokenInput!): SetDeviceTokenResult! + setFavoriteArticle(id: ID!): SetFavoriteArticleResult! + setIntegration(input: SetIntegrationInput!): SetIntegrationResult! + setLabels(input: SetLabelsInput!): SetLabelsResult! + setLabelsForHighlight(input: SetLabelsForHighlightInput!): SetLabelsResult! + setLinkArchived(input: ArchiveLinkInput!): ArchiveLinkResult! + setRule(input: SetRuleInput!): SetRuleResult! + setUserPersonalization(input: SetUserPersonalizationInput!): SetUserPersonalizationResult! + setWebhook(input: SetWebhookInput!): SetWebhookResult! + subscribe(input: SubscribeInput!): SubscribeResult! + unsubscribe(name: String!, subscriptionId: ID): UnsubscribeResult! + updateEmail(input: UpdateEmailInput!): UpdateEmailResult! + updateFilter(input: UpdateFilterInput!): UpdateFilterResult! + updateFolderPolicy(input: UpdateFolderPolicyInput!): UpdateFolderPolicyResult! + updateHighlight(input: UpdateHighlightInput!): UpdateHighlightResult! + updateLabel(input: UpdateLabelInput!): UpdateLabelResult! + updateNewsletterEmail(input: UpdateNewsletterEmailInput!): UpdateNewsletterEmailResult! + updatePage(input: UpdatePageInput!): UpdatePageResult! + updatePost(input: UpdatePostInput!): UpdatePostResult! + updateSubscription(input: UpdateSubscriptionInput!): UpdateSubscriptionResult! + updateUser(input: UpdateUserInput!): UpdateUserResult! + updateUserProfile(input: UpdateUserProfileInput!): UpdateUserProfileResult! + uploadFileRequest(input: UploadFileRequestInput!): UploadFileRequestResult! + uploadImportFile(contentType: String!, type: UploadImportFileType!): UploadImportFileResult! +} + +type NewsletterEmail { + address: String! + confirmationCode: String + createdAt: Date! + description: String + folder: String! + id: ID! + name: String + subscriptionCount: Int! +} + +type NewsletterEmailsError { + errorCodes: [NewsletterEmailsErrorCode!]! +} + +enum NewsletterEmailsErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union NewsletterEmailsResult = NewsletterEmailsError | NewsletterEmailsSuccess + +type NewsletterEmailsSuccess { + newsletterEmails: [NewsletterEmail!]! +} + +type OptInFeatureError { + errorCodes: [OptInFeatureErrorCode!]! +} + +enum OptInFeatureErrorCode { + BAD_REQUEST + INELIGIBLE + NOT_FOUND +} + +input OptInFeatureInput { + name: String! +} + +union OptInFeatureResult = OptInFeatureError | OptInFeatureSuccess + +type OptInFeatureSuccess { + feature: Feature! +} + +type Page { + author: String + createdAt: Date! + description: String + hash: String! + id: ID! + image: String! + originalHtml: String! + originalUrl: String! + publishedAt: Date + readableHtml: String! + title: String! + type: PageType! + updatedAt: Date + url: String! +} + +type PageInfo { + endCursor: String + hasNextPage: Boolean! + hasPreviousPage: Boolean! + startCursor: String + totalCount: Int +} + +input PageInfoInput { + author: String + canonicalUrl: String + contentType: String + description: String + previewImage: String + publishedAt: Date + title: String +} + +enum PageType { + ARTICLE + BOOK + FILE + HIGHLIGHTS + IMAGE + PROFILE + TWEET + UNKNOWN + VIDEO + WEBSITE +} + +input ParseResult { + byline: String + content: String! + dir: String + excerpt: String! + language: String + length: Int! + previewImage: String + publishedDate: Date + siteIcon: String + siteName: String + textContent: String! + title: String! +} + +type Post { + author: String! + content: String! + createdAt: Date! + highlights: [Highlight!] + id: ID! + libraryItems: [Article!] + ownedByViewer: Boolean! + thought: String + thumbnail: String + title: String! + updatedAt: Date! +} + +type PostEdge { + cursor: String! + node: Post! +} + +type PostError { + errorCodes: [PostErrorCode!]! +} + +enum PostErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union PostResult = PostError | PostSuccess + +type PostSuccess { + post: Post! +} + +type PostsError { + errorCodes: [PostsErrorCode!]! +} + +enum PostsErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union PostsResult = PostsError | PostsSuccess + +type PostsSuccess { + edges: [PostEdge!]! + pageInfo: PageInfo! +} + +input PreparedDocumentInput { + document: String! + pageInfo: PageInfoInput! +} + +type Profile { + bio: String + id: ID! + pictureUrl: String + private: Boolean! + username: String! +} + +type Query { + apiKeys: ApiKeysResult! + article(format: String, slug: String!, username: String!): ArticleResult! + articleSavingRequest(id: ID, url: String): ArticleSavingRequestResult! + deviceTokens: DeviceTokensResult! + discoverFeeds: DiscoverFeedResult! + discoverTopics: GetDiscoverTopicResults! + feeds(input: FeedsInput!): FeedsResult! + filters: FiltersResult! + folderPolicies: FolderPoliciesResult! + getDiscoverFeedArticles(after: String, discoverTopicId: String!, feedId: ID, first: Int): GetDiscoverFeedArticleResults! + getUserPersonalization: GetUserPersonalizationResult! + groups: GroupsResult! + hello: String + hiddenHomeSection: HiddenHomeSectionResult! + highlights(after: String, first: Int, query: String): HighlightsResult! + home(after: String, first: Int): HomeResult! + integration(name: String!): IntegrationResult! + integrations: IntegrationsResult! + labels: LabelsResult! + me: User + newsletterEmails: NewsletterEmailsResult! + post(id: ID!): PostResult! + posts(after: String, first: Int, userId: ID!): PostsResult! + recentEmails: RecentEmailsResult! + recentSearches: RecentSearchesResult! + rules(enabled: Boolean): RulesResult! + scanFeeds(input: ScanFeedsInput!): ScanFeedsResult! + search(after: String, first: Int, format: String, includeContent: Boolean, query: String): SearchResult! + sendInstallInstructions: SendInstallInstructionsResult! + subscription(id: ID!): SubscriptionResult! + subscriptions(sort: SortParams, type: SubscriptionType): SubscriptionsResult! + typeaheadSearch(first: Int, query: String!): TypeaheadSearchResult! + updatesSince(after: String, first: Int, folder: String, since: Date!, sort: SortParams): UpdatesSinceResult! + user(userId: ID, username: String): UserResult! + users: UsersResult! + validateUsername(username: String!): Boolean! + webhook(id: ID!): WebhookResult! + webhooks: WebhooksResult! +} + +type Reaction { + code: ReactionType! + createdAt: Date! + id: ID! + updatedAt: Date + user: User! +} + +enum ReactionType { + CRYING + HEART + HUSHED + LIKE + POUT + SMILE +} + +type ReadState { + progressAnchorIndex: Int! + progressPercent: Float! + reading: Boolean + readingTime: Int +} + +type RecentEmail { + createdAt: Date! + from: String! + html: String + id: ID! + reply: String + replyTo: String + subject: String! + text: String! + to: String! + type: String! +} + +type RecentEmailsError { + errorCodes: [RecentEmailsErrorCode!]! +} + +enum RecentEmailsErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union RecentEmailsResult = RecentEmailsError | RecentEmailsSuccess + +type RecentEmailsSuccess { + recentEmails: [RecentEmail!]! +} + +type RecentSearch { + createdAt: Date! + id: ID! + term: String! +} + +type RecentSearchesError { + errorCodes: [RecentSearchesErrorCode!]! +} + +enum RecentSearchesErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union RecentSearchesResult = RecentSearchesError | RecentSearchesSuccess + +type RecentSearchesSuccess { + searches: [RecentSearch!]! +} + +type RecommendError { + errorCodes: [RecommendErrorCode!]! +} + +enum RecommendErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +type RecommendHighlightsError { + errorCodes: [RecommendHighlightsErrorCode!]! +} + +enum RecommendHighlightsErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input RecommendHighlightsInput { + groupIds: [ID!]! + highlightIds: [ID!]! + note: String + pageId: ID! +} + +union RecommendHighlightsResult = RecommendHighlightsError | RecommendHighlightsSuccess + +type RecommendHighlightsSuccess { + success: Boolean! +} + +input RecommendInput { + groupIds: [ID!]! + note: String + pageId: ID! + recommendedWithHighlights: Boolean +} + +union RecommendResult = RecommendError | RecommendSuccess + +type RecommendSuccess { + success: Boolean! +} + +type Recommendation { + id: ID! + name: String! + note: String + recommendedAt: Date! + user: RecommendingUser +} + +type RecommendationGroup { + admins: [User!]! + canPost: Boolean! + canSeeMembers: Boolean! + createdAt: Date! + description: String + id: ID! + inviteUrl: String! + members: [User!]! + name: String! + topics: [String!] + updatedAt: Date +} + +type RecommendingUser { + name: String! + profileImageURL: String + userId: String! + username: String! +} + +type RefreshHomeError { + errorCodes: [RefreshHomeErrorCode!]! +} + +enum RefreshHomeErrorCode { + PENDING +} + +union RefreshHomeResult = RefreshHomeError | RefreshHomeSuccess + +type RefreshHomeSuccess { + success: Boolean! +} + +type Reminder { + archiveUntil: Boolean! + id: ID! + remindAt: Date! + sendNotification: Boolean! +} + +type ReminderError { + errorCodes: [ReminderErrorCode!]! +} + +enum ReminderErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union ReminderResult = ReminderError | ReminderSuccess + +type ReminderSuccess { + reminder: Reminder! +} + +type ReplyToEmailError { + errorCodes: [ReplyToEmailErrorCode!]! +} + +enum ReplyToEmailErrorCode { + UNAUTHORIZED +} + +union ReplyToEmailResult = ReplyToEmailError | ReplyToEmailSuccess + +type ReplyToEmailSuccess { + success: Boolean! +} + +input ReportItemInput { + itemUrl: String! + pageId: ID! + reportComment: String! + reportTypes: [ReportType!]! + sharedBy: ID +} + +type ReportItemResult { + message: String! +} + +enum ReportType { + ABUSIVE + CONTENT_DISPLAY + CONTENT_VIOLATION + SPAM +} + +enum RepresentationType { + CONTENT + FEED_CONTENT +} + +type RevokeApiKeyError { + errorCodes: [RevokeApiKeyErrorCode!]! +} + +enum RevokeApiKeyErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union RevokeApiKeyResult = RevokeApiKeyError | RevokeApiKeySuccess + +type RevokeApiKeySuccess { + apiKey: ApiKey! +} + +type Rule { + actions: [RuleAction!]! + createdAt: Date! + enabled: Boolean! + eventTypes: [RuleEventType!]! + failedAt: Date + filter: String! + id: ID! + name: String! + updatedAt: Date +} + +type RuleAction { + params: [String!]! + type: RuleActionType! +} + +input RuleActionInput { + params: [String!]! + type: RuleActionType! +} + +enum RuleActionType { + ADD_LABEL + ARCHIVE + DELETE + EXPORT + MARK_AS_READ + SEND_NOTIFICATION + WEBHOOK +} + +enum RuleEventType { + HIGHLIGHT_CREATED + HIGHLIGHT_UPDATED + LABEL_CREATED + PAGE_CREATED + PAGE_UPDATED +} + +type RulesError { + errorCodes: [RulesErrorCode!]! +} + +enum RulesErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union RulesResult = RulesError | RulesSuccess + +type RulesSuccess { + rules: [Rule!]! +} + +type SaveArticleReadingProgressError { + errorCodes: [SaveArticleReadingProgressErrorCode!]! +} + +enum SaveArticleReadingProgressErrorCode { + BAD_DATA + NOT_FOUND + UNAUTHORIZED +} + +input SaveArticleReadingProgressInput { + force: Boolean + id: ID! + readingProgressAnchorIndex: Int + readingProgressPercent: Float! + readingProgressTopPercent: Float +} + +union SaveArticleReadingProgressResult = SaveArticleReadingProgressError | SaveArticleReadingProgressSuccess + +type SaveArticleReadingProgressSuccess { + updatedArticle: Article! +} + +type SaveDiscoverArticleError { + errorCodes: [SaveDiscoverArticleErrorCode!]! +} + +enum SaveDiscoverArticleErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input SaveDiscoverArticleInput { + discoverArticleId: ID! + locale: String + timezone: String +} + +union SaveDiscoverArticleResult = SaveDiscoverArticleError | SaveDiscoverArticleSuccess + +type SaveDiscoverArticleSuccess { + saveId: String! + url: String! +} + +type SaveError { + errorCodes: [SaveErrorCode!]! + message: String +} + +enum SaveErrorCode { + EMBEDDED_HIGHLIGHT_FAILED + UNAUTHORIZED + UNKNOWN +} + +input SaveFileInput { + clientRequestId: ID! + folder: String + labels: [CreateLabelInput!] + publishedAt: Date + savedAt: Date + source: String! + state: ArticleSavingRequestStatus + subscription: String + uploadFileId: ID! + url: String! +} + +type SaveFilterError { + errorCodes: [SaveFilterErrorCode!]! +} + +enum SaveFilterErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input SaveFilterInput { + category: String + description: String + filter: String! + folder: String + name: String! + position: Int +} + +union SaveFilterResult = SaveFilterError | SaveFilterSuccess + +type SaveFilterSuccess { + filter: Filter! +} + +input SavePageInput { + clientRequestId: ID! + folder: String + labels: [CreateLabelInput!] + originalContent: String! + parseResult: ParseResult + publishedAt: Date + rssFeedUrl: String + savedAt: Date + source: String! + state: ArticleSavingRequestStatus + title: String + url: String! +} + +union SaveResult = SaveError | SaveSuccess + +type SaveSuccess { + clientRequestId: ID! + url: String! +} + +input SaveUrlInput { + clientRequestId: ID! + folder: String + labels: [CreateLabelInput!] + locale: String + publishedAt: Date + savedAt: Date + source: String! + state: ArticleSavingRequestStatus + timezone: String + url: String! +} + +type ScanFeedsError { + errorCodes: [ScanFeedsErrorCode!]! +} + +enum ScanFeedsErrorCode { + BAD_REQUEST +} + +input ScanFeedsInput { + opml: String + url: String +} + +union ScanFeedsResult = ScanFeedsError | ScanFeedsSuccess + +type ScanFeedsSuccess { + feeds: [Feed!]! +} + +type SearchError { + errorCodes: [SearchErrorCode!]! +} + +enum SearchErrorCode { + QUERY_TOO_LONG + UNAUTHORIZED +} + +type SearchItem { + aiSummary: String + annotation: String + archivedAt: Date + author: String + color: String + content: String + contentReader: ContentReader! + createdAt: Date! + description: String + directionality: DirectionalityType + feedContent: String + folder: String! + format: String + highlights: [Highlight!] + highlightsCount: Int + id: ID! + image: String + isArchived: Boolean! + labels: [Label!] + language: String + links: JSON + originalArticleUrl: String + ownedByViewer: Boolean + pageId: ID + pageType: PageType! + previewContentType: String + publishedAt: Date + quote: String + readAt: Date + readingProgressAnchorIndex: Int! + readingProgressPercent: Float! + readingProgressTopPercent: Float + recommendations: [Recommendation!] + savedAt: Date! + score: Float + seenAt: Date + shortId: String + siteIcon: String + siteName: String + slug: String! + state: ArticleSavingRequestStatus + subscription: String + title: String! + unsubHttpUrl: String + unsubMailTo: String + updatedAt: Date + uploadFileId: ID + url: String! + wordsCount: Int +} + +type SearchItemEdge { + cursor: String! + node: SearchItem! +} + +union SearchResult = SearchError | SearchSuccess + +type SearchSuccess { + edges: [SearchItemEdge!]! + pageInfo: PageInfo! +} + +type SendInstallInstructionsError { + errorCodes: [SendInstallInstructionsErrorCode!]! +} + +enum SendInstallInstructionsErrorCode { + BAD_REQUEST + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +union SendInstallInstructionsResult = SendInstallInstructionsError | SendInstallInstructionsSuccess + +type SendInstallInstructionsSuccess { + sent: Boolean! +} + +type SetBookmarkArticleError { + errorCodes: [SetBookmarkArticleErrorCode!]! +} + +enum SetBookmarkArticleErrorCode { + BOOKMARK_EXISTS + NOT_FOUND +} + +input SetBookmarkArticleInput { + articleID: ID! + bookmark: Boolean! +} + +union SetBookmarkArticleResult = SetBookmarkArticleError | SetBookmarkArticleSuccess + +type SetBookmarkArticleSuccess { + bookmarkedArticle: Article! +} + +type SetDeviceTokenError { + errorCodes: [SetDeviceTokenErrorCode!]! +} + +enum SetDeviceTokenErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input SetDeviceTokenInput { + id: ID + token: String +} + +union SetDeviceTokenResult = SetDeviceTokenError | SetDeviceTokenSuccess + +type SetDeviceTokenSuccess { + deviceToken: DeviceToken! +} + +type SetFavoriteArticleError { + errorCodes: [SetFavoriteArticleErrorCode!]! +} + +enum SetFavoriteArticleErrorCode { + ALREADY_EXISTS + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union SetFavoriteArticleResult = SetFavoriteArticleError | SetFavoriteArticleSuccess + +type SetFavoriteArticleSuccess { + success: Boolean! +} + +type SetFollowError { + errorCodes: [SetFollowErrorCode!]! +} + +enum SetFollowErrorCode { + NOT_FOUND + UNAUTHORIZED +} + +input SetFollowInput { + follow: Boolean! + userId: ID! +} + +union SetFollowResult = SetFollowError | SetFollowSuccess + +type SetFollowSuccess { + updatedUser: User! +} + +type SetIntegrationError { + errorCodes: [SetIntegrationErrorCode!]! +} + +enum SetIntegrationErrorCode { + ALREADY_EXISTS + BAD_REQUEST + INVALID_TOKEN + NOT_FOUND + UNAUTHORIZED +} + +input SetIntegrationInput { + enabled: Boolean! + id: ID + importItemState: ImportItemState + name: String! + settings: JSON + syncedAt: Date + taskName: String + token: String! + type: IntegrationType +} + +union SetIntegrationResult = SetIntegrationError | SetIntegrationSuccess + +type SetIntegrationSuccess { + integration: Integration! +} + +type SetLabelsError { + errorCodes: [SetLabelsErrorCode!]! +} + +enum SetLabelsErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input SetLabelsForHighlightInput { + highlightId: ID! + labelIds: [ID!] + labels: [CreateLabelInput!] +} + +input SetLabelsInput { + labelIds: [ID!] + labels: [CreateLabelInput!] + pageId: ID! + source: String +} + +union SetLabelsResult = SetLabelsError | SetLabelsSuccess + +type SetLabelsSuccess { + labels: [Label!]! +} + +type SetRuleError { + errorCodes: [SetRuleErrorCode!]! +} + +enum SetRuleErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input SetRuleInput { + actions: [RuleActionInput!]! + description: String + enabled: Boolean! + eventTypes: [RuleEventType!]! + filter: String! + id: ID + name: String! +} + +union SetRuleResult = SetRuleError | SetRuleSuccess + +type SetRuleSuccess { + rule: Rule! +} + +type SetShareArticleError { + errorCodes: [SetShareArticleErrorCode!]! +} + +enum SetShareArticleErrorCode { + NOT_FOUND + UNAUTHORIZED +} + +input SetShareArticleInput { + articleID: ID! + share: Boolean! + sharedComment: String + sharedWithHighlights: Boolean +} + +union SetShareArticleResult = SetShareArticleError | SetShareArticleSuccess + +type SetShareArticleSuccess { + updatedArticle: Article! + updatedFeedArticle: FeedArticle + updatedFeedArticleId: String +} + +type SetShareHighlightError { + errorCodes: [SetShareHighlightErrorCode!]! +} + +enum SetShareHighlightErrorCode { + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +input SetShareHighlightInput { + id: ID! + share: Boolean! +} + +union SetShareHighlightResult = SetShareHighlightError | SetShareHighlightSuccess + +type SetShareHighlightSuccess { + highlight: Highlight! +} + +type SetUserPersonalizationError { + errorCodes: [SetUserPersonalizationErrorCode!]! +} + +enum SetUserPersonalizationErrorCode { + NOT_FOUND + UNAUTHORIZED +} + +input SetUserPersonalizationInput { + digestConfig: DigestConfigInput + fields: JSON + fontFamily: String + fontSize: Int + libraryLayoutType: String + librarySortOrder: SortOrder + margin: Int + speechRate: String + speechSecondaryVoice: String + speechVoice: String + speechVolume: String + theme: String +} + +union SetUserPersonalizationResult = SetUserPersonalizationError | SetUserPersonalizationSuccess + +type SetUserPersonalizationSuccess { + updatedUserPersonalization: UserPersonalization! +} + +type SetWebhookError { + errorCodes: [SetWebhookErrorCode!]! +} + +enum SetWebhookErrorCode { + ALREADY_EXISTS + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input SetWebhookInput { + contentType: String + enabled: Boolean + eventTypes: [WebhookEvent!]! + id: ID + method: String + url: String! +} + +union SetWebhookResult = SetWebhookError | SetWebhookSuccess + +type SetWebhookSuccess { + webhook: Webhook! +} + +type ShareStats { + readDuration: Int! + saveCount: Int! + viewCount: Int! +} + +type SharedArticleError { + errorCodes: [SharedArticleErrorCode!]! +} + +enum SharedArticleErrorCode { + NOT_FOUND +} + +union SharedArticleResult = SharedArticleError | SharedArticleSuccess + +type SharedArticleSuccess { + article: Article! +} + +enum SignupErrorCode { + ACCESS_DENIED + EXPIRED_TOKEN + GOOGLE_AUTH_ERROR + INVALID_EMAIL + INVALID_PASSWORD + INVALID_USERNAME + UNKNOWN + USER_EXISTS +} + +enum SortBy { + PUBLISHED_AT + SAVED_AT + SCORE + UPDATED_TIME +} + +enum SortOrder { + ASCENDING + DESCENDING +} + +input SortParams { + by: SortBy! + order: SortOrder +} + +type SubscribeError { + errorCodes: [SubscribeErrorCode!]! +} + +enum SubscribeErrorCode { + ALREADY_SUBSCRIBED + BAD_REQUEST + EXCEEDED_MAX_SUBSCRIPTIONS + NOT_FOUND + UNAUTHORIZED +} + +input SubscribeInput { + autoAddToLibrary: Boolean + fetchContent: Boolean + fetchContentType: FetchContentType + folder: String + isPrivate: Boolean + subscriptionType: SubscriptionType + url: String! +} + +union SubscribeResult = SubscribeError | SubscribeSuccess + +type SubscribeSuccess { + subscriptions: [Subscription!]! +} + +type Subscription { + autoAddToLibrary: Boolean + count: Int! + createdAt: Date! + description: String + failedAt: Date + fetchContent: Boolean! + fetchContentType: FetchContentType! + folder: String! + icon: String + id: ID! + isPrivate: Boolean + lastFetchedAt: Date + mostRecentItemDate: Date + name: String! + newsletterEmail: String + refreshedAt: Date + status: SubscriptionStatus! + type: SubscriptionType! + unsubscribeHttpUrl: String + unsubscribeMailTo: String + updatedAt: Date + url: String +} + +type SubscriptionError { + errorCodes: [ErrorCode!]! +} + +union SubscriptionResult = SubscriptionError | SubscriptionSuccess + +type SubscriptionRootType { + hello: String +} + +enum SubscriptionStatus { + ACTIVE + DELETED + UNSUBSCRIBED +} + +type SubscriptionSuccess { + subscription: Subscription! +} + +enum SubscriptionType { + NEWSLETTER + RSS +} + +type SubscriptionsError { + errorCodes: [SubscriptionsErrorCode!]! +} + +enum SubscriptionsErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union SubscriptionsResult = SubscriptionsError | SubscriptionsSuccess + +type SubscriptionsSuccess { + subscriptions: [Subscription!]! +} + +type SyncUpdatedItemEdge { + cursor: String! + itemID: ID! + node: SearchItem + updateReason: UpdateReason! +} + +type Task { + cancellable: Boolean + createdAt: Date! + failedReason: String + id: ID! + name: String! + progress: Float + runningTime: Int + state: TaskState! +} + +enum TaskState { + CANCELLED + FAILED + PENDING + RUNNING + SUCCEEDED +} + +type TypeaheadSearchError { + errorCodes: [TypeaheadSearchErrorCode!]! +} + +enum TypeaheadSearchErrorCode { + UNAUTHORIZED +} + +type TypeaheadSearchItem { + contentReader: ContentReader! + id: ID! + siteName: String + slug: String! + title: String! +} + +union TypeaheadSearchResult = TypeaheadSearchError | TypeaheadSearchSuccess + +type TypeaheadSearchSuccess { + items: [TypeaheadSearchItem!]! +} + +type UnsubscribeError { + errorCodes: [UnsubscribeErrorCode!]! +} + +enum UnsubscribeErrorCode { + ALREADY_UNSUBSCRIBED + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED + UNSUBSCRIBE_METHOD_NOT_FOUND +} + +union UnsubscribeResult = UnsubscribeError | UnsubscribeSuccess + +type UnsubscribeSuccess { + subscription: Subscription! +} + +type UpdateEmailError { + errorCodes: [UpdateEmailErrorCode!]! +} + +enum UpdateEmailErrorCode { + BAD_REQUEST + EMAIL_ALREADY_EXISTS + UNAUTHORIZED +} + +input UpdateEmailInput { + email: String! +} + +union UpdateEmailResult = UpdateEmailError | UpdateEmailSuccess + +type UpdateEmailSuccess { + email: String! + verificationEmailSent: Boolean +} + +type UpdateFilterError { + errorCodes: [UpdateFilterErrorCode!]! +} + +enum UpdateFilterErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input UpdateFilterInput { + category: String + description: String + filter: String + folder: String + id: String! + name: String + position: Int + visible: Boolean +} + +union UpdateFilterResult = UpdateFilterError | UpdateFilterSuccess + +type UpdateFilterSuccess { + filter: Filter! +} + +type UpdateFolderPolicyError { + errorCodes: [UpdateFolderPolicyErrorCode!]! +} + +enum UpdateFolderPolicyErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input UpdateFolderPolicyInput { + action: FolderPolicyAction + afterDays: Int + id: ID! +} + +union UpdateFolderPolicyResult = UpdateFolderPolicyError | UpdateFolderPolicySuccess + +type UpdateFolderPolicySuccess { + policy: FolderPolicy! +} + +type UpdateHighlightError { + errorCodes: [UpdateHighlightErrorCode!]! +} + +enum UpdateHighlightErrorCode { + BAD_DATA + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +input UpdateHighlightInput { + annotation: String + color: String + highlightId: ID! + html: String + quote: String + sharedAt: Date +} + +type UpdateHighlightReplyError { + errorCodes: [UpdateHighlightReplyErrorCode!]! +} + +enum UpdateHighlightReplyErrorCode { + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +input UpdateHighlightReplyInput { + highlightReplyId: ID! + text: String! +} + +union UpdateHighlightReplyResult = UpdateHighlightReplyError | UpdateHighlightReplySuccess + +type UpdateHighlightReplySuccess { + highlightReply: HighlightReply! +} + +union UpdateHighlightResult = UpdateHighlightError | UpdateHighlightSuccess + +type UpdateHighlightSuccess { + highlight: Highlight! +} + +type UpdateLabelError { + errorCodes: [UpdateLabelErrorCode!]! +} + +enum UpdateLabelErrorCode { + BAD_REQUEST + FORBIDDEN + NOT_FOUND + UNAUTHORIZED +} + +input UpdateLabelInput { + color: String! + description: String + labelId: ID! + name: String! +} + +union UpdateLabelResult = UpdateLabelError | UpdateLabelSuccess + +type UpdateLabelSuccess { + label: Label! +} + +type UpdateLinkShareInfoError { + errorCodes: [UpdateLinkShareInfoErrorCode!]! +} + +enum UpdateLinkShareInfoErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input UpdateLinkShareInfoInput { + description: String! + linkId: ID! + title: String! +} + +union UpdateLinkShareInfoResult = UpdateLinkShareInfoError | UpdateLinkShareInfoSuccess + +type UpdateLinkShareInfoSuccess { + message: String! +} + +type UpdateNewsletterEmailError { + errorCodes: [UpdateNewsletterEmailErrorCode!]! +} + +enum UpdateNewsletterEmailErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input UpdateNewsletterEmailInput { + description: String + folder: String + id: ID! + name: String +} + +union UpdateNewsletterEmailResult = UpdateNewsletterEmailError | UpdateNewsletterEmailSuccess + +type UpdateNewsletterEmailSuccess { + newsletterEmail: NewsletterEmail! +} + +type UpdatePageError { + errorCodes: [UpdatePageErrorCode!]! +} + +enum UpdatePageErrorCode { + BAD_REQUEST + FORBIDDEN + NOT_FOUND + UNAUTHORIZED + UPDATE_FAILED +} + +input UpdatePageInput { + byline: String + description: String + pageId: ID! + previewImage: String + publishedAt: Date + savedAt: Date + state: ArticleSavingRequestStatus + title: String +} + +union UpdatePageResult = UpdatePageError | UpdatePageSuccess + +type UpdatePageSuccess { + updatedPage: Article! +} + +type UpdatePostError { + errorCodes: [UpdatePostErrorCode!]! +} + +enum UpdatePostErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input UpdatePostInput { + content: String + highlightIds: [ID!] + id: ID! + libraryItemIds: [ID!] + thought: String + thumbnail: String + title: String +} + +union UpdatePostResult = UpdatePostError | UpdatePostSuccess + +type UpdatePostSuccess { + post: Post! +} + +enum UpdateReason { + CREATED + DELETED + UPDATED +} + +type UpdateReminderError { + errorCodes: [UpdateReminderErrorCode!]! +} + +enum UpdateReminderErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input UpdateReminderInput { + archiveUntil: Boolean! + id: ID! + remindAt: Date! + sendNotification: Boolean! +} + +union UpdateReminderResult = UpdateReminderError | UpdateReminderSuccess + +type UpdateReminderSuccess { + reminder: Reminder! +} + +type UpdateSharedCommentError { + errorCodes: [UpdateSharedCommentErrorCode!]! +} + +enum UpdateSharedCommentErrorCode { + NOT_FOUND + UNAUTHORIZED +} + +input UpdateSharedCommentInput { + articleID: ID! + sharedComment: String! +} + +union UpdateSharedCommentResult = UpdateSharedCommentError | UpdateSharedCommentSuccess + +type UpdateSharedCommentSuccess { + articleID: ID! + sharedComment: String! +} + +type UpdateSubscriptionError { + errorCodes: [UpdateSubscriptionErrorCode!]! +} + +enum UpdateSubscriptionErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input UpdateSubscriptionInput { + autoAddToLibrary: Boolean + description: String + failedAt: Date + fetchContent: Boolean + fetchContentType: FetchContentType + folder: String + id: ID! + isPrivate: Boolean + lastFetchedChecksum: String + mostRecentItemDate: Date + name: String + refreshedAt: Date + scheduledAt: Date + status: SubscriptionStatus +} + +union UpdateSubscriptionResult = UpdateSubscriptionError | UpdateSubscriptionSuccess + +type UpdateSubscriptionSuccess { + subscription: Subscription! +} + +type UpdateUserError { + errorCodes: [UpdateUserErrorCode!]! +} + +enum UpdateUserErrorCode { + BIO_TOO_LONG + EMPTY_NAME + UNAUTHORIZED + USER_NOT_FOUND +} + +input UpdateUserInput { + bio: String + name: String! +} + +type UpdateUserProfileError { + errorCodes: [UpdateUserProfileErrorCode!]! +} + +enum UpdateUserProfileErrorCode { + BAD_DATA + BAD_USERNAME + FORBIDDEN + UNAUTHORIZED + USERNAME_EXISTS +} + +input UpdateUserProfileInput { + bio: String + pictureUrl: String + userId: ID! + username: String +} + +union UpdateUserProfileResult = UpdateUserProfileError | UpdateUserProfileSuccess + +type UpdateUserProfileSuccess { + user: User! +} + +union UpdateUserResult = UpdateUserError | UpdateUserSuccess + +type UpdateUserSuccess { + user: User! +} + +type UpdatesSinceError { + errorCodes: [UpdatesSinceErrorCode!]! +} + +enum UpdatesSinceErrorCode { + UNAUTHORIZED +} + +union UpdatesSinceResult = UpdatesSinceError | UpdatesSinceSuccess + +type UpdatesSinceSuccess { + edges: [SyncUpdatedItemEdge!]! + pageInfo: PageInfo! +} + +type UploadFileRequestError { + errorCodes: [UploadFileRequestErrorCode!]! +} + +enum UploadFileRequestErrorCode { + BAD_INPUT + FAILED_CREATE + UNAUTHORIZED +} + +input UploadFileRequestInput { + clientRequestId: String + contentType: String! + createPageEntry: Boolean + url: String! +} + +union UploadFileRequestResult = UploadFileRequestError | UploadFileRequestSuccess + +type UploadFileRequestSuccess { + createdPageId: String + id: ID! + uploadFileId: ID + uploadSignedUrl: String +} + +enum UploadFileStatus { + COMPLETED + INITIALIZED +} + +type UploadImportFileError { + errorCodes: [UploadImportFileErrorCode!]! +} + +enum UploadImportFileErrorCode { + BAD_REQUEST + UNAUTHORIZED + UPLOAD_DAILY_LIMIT_EXCEEDED +} + +union UploadImportFileResult = UploadImportFileError | UploadImportFileSuccess + +type UploadImportFileSuccess { + uploadSignedUrl: String +} + +enum UploadImportFileType { + MATTER + POCKET + URL_LIST +} + +type User { + createdAt: Date! + email: String + featureList: [Feature!] + features: [String] + followersCount: Int + friendsCount: Int + id: ID! + intercomHash: String + isFriend: Boolean @deprecated(reason: "isFriend has been replaced with viewerIsFollowing") + isFullUser: Boolean + name: String! + picture: String + profile: Profile! + sharedArticles: [FeedArticle!]! + sharedArticlesCount: Int + sharedHighlightsCount: Int + sharedNotesCount: Int + source: String + viewerIsFollowing: Boolean +} + +type UserError { + errorCodes: [UserErrorCode!]! +} + +enum UserErrorCode { + BAD_REQUEST + UNAUTHORIZED + USER_NOT_FOUND +} + +type UserPersonalization { + digestConfig: DigestConfig + fields: JSON + fontFamily: String + fontSize: Int + id: ID + libraryLayoutType: String + librarySortOrder: SortOrder + margin: Int + speechRate: String + speechSecondaryVoice: String + speechVoice: String + speechVolume: String + theme: String +} + +union UserResult = UserError | UserSuccess + +type UserSuccess { + user: User! +} + +type UsersError { + errorCodes: [UsersErrorCode!]! +} + +enum UsersErrorCode { + UNAUTHORIZED +} + +union UsersResult = UsersError | UsersSuccess + +type UsersSuccess { + users: [User!]! +} + +type Webhook { + contentType: String! + createdAt: Date! + enabled: Boolean! + eventTypes: [WebhookEvent!]! + id: ID! + method: String! + updatedAt: Date + url: String! +} + +type WebhookError { + errorCodes: [WebhookErrorCode!]! +} + +enum WebhookErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +enum WebhookEvent { + HIGHLIGHT_CREATED + HIGHLIGHT_DELETED + HIGHLIGHT_UPDATED + LABEL_CREATED + LABEL_DELETED + LABEL_UPDATED + PAGE_CREATED + PAGE_DELETED + PAGE_UPDATED +} + +union WebhookResult = WebhookError | WebhookSuccess + +type WebhookSuccess { + webhook: Webhook! +} + +type WebhooksError { + errorCodes: [WebhooksErrorCode!]! +} + +enum WebhooksErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union WebhooksResult = WebhooksError | WebhooksSuccess + +type WebhooksSuccess { + webhooks: [Webhook!]! +} diff --git a/self-hosting/omc/esbuild.config.mjs b/self-hosting/omc/esbuild.config.mjs new file mode 100755 index 000000000..11a347e0c --- /dev/null +++ b/self-hosting/omc/esbuild.config.mjs @@ -0,0 +1,77 @@ +#!/usr/bin/env node +/** + * ESBuild Configuration for OMC CLI + * AIDEV-NOTE: Bundles OCLIF CLI with ESM support and path aliases + */ + +import { build } from 'esbuild'; +import { glob } from 'glob'; +import { copyFile, mkdir } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +// Find all command files +const commandFiles = await glob('src/commands/**/*.ts', { cwd: __dirname }); +const libFiles = await glob('src/lib/**/*.ts', { cwd: __dirname }); +const storageFiles = await glob('src/storage/**/*.ts', { cwd: __dirname }); +const rootLibFiles = await glob('lib/**/*.js', { cwd: __dirname }); + +// Build configuration +const buildConfig = { + entryPoints: [ + 'bin/omc.ts', + 'index.ts', + ...commandFiles, + ...libFiles, + ...storageFiles, + ...rootLibFiles, + ], + bundle: true, + format: 'esm', + platform: 'node', + target: 'node18', + outdir: 'dist', + packages: 'external', + splitting: true, + sourcemap: true, + // Path alias resolution + alias: { + '@lib': path.join(__dirname, 'src/lib'), + '@storage': path.join(__dirname, 'src/storage'), + '@analysis': path.join(__dirname, 'src/analysis'), + '@generation': path.join(__dirname, 'src/generation'), + '@publishing': path.join(__dirname, 'src/publishing'), + '@workflows': path.join(__dirname, 'src/workflows'), + '@utils': path.join(__dirname, 'src/utils'), + '@omc-types': path.join(__dirname, 'src/types'), + }, + // Preserve directory structure + outbase: '.', +}; + +try { + await build(buildConfig); + + // Copy schema files to dist (needed by chunk files) + const schemaDir = path.join(__dirname, 'dist/schema'); + await mkdir(schemaDir, { recursive: true }); + await copyFile( + path.join(__dirname, 'src/storage/schema/tracking-schema.sql'), + path.join(schemaDir, 'tracking-schema.sql') + ); + + // Copy analysis prompt files (used at runtime by ContentAnalyzer) + const promptDir = path.join(__dirname, 'dist/src/analysis/prompts'); + await mkdir(promptDir, { recursive: true }); + await copyFile( + path.join(__dirname, 'src/analysis/prompts/analyze-article.md'), + path.join(promptDir, 'analyze-article.md') + ); + + console.log('✅ Build complete'); +} catch (error) { + console.error('❌ Build failed:', error); + process.exit(1); +} diff --git a/self-hosting/omc/index.ts b/self-hosting/omc/index.ts new file mode 100755 index 000000000..d553395a6 --- /dev/null +++ b/self-hosting/omc/index.ts @@ -0,0 +1,9 @@ +#!/usr/bin/env node +/** + * OMC Main CLI Entry Point + * AIDEV-NOTE: OCLIF CLI runner for OMC commands + */ + +import { run } from '@oclif/core'; + +await run(process.argv.slice(2), import.meta.url); diff --git a/self-hosting/omc/lib/omnivore/client.d.ts b/self-hosting/omc/lib/omnivore/client.d.ts new file mode 100644 index 000000000..d57fb599a --- /dev/null +++ b/self-hosting/omc/lib/omnivore/client.d.ts @@ -0,0 +1,96 @@ +export interface OmnivoreHighlight { + id: string; + quote: string; + annotation?: string | null; + createdAt: string; + updatedAt?: string; + type?: string; +} + +export interface OmnivoreLabel { + id: string; + name: string; + color: string; +} + +export interface OmnivoreArticle { + id: string; + title: string; + url: string; + content?: string | null; + description?: string | null; + author?: string | null; + publishedAt?: string | null; + slug?: string; + wordCount?: number; + createdAt: string; + savedAt: string; + updatedAt: string; + isArchived?: boolean; + folder?: string; + labels: OmnivoreLabel[]; + highlights: OmnivoreHighlight[]; +} + +export interface GetMeResult { + id: string; + name?: string; + email?: string; + profile?: { username?: string }; +} + +export interface GetArticleResult { + article?: OmnivoreArticle; + errorCodes?: string[]; +} + +export function getMe(): Promise; +export function searchArticles(args?: Record): Promise; +export function getArticle(slug: string, username: string): Promise; +export function getArticlesByDate(args?: Record): Promise; +export function getArticlesByLabel(labelName: string, first?: number): Promise; +export function getRecentArticles(hours?: number, first?: number): Promise; +export function searchByTopic(topic: string, first?: number): Promise; +export function getUnreadArticles(first?: number): Promise; +export function getLabels(): Promise; +export function getHighlights(slug: string, username: string): Promise; + +export function updatePage(args: { + pageId: string; + description?: string; + title?: string; + byline?: string; + publishedAt?: string; + savedAt?: string; +}): Promise; + +export function createLabel(args: { + name: string; + color?: string; + description?: string; +}): Promise; + +export function setLabels(args: { + pageId: string; + labelIds?: string[]; + labels?: Array<{ name: string; color?: string; description?: string }>; + source?: string; +}): Promise; + +export function saveUrl(args: { + url: string; + source: string; + clientRequestId: string; + folder?: string; + savedAt?: string; + publishedAt?: string; + labels?: Array<{ name: string; color?: string; description?: string }>; + timezone?: string; + locale?: string; + state?: string; +}): Promise; + +export function createHighlight(args: Record): Promise; +export function updateHighlight(args: Record): Promise; +export function deleteHighlight(highlightId: string): Promise; +export function testConnection(): Promise; diff --git a/self-hosting/omc/lib/omnivore/client.js b/self-hosting/omc/lib/omnivore/client.js new file mode 100644 index 000000000..92c7d3b01 --- /dev/null +++ b/self-hosting/omc/lib/omnivore/client.js @@ -0,0 +1,585 @@ +#!/usr/bin/env node + +/** + * Omnivore API Client + * GraphQL client for interacting with Omnivore API + */ + +import fetch from 'node-fetch'; +import { config } from 'dotenv'; + +// Load environment variables +config(); + +const API_URL = process.env.OMNIVORE_API_URL || 'https://api-prod.omnivore.app/api/graphql'; +const API_KEY = process.env.OMNIVORE_API_KEY; + +if (!API_KEY) { + console.error('❌ OMNIVORE_API_KEY not set in environment'); + process.exit(1); +} + +/** + * Execute GraphQL query against Omnivore API + */ +async function graphqlRequest(query, variables = {}) { + try { + const response = await fetch(API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + // Omnivore self-hosted commonly expects `Omnivore-Authorization`, while some deployments accept `Authorization`. + // Sending both keeps the CLI compatible across environments. + 'Authorization': API_KEY, + 'Omnivore-Authorization': API_KEY, + }, + body: JSON.stringify({ query, variables }), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const result = await response.json(); + + if (result.errors) { + throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`); + } + + return result.data; + } catch (error) { + console.error('GraphQL Request Error:', error.message); + throw error; + } +} + +/** + * Get current authenticated user + */ +export async function getMe() { + const query = ` + query { + me { + id + name + email + profile { + username + } + } + } + `; + + const data = await graphqlRequest(query); + return data.me; +} + +/** + * Search articles with filters + */ +export async function searchArticles({ + query = 'in:all', + first = 10, + after = '', + includeContent = false, +} = {}) { + const searchQuery = ` + query Search($query: String!, $first: Int, $after: String, $includeContent: Boolean) { + search(query: $query, first: $first, after: $after, includeContent: $includeContent) { + ... on SearchSuccess { + pageInfo { + totalCount + hasNextPage + endCursor + } + edges { + cursor + node { + id + slug + title + url + originalArticleUrl + createdAt + updatedAt + publishedAt + savedAt + author + description + image + siteName + pageType + wordsCount + readingProgressTopPercent + isArchived + folder + content + labels { + id + name + color + } + highlights { + id + quote + annotation + createdAt + } + } + } + } + ... on SearchError { + errorCodes + } + } + } + `; + + const data = await graphqlRequest(searchQuery, { query, first, after, includeContent }); + + if (data.search.errorCodes) { + throw new Error(`Search Error: ${data.search.errorCodes.join(', ')}`); + } + + return data.search; +} + +/** + * Get article by ID + */ +export async function getArticle(slug, username) { + const query = ` + query GetArticle($slug: String!, $username: String!) { + article(slug: $slug, username: $username) { + ... on ArticleSuccess { + article { + id + title + url + content + author + description + publishedAt + updatedAt + createdAt + savedAt + highlights(input: {}) { + id + quote + annotation + } + labels { + id + name + color + } + } + } + ... on ArticleError { + errorCodes + } + } + } + `; + + const data = await graphqlRequest(query, { slug, username }); + return data.article; +} + +/** + * Get articles saved in a time range + */ +export async function getArticlesByDate({ + startDate, + endDate, + first = 100, +} = {}) { + const start = startDate || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); + const end = endDate || new Date().toISOString(); + + const query = `saved:>${start.split('T')[0]} saved:<${end.split('T')[0]} sort:saved-desc`; + + return await searchArticles({ query, first, includeContent: false }); +} + +/** + * Get articles with specific labels + */ +export async function getArticlesByLabel(labelName, first = 50) { + const query = `label:${labelName} sort:saved-desc`; + return await searchArticles({ query, first }); +} + +/** + * Get recently saved articles + */ +export async function getRecentArticles(hours = 24, first = 50) { + let query; + + if (hours <= 24) { + query = 'saved:last24hrs'; + } else if (hours <= 168) { + query = 'saved:last7days'; + } else { + const days = Math.ceil(hours / 24); + const date = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString().split('T')[0]; + query = `saved:>${date}`; + } + + query += ' sort:saved-desc'; + return await searchArticles({ query, first, includeContent: false }); +} + +/** + * Get articles by topic/keyword + */ +export async function searchByTopic(topic, first = 50) { + const query = `${topic} sort:saved-desc`; + return await searchArticles({ query, first, includeContent: false }); +} + +/** + * Get unread articles + */ +export async function getUnreadArticles(first = 50) { + const query = 'in:inbox is:unread sort:saved-desc'; + return await searchArticles({ query, first }); +} + +/** + * Get all labels + */ +export async function getLabels() { + const query = ` + query { + labels { + ... on LabelsSuccess { + labels { + id + name + color + description + createdAt + } + } + ... on LabelsError { + errorCodes + } + } + } + `; + + const data = await graphqlRequest(query); + + if (data.labels.errorCodes) { + throw new Error(`Labels Error: ${data.labels.errorCodes.join(', ')}`); + } + + return data.labels.labels; +} + +/** + * Get highlights for an article + */ +export async function getHighlights(slug, username) { + const query = ` + query GetHighlights($slug: String!, $username: String!) { + article(slug: $slug, username: $username) { + ... on ArticleSuccess { + article { + highlights { + id + quote + annotation + createdAt + updatedAt + type + } + } + } + ... on ArticleError { + errorCodes + } + } + } + `; + + const data = await graphqlRequest(query, { slug, username }); + return data.article?.article?.highlights || []; +} + +/** + * Update article/page metadata + */ +export async function updatePage({ pageId, description, title, byline, publishedAt, savedAt }) { + const mutation = ` + mutation UpdatePage($input: UpdatePageInput!) { + updatePage(input: $input) { + ... on UpdatePageSuccess { + updatedPage { + id + title + description + author + } + } + ... on UpdatePageError { + errorCodes + } + } + } + `; + + const input = { pageId }; + if (description !== undefined) input.description = description; + if (title !== undefined) input.title = title; + if (byline !== undefined) input.byline = byline; + if (publishedAt !== undefined) input.publishedAt = publishedAt; + if (savedAt !== undefined) input.savedAt = savedAt; + + const data = await graphqlRequest(mutation, { input }); + return data.updatePage; +} + +/** + * Create a label. + */ +export async function createLabel({ name, color, description }) { + const mutation = ` + mutation CreateLabel($input: CreateLabelInput!) { + createLabel(input: $input) { + ... on CreateLabelSuccess { + label { + id + name + color + description + createdAt + } + } + ... on CreateLabelError { + errorCodes + } + } + } + `; + + const input = { name }; + if (color !== undefined) input.color = color; + if (description !== undefined) input.description = description; + + const data = await graphqlRequest(mutation, { input }); + return data.createLabel; +} + +/** + * Set labels for a page (replaces existing labels). + * Use either `labelIds` or `labels` (CreateLabelInput[]). If `labels` is used, Omnivore may create missing labels. + */ +export async function setLabels({ pageId, labelIds, labels, source }) { + const mutation = ` + mutation SetLabels($input: SetLabelsInput!) { + setLabels(input: $input) { + ... on SetLabelsSuccess { + labels { + id + name + color + } + } + ... on SetLabelsError { + errorCodes + } + } + } + `; + + const input = { pageId }; + if (labelIds !== undefined) input.labelIds = labelIds; + if (labels !== undefined) input.labels = labels; + if (source !== undefined) input.source = source; + + const data = await graphqlRequest(mutation, { input }); + return data.setLabels; +} + +/** + * Save a URL into Omnivore (creates a new library item / saving request). + */ +export async function saveUrl({ url, source, folder, savedAt, publishedAt, labels, timezone, locale, state, clientRequestId }) { + const mutation = ` + mutation SaveUrl($input: SaveUrlInput!) { + saveUrl(input: $input) { + ... on SaveSuccess { + url + clientRequestId + } + ... on SaveError { + errorCodes + } + } + } + `; + + const input = { url, source, clientRequestId }; + if (folder !== undefined) input.folder = folder; + if (savedAt !== undefined) input.savedAt = savedAt; + if (publishedAt !== undefined) input.publishedAt = publishedAt; + if (labels !== undefined) input.labels = labels; + if (timezone !== undefined) input.timezone = timezone; + if (locale !== undefined) input.locale = locale; + if (state !== undefined) input.state = state; + + const data = await graphqlRequest(mutation, { input }); + return data.saveUrl; +} + +/** + * Create highlight with annotation (or standalone NOTE) + */ +export async function createHighlight({ id, shortId, articleId, quote = '', annotation, patch = '', prefix = '', suffix = '', color = '#FFD700', type = 'HIGHLIGHT' }) { + const mutation = ` + mutation CreateHighlight($input: CreateHighlightInput!) { + createHighlight(input: $input) { + ... on CreateHighlightSuccess { + highlight { + id + quote + annotation + color + type + createdAt + } + } + ... on CreateHighlightError { + errorCodes + } + } + } + `; + + const input = { + id, + shortId, + articleId, + patch, + prefix, + suffix, + color, + type + }; + + if (quote) input.quote = quote; + if (annotation) input.annotation = annotation; + + const data = await graphqlRequest(mutation, { input }); + return data.createHighlight; +} + +/** + * Update highlight/note annotation + */ +export async function updateHighlight({ highlightId, annotation, quote, html, color }) { + const mutation = ` + mutation UpdateHighlight($input: UpdateHighlightInput!) { + updateHighlight(input: $input) { + ... on UpdateHighlightSuccess { + highlight { + id + annotation + updatedAt + } + } + ... on UpdateHighlightError { + errorCodes + } + } + } + `; + + const input = { highlightId }; + if (annotation !== undefined) input.annotation = annotation; + if (quote !== undefined) input.quote = quote; + if (html !== undefined) input.html = html; + if (color !== undefined) input.color = color; + + const data = await graphqlRequest(mutation, { input }); + return data.updateHighlight; +} + +/** + * Delete highlight/note + */ +export async function deleteHighlight(highlightId) { + const mutation = ` + mutation DeleteHighlight($highlightId: ID!) { + deleteHighlight(highlightId: $highlightId) { + ... on DeleteHighlightSuccess { + highlight { + id + } + } + ... on DeleteHighlightError { + errorCodes + } + } + } + `; + + const data = await graphqlRequest(mutation, { highlightId }); + return data.deleteHighlight; +} + +/** + * Test API connection + */ +export async function testConnection() { + try { + const user = await getMe(); + console.log('✅ Connected to Omnivore API'); + console.log(` User: ${user.name} (${user.email})`); + console.log(` Username: ${user.profile.username}`); + return true; + } catch (error) { + console.error('❌ Failed to connect to Omnivore API'); + console.error(` ${error.message}`); + return false; + } +} + +// CLI usage +if (import.meta.url === `file://${process.argv[1]}`) { + const args = process.argv.slice(2); + + if (args.includes('--test')) { + await testConnection(); + } else if (args.includes('--recent')) { + const hours = parseInt(args[args.indexOf('--recent') + 1]) || 24; + const result = await getRecentArticles(hours); + console.log(`Found ${result.pageInfo.totalCount} articles from last ${hours} hours`); + result.edges.forEach(({ node }) => { + console.log(` - ${node.title}`); + console.log(` ${node.url}`); + console.log(` Saved: ${node.savedAt}`); + }); + } else { + console.log('Usage:'); + console.log(' node client.js --test # Test API connection'); + console.log(' node client.js --recent [hours] # Get recent articles'); + } +} + +export default { + graphqlRequest, + getMe, + searchArticles, + getArticle, + getArticlesByDate, + getArticlesByLabel, + getRecentArticles, + searchByTopic, + getUnreadArticles, + getLabels, + getHighlights, + testConnection, +}; diff --git a/self-hosting/omc/lib/omnivore/queries.js b/self-hosting/omc/lib/omnivore/queries.js new file mode 100644 index 000000000..a508230fe --- /dev/null +++ b/self-hosting/omc/lib/omnivore/queries.js @@ -0,0 +1,369 @@ +/** + * Pre-built GraphQL Queries for Omnivore API + * Collection of common query patterns for content analysis + */ + +/** + * Search query with all fields needed for content analysis + */ +export const SEARCH_ARTICLES_FULL = ` + query SearchArticlesFull($query: String!, $first: Int, $after: String) { + search(query: $query, first: $first, after: $after, includeContent: false) { + ... on SearchSuccess { + pageInfo { + totalCount + hasNextPage + endCursor + } + edges { + cursor + node { + id + title + url + originalArticleUrl + slug + createdAt + updatedAt + publishedAt + savedAt + author + description + image + siteName + pageType + wordCount + readingProgressTopPercent + readingProgressBottomPercent + isArchived + folder + labels { + id + name + color + } + highlights { + id + quote + annotation + createdAt + } + } + } + } + ... on SearchError { + errorCodes + } + } + } +`; + +/** + * Get article with full content + */ +export const GET_ARTICLE_FULL = ` + query GetArticleFull($slug: String!, $username: String!) { + article(slug: $slug, username: $username) { + ... on ArticleSuccess { + article { + id + title + url + slug + content + author + description + image + publishedAt + createdAt + savedAt + updatedAt + wordCount + siteName + originalArticleUrl + highlights { + id + quote + annotation + createdAt + } + labels { + id + name + color + description + } + } + } + ... on ArticleError { + errorCodes + } + } + } +`; + +/** + * Get user profile + */ +export const GET_USER_PROFILE = ` + query GetUserProfile { + me { + id + name + email + profile { + username + pictureUrl + bio + } + } + } +`; + +/** + * Get all labels + */ +export const GET_LABELS = ` + query GetLabels { + labels { + ... on LabelsSuccess { + labels { + id + name + color + description + createdAt + } + } + ... on LabelsError { + errorCodes + } + } + } +`; + +/** + * Search with content included (for analysis) + */ +export const SEARCH_WITH_CONTENT = ` + query SearchWithContent($query: String!, $first: Int, $after: String) { + search(query: $query, first: $first, after: $after, includeContent: true) { + ... on SearchSuccess { + pageInfo { + totalCount + hasNextPage + endCursor + } + edges { + cursor + node { + id + title + url + content + author + description + publishedAt + savedAt + siteName + labels { + name + } + highlights { + quote + annotation + } + } + } + } + ... on SearchError { + errorCodes + } + } + } +`; + +/** + * Common search query patterns + */ +export const QUERY_PATTERNS = { + // Time-based + LAST_24_HOURS: 'saved:last24hrs sort:saved-desc', + LAST_7_DAYS: 'saved:last7days sort:saved-desc', + LAST_30_DAYS: 'saved:last30days sort:saved-desc', + + // Status-based + INBOX: 'in:inbox sort:saved-desc', + ARCHIVED: 'in:archive sort:saved-desc', + UNREAD: 'is:unread sort:saved-desc', + READ: 'is:read sort:saved-desc', + + // Type-based + ARTICLES: 'type:article sort:saved-desc', + HIGHLIGHTS: 'type:highlights sort:saved-desc', + + // Content-based + WITH_HIGHLIGHTS: 'has:highlights sort:saved-desc', + WITH_LABELS: 'has:labels sort:saved-desc', + NO_LABELS: 'no:label sort:saved-desc', + + // Topic-based (AI/Tech) + AI_ML: '(ai OR "machine learning" OR llm OR "language model") sort:saved-desc', + DEVOPS: '(devops OR kubernetes OR docker OR "cloud native") sort:saved-desc', + DATABASES: '(database OR postgres OR mongodb OR redis) sort:saved-desc', + PROGRAMMING: '(programming OR coding OR "software engineering") sort:saved-desc', + STARTUPS: '(startup OR founder OR "venture capital" OR product) sort:saved-desc', + + // Combined patterns + RECENT_AI: 'saved:last7days (ai OR llm OR "machine learning") sort:saved-desc', + RECENT_TECH: 'saved:last7days (programming OR devops OR database) sort:saved-desc', + UNREAD_ARTICLES: 'in:inbox is:unread type:article sort:saved-desc', +}; + +/** + * Build date range query + */ +export function buildDateRangeQuery(startDate, endDate, sortBy = 'saved-desc') { + const start = typeof startDate === 'string' ? startDate : startDate.toISOString().split('T')[0]; + const end = typeof endDate === 'string' ? endDate : endDate.toISOString().split('T')[0]; + + return `saved:>${start} saved:<${end} sort:${sortBy}`; +} + +/** + * Build label query + */ +export function buildLabelQuery(labels, operator = 'OR') { + if (Array.isArray(labels)) { + return labels.map(l => `label:${l}`).join(` ${operator} `); + } + return `label:${labels}`; +} + +/** + * Build keyword query + */ +export function buildKeywordQuery(keywords, operator = 'OR') { + if (Array.isArray(keywords)) { + const quoted = keywords.map(k => `"${k}"`); + return `(${quoted.join(` ${operator} `)})`; + } + return `"${keywords}"`; +} + +/** + * Build complex query + */ +export function buildComplexQuery({ + keywords = [], + labels = [], + timeRange = 'last7days', + status = null, + hasHighlights = null, + sortBy = 'saved-desc', +} = {}) { + const parts = []; + + // Time range + if (timeRange === 'last24hrs' || timeRange === 'last7days' || timeRange === 'last30days') { + parts.push(`saved:${timeRange}`); + } else if (timeRange.includes('>') || timeRange.includes('<')) { + parts.push(timeRange); + } + + // Keywords + if (keywords.length > 0) { + parts.push(buildKeywordQuery(keywords)); + } + + // Labels + if (labels.length > 0) { + parts.push(buildLabelQuery(labels)); + } + + // Status + if (status) { + if (status === 'inbox') parts.push('in:inbox'); + else if (status === 'archived') parts.push('in:archive'); + else if (status === 'unread') parts.push('is:unread'); + else if (status === 'read') parts.push('is:read'); + } + + // Highlights + if (hasHighlights === true) parts.push('has:highlights'); + else if (hasHighlights === false) parts.push('no:highlights'); + + // Sort + if (sortBy) parts.push(`sort:${sortBy}`); + + return parts.join(' '); +} + +/** + * Topic-based query builders + */ +export const TOPIC_QUERIES = { + ai: { + keywords: ['ai', 'artificial intelligence', 'machine learning', 'llm', 'language model', 'gpt', 'claude', 'neural network'], + description: 'AI and Machine Learning', + }, + devops: { + keywords: ['devops', 'kubernetes', 'docker', 'ci/cd', 'cloud native', 'infrastructure', 'terraform'], + description: 'DevOps and Infrastructure', + }, + programming: { + keywords: ['programming', 'coding', 'software engineering', 'algorithm', 'data structure', 'clean code'], + description: 'Programming and Software Engineering', + }, + databases: { + keywords: ['database', 'sql', 'nosql', 'postgres', 'mongodb', 'redis', 'query optimization'], + description: 'Databases and Data Engineering', + }, + web: { + keywords: ['web development', 'javascript', 'typescript', 'react', 'vue', 'frontend', 'backend'], + description: 'Web Development', + }, + cloud: { + keywords: ['aws', 'azure', 'gcp', 'cloud computing', 'serverless', 'lambda'], + description: 'Cloud Computing', + }, + startup: { + keywords: ['startup', 'founder', 'entrepreneurship', 'venture capital', 'product market fit', 'growth'], + description: 'Startups and Business', + }, + security: { + keywords: ['security', 'cybersecurity', 'authentication', 'encryption', 'vulnerability'], + description: 'Security', + }, +}; + +/** + * Build query for specific topic + */ +export function buildTopicQuery(topicKey, timeRange = 'last7days') { + const topic = TOPIC_QUERIES[topicKey]; + if (!topic) { + throw new Error(`Unknown topic: ${topicKey}`); + } + + return buildComplexQuery({ + keywords: topic.keywords, + timeRange, + sortBy: 'saved-desc', + }); +} + +export default { + SEARCH_ARTICLES_FULL, + GET_ARTICLE_FULL, + GET_USER_PROFILE, + GET_LABELS, + SEARCH_WITH_CONTENT, + QUERY_PATTERNS, + buildDateRangeQuery, + buildLabelQuery, + buildKeywordQuery, + buildComplexQuery, + buildTopicQuery, + TOPIC_QUERIES, +}; diff --git a/self-hosting/omc/package.json b/self-hosting/omc/package.json new file mode 100644 index 000000000..4f0e83815 --- /dev/null +++ b/self-hosting/omc/package.json @@ -0,0 +1,92 @@ +{ + "name": "omnivore-content-system", + "version": "1.0.0", + "description": "AI-powered content monetization system for Omnivore reading - generates blog posts, newsletters, and tracks trends", + "packageManager": "pnpm@10.14.0", + "type": "module", + "bin": { + "omc": "./dist/bin/omc.js" + }, + "scripts": { + "build": "node esbuild.config.mjs", + "prebuild": "rm -rf dist", + "build:watch": "tsc -p tsconfig.json --watch", + "codegen": "graphql-codegen --config codegen.yml", + "codegen:watch": "graphql-codegen --config codegen.yml --watch", + "dev": "tsx watch index.ts", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest --watch", + "fetch": "corepack pnpm -s run build && node dist/bin/omc.js queue add --hours 24", + "analyze:auto": "corepack pnpm -s run build && node dist/bin/omc.js analyze auto --batch-size 5", + "analyze:retry": "corepack pnpm -s run build && node dist/bin/omc.js analyze retry --failed", + "report:corpus": "corepack pnpm -s run build && node dist/bin/omc.js report corpus" + }, + "keywords": [ + "omnivore", + "content", + "blog", + "newsletter", + "ai", + "claude", + "agent" + ], + "author": "", + "license": "MIT", + "dependencies": { + "@anthropic-ai/claude-agent-sdk": "^0.1.1", + "@anthropic-ai/sdk": "^0.20.0", + "@graphql-typed-document-node/core": "^3.2.0", + "@oclif/core": "^3.26.6", + "@oclif/plugin-help": "^6.0.21", + "better-sqlite3": "^12.4.1", + "chalk": "^5.3.0", + "csv-parse": "^6.1.0", + "dotenv": "^16.4.0", + "glob": "^11.0.3", + "graphql": "^16.11.0", + "graphql-tag": "^2.12.6", + "gray-matter": "^4.0.3", + "markdown-it": "^14.0.0", + "node-fetch": "^3.3.2", + "p-limit": "^5.0.0" + }, + "devDependencies": { + "@graphql-codegen/cli": "^6.0.0", + "@graphql-codegen/typed-document-node": "^6.0.1", + "@graphql-codegen/typescript": "^5.0.1", + "@graphql-codegen/typescript-operations": "^5.0.1", + "@types/better-sqlite3": "^7.6.13", + "@types/node": "^20.0.0", + "esbuild": "^0.19.0", + "nodemon": "^3.0.0", + "tsx": "^4.0.0", + "typescript": "^5.4.0", + "vitest": "^1.0.0" + }, + "engines": { + "node": ">=18.0.0" + }, + "oclif": { + "commands": "./dist/src/commands", + "bin": "omc", + "dirname": "omc", + "plugins": [ + "@oclif/plugin-help" + ], + "topicSeparator": " ", + "additionalHelpFlags": [ + "-h" + ], + "additionalVersionFlags": [ + "-v" + ] + }, + "pnpm": { + "onlyBuiltDependencies": [ + "better-sqlite3", + "esbuild" + ] + } +} diff --git a/self-hosting/omc/scripts/cleanup-duplicate-notes.mjs b/self-hosting/omc/scripts/cleanup-duplicate-notes.mjs new file mode 100755 index 000000000..42444cbbd --- /dev/null +++ b/self-hosting/omc/scripts/cleanup-duplicate-notes.mjs @@ -0,0 +1,47 @@ +#!/usr/bin/env node +import { execSync } from 'child_process'; +import { getHighlights, deleteHighlight, getMe } from '../lib/omnivore/client.js'; +import Database from 'better-sqlite3'; + +const articleId = process.argv[2]; +if (!articleId) { + console.error('Usage: node scripts/cleanup-duplicate-notes.mjs '); + process.exit(1); +} + +// Use CLI to get article slug +const db = Database(process.env.DATABASE_PATH || 'data/omnivore-content.db'); +const job = db.prepare('SELECT article_slug FROM analysis_queue WHERE article_id = ?').get(articleId); +db.close(); + +if (!job) { + console.error(`Article not found: ${articleId}`); + process.exit(1); +} + +// Get username from Omnivore API +const user = await getMe(); +const username = user.profile.username; + +const highlights = await getHighlights(job.article_slug, username); +const notes = highlights.filter(h => h.type === 'NOTE'); + +console.log(`Found ${notes.length} notes total`); + +if (notes.length > 0) { + console.log(`\nFirst note content (first 500 chars):`); + console.log(notes[0].annotation?.substring(0, 500)); +} + +const analysisNotes = notes.filter(h => h.annotation?.includes('_Analysis generated by Omnivore Content System_')); +const duplicates = notes.filter(h => !h.annotation?.includes('_Analysis generated by Omnivore Content System_')); + +console.log(`\nAnalysis notes with marker: ${analysisNotes.length}`); +console.log(`Duplicate notes to delete: ${duplicates.length}`); + +for (const note of duplicates) { + console.log(`Deleting note ${note.id}...`); + await deleteHighlight(note.id); +} + +console.log('✓ Cleanup complete'); diff --git a/self-hosting/omc/scripts/daily-analysis.sh b/self-hosting/omc/scripts/daily-analysis.sh new file mode 100755 index 000000000..0c7f5223f --- /dev/null +++ b/self-hosting/omc/scripts/daily-analysis.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_DIR="${REPO_DIR:-/Volumes/devel/personal/omnivore-content-system}" +BATCH_SIZE="${BATCH_SIZE:-5}" +HOURS="${HOURS:-24}" + +cd "$REPO_DIR" + +mkdir -p content/corpus-reports + +# Ensure dependencies/build are available. Prefer Corepack-managed pnpm so native modules +# (better-sqlite3) are built for the active Node version. +if command -v corepack >/dev/null 2>&1; then + corepack install >/dev/null 2>&1 || true +fi + +corepack pnpm -s run build + +node dist/bin/omc.js queue add --hours "$HOURS" +node dist/bin/omc.js analyze auto --batch-size "$BATCH_SIZE" +node dist/bin/omc.js analyze retry --failed + +node dist/bin/omc.js report corpus > "content/corpus-reports/$(date +%Y-%m-%d)-daily.md" + diff --git a/self-hosting/omc/src/analysis/ContentAnalyzer.ts b/self-hosting/omc/src/analysis/ContentAnalyzer.ts new file mode 100644 index 000000000..1a5c2a2a6 --- /dev/null +++ b/self-hosting/omc/src/analysis/ContentAnalyzer.ts @@ -0,0 +1,85 @@ +import { readFileSync, existsSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import type { AnalysisRequest, ContentAnalysis } from '@omc-types/analysis.js'; +import { analyzeArticleWithCodexCli } from '@lib/ai/codex-cli-client.js'; + +export interface AnalyzeArticleInput extends AnalysisRequest { + articleId: string; +} + +export interface ContentAnalyzerOptions { + promptPath?: string; + model?: string; + timeoutMs?: number; + maxAttempts?: number; +} + +export class ContentAnalyzer { + private readonly prompt: string; + private readonly options: ContentAnalyzerOptions; + + constructor(options: ContentAnalyzerOptions = {}) { + this.options = options; + this.prompt = loadPromptText(options.promptPath); + } + + async analyzeArticle(input: AnalyzeArticleInput): Promise { + const { articleId, ...request } = input; + const { analysis } = await analyzeArticleWithCodexCli(request, this.prompt, { + model: this.options.model, + timeoutMs: this.options.timeoutMs, + maxAttempts: this.options.maxAttempts, + }); + return normalizeAnalysis(articleId, analysis); + } +} + +function loadPromptText(explicitPath?: string): string { + const candidates = [ + explicitPath, + process.env.OMC_ANALYZE_PROMPT_PATH, + promptPathNearThisFile(), + join(process.cwd(), 'src/analysis/prompts/analyze-article.md'), + ] + .filter(Boolean) as string[]; + for (const p of candidates) { + if (existsSync(p)) return readFileSync(p, 'utf-8'); + } + throw new Error('Could not locate analyze-article.md prompt (set OMC_ANALYZE_PROMPT_PATH)'); +} + +function promptPathNearThisFile(): string { + const here = dirname(fileURLToPath(import.meta.url)); + return join(here, 'prompts', 'analyze-article.md'); +} + +function normalizeAnalysis(articleId: string, analysis: ContentAnalysis): ContentAnalysis { + return { + ...fillDefaults(analysis), + articleId, + analyzedAt: analysis.analyzedAt || new Date().toISOString(), + }; +} + +function fillDefaults(analysis: ContentAnalysis): ContentAnalysis { + return { + ...analysis, + topics: analysis.topics ?? [], + topicScores: analysis.topicScores ?? {}, + keyPoints: analysis.keyPoints ?? [], + contentType: analysis.contentType || 'N/A', + problemStatement: analysis.problemStatement || 'N/A', + audienceLevel: analysis.audienceLevel || 'N/A', + technologiesMentioned: analysis.technologiesMentioned ?? [], + companiesMentioned: analysis.companiesMentioned ?? [], + peopleMentioned: analysis.peopleMentioned ?? [], + conceptsExplained: analysis.conceptsExplained ?? [], + relatedTechnologies: analysis.relatedTechnologies ?? [], + useCases: analysis.useCases ?? [], + targetKeywords: analysis.targetKeywords ?? [], + searchQuestions: analysis.searchQuestions ?? [], + githubRepo: analysis.githubRepo || 'N/A', + releaseInfo: analysis.releaseInfo || 'N/A', + }; +} diff --git a/self-hosting/omc/src/analysis/analyze-auto-runner.ts b/self-hosting/omc/src/analysis/analyze-auto-runner.ts new file mode 100644 index 000000000..11144ca07 --- /dev/null +++ b/self-hosting/omc/src/analysis/analyze-auto-runner.ts @@ -0,0 +1,160 @@ +import { existsSync, mkdirSync, writeFileSync, unlinkSync } from 'node:fs'; +import { join } from 'node:path'; +import type { AnalysisJob, AnalysisQueueRepository } from '@storage/AnalysisQueueRepository.js'; +import type { ContentAnalyzer, AnalyzeArticleInput } from '@analysis/ContentAnalyzer.js'; +import type { AnalysisWriter } from '@storage/AnalysisWriter.js'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; +import type { OmnivoreArticle } from '@omc-types/omnivore.js'; +import { getArticle } from '@lib/omnivore/client.js'; +import type { AnalysisJsonlRecord } from '@storage/AnalysisWriter.js'; + +export interface AnalyzeAutoRunnerDeps { + jobs: AnalysisJob[]; + username: string; + analyzer: ContentAnalyzer; + writer: AnalysisWriter; + repo: AnalysisQueueRepository; + keepTemp: boolean; + jsonlPath?: string; +} + +export interface AnalyzeAutoRunnerResult { + saved: number; + failed: number; + skipped: number; +} + +export async function runAnalyzeAuto(deps: AnalyzeAutoRunnerDeps): Promise { + ensureTempDir(); + const totals = { saved: 0, failed: 0, skipped: 0 }; + for (const job of deps.jobs) { + const outcome = await runOne(job, deps); + totals[outcome]++; + } + return totals; +} + +async function runOne(job: AnalysisJob, deps: AnalyzeAutoRunnerDeps): Promise<'saved' | 'failed' | 'skipped'> { + const article = await fetchArticle(job.articleSlug, deps.username); + if (!article) return markFailure(deps.repo, job.articleId, 'Failed to fetch article from Omnivore'); + if (!article.content) return markFailure(deps.repo, job.articleId, 'Omnivore returned no content for article'); + + const input = buildAnalyzeInput(job.articleId, article); + const analysis = await deps.analyzer.analyzeArticle(input); + const tempPath = writeTempRecord(job, article, deps.username, analysis); + return await saveAndMarkComplete(job, article, analysis, tempPath, deps); +} + +async function fetchArticle(slug: string, username: string): Promise { + try { + const result = await getArticle(slug, username); + if (result.errorCodes?.length) return null; + return result.article ?? null; + } catch { + return null; + } +} + +function buildAnalyzeInput(articleId: string, article: OmnivoreArticle): AnalyzeArticleInput { + const content = String(article.content ?? ''); + return { + articleId, + title: article.title, + author: article.author ?? undefined, + url: article.url, + content, + wordCount: estimateWordCount(content), + highlights: (article.highlights ?? []).map((h) => ({ + quote: h.quote, + annotation: h.annotation ?? undefined, + })), + publishedAt: article.publishedAt ?? undefined, + }; +} + +function writeTempRecord(job: AnalysisJob, article: OmnivoreArticle, username: string, analysis: ContentAnalysis): string { + const path = join('temp', `${job.articleSlug}.jsonl`); + const record = { + articleId: job.articleId, + articleSlug: job.articleSlug, + username, + articleUrl: article.url, + articleTitle: article.title, + savedAt: article.savedAt, + publishedAt: article.publishedAt ?? null, + updatedAt: article.updatedAt ?? null, + analysis, + }; + writeFileSync(path, JSON.stringify(record) + '\n', 'utf-8'); + return path; +} + +async function saveAndMarkComplete( + job: AnalysisJob, + article: OmnivoreArticle, + analysis: ContentAnalysis, + tempPath: string, + deps: AnalyzeAutoRunnerDeps +): Promise<'saved' | 'failed'> { + try { + const mdPath = await deps.writer.write(job.articleId, article.url, article.title, article.savedAt, analysis, job.articleSlug); + deps.repo.storeAnalysis(job.articleId, article.publishedAt ?? null, article.updatedAt ?? null, JSON.stringify(analysis), mdPath); + if (deps.jsonlPath) { + await deps.writer.appendToJsonl(deps.jsonlPath, buildJsonlRecord(job, article, analysis, mdPath)); + } + if (!deps.keepTemp) unlinkSync(tempPath); + return 'saved'; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + deps.repo.markFailed(job.articleId, `Save failed: ${message}`); + return 'failed'; + } +} + +function buildJsonlRecord(job: AnalysisJob, article: OmnivoreArticle, analysis: ContentAnalysis, markdownPath: string): AnalysisJsonlRecord { + return { + articleId: job.articleId, + articleSlug: job.articleSlug, + articleUrl: article.url, + articleTitle: article.title, + savedAt: article.savedAt, + publishedAt: article.publishedAt ?? null, + updatedAt: article.updatedAt ?? null, + markdownPath, + analyzedAt: analysis.analyzedAt, + topics: analysis.topics, + topicScores: analysis.topicScores, + sentiment: analysis.sentiment, + summary: analysis.summary, + keyPoints: analysis.keyPoints, + monetizationAngle: analysis.monetizationAngle, + contentType: analysis.contentType, + problemStatement: analysis.problemStatement, + audienceLevel: analysis.audienceLevel, + technologiesMentioned: analysis.technologiesMentioned, + companiesMentioned: analysis.companiesMentioned, + peopleMentioned: analysis.peopleMentioned, + conceptsExplained: analysis.conceptsExplained, + relatedTechnologies: analysis.relatedTechnologies, + useCases: analysis.useCases, + targetKeywords: analysis.targetKeywords, + searchQuestions: analysis.searchQuestions, + githubRepo: analysis.githubRepo, + releaseInfo: analysis.releaseInfo, + }; +} + +function markFailure(repo: AnalysisQueueRepository, articleId: string, message: string): 'failed' { + repo.markFailed(articleId, message); + return 'failed'; +} + +function ensureTempDir(): void { + if (!existsSync('temp')) mkdirSync('temp', { recursive: true }); +} + +function estimateWordCount(text: string): number { + const trimmed = text.trim(); + if (!trimmed) return 0; + return trimmed.split(/\s+/).length; +} diff --git a/self-hosting/omc/src/analysis/prompts/analyze-article.md b/self-hosting/omc/src/analysis/prompts/analyze-article.md new file mode 100644 index 000000000..0c835a617 --- /dev/null +++ b/self-hosting/omc/src/analysis/prompts/analyze-article.md @@ -0,0 +1,129 @@ +# Article Content Analysis Prompt + +You are an elite content strategist and knowledge graph architect specializing in evidence-based article analysis for content monetization. Your dual expertise lies in extracting monetizable insights while building structured metadata for corpus-wide knowledge linking. + +## Primary Directive: Anti-Hallucination Rule + +This rule applies to EVERY field you extract - it is non-negotiable: + +- If information is NOT explicitly present in the article, use "N/A" (strings) or ["N/A"] (arrays) +- DO NOT infer, guess, estimate, or use your training data +- Only extract what is directly stated in the article text +- When uncertain, always use "N/A" - data integrity trumps completeness + +**Why this matters**: Hallucinated data corrupts the knowledge graph and misleads content strategy. "N/A" enables accurate corpus analysis; invented data causes systematic failures. + +## Task + +Analyze the provided article content and extract structured insights in JSON format. You are PURELY ADDITIVE - you will receive article metadata and ONLY add the `analysis` field. + +## Analysis Fields + +### Core Analysis (Always Required) + +**topics** (array, 2-5 items): Main topics from approved categories: +- AI & Machine Learning, Developer Tools, Software Engineering, DevOps & Infrastructure +- Databases, Cloud, Startups & Business, Security + +**topicScores** (object): Confidence 0-1 for each topic +- 0.90-1.0: Core focus +- 0.70-0.89: Significant discussion +- 0.50-0.69: Mentioned but not central +- <0.50: Exclude + +**summary** (string, 2-3 sentences): Focus on "so what?" - why this matters to developers/tech professionals. **Chain-of-thought**: (1) What's the main point? (2) Why does it matter? (3) What's the implication? + +**keyPoints** (array, 3-5 items): Actionable insights, surprising facts, or important takeaways readers can learn or apply. Not generic observations. + +**sentiment** (string): "positive" (optimistic, solutions-focused), "neutral" (informative, balanced), "negative" (critical, problems-focused) + +**monetizationAngle** (string): Specific content opportunity - comparison post, tutorial, weekly roundup, deep dive, contrarian take + +### Content Planning (Evidence-Based) + +**contentType** (string): Open-ended description of article type - "getting started guide", "comparison review", "release announcement", "case study", "technical deep dive", "news article", "tool documentation", etc. Use "N/A" if unclear. + +**problemStatement** (string): What specific problem does article address? Use "N/A" if not explicitly stated. + +**audienceLevel** (string): Based on technical depth - "beginner", "intermediate", "advanced", or "N/A" if unclear + +### Knowledge Graph (Corpus Linking) + +Extract ONLY what's explicitly mentioned - these fields enable future corpus-wide analysis and RAG: + +**technologiesMentioned** (array): Specific tools, frameworks, languages named (e.g., ["Ray", "Python", "Dask"]) + +**companiesMentioned** (array): Organizations, companies mentioned (e.g., ["Anyscale", "OpenAI"]) + +**peopleMentioned** (array): Notable people if relevant (founders, researchers, etc.), else ["N/A"] + +**conceptsExplained** (array): Technical concepts or techniques explained (e.g., ["distributed computing", "actor model"]) + +**relatedTechnologies** (array): Technologies this compares to or builds upon (e.g., ["Spark", "Dask"]) + +**useCases** (array): Specific scenarios described (e.g., ["ML training", "data processing"]) + +### SEO Signals (Article-Based Only) + +**targetKeywords** (array): Keywords that appear emphasized/repeated in article, or ["N/A"] if none obvious + +**searchQuestions** (array): If article answers specific questions, list them, else ["N/A"] + +### Trend Signals (If Present) + +**githubRepo** (string): GitHub URL if article is about/links to repo, else "N/A" + +**releaseInfo** (string): Version/release info if article announces one, else "N/A" + +## Output Format + +**Input stub (what you receive):** +```json +{"articleId":"...","articleSlug":"...","username":"...","articleUrl":"...","articleTitle":"...","savedAt":"...","publishedAt":"...","updatedAt":"..."} +``` + +**Output (what you return):** +```json +{"articleId":"...","articleSlug":"...","username":"...","articleUrl":"...","articleTitle":"...","savedAt":"...","publishedAt":"...","updatedAt":"...","analysis":{"topics":[],"topicScores":{},"summary":"","keyPoints":[],"sentiment":"","monetizationAngle":"","contentType":"","problemStatement":"","audienceLevel":"","technologiesMentioned":[],"companiesMentioned":[],"peopleMentioned":[],"conceptsExplained":[],"relatedTechnologies":[],"useCases":[],"targetKeywords":[],"searchQuestions":[],"githubRepo":"","releaseInfo":"","analyzedAt":"ISO-timestamp"}} +``` + +**What you do**: Add the `analysis` field ONLY. All other fields are pass-through from stub. + +## Decision-Making Framework + +**For each field, ask**: +1. Is this information explicitly stated in the article? +2. If yes: Extract verbatim or in standardized form +3. If no: Use "N/A" or ["N/A"] +4. If uncertain: Default to "N/A" + +**Quality checks**: +- All topic labels from approved categories? +- Summary captures "so what?", not just "what?" +- Key points actionable and specific? +- Knowledge graph fields evidence-based? +- No hallucinated dates, companies, or technologies? + +You are precise, evidence-driven, and committed to data integrity over completeness. + +## Instructions for LLM Integration + +When using this prompt: + +1. **Load article content**: Fetch the full article text from your content source +2. **Provide article metadata**: Pass in the stub JSON with article metadata +3. **Request analysis**: Ask the LLM to analyze the content and return the complete JSON with added `analysis` field +4. **Validate output**: Ensure the returned JSON is valid and contains all required fields +5. **Save result**: Write the enriched JSON to your output destination + +**Example request format**: +``` +Using the prompt from analyze-article.md, analyze this article: + +[Article content here] + +Article metadata: +[Stub JSON here] + +Return the complete JSON with analysis field added. +``` diff --git a/self-hosting/omc/src/analysis/prompts/analyze.md b/self-hosting/omc/src/analysis/prompts/analyze.md new file mode 100644 index 000000000..dc13a57a6 --- /dev/null +++ b/self-hosting/omc/src/analysis/prompts/analyze.md @@ -0,0 +1,147 @@ +# Article Content Analysis Prompt + +You are analyzing an article for content monetization opportunities. Extract structured insights that can be used to create valuable blog posts, newsletters, and social media content. + +⚠️ **CRITICAL ANTI-HALLUCINATION RULE** + +This rule applies to EVERY field in your analysis: +- If information is NOT explicitly present in the article content, use "N/A" (for strings) or ["N/A"] (for arrays) +- DO NOT infer, guess, estimate, or use knowledge from your training data +- Only extract information that is directly stated in the article text +- When in doubt, use "N/A" - it's better to have missing data than invented data + +## Article Details + +**Title**: {{title}} +**Author**: {{author}} +**URL**: {{url}} +**Word Count**: {{wordCount}} +**Published**: {{publishedAt}} + +**Content**: +{{content}} + +{{#if highlights}} +**User Highlights**: +{{#each highlights}} +- "{{quote}}"{{#if annotation}} — Note: {{annotation}}{{/if}} +{{/each}} +{{/if}} + +## Analysis Task + +Extract the following information and return as JSON: + +```json +{ + "topics": ["topic1", "topic2"], + "topicScores": { + "topic1": 0.95, + "topic2": 0.88 + }, + "summary": "2-3 sentence summary capturing main points and why this matters", + "keyPoints": [ + "First key takeaway", + "Second key takeaway", + "Third key takeaway" + ], + "sentiment": "positive|neutral|negative", + "monetizationAngle": "How to turn this into valuable content", + + "contentType": "tutorial|comparison|announcement|case-study|deep-dive|news|tool-review|other", + "publishedDate": "2024-03-15 OR N/A", + "updatedDate": "2024-03-20 OR N/A", + "problemStatement": "What problem does this article address OR N/A", + "audienceLevel": "beginner|intermediate|advanced OR N/A", + + "technologiesMentioned": ["Ray", "Python", "Dask"], + "companiesMentioned": ["Anyscale", "OpenAI"], + "peopleMentioned": ["author name if notable"], + "conceptsExplained": ["distributed computing", "actor model"], + "relatedTechnologies": ["Spark", "Dask"], + "useCases": ["ML training", "data processing"], + + "targetKeywords": ["distributed python", "ray framework"], + "searchQuestions": ["how to scale python workloads"], + + "githubRepo": "https://github.com/org/repo OR N/A", + "releaseInfo": "v2.0 released OR N/A" +} +``` + +## Guidelines + +### Topics (2-5 main topics) +Prioritize these categories that align with the content strategy: +- **AI & Machine Learning**: ai, machine-learning, llm, neural-networks, training, inference +- **Developer Tools**: developer-tools, ide, debugging, testing, ci-cd +- **Software Engineering**: software-engineering, architecture, design-patterns, code-quality +- **DevOps & Infrastructure**: devops, kubernetes, docker, cloud-native, infrastructure +- **Databases**: databases, sql, nosql, data-modeling, performance +- **Cloud**: cloud, aws, azure, gcp, serverless +- **Startups & Business**: startups, product, growth, monetization, business-strategy +- **Security**: security, auth, encryption, vulnerabilities + +### Topic Scores (0-1 confidence) +- 0.90-1.0: Core focus of article +- 0.70-0.89: Significant discussion +- 0.50-0.69: Mentioned/relevant but not central +- Below 0.50: Don't include + +### Summary +Focus on the "so what?" - why does this matter to developers/tech professionals? + +### Key Points +Extract actionable insights, surprising facts, or important takeaways. Focus on what readers can learn or apply. + +### Sentiment +- **positive**: Optimistic, solutions-focused, celebrates innovation +- **neutral**: Informative, balanced, educational +- **negative**: Critical, identifies problems, warnings + +### Monetization Angle +Identify content opportunities: +- "Compare with 2-3 similar tools for comparison post" +- "Tutorial on implementing this technique" +- "Weekly roundup: combine with X other articles on topic Y" +- "Deep dive: expand on concept Z with 10+ articles" +- "Contrarian take: why this approach has limitations" + +### Content Type +Describe what kind of article this is using open-ended language: +- "getting started guide", "comparison review", "release announcement", "case study", "technical deep dive", "news article", "tool documentation", etc. +- Use N/A if unclear from content + +### Dates +- Extract publishedDate and updatedDate if shown in article (use ISO format YYYY-MM-DD) +- Use "N/A" if dates not present + +### Problem Statement +- What specific problem does the article address? +- Use N/A if not explicitly stated + +### Audience Level +- Based on technical depth: "beginner", "intermediate", "advanced" +- Use N/A if unclear + +### Knowledge Graph Fields +Extract ONLY what's explicitly mentioned: +- **technologiesMentioned**: Specific tools, frameworks, languages named in article +- **companiesMentioned**: Organizations, companies mentioned +- **peopleMentioned**: Notable people mentioned (founders, researchers, etc.) - use N/A if none +- **conceptsExplained**: Technical concepts or techniques explained +- **relatedTechnologies**: Technologies this compares to or builds upon +- **useCases**: Specific use cases or scenarios described + +### SEO Signals +- **targetKeywords**: Keywords that appear emphasized/repeated in article (N/A if none obvious) +- **searchQuestions**: If article answers specific questions, list them (N/A if not question-format) + +### Trend Signals +- **githubRepo**: If article is about or links to a GitHub repository +- **releaseInfo**: If article announces a new version or release +- Use N/A for both if not applicable + +## Output Format + +Return ONLY valid JSON matching the schema above. No markdown code blocks, no additional text. diff --git a/self-hosting/omc/src/commands/analyze/auto.ts b/self-hosting/omc/src/commands/analyze/auto.ts new file mode 100644 index 000000000..108b66d2c --- /dev/null +++ b/self-hosting/omc/src/commands/analyze/auto.ts @@ -0,0 +1,128 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { fetchUsername } from '@lib/cli/graphql.js'; +import { formatHeader, formatSuccess } from '@lib/cli/formatters.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { displayQueueStats } from '@lib/cli/queue-display.js'; +import { AnalysisWriter } from '@storage/AnalysisWriter.js'; +import type { AnalysisJob, QueueStats, AnalysisQueueRepository } from '@storage/AnalysisQueueRepository.js'; +import { ContentAnalyzer } from '@analysis/ContentAnalyzer.js'; +import { runAnalyzeAuto } from '@analysis/analyze-auto-runner.js'; + +interface AnalyzeAutoFlags { + 'batch-size': number; + 'article-id'?: string; + all: boolean; + model?: string; + 'timeout-ms': number; + 'keep-temp': boolean; + jsonl: boolean; + 'jsonl-path': string; + json: boolean; +} + +export default class AnalyzeAuto extends BaseCommand { + static override description = 'Analyze queued articles end-to-end (non-interactive)'; + + static override examples = [ + '$ omc analyze auto --batch-size 5', + '$ omc analyze auto --article-id abc123', + '$ omc analyze auto --json', + ]; + + static override flags = { + 'batch-size': Flags.integer({ + char: 'b', + description: 'Number of articles to analyze', + default: 5, + }), + 'article-id': Flags.string({ + description: 'Analyze specific article by ID', + exclusive: ['all'], + }), + all: Flags.boolean({ + description: 'Analyze pending + failed jobs (up to batch size)', + exclusive: ['article-id'], + }), + model: Flags.string({ + description: 'Codex model override (passed to codex exec -m)', + }), + 'timeout-ms': Flags.integer({ + description: 'Timeout per article analysis (milliseconds)', + default: 10 * 60 * 1000, + }), + 'keep-temp': Flags.boolean({ + description: 'Keep temp/*.jsonl files after successful save', + default: false, + }), + jsonl: Flags.boolean({ + description: 'Append each completed analysis to content/analysis/analyses.jsonl', + default: false, + }), + 'jsonl-path': Flags.string({ + description: 'Path to JSONL output (requires --jsonl)', + default: 'content/analysis/analyses.jsonl', + }), + json: jsonFlag(), + }; + + protected async execute(flags: AnalyzeAutoFlags): Promise { + return await withDatabase(async ({ repo }) => { + const jobs = this.selectJobs(repo, flags); + if (jobs.length === 0) return this.outputEmpty(flags.json); + + if (!flags.json) { + this.log(formatHeader('Analysis Queue Status')); + displayQueueStats(repo.getStats()); + } + + this.markJobsInProgress(repo, jobs); + + const username = await fetchUsername(); + const analyzer = new ContentAnalyzer({ model: flags.model, timeoutMs: flags['timeout-ms'] }); + const writer = new AnalysisWriter({ outputDir: 'content/analysis' }); + + const jsonlPath = flags.jsonl ? flags['jsonl-path'] : undefined; + const results = await runAnalyzeAuto({ jobs, username, analyzer, writer, repo, keepTemp: flags['keep-temp'], jsonlPath }); + this.outputResults(results, repo.getStats(), flags.json); + }); + } + + private selectJobs(repo: AnalysisQueueRepository, flags: AnalyzeAutoFlags): AnalysisJob[] { + if (flags['article-id']) { + const job = repo.getByArticleId(flags['article-id']); + if (!job) throw new Error(`Article with ID ${flags['article-id']} not found`); + return [job]; + } + if (flags.all) { + const combined = [...repo.getByStatus('pending'), ...repo.getByStatus('failed')]; + return combined.slice(0, flags['batch-size'] ?? combined.length); + } + return repo.getPending(flags['batch-size']); + } + + private outputEmpty(jsonMode: boolean): void { + if (jsonMode) this.log(JSON.stringify({ saved: 0, failed: 0, skipped: 0 }, null, 2)); + else this.log('No jobs to process'); + } + + private markJobsInProgress(repo: AnalysisQueueRepository, jobs: AnalysisJob[]): void { + for (const job of jobs) repo.markInProgress(job.articleId); + } + + private outputResults(results: { saved: number; failed: number; skipped: number }, stats: QueueStats, jsonMode: boolean): void { + if (jsonMode) { + this.log(JSON.stringify({ ...results, stats }, null, 2)); + return; + } + this.log(''); + this.log(formatSuccess(`Saved ${results.saved} article(s)`)); + if (results.failed) this.log(`Failed: ${results.failed}`); + if (results.skipped) this.log(`Skipped: ${results.skipped}`); + this.log(''); + this.log(formatHeader('Updated Queue Status')); + displayQueueStats(stats); + } + +} diff --git a/self-hosting/omc/src/commands/analyze/complete.ts b/self-hosting/omc/src/commands/analyze/complete.ts new file mode 100644 index 000000000..b60dafaef --- /dev/null +++ b/self-hosting/omc/src/commands/analyze/complete.ts @@ -0,0 +1,212 @@ +import { Flags } from '@oclif/core'; +import { readFileSync, unlinkSync, existsSync } from 'node:fs'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader, formatSuccess } from '@lib/cli/formatters.js'; +import { parseJsonSafely } from '@lib/cli/command-utils.js'; +import { displayQueueStats } from '@lib/cli/queue-display.js'; +import { AnalysisWriter } from '@storage/AnalysisWriter.js'; +import { glob } from 'glob'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; +import type { AnalysisJsonlRecord } from '@storage/AnalysisWriter.js'; +import type { AnalysisQueueRepository, QueueStats } from '@storage/AnalysisQueueRepository.js'; + +interface EnrichedResult { + articleId: string; + articleSlug: string; + username: string; + articleUrl: string; + articleTitle: string; + savedAt: string; + publishedAt: string | null; + updatedAt: string | null; + analysis?: ContentAnalysis; +} + +interface AnalyzeCompleteFlags { + 'keep-temp': boolean; + jsonl: boolean; + 'jsonl-path': string; + json: boolean; +} + +interface SaveResults { + saved: number; + failed: number; + stats: QueueStats; +} + +function buildJsonlRecord(result: EnrichedResult, analysis: ContentAnalysis, markdownPath: string): AnalysisJsonlRecord { + return { + articleId: result.articleId, + articleSlug: result.articleSlug, + articleUrl: result.articleUrl, + articleTitle: result.articleTitle, + savedAt: result.savedAt, + publishedAt: result.publishedAt, + updatedAt: result.updatedAt, + markdownPath, + analyzedAt: analysis.analyzedAt, + topics: analysis.topics, + topicScores: analysis.topicScores, + sentiment: analysis.sentiment, + summary: analysis.summary, + keyPoints: analysis.keyPoints, + monetizationAngle: analysis.monetizationAngle, + contentType: analysis.contentType, + problemStatement: analysis.problemStatement, + audienceLevel: analysis.audienceLevel, + technologiesMentioned: analysis.technologiesMentioned, + companiesMentioned: analysis.companiesMentioned, + peopleMentioned: analysis.peopleMentioned, + conceptsExplained: analysis.conceptsExplained, + relatedTechnologies: analysis.relatedTechnologies, + useCases: analysis.useCases, + targetKeywords: analysis.targetKeywords, + searchQuestions: analysis.searchQuestions, + githubRepo: analysis.githubRepo, + releaseInfo: analysis.releaseInfo, + }; +} + +/** + * OCLIF command: omc analyze complete + * Saves agent-analyzed results from temp/ to database and marks jobs complete. + * AIDEV-NOTE: analysis-complete - final step in analyze workflow after agents finish + */ +export default class AnalyzeComplete extends BaseCommand { + static override description = 'Save analyzed results and mark jobs complete'; + + static override examples = [ + '$ omc analyze complete', + '$ omc analyze complete --keep-temp', + ]; + + static override flags = { + 'keep-temp': Flags.boolean({ + description: 'Keep temp files after saving', + default: false, + }), + jsonl: Flags.boolean({ + description: 'Append each completed analysis to content/analysis/analyses.jsonl', + default: false, + }), + 'jsonl-path': Flags.string({ + description: 'Path to JSONL output (requires --jsonl)', + default: 'content/analysis/analyses.jsonl', + }), + json: jsonFlag(), + }; + + protected async execute(flags: AnalyzeCompleteFlags): Promise { + this.validateFlags(flags); + const files = await this.findAnalyzedFiles(); + + if (files.length === 0) { + this.log('No analyzed files found in temp/'); + return; + } + + await withDatabase(async ({ repo }) => { + const writer = new AnalysisWriter({ outputDir: 'content/analysis' }); + const results = await this.saveResults(files, repo, writer, { + keepTemp: flags['keep-temp'], + writeJsonl: flags.jsonl, + jsonlPath: flags['jsonl-path'], + }); + this.displayResults(results, flags.json); + }); + } + + private validateFlags(flags: AnalyzeCompleteFlags): void { + if (flags['jsonl-path'] !== 'content/analysis/analyses.jsonl' && !flags.jsonl) { + throw new Error('--jsonl-path requires --jsonl'); + } + } + + private async findAnalyzedFiles(): Promise { + const allFiles = await glob('temp/*.jsonl'); + return allFiles.filter((file) => { + if (!existsSync(file)) return false; + const content = readFileSync(file, 'utf-8').trim(); + if (!content) return false; + const data = parseJsonSafely(content); + return data?.analysis !== undefined; + }); + } + + private async saveResults( + files: string[], + repo: AnalysisQueueRepository, + writer: AnalysisWriter, + options: { keepTemp: boolean; writeJsonl: boolean; jsonlPath: string } + ): Promise { + let saved = 0; + let failed = 0; + + for (const file of files) { + const success = await this.saveFile(file, repo, writer, options); + if (success) saved++; + else failed++; + } + + return { saved, failed, stats: repo.getStats() }; + } + + private async saveFile( + file: string, + repo: AnalysisQueueRepository, + writer: AnalysisWriter, + options: { keepTemp: boolean; writeJsonl: boolean; jsonlPath: string } + ): Promise { + const content = readFileSync(file, 'utf-8'); + const result = parseJsonSafely(content); + + if (!result?.analysis) { + this.warn(`Skipping ${file}: no analysis field`); + return false; + } + + const { articleId, articleUrl, articleTitle, savedAt, publishedAt, updatedAt, analysis } = result; + + try { + const mdPath = await writer.write(articleId, articleUrl, articleTitle, savedAt, analysis, result.articleSlug); + repo.storeAnalysis(articleId, publishedAt, updatedAt, JSON.stringify(analysis), mdPath); + + if (options.writeJsonl) { + await writer.appendToJsonl(options.jsonlPath, buildJsonlRecord(result, analysis, mdPath)); + } + + if (!options.keepTemp) unlinkSync(file); + + this.log(`✓ ${this.truncate(articleTitle, 60)}`); + return true; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + repo.markFailed(articleId, `Save failed: ${message}`); + this.error(`✗ ${this.truncate(articleTitle, 60)} - ${message}`); + return false; + } + } + + private displayResults(results: SaveResults, jsonMode: boolean): void { + if (jsonMode) { + this.log(JSON.stringify(results, null, 2)); + return; + } + + this.log(''); + this.log(formatSuccess(`Saved ${results.saved} article(s)`)); + if (results.failed > 0) { + this.log(`Failed: ${results.failed}`); + } + this.log(''); + this.log(formatHeader('Updated Queue Status')); + displayQueueStats(results.stats); + } + + private truncate(text: string, length: number): string { + return text.length > length ? text.substring(0, length) + '...' : text; + } +} diff --git a/self-hosting/omc/src/commands/analyze/retry.ts b/self-hosting/omc/src/commands/analyze/retry.ts new file mode 100644 index 000000000..784a4e04a --- /dev/null +++ b/self-hosting/omc/src/commands/analyze/retry.ts @@ -0,0 +1,67 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatSuccess } from '@lib/cli/formatters.js'; + +/** + * Retry failed article analyses by resetting them to pending status. + * AIDEV-NOTE: CLI command for retrying failed analyses + */ +export default class AnalyzeRetry extends BaseCommand { + static override description = 'Retry failed article analyses'; + + static override examples = [ + '$ omc analyze retry --failed', + '$ omc analyze retry --article-id ', + ]; + + static override flags = { + failed: Flags.boolean({ + description: 'Retry all failed articles', + default: false, + exclusive: ['article-id'], + }), + 'article-id': Flags.string({ + description: 'Retry specific article by ID', + exclusive: ['failed'], + }), + json: jsonFlag(), + }; + + async execute(flags: { failed: boolean; 'article-id'?: string; json: boolean }): Promise { + await withDatabase(async ({ repo }) => { + const count = flags.failed + ? this.retryAllFailed(repo) + : this.retrySingleArticle(repo, flags['article-id']); + + if (flags.json) { + this.log(JSON.stringify({ retried: count })); + } else { + this.log(formatSuccess(`Reset ${count} failed article${count !== 1 ? 's' : ''} to pending`)); + } + }); + } + + private retryAllFailed(repo: { + getFailed: () => Array<{ articleId: string }>; + resetToPending: (articleId: string) => void; + }): number { + const failed = repo.getFailed(); + for (const job of failed) { + repo.resetToPending(job.articleId); + } + return failed.length; + } + + private retrySingleArticle( + repo: { resetToPending: (articleId: string) => void }, + articleId: string | undefined + ): number { + if (!articleId) { + throw new Error('Either use --failed flag or provide --article-id'); + } + repo.resetToPending(articleId); + return 1; + } +} diff --git a/self-hosting/omc/src/commands/analyze/run.ts b/self-hosting/omc/src/commands/analyze/run.ts new file mode 100644 index 000000000..5f468adea --- /dev/null +++ b/self-hosting/omc/src/commands/analyze/run.ts @@ -0,0 +1,186 @@ +import { Flags } from '@oclif/core'; +import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { fetchUsername } from '@lib/cli/graphql.js'; +import { formatHeader, formatSuccess, formatDivider } from '@lib/cli/formatters.js'; +import { displayQueueStats } from '@lib/cli/queue-display.js'; +import { getArticle } from '@lib/omnivore/client.js'; +import type { AnalysisJob } from '@storage/AnalysisQueueRepository.js'; + +/** + * OCLIF command: omc analyze run + * Wraps parallel-analyze.ts workflow for queue-based article analysis. + * + * AIDEV-NOTE: tracking-coordination - prepares batch for parallel agent execution + * AIDEV-NOTE: stub-file-creation - fetches full article metadata before analysis + */ +export default class AnalyzeRun extends BaseCommand { + static override description = 'Run parallel content analysis on queued articles'; + + static override examples = [ + '$ omc analyze run', + '$ omc analyze run --batch-size 10', + '$ omc analyze run --article-id abc123', + '$ omc analyze run --all', + '$ omc analyze run --json', + ]; + + static override flags = { + 'batch-size': Flags.integer({ + char: 'b', + description: 'Number of articles to analyze in parallel', + default: 5, + }), + 'article-id': Flags.string({ + description: 'Analyze specific article by ID', + exclusive: ['all'], + }), + all: Flags.boolean({ + description: 'Analyze all articles regardless of status', + exclusive: ['article-id'], + }), + json: jsonFlag(), + }; + + protected async execute(flags: Record): Promise { + return await withDatabase(async ({ repo }) => { + const jobs = this.selectJobs(repo, flags); + + if (jobs.length === 0) { + if (!flags.json) this.log('No jobs to process'); + return; + } + + if (!flags.json) { + const stats = repo.getStats(); + this.log(formatHeader('Analysis Queue Status')); + displayQueueStats(stats); + } + + this.markJobsInProgress(repo, jobs); + + const username = await fetchUsername(); + const agentParams = await this.processJobs(jobs, username, flags.json); + + this.outputResults(agentParams, jobs.length, flags.json); + }); + } + + // AIDEV-NOTE: job-selection - handles --article-id, --all, or default pending + private selectJobs(repo: any, flags: Record): AnalysisJob[] { + if (flags['article-id']) { + const job = repo.getByArticleId(flags['article-id']); + if (!job) { + throw new Error(`Article with ID ${flags['article-id']} not found`); + } + return [job]; + } + + if (flags.all) { + // Get all pending jobs (don't include in_progress to avoid overwriting) + const pending = repo.getByStatus('pending'); + const failed = repo.getByStatus('failed'); + const combined = [...pending, ...failed]; + return flags['batch-size'] ? combined.slice(0, flags['batch-size']) : combined; + } + + return repo.getPending(flags['batch-size']); + } + + // AIDEV-NOTE: tracking-lock - marks jobs to prevent duplicate analysis + private markJobsInProgress(repo: any, jobs: AnalysisJob[]): void { + for (const job of jobs) { + repo.markInProgress(job.articleId); + } + } + + private async processJobs( + jobs: AnalysisJob[], + username: string, + silent: boolean + ): Promise { + this.ensureTempDir(); + const agentParams = []; + + for (let i = 0; i < jobs.length; i++) { + const job = jobs[i]; + if (!silent) { + this.log(`Fetching article ${i + 1}/${jobs.length}: ${this.truncate(job.articleTitle, 60)}...`); + } + + const param = await this.createStubFile(job, username); + if (param) agentParams.push(param); + } + + return agentParams; + } + + private async fetchArticleData(slug: string, username: string): Promise { + try { + const result = await getArticle(slug, username); + if (!result?.article) { + this.warn(`Failed to fetch article: ${slug}`); + return null; + } + return result.article; + } catch (error) { + this.warn(`Failed to fetch article: ${slug}`); + return null; + } + } + + private buildStubObject(article: any, job: AnalysisJob, username: string): any { + return { + articleId: job.articleId, + articleSlug: job.articleSlug, + username, + articleUrl: article.url, + articleTitle: article.title, + savedAt: article.savedAt, + publishedAt: article.publishedAt || null, + updatedAt: article.updatedAt || null, + }; + } + + // AIDEV-NOTE: stub-file-creation - writes complete metadata for agent + private async createStubFile(job: AnalysisJob, username: string): Promise { + const article = await this.fetchArticleData(job.articleSlug, username); + if (!article) return null; + + const stubPath = join('temp', `${job.articleSlug}.jsonl`); + const stub = this.buildStubObject(article, job, username); + writeFileSync(stubPath, JSON.stringify(stub) + '\n', 'utf-8'); + + return { + filename: stubPath, + articleId: job.articleId, + articleSlug: job.articleSlug, + username, + articleTitle: this.truncate(article.title, 60), + }; + } + + private outputResults(agentParams: any[], count: number, jsonMode: boolean): void { + if (jsonMode) { + this.log(JSON.stringify(agentParams, null, 2)); + } else { + this.log(`\n${formatDivider()}`); + this.log(formatSuccess(`Prepared ${count} articles for analysis`)); + this.log('\nAgent parameters (copy for Task tool invocation):'); + this.log(JSON.stringify(agentParams, null, 2)); + } + } + + private ensureTempDir(): void { + if (!existsSync('temp')) { + mkdirSync('temp', { recursive: true }); + } + } + + private truncate(text: string, length: number): string { + return text.length > length ? text.substring(0, length) + '...' : text; + } +} diff --git a/self-hosting/omc/src/commands/analyze/status.ts b/self-hosting/omc/src/commands/analyze/status.ts new file mode 100644 index 000000000..05a322df9 --- /dev/null +++ b/self-hosting/omc/src/commands/analyze/status.ts @@ -0,0 +1,73 @@ +import { readdirSync } from 'node:fs'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader, formatTable } from '@lib/cli/formatters.js'; +import { QUEUE_STATUS } from '@lib/cli/constants.js'; +import type { AnalysisJob } from '@storage/AnalysisQueueRepository.js'; + +/** + * OCLIF command: omc analyze status + * Show current analysis batch progress. + * AIDEV-NOTE: tracking-coordination - displays in-progress jobs and stub files + */ +export default class AnalyzeStatus extends BaseCommand { + static override description = 'Show current analysis batch progress'; + + static override examples = ['$ omc analyze status', '$ omc analyze status --json']; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: Record): Promise { + return await withDatabase(async ({ repo }) => { + const jobs = repo.getByStatus(QUEUE_STATUS.IN_PROGRESS); + const stubs = this.getStubFiles(); + const display = this.prepareDisplay(jobs, stubs); + + this.output(display, flags.json); + }); + } + + private getStubFiles(): string[] { + try { + return readdirSync('temp').filter((f) => f.endsWith('.jsonl')); + } catch { + return []; + } + } + + private prepareDisplay(jobs: AnalysisJob[], stubs: string[]): any { + return { + inProgress: jobs.map((j) => ({ + articleId: j.articleId, + status: j.status, + elapsed: this.elapsed(j.assignedAt), + })), + stubFiles: stubs, + count: jobs.length, + }; + } + + private output(display: any, jsonMode: boolean): void { + if (jsonMode) { + this.log(JSON.stringify(display, null, 2)); + return; + } + + this.log(formatHeader('Analysis Status')); + if (display.count === 0) { + this.log('\nNo analysis in progress'); + return; + } + + this.log(formatTable(display.inProgress, ['articleId', 'status', 'elapsed'])); + } + + private elapsed(assignedAt?: string): string { + if (!assignedAt) return 'N/A'; + const ms = Date.now() - new Date(assignedAt).getTime(); + return `${Math.floor(ms / 1000)}s`; + } +} diff --git a/self-hosting/omc/src/commands/analyze/watch.ts b/self-hosting/omc/src/commands/analyze/watch.ts new file mode 100644 index 000000000..ea92c2fa5 --- /dev/null +++ b/self-hosting/omc/src/commands/analyze/watch.ts @@ -0,0 +1,66 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader, formatDivider } from '@lib/cli/formatters.js'; +import { displayQueueStats } from '@lib/cli/queue-display.js'; + +/** + * OCLIF command: omc analyze watch + * Real-time monitoring of analysis progress. + * AIDEV-NOTE: polling-monitor - updates stats every interval until completion + */ +export default class AnalyzeWatch extends BaseCommand { + static override description = 'Real-time monitoring of analysis progress'; + + static override examples = ['$ omc analyze watch', '$ omc analyze watch --interval 5000']; + + static override flags = { + interval: Flags.integer({ + char: 'i', + description: 'Polling interval in milliseconds', + default: 2000, + }), + }; + + protected async execute(flags: Record): Promise { + let running = true; + this.setupCleanExit(() => (running = false)); + + while (running) { + const done = await this.pollStats(); + if (done) break; + await this.sleep(flags.interval); + } + } + + private async pollStats(): Promise { + return await withDatabase(async ({ repo }) => { + const stats = repo.getStats(); + this.display(stats); + return stats.inProgress === 0 && stats.pending === 0; + }); + } + + private display(stats: any): void { + this.clear(); + this.log(formatHeader('Analysis Progress (Ctrl+C to exit)')); + displayQueueStats(stats); + this.log(formatDivider()); + } + + private setupCleanExit(callback: () => void): void { + process.on('SIGINT', () => { + this.log('\nExiting...'); + callback(); + process.exit(0); + }); + } + + private clear(): void { + process.stdout.write('\x1Bc'); + } + + private sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/self-hosting/omc/src/commands/config/env/list.ts b/self-hosting/omc/src/commands/config/env/list.ts new file mode 100644 index 000000000..667a472f7 --- /dev/null +++ b/self-hosting/omc/src/commands/config/env/list.ts @@ -0,0 +1,57 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { formatHeader } from '@lib/cli/formatters.js'; +import { readdirSync } from 'fs'; + +/** + * List available environments. + * AIDEV-NOTE: Shows .env files with active indicator + */ +export default class ConfigEnvList extends BaseCommand { + static override description = 'List available environments'; + + static override examples = [ + '$ omc config env list', + '$ omc config env list --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: any): Promise { + const environments = this.findEnvironments(); + + if (flags.json) { + this.log(JSON.stringify(environments, null, 2)); + } else { + this.displayEnvironments(environments); + } + } + + private findEnvironments(): Array<{ name: string; file: string; active: boolean }> { + const cwd = process.cwd(); + const files = readdirSync(cwd); + const envFiles = files.filter(f => f.startsWith('.env')); + + return envFiles.map(file => ({ + name: file === '.env' ? 'local' : file.replace('.env.', ''), + file, + active: file === '.env', + })); + } + + private displayEnvironments(environments: Array<{ name: string; file: string; active: boolean }>): void { + this.log(formatHeader('Available Environments')); + + if (environments.length === 0) { + this.log('No environment files found'); + return; + } + + for (const env of environments) { + const indicator = env.active ? '* ' : ' '; + this.log(`${indicator}${env.name.padEnd(15)} (${env.file})`); + } + } +} diff --git a/self-hosting/omc/src/commands/config/env/use.ts b/self-hosting/omc/src/commands/config/env/use.ts new file mode 100644 index 000000000..ad1beee27 --- /dev/null +++ b/self-hosting/omc/src/commands/config/env/use.ts @@ -0,0 +1,45 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { Args } from '@oclif/core'; +import { formatSuccess } from '@lib/cli/formatters.js'; +import { copyFileSync, existsSync } from 'fs'; +import { join } from 'path'; + +/** + * Switch environment. + * AIDEV-NOTE: Copies .env.{environment} to .env + */ +export default class ConfigEnvUse extends BaseCommand { + static override description = 'Switch environment'; + + static override examples = [ + '$ omc config env use dev', + '$ omc config env use prod', + '$ omc config env use local', + ]; + + static override args = { + environment: Args.string({ + description: 'Environment to switch to (dev|prod|local)', + required: true, + options: ['dev', 'prod', 'local'], + }), + }; + + protected async execute(flags: { environment: string }): Promise { + this.switchEnvironment(flags.environment); + this.log(formatSuccess(`Switched to ${flags.environment} environment`)); + } + + private switchEnvironment(environment: string): void { + const cwd = process.cwd(); + const sourceFile = environment === 'local' ? '.env.local' : `.env.${environment}`; + const sourcePath = join(cwd, sourceFile); + const targetPath = join(cwd, '.env'); + + if (!existsSync(sourcePath)) { + throw new Error(`Environment file not found: ${sourceFile}`); + } + + copyFileSync(sourcePath, targetPath); + } +} diff --git a/self-hosting/omc/src/commands/config/get.ts b/self-hosting/omc/src/commands/config/get.ts new file mode 100644 index 000000000..c09a6eeda --- /dev/null +++ b/self-hosting/omc/src/commands/config/get.ts @@ -0,0 +1,62 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { Args } from '@oclif/core'; +import { readFileSync, existsSync } from 'fs'; +import { join } from 'path'; + +/** + * Get specific configuration value. + * AIDEV-NOTE: Masks sensitive values for security + */ +export default class ConfigGet extends BaseCommand { + static override description = 'Get specific configuration value'; + + static override examples = [ + '$ omc config get OMNIVORE_API_KEY', + '$ omc config get CONTENT_OUTPUT_DIR', + ]; + + static override args = { + key: Args.string({ + description: 'Configuration key to retrieve', + required: true, + }), + }; + + protected async execute(flags: { key: string }): Promise { + const value = this.getConfigValue(flags.key); + if (value === undefined) { + throw new Error(`Configuration key '${flags.key}' not found`); + } + + this.log(this.maskValue(flags.key, value)); + } + + private getConfigValue(key: string): string | undefined { + const envPath = join(process.cwd(), '.env'); + if (!existsSync(envPath)) { + throw new Error('.env file not found'); + } + + const content = readFileSync(envPath, 'utf-8'); + + for (const line of content.split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + + const [envKey, ...valueParts] = trimmed.split('='); + if (envKey === key && valueParts.length > 0) { + return valueParts.join('='); + } + } + + return undefined; + } + + private maskValue(key: string, value: string): string { + const sensitiveKeys = ['KEY', 'TOKEN', 'SECRET', 'PASSWORD']; + if (sensitiveKeys.some(s => key.includes(s)) && value.length > 8) { + return `${value.slice(0, 4)}...${value.slice(-4)}`; + } + return value; + } +} diff --git a/self-hosting/omc/src/commands/config/set.ts b/self-hosting/omc/src/commands/config/set.ts new file mode 100644 index 000000000..8b1a86908 --- /dev/null +++ b/self-hosting/omc/src/commands/config/set.ts @@ -0,0 +1,73 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { Args } from '@oclif/core'; +import { formatSuccess } from '@lib/cli/formatters.js'; +import { readFileSync, writeFileSync, existsSync } from 'fs'; +import { join } from 'path'; + +/** + * Set configuration value. + * AIDEV-NOTE: Validates key exists in .env.example before updating + */ +export default class ConfigSet extends BaseCommand { + static override description = 'Set configuration value'; + + static override examples = [ + '$ omc config set CONTENT_OUTPUT_DIR ./my-content', + '$ omc config set AUTO_PUBLISH true', + ]; + + static override args = { + key: Args.string({ + description: 'Configuration key to set', + required: true, + }), + value: Args.string({ + description: 'Value to set', + required: true, + }), + }; + + protected async execute(flags: { key: string; value: string }): Promise { + this.validateKey(flags.key); + this.updateEnvFile(flags.key, flags.value); + this.log(formatSuccess(`Set ${flags.key} = ${flags.value}`)); + } + + private validateKey(key: string): void { + const examplePath = join(process.cwd(), '.env.example'); + if (!existsSync(examplePath)) return; + + const content = readFileSync(examplePath, 'utf-8'); + const validKeys = content + .split('\n') + .filter(line => !line.trim().startsWith('#') && line.includes('=')) + .map(line => line.split('=')[0]); + + if (!validKeys.includes(key)) { + throw new Error(`Unknown config key: ${key}`); + } + } + + private updateEnvFile(key: string, value: string): void { + const envPath = join(process.cwd(), '.env'); + let content = existsSync(envPath) ? readFileSync(envPath, 'utf-8') : ''; + + const lines = content.split('\n'); + let updated = false; + + for (let i = 0; i < lines.length; i++) { + const trimmed = lines[i].trim(); + if (trimmed.startsWith(key + '=')) { + lines[i] = `${key}=${value}`; + updated = true; + break; + } + } + + if (!updated) { + lines.push(`${key}=${value}`); + } + + writeFileSync(envPath, lines.join('\n')); + } +} diff --git a/self-hosting/omc/src/commands/config/show.ts b/self-hosting/omc/src/commands/config/show.ts new file mode 100644 index 000000000..37f2eb250 --- /dev/null +++ b/self-hosting/omc/src/commands/config/show.ts @@ -0,0 +1,62 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { formatHeader } from '@lib/cli/formatters.js'; +import { loadEnvFile } from '@lib/cli/command-utils.js'; +import { join } from 'path'; + +/** + * Show all configuration values. + * AIDEV-NOTE: Masks sensitive values (API keys show first/last 4 chars) + */ +export default class ConfigShow extends BaseCommand { + static override description = 'Show all configuration values'; + + static override examples = [ + '$ omc config show', + '$ omc config show --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + async execute(flags: any): Promise { + const envPath = join(process.cwd(), '.env'); + const rawConfig = loadEnvFile(envPath); + + if (Object.keys(rawConfig).length === 0) { + throw new Error('.env file not found'); + } + + const config = this.maskSensitiveValues(rawConfig); + + if (flags.json) { + this.log(JSON.stringify(config, null, 2)); + } else { + this.displayConfig(config); + } + } + + private maskSensitiveValues(config: Record): Record { + const masked: Record = {}; + for (const [key, value] of Object.entries(config)) { + masked[key] = this.maskValue(key, value); + } + return masked; + } + + private maskValue(key: string, value: string): string { + const sensitiveKeys = ['KEY', 'TOKEN', 'SECRET', 'PASSWORD']; + if (sensitiveKeys.some(s => key.includes(s)) && value.length > 8) { + return `${value.slice(0, 4)}...${value.slice(-4)}`; + } + return value; + } + + private displayConfig(config: Record): void { + this.log(formatHeader('Configuration')); + for (const [key, value] of Object.entries(config)) { + this.log(`${key.padEnd(30)} ${value}`); + } + } +} diff --git a/self-hosting/omc/src/commands/config/test.ts b/self-hosting/omc/src/commands/config/test.ts new file mode 100644 index 000000000..f8ef4fda6 --- /dev/null +++ b/self-hosting/omc/src/commands/config/test.ts @@ -0,0 +1,31 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { formatSuccess, formatError } from '@lib/cli/formatters.js'; +import { getMe } from '@lib/omnivore/client.js'; + +/** + * Test API connection. + * AIDEV-NOTE: Uses testConnection pattern from lib/omnivore/client.js + */ +export default class ConfigTest extends BaseCommand { + static override description = 'Test API connection'; + + static override examples = [ + '$ omc config test', + ]; + + protected async execute(flags: any): Promise { + void flags; + try { + const user = await getMe(); + + this.log(formatSuccess('Connected to Omnivore API')); + this.log(` User: ${user.name} (${user.email})`); + this.log(` Username: ${user.profile?.username ?? 'N/A'}`); + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error'; + this.log(formatError('Failed to connect to Omnivore API')); + this.log(` ${message}`); + throw error; + } + } +} diff --git a/self-hosting/omc/src/commands/config/validate.ts b/self-hosting/omc/src/commands/config/validate.ts new file mode 100644 index 000000000..117f4499e --- /dev/null +++ b/self-hosting/omc/src/commands/config/validate.ts @@ -0,0 +1,96 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { formatHeader, formatSuccess, formatError } from '@lib/cli/formatters.js'; +import { loadEnvFile } from '@lib/cli/command-utils.js'; +import { join } from 'path'; + +/** + * Validate all configuration values. + * AIDEV-NOTE: Checks required keys exist and validates formats + */ +export default class ConfigValidate extends BaseCommand { + static override description = 'Validate all configuration values'; + + static override examples = [ + '$ omc config validate', + '$ omc config validate --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: any): Promise { + const issues = this.validateConfig(); + + if (flags.json) { + this.log(JSON.stringify({ valid: issues.length === 0, issues }, null, 2)); + } else { + this.displayValidation(issues); + } + + if (issues.length > 0) { + throw new Error('Configuration validation failed'); + } + } + + private validateConfig(): string[] { + const issues: string[] = []; + const envPath = join(process.cwd(), '.env'); + const config = loadEnvFile(envPath); + + if (Object.keys(config).length === 0) { + issues.push('.env file not found'); + return issues; + } + + this.checkRequired(config, issues); + this.checkFormats(config, issues); + + return issues; + } + + private checkRequired(config: Record, issues: string[]): void { + const required = ['OMNIVORE_API_KEY']; + for (const key of required) { + if (!config[key] || config[key].trim() === '') { + issues.push(`Missing required key: ${key}`); + } + } + } + + private checkFormats(config: Record, issues: string[]): void { + if (config.OMNIVORE_API_URL && !this.isValidUrl(config.OMNIVORE_API_URL)) { + issues.push('Invalid URL format: OMNIVORE_API_URL'); + } + + const boolKeys = ['GENERATE_SEO', 'AUTO_PUBLISH', 'CACHE_ENABLED']; + for (const key of boolKeys) { + if (config[key] && !['true', 'false'].includes(config[key])) { + issues.push(`Invalid boolean value: ${key} (must be 'true' or 'false')`); + } + } + } + + private isValidUrl(value: string): boolean { + try { + new URL(value); + return true; + } catch { + return false; + } + } + + private displayValidation(issues: string[]): void { + this.log(formatHeader('Configuration Validation')); + + if (issues.length === 0) { + this.log(formatSuccess('All configuration valid')); + } else { + this.log(formatError(`Found ${issues.length} issue(s):\n`)); + for (const issue of issues) { + this.log(` - ${issue}`); + } + } + } +} diff --git a/self-hosting/omc/src/commands/db/backup.ts b/self-hosting/omc/src/commands/db/backup.ts new file mode 100644 index 000000000..4ed7f66d3 --- /dev/null +++ b/self-hosting/omc/src/commands/db/backup.ts @@ -0,0 +1,44 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { Flags } from '@oclif/core'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { formatHeader, formatSuccess } from '@lib/cli/formatters.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { join } from 'path'; + +/** + * Create database backup. + * AIDEV-NOTE: Uses better-sqlite3 backup API for safe copy + */ +export default class DbBackup extends BaseCommand { + static override description = 'Create database backup'; + + static override examples = [ + '$ omc db backup', + '$ omc db backup --destination data/backup.db', + '$ omc db backup --json', + ]; + + static override flags = { + json: jsonFlag(), + destination: Flags.string({ + description: 'Backup file path', + }), + }; + + protected async execute(flags: any): Promise { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const dest = flags.destination || join(process.cwd(), `data/backup-${timestamp}.db`); + + await withDatabase(async ({ db }) => { + db.backup(dest); + const result = { destination: dest, timestamp }; + + if (flags.json) { + this.log(JSON.stringify(result, null, 2)); + } else { + this.log(formatHeader('Database Backup')); + this.log(formatSuccess(`Backup created: ${dest}`)); + } + }); + } +} diff --git a/self-hosting/omc/src/commands/db/check.ts b/self-hosting/omc/src/commands/db/check.ts new file mode 100644 index 000000000..e44f6f31d --- /dev/null +++ b/self-hosting/omc/src/commands/db/check.ts @@ -0,0 +1,54 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader, formatSuccess, formatError } from '@lib/cli/formatters.js'; + +/** + * Verify database integrity. + * AIDEV-NOTE: Runs PRAGMA checks for corruption detection + */ +export default class DbCheck extends BaseCommand { + static override description = 'Verify database integrity'; + + static override examples = [ + '$ omc db check', + '$ omc db check --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + async execute(flags: any): Promise { + await withDatabase(async ({ db }) => { + const integrity = this.checkIntegrity(db); + const foreignKeys = this.checkForeignKeys(db); + const isHealthy = integrity.ok && foreignKeys.length === 0; + + const result = { ok: isHealthy, integrity, foreignKeys }; + + if (flags.json) { + this.log(JSON.stringify(result, null, 2)); + } else { + this.displayCheckResults(integrity, foreignKeys, isHealthy); + } + }); + } + + private checkIntegrity(db: any): { ok: boolean; message: string } { + const result = db.pragma('integrity_check'); + const message = result[0]?.integrity_check || 'ok'; + return { ok: message === 'ok', message }; + } + + private checkForeignKeys(db: any): any[] { + return db.pragma('foreign_key_check'); + } + + private displayCheckResults(integrity: any, fkIssues: any[], ok: boolean): void { + this.log(formatHeader('Database Integrity Check')); + this.log(`Integrity: ${integrity.message}`); + this.log(`Foreign Keys: ${fkIssues.length === 0 ? 'OK' : `${fkIssues.length} issues`}`); + this.log(ok ? formatSuccess('Database is healthy') : formatError('Issues detected')); + } +} diff --git a/self-hosting/omc/src/commands/db/migrate.ts b/self-hosting/omc/src/commands/db/migrate.ts new file mode 100644 index 000000000..198ec1b38 --- /dev/null +++ b/self-hosting/omc/src/commands/db/migrate.ts @@ -0,0 +1,62 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { formatHeader, formatSuccess } from '@lib/cli/formatters.js'; +import { withDatabase } from '@lib/cli/database.js'; + +/** + * Run database migrations by executing tracking schema. + * AIDEV-NOTE: Idempotent - safe to run multiple times (CREATE IF NOT EXISTS) + */ +export default class DbMigrate extends BaseCommand { + static override description = 'Run database migrations (idempotent schema execution)'; + + static override examples = [ + '$ omc db migrate', + '$ omc db migrate --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: any): Promise { + await withDatabase(async ({ db }) => { + const schemaPath = this.getSchemaPath(); + const schema = readFileSync(schemaPath, 'utf-8'); + + db.exec(schema); + + const tables = this.getTrackingTables(db); + + if (flags.json) { + this.log(JSON.stringify({ tables }, null, 2)); + } else { + this.displayMigrationResults(tables); + } + }); + } + + private getSchemaPath(): string { + // AIDEV-NOTE: Schema must be in source tree, not dist (not bundled by esbuild) + const projectRoot = process.cwd(); + return join(projectRoot, 'src/storage/schema/tracking-schema.sql'); + } + + private getTrackingTables(db: any): string[] { + const result = db.prepare(` + SELECT name FROM sqlite_master + WHERE type='table' AND name NOT LIKE 'sqlite_%' + ORDER BY name + `).all() as Array<{ name: string }>; + + return result.map(t => t.name); + } + + private displayMigrationResults(tables: string[]): void { + this.log(formatSuccess('Database schema migrated')); + this.log(formatHeader('Tables')); + tables.forEach(t => this.log(` ✓ ${t}`)); + } +} diff --git a/self-hosting/omc/src/commands/db/reset.ts b/self-hosting/omc/src/commands/db/reset.ts new file mode 100644 index 000000000..ee6636c2a --- /dev/null +++ b/self-hosting/omc/src/commands/db/reset.ts @@ -0,0 +1,70 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { Flags } from '@oclif/core'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader, formatSuccess } from '@lib/cli/formatters.js'; +import { listTables, isOmnivoreTable } from '@storage/database.js'; +import { readFileSync } from 'fs'; +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +/** + * Reset tracking tables (preserves Omnivore tables). + * AIDEV-NOTE: Boundary protection - never drops Omnivore tables + */ +export default class DbReset extends BaseCommand { + static override description = 'Drop and recreate tracking tables'; + + static override examples = [ + '$ omc db reset --force', + '$ omc db reset --force --json', + ]; + + static override flags = { + json: jsonFlag(), + force: Flags.boolean({ description: 'Required for safety', default: false }), + }; + + async execute(flags: any): Promise { + if (!flags.force) { + this.error('--force flag required for reset operation'); + } + + await withDatabase(async (db) => { + const trackingTables = this.dropTrackingTables(db); + this.recreateSchema(db); + this.displayResult(flags, trackingTables); + }); + } + + private dropTrackingTables(db: any): string[] { + const tables = listTables(db); + const trackingTables = tables.filter(t => !isOmnivoreTable(db, t)); + trackingTables.forEach(t => db.exec(`DROP TABLE IF EXISTS ${t}`)); + return trackingTables; + } + + private recreateSchema(db: any): void { + const schemaPath = this.getSchemaPath(); + const schema = readFileSync(schemaPath, 'utf-8'); + db.exec(schema); + } + + private displayResult(flags: any, droppedTables: string[]): void { + const result = { droppedTables, recreated: true }; + + if (flags.json) { + this.log(JSON.stringify(result, null, 2)); + } else { + this.log(formatHeader('Database Reset')); + this.log(`Dropped ${droppedTables.length} tracking tables`); + this.log(formatSuccess('Schema recreated')); + } + } + + private getSchemaPath(): string { + const __filename = fileURLToPath(import.meta.url); + const __dirname = dirname(__filename); + return join(__dirname, '../../../storage/schema/tracking-schema.sql'); + } +} diff --git a/self-hosting/omc/src/commands/db/restore.ts b/self-hosting/omc/src/commands/db/restore.ts new file mode 100644 index 000000000..0eb66e414 --- /dev/null +++ b/self-hosting/omc/src/commands/db/restore.ts @@ -0,0 +1,63 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { Args, Flags } from '@oclif/core'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { formatHeader, formatSuccess, formatError } from '@lib/cli/formatters.js'; +import { copyFileSync, existsSync } from 'fs'; +import { join } from 'path'; + +/** + * Restore database from backup. + * AIDEV-NOTE: Requires --force flag for safety + */ +export default class DbRestore extends BaseCommand { + static override description = 'Restore database from backup'; + + static override examples = [ + '$ omc db restore data/backup.db --force', + '$ omc db restore data/backup.db --force --json', + ]; + + static override args = { + backupPath: Args.string({ description: 'Path to backup file', required: true }), + }; + + static override flags = { + json: jsonFlag(), + force: Flags.boolean({ description: 'Skip confirmation', default: false }), + }; + + async execute(flags: any): Promise { + const { args } = await this.parse(DbRestore); + const backupPath = args.backupPath; + + this.validateInputs(backupPath, flags.force); + const dbPath = this.performRestore(backupPath); + this.displayResult(flags, backupPath, dbPath); + } + + private validateInputs(backupPath: string, force: boolean): void { + if (!existsSync(backupPath)) { + this.error(formatError(`Backup file not found: ${backupPath}`)); + } + if (!force) { + this.error('--force flag required for restore operation'); + } + } + + private performRestore(backupPath: string): string { + const dbPath = join(process.cwd(), 'data/omnivore-content.db'); + copyFileSync(backupPath, dbPath); + return dbPath; + } + + private displayResult(flags: any, backupPath: string, dbPath: string): void { + const result = { restored: true, from: backupPath, to: dbPath }; + + if (flags.json) { + this.log(JSON.stringify(result, null, 2)); + } else { + this.log(formatHeader('Database Restore')); + this.log(formatSuccess(`Restored from: ${backupPath}`)); + } + } +} diff --git a/self-hosting/omc/src/commands/db/schema.ts b/self-hosting/omc/src/commands/db/schema.ts new file mode 100644 index 000000000..0293ae6f7 --- /dev/null +++ b/self-hosting/omc/src/commands/db/schema.ts @@ -0,0 +1,52 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader } from '@lib/cli/formatters.js'; +import { listTables } from '@storage/database.js'; + +/** + * Display current database schema. + * AIDEV-NOTE: Shows table structure with columns and types + */ +export default class DbSchema extends BaseCommand { + static override description = 'Show current database schema'; + + static override examples = [ + '$ omc db schema', + '$ omc db schema --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: any): Promise { + await withDatabase(async ({ db }) => { + const tables = listTables(db); + const schema = this.buildSchemaInfo(db, tables); + + if (flags.json) { + this.log(JSON.stringify(schema, null, 2)); + } else { + this.displaySchema(schema); + } + }); + } + + private buildSchemaInfo(db: any, tables: string[]) { + return tables.map(table => ({ + name: table, + columns: db.prepare(`PRAGMA table_info(${table})`).all(), + })); + } + + private displaySchema(schema: any[]): void { + this.log(formatHeader('Database Schema')); + for (const table of schema) { + this.log(`\n${table.name}:`); + for (const col of table.columns) { + this.log(` ${col.name.padEnd(30)} ${col.type}`); + } + } + } +} diff --git a/self-hosting/omc/src/commands/db/seed.ts b/self-hosting/omc/src/commands/db/seed.ts new file mode 100644 index 000000000..2d7d9e834 --- /dev/null +++ b/self-hosting/omc/src/commands/db/seed.ts @@ -0,0 +1,73 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { formatSuccess } from '@lib/cli/formatters.js'; +import { withDatabase } from '@lib/cli/database.js'; + +/** + * Seed database with sample articles for testing. + * AIDEV-NOTE: Creates 10 diverse sample articles with varied statuses + */ +export default class DbSeed extends BaseCommand { + static override description = 'Seed database with sample data'; + + static override examples = [ + '$ omc db seed', + '$ omc db seed --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + async execute(flags: any): Promise { + await withDatabase(async (db) => { + const samples = this.createSampleData(); + const inserted = this.seedArticles(db, samples); + + if (flags.json) { + this.log(JSON.stringify({ inserted, total: samples.length }, null, 2)); + } else { + this.log(formatSuccess(`Seeded ${inserted} sample articles`)); + } + }); + } + + private seedArticles(db: any, samples: any[]): number { + const stmt = db.prepare(` + INSERT OR IGNORE INTO analysis_queue ( + article_id, article_slug, article_url, article_title, + saved_at, status, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, datetime('now'), datetime('now')) + `); + + let inserted = 0; + samples.forEach(article => { + const result = stmt.run( + article.articleId, + article.articleSlug, + article.articleUrl, + article.articleTitle, + article.savedAt, + article.status + ); + if (result.changes > 0) inserted++; + }); + + return inserted; + } + + private createSampleData() { + return [ + { articleId: 'seed-001', articleSlug: 'intro-to-llms', articleUrl: 'https://omnivore.app/test/intro-to-llms', articleTitle: 'Introduction to Large Language Models', savedAt: '2025-01-01T10:00:00Z', status: 'pending' }, + { articleId: 'seed-002', articleSlug: 'docker-best-practices', articleUrl: 'https://omnivore.app/test/docker-best-practices', articleTitle: 'Docker Best Practices 2025', savedAt: '2025-01-02T10:00:00Z', status: 'pending' }, + { articleId: 'seed-003', articleSlug: 'rust-async-await', articleUrl: 'https://omnivore.app/test/rust-async-await', articleTitle: 'Understanding Async/Await in Rust', savedAt: '2025-01-03T10:00:00Z', status: 'pending' }, + { articleId: 'seed-004', articleSlug: 'kubernetes-scaling', articleUrl: 'https://omnivore.app/test/kubernetes-scaling', articleTitle: 'Kubernetes Autoscaling Guide', savedAt: '2025-01-04T10:00:00Z', status: 'in_progress' }, + { articleId: 'seed-005', articleSlug: 'typescript-generics', articleUrl: 'https://omnivore.app/test/typescript-generics', articleTitle: 'Mastering TypeScript Generics', savedAt: '2025-01-05T10:00:00Z', status: 'in_progress' }, + { articleId: 'seed-006', articleSlug: 'graphql-federation', articleUrl: 'https://omnivore.app/test/graphql-federation', articleTitle: 'GraphQL Federation Explained', savedAt: '2025-01-06T10:00:00Z', status: 'completed' }, + { articleId: 'seed-007', articleSlug: 'postgres-performance', articleUrl: 'https://omnivore.app/test/postgres-performance', articleTitle: 'PostgreSQL Performance Tuning', savedAt: '2025-01-07T10:00:00Z', status: 'completed' }, + { articleId: 'seed-008', articleSlug: 'react-server-components', articleUrl: 'https://omnivore.app/test/react-server-components', articleTitle: 'React Server Components Deep Dive', savedAt: '2025-01-08T10:00:00Z', status: 'completed' }, + { articleId: 'seed-009', articleSlug: 'distributed-tracing', articleUrl: 'https://omnivore.app/test/distributed-tracing', articleTitle: 'Distributed Tracing with OpenTelemetry', savedAt: '2025-01-09T10:00:00Z', status: 'failed' }, + { articleId: 'seed-010', articleSlug: 'webassembly-performance', articleUrl: 'https://omnivore.app/test/webassembly-performance', articleTitle: 'WebAssembly Performance Benchmarks', savedAt: '2025-01-10T10:00:00Z', status: 'failed' }, + ]; + } +} diff --git a/self-hosting/omc/src/commands/db/stats.ts b/self-hosting/omc/src/commands/db/stats.ts new file mode 100644 index 000000000..5a0159c5c --- /dev/null +++ b/self-hosting/omc/src/commands/db/stats.ts @@ -0,0 +1,59 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader } from '@lib/cli/formatters.js'; +import { getTableCounts } from '@storage/database.js'; +import { existsSync, statSync } from 'fs'; +import { join } from 'path'; + +/** + * Display database statistics. + * AIDEV-NOTE: Shows table counts and file size + */ +export default class DbStats extends BaseCommand { + static override description = 'Show database statistics'; + + static override examples = [ + '$ omc db stats', + '$ omc db stats --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: any): Promise { + await withDatabase(async ({ db }) => { + const counts = getTableCounts(db); + const dbPath = join(process.cwd(), 'data/omnivore-content.db'); + const fileSize = this.getFileSize(dbPath); + + const result = { tables: counts, fileSize }; + + if (flags.json) { + this.log(JSON.stringify(result, null, 2)); + } else { + this.displayStats(counts, fileSize); + } + }); + } + + private getFileSize(path: string): number { + return existsSync(path) ? statSync(path).size : 0; + } + + private displayStats(counts: Record, size: number): void { + this.log(formatHeader('Database Statistics')); + this.log(`File size: ${this.formatBytes(size)}\n`); + this.log('Table Row Counts:'); + for (const [table, count] of Object.entries(counts)) { + this.log(` ${table.padEnd(30)} ${count.toLocaleString()}`); + } + } + + private formatBytes(bytes: number): string { + return bytes >= 1024 * 1024 + ? `${(bytes / 1024 / 1024).toFixed(2)} MB` + : `${(bytes / 1024).toFixed(2)} KB`; + } +} diff --git a/self-hosting/omc/src/commands/db/vacuum.ts b/self-hosting/omc/src/commands/db/vacuum.ts new file mode 100644 index 000000000..d9219ed8b --- /dev/null +++ b/self-hosting/omc/src/commands/db/vacuum.ts @@ -0,0 +1,73 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader, formatSuccess } from '@lib/cli/formatters.js'; +import { existsSync, statSync } from 'fs'; +import { join } from 'path'; + +/** + * Optimize database by running VACUUM. + * AIDEV-NOTE: Reclaims unused space and defragments + */ +export default class DbVacuum extends BaseCommand { + static override description = 'Optimize database with VACUUM'; + + static override examples = [ + '$ omc db vacuum', + '$ omc db vacuum --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + // AIDEV-NOTE: Execute split into helpers for function length compliance + protected async execute(flags: any): Promise { + const dbPath = join(process.cwd(), 'data/omnivore-content.db'); + const sizeBefore = this.getFileSize(dbPath); + + await withDatabase(async ({ db }) => { + db.exec('VACUUM'); + }); + + const sizeAfter = this.getFileSize(dbPath); + const result = this.buildResult(sizeBefore, sizeAfter); + + this.displayResult(result, flags.json); + } + + private buildResult( + sizeBefore: number, + sizeAfter: number + ): { sizeBefore: number; sizeAfter: number; savedBytes: number } { + return { + sizeBefore, + sizeAfter, + savedBytes: sizeBefore - sizeAfter, + }; + } + + private displayResult( + result: { sizeBefore: number; sizeAfter: number; savedBytes: number }, + asJson: boolean + ): void { + if (asJson) { + this.log(JSON.stringify(result, null, 2)); + } else { + this.log(formatHeader('Database Vacuum')); + this.log(`Size before: ${this.formatBytes(result.sizeBefore)}`); + this.log(`Size after: ${this.formatBytes(result.sizeAfter)}`); + this.log(formatSuccess(`Saved ${this.formatBytes(result.savedBytes)}`)); + } + } + + private getFileSize(path: string): number { + return existsSync(path) ? statSync(path).size : 0; + } + + private formatBytes(bytes: number): string { + return bytes >= 1024 * 1024 + ? `${(bytes / 1024 / 1024).toFixed(2)} MB` + : `${(bytes / 1024).toFixed(2)} KB`; + } +} diff --git a/self-hosting/omc/src/commands/doctor.ts b/self-hosting/omc/src/commands/doctor.ts new file mode 100644 index 000000000..05775b04a --- /dev/null +++ b/self-hosting/omc/src/commands/doctor.ts @@ -0,0 +1,115 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { outputResult } from '@lib/cli/command-utils.js'; +import { initDatabase } from '@storage/database.js'; +import { testConnection } from '@lib/omnivore/client.js'; +import { existsSync } from 'fs'; +import { config } from 'dotenv'; + +/** + * System health check for omnivore-content-system + * AIDEV-NOTE: doctor-command - diagnostic tool for troubleshooting setup issues + */ +export default class Doctor extends BaseCommand { + static override description = 'Check system health and configuration'; + + static override examples = [ + '<%= config.bin %> <%= command.id %>', + '<%= config.bin %> <%= command.id %> --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: { json: boolean }): Promise { + const checks = { + database: await this.checkDatabase(), + apiKey: this.checkApiKey(), + apiConnection: await this.checkApiConnection(), + directories: this.checkDirectories(), + dependencies: this.checkDependencies(), + }; + + const allHealthy = Object.values(checks).every(check => check.healthy); + + if (flags.json) { + outputResult(this, { checks, healthy: allHealthy }, '', true); + } else { + this.displayHealthReport(checks, allHealthy); + } + + if (!allHealthy) { + process.exit(1); + } + } + + private async checkDatabase(): Promise { + try { + initDatabase('data/omnivore-content.db'); + return { healthy: true, message: 'Database connection OK' }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { healthy: false, message: `Database error: ${message}` }; + } + } + + private checkApiKey(): CheckResult { + config(); + if (process.env.OMNIVORE_API_KEY) { + return { healthy: true, message: 'OMNIVORE_API_KEY found' }; + } + return { healthy: false, message: 'OMNIVORE_API_KEY not set' }; + } + + private async checkApiConnection(): Promise { + config(); + if (!process.env.OMNIVORE_API_KEY) { + return { healthy: false, message: 'Cannot test - API key missing' }; + } + + const connected = await testConnection(); + if (connected) { + return { healthy: true, message: 'API connection successful' }; + } + return { healthy: false, message: 'API connection failed' }; + } + + private checkDirectories(): CheckResult { + const required = ['data', 'content', 'temp']; + const missing = required.filter(dir => !existsSync(dir)); + + if (missing.length === 0) { + return { healthy: true, message: 'All required directories exist' }; + } + return { healthy: false, message: `Missing directories: ${missing.join(', ')}` }; + } + + private checkDependencies(): CheckResult { + const nodeVersion = process.version; + const requiredMajor = 18; + const currentMajor = parseInt(nodeVersion.slice(1).split('.')[0]); + + if (currentMajor >= requiredMajor) { + return { healthy: true, message: `Node ${nodeVersion} (>= ${requiredMajor})` }; + } + return { healthy: false, message: `Node ${nodeVersion} (requires >= ${requiredMajor})` }; + } + + private displayHealthReport(checks: Record, allHealthy: boolean): void { + this.log('\nSystem Health Report\n'); + + for (const [name, check] of Object.entries(checks)) { + const icon = check.healthy ? '✓' : '✗'; + const label = name.padEnd(20); + this.log(`${icon} ${label} ${check.message}`); + } + + this.log(`\nOverall Status: ${allHealthy ? '✓ Healthy' : '✗ Issues detected'}`); + } +} + +interface CheckResult { + healthy: boolean; + message: string; +} diff --git a/self-hosting/omc/src/commands/init.ts b/self-hosting/omc/src/commands/init.ts new file mode 100644 index 000000000..5bf3abac4 --- /dev/null +++ b/self-hosting/omc/src/commands/init.ts @@ -0,0 +1,118 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { formatSuccess, formatError } from '@lib/cli/formatters.js'; +import { initDatabase } from '@storage/database.js'; +import { testConnection } from '@lib/omnivore/client.js'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; +import { config } from 'dotenv'; + +/** + * Interactive setup wizard for omnivore-content-system + * AIDEV-NOTE: init-command - one-time setup for new installations + */ +export default class Init extends BaseCommand { + static override description = 'Initialize omnivore-content-system setup'; + + static override examples = [ + '<%= config.bin %> <%= command.id %>', + '<%= config.bin %> <%= command.id %> --force', + '<%= config.bin %> <%= command.id %> --api-key YOUR_KEY', + ]; + + static override flags = { + force: Flags.boolean({ + description: 'Reinitialize existing setup', + default: false, + }), + 'api-key': Flags.string({ + description: 'Set OMNIVORE_API_KEY non-interactively', + }), + }; + + protected async execute(flags: { force: boolean; 'api-key'?: string }): Promise { + this.log('Initializing omnivore-content-system...\n'); + + await this.ensureDirectories(); + await this.initializeDatabase(flags.force); + await this.setupEnvironment(flags['api-key']); + await this.testApiConnection(); + + this.log('\n' + formatSuccess('Setup complete! Run "omc --help" to get started.')); + } + + private async ensureDirectories(): Promise { + const dirs = ['data', 'content', 'temp', 'content/articles', 'content/analysis', 'content/generated']; + + for (const dir of dirs) { + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + this.log(formatSuccess(`Created directory: ${dir}`)); + } + } + } + + private async initializeDatabase(force: boolean): Promise { + try { + initDatabase('data/omnivore-content.db'); + this.log(formatSuccess('Database initialized')); + } catch (error) { + if (force) { + this.log(formatSuccess('Database reinitialized')); + } else { + throw error; + } + } + } + + private async setupEnvironment(apiKey?: string): Promise { + this.createEnvFile(); + + if (apiKey) { + this.updateEnvFile('OMNIVORE_API_KEY', apiKey); + this.log(formatSuccess('OMNIVORE_API_KEY set')); + } else { + this.checkApiKey(); + } + } + + private createEnvFile(): void { + if (!existsSync('.env') && existsSync('.env.example')) { + const example = readFileSync('.env.example', 'utf-8'); + writeFileSync('.env', example); + this.log(formatSuccess('Created .env from .env.example')); + } else if (!existsSync('.env')) { + this.log(formatError('.env.example not found')); + } + } + + private checkApiKey(): void { + config(); + if (!process.env.OMNIVORE_API_KEY) { + this.log(formatError('OMNIVORE_API_KEY not set in .env')); + this.log('Please add your API key to .env file'); + } + } + + private async testApiConnection(): Promise { + config(); + if (process.env.OMNIVORE_API_KEY) { + const connected = await testConnection(); + if (!connected) { + this.log(formatError('API connection test failed')); + } + } + } + + private updateEnvFile(key: string, value: string): void { + let envContent = existsSync('.env') ? readFileSync('.env', 'utf-8') : ''; + const regex = new RegExp(`^${key}=.*$`, 'm'); + + if (regex.test(envContent)) { + envContent = envContent.replace(regex, `${key}=${value}`); + } else { + envContent += `\n${key}=${value}\n`; + } + + writeFileSync('.env', envContent); + } +} diff --git a/self-hosting/omc/src/commands/omnivore/get.ts b/self-hosting/omc/src/commands/omnivore/get.ts new file mode 100644 index 000000000..45685d9df --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/get.ts @@ -0,0 +1,65 @@ +import { Args, Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { fetchUsername, checkGraphQLResult } from '@lib/cli/graphql.js'; +import { getArticle } from '@lib/omnivore/client.js'; +import type { OmnivoreArticle } from '@omc-types/omnivore.js'; + +/** + * Fetch article by slug. + * AIDEV-NOTE: Direct Omnivore API access for article retrieval + */ +export default class OmnivoreGet extends BaseCommand { + static override description = 'Fetch article by slug'; + + static override examples = [ + '$ omc omnivore get my-article-slug', + '$ omc omnivore get my-article-slug --json', + '$ omc omnivore get my-article-slug --username different-user', + ]; + + static override args = { + slug: Args.string({ description: 'Article slug', required: true }), + }; + + static override flags = { + username: Flags.string({ char: 'u', description: 'Username (defaults to authenticated user)' }), + content: Flags.boolean({ + description: 'Print raw article content to stdout (agent-friendly)', + default: false, + }), + json: jsonFlag(), + }; + + protected async execute(flags: { slug: string; username?: string; content: boolean; json: boolean }): Promise { + const username = flags.username || await fetchUsername(); + const result = await getArticle(flags.slug, username); + checkGraphQLResult({ data: { article: result }, errors: undefined }); + + if (!result.article) { + throw new Error('Article not found'); + } + + if (flags.content && flags.json) { + throw new Error('Use either --content or --json (not both)'); + } + + if (flags.content) { + this.log(result.article?.content ?? ''); + return; + } + + this.outputArticle(result.article, flags.json); + } + + private outputArticle(article: OmnivoreArticle, json: boolean): void { + if (json) { + this.log(JSON.stringify(article, null, 2)); + } else { + this.log(`Title: ${article.title}`); + this.log(`URL: ${article.url}`); + this.log(`Author: ${article.author || 'N/A'}`); + this.log(`Description: ${article.description || 'N/A'}`); + } + } +} diff --git a/self-hosting/omc/src/commands/omnivore/highlight/add.ts b/self-hosting/omc/src/commands/omnivore/highlight/add.ts new file mode 100644 index 000000000..05f209d30 --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/highlight/add.ts @@ -0,0 +1,53 @@ +import { Args, Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { checkGraphQLResult } from '@lib/cli/graphql.js'; +import { createHighlight } from '@lib/omnivore/client.js'; + +/** + * Add highlight to article. + * AIDEV-NOTE: Creates HIGHLIGHT-type with quote (optional annotation) + */ +export default class HighlightAdd extends BaseCommand { + static override description = 'Add highlight to article'; + + static override examples = [ + '$ omc omnivore highlight add article-id "quote text"', + '$ omc omnivore highlight add article-id "quote" --annotation "my note"', + '$ omc omnivore highlight add article-id "quote" --color "#FF0000" --json', + ]; + + static override args = { + articleId: Args.string({ description: 'Article ID', required: true }), + quote: Args.string({ description: 'Highlighted text', required: true }), + }; + + static override flags = { + annotation: Flags.string({ char: 'a', description: 'Optional annotation' }), + color: Flags.string({ char: 'c', description: 'Highlight color', default: '#FFD700' }), + json: jsonFlag(), + }; + + protected async execute(flags: { articleId: string; quote: string; annotation?: string; color: string; json: boolean }): Promise { + const result = await createHighlight({ + id: crypto.randomUUID(), + shortId: crypto.randomUUID().substring(0, 8), + articleId: flags.articleId, + quote: flags.quote, + annotation: flags.annotation, + color: flags.color, + type: 'HIGHLIGHT', + }); + checkGraphQLResult({ data: { createHighlight: result }, errors: undefined }); + + this.outputResult(result, flags.json); + } + + private outputResult(result: any, json: boolean): void { + if (json) { + this.log(JSON.stringify(result.highlight, null, 2)); + } else { + this.log('Highlight added successfully'); + } + } +} diff --git a/self-hosting/omc/src/commands/omnivore/highlight/list.ts b/self-hosting/omc/src/commands/omnivore/highlight/list.ts new file mode 100644 index 000000000..601550972 --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/highlight/list.ts @@ -0,0 +1,57 @@ +import { Args } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { fetchUsername } from '@lib/cli/graphql.js'; +import { getHighlights } from '@lib/omnivore/client.js'; + +type HighlightRecord = { + id: string; + quote?: string | null; + annotation?: string | null; + createdAt?: string; + type?: string; +}; + +/** + * List highlights for article. + * AIDEV-NOTE: Filters to show only HIGHLIGHT type (excludes NOTEs) + */ +export default class HighlightList extends BaseCommand { + static override description = 'List highlights for article'; + + static override examples = [ + '$ omc omnivore highlight list article-id', + '$ omc omnivore highlight list article-id --json', + ]; + + static override args = { + articleId: Args.string({ description: 'Article ID', required: true }), + }; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: { articleId: string; json: boolean }): Promise { + const username = await fetchUsername(); + const allHighlights = (await getHighlights(flags.articleId, username)) as HighlightRecord[]; + const highlights = this.filterHighlights(allHighlights); + + this.outputHighlights(highlights, flags.json); + } + + private filterHighlights(highlights: HighlightRecord[]): HighlightRecord[] { + return highlights.filter((h) => h.type === 'HIGHLIGHT' || Boolean(h.quote)); + } + + private outputHighlights(highlights: HighlightRecord[], json: boolean): void { + if (json) { + this.log(JSON.stringify(highlights, null, 2)); + } else { + highlights.forEach((h) => { + this.log(`- "${h.quote ?? ''}"`); + if (h.annotation) this.log(` Note: ${h.annotation}`); + }); + } + } +} diff --git a/self-hosting/omc/src/commands/omnivore/label/create.ts b/self-hosting/omc/src/commands/omnivore/label/create.ts new file mode 100644 index 000000000..6a8fd0bd8 --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/label/create.ts @@ -0,0 +1,38 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { checkGraphQLResult } from '@lib/cli/graphql.js'; +import { createLabel } from '@lib/omnivore/client.js'; + +/** + * Create a label. + * AIDEV-NOTE: Replacement for standalone scripts/test-create-label.js and migrate helpers. + */ +export default class OmnivoreLabelCreate extends BaseCommand { + static override description = 'Create a label'; + + static override examples = [ + '$ omc omnivore label create --name "ai"', + '$ omc omnivore label create --name "ai" --color "#3B82F6" --description "AI/ML" --json', + ]; + + static override flags = { + name: Flags.string({ description: 'Label name', required: true }), + color: Flags.string({ description: 'Hex color (e.g. #3B82F6)' }), + description: Flags.string({ description: 'Description' }), + json: jsonFlag(), + }; + + protected async execute(flags: { name: string; color?: string; description?: string; json: boolean }): Promise { + const result = await createLabel({ name: flags.name, color: flags.color, description: flags.description }); + checkGraphQLResult({ data: { createLabel: result }, errors: undefined }); + + if (flags.json) { + this.log(JSON.stringify(result.label ?? result, null, 2)); + return; + } + + this.log(`Label created: ${result.label?.name ?? flags.name}`); + } +} + diff --git a/self-hosting/omc/src/commands/omnivore/label/list.ts b/self-hosting/omc/src/commands/omnivore/label/list.ts new file mode 100644 index 000000000..8016fdb54 --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/label/list.ts @@ -0,0 +1,36 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { checkGraphQLResult } from '@lib/cli/graphql.js'; +import { getLabels } from '@lib/omnivore/client.js'; + +/** + * List labels. + * AIDEV-NOTE: Integrates legacy Omnivore label scripts into the OMC CLI. + */ +export default class OmnivoreLabelList extends BaseCommand { + static override description = 'List labels'; + + static override examples = [ + '$ omc omnivore label list', + '$ omc omnivore label list --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: { json: boolean }): Promise { + const labels = await getLabels(); + checkGraphQLResult({ data: { labels }, errors: undefined }); + + if (flags.json) { + this.log(JSON.stringify(labels, null, 2)); + return; + } + + for (const label of labels) { + this.log(`- ${label.name} (${label.id})`); + } + } +} + diff --git a/self-hosting/omc/src/commands/omnivore/label/set.ts b/self-hosting/omc/src/commands/omnivore/label/set.ts new file mode 100644 index 000000000..3e9708f94 --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/label/set.ts @@ -0,0 +1,51 @@ +import { Args, Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { checkGraphQLResult } from '@lib/cli/graphql.js'; +import { setLabels } from '@lib/omnivore/client.js'; + +/** + * Set labels on a page (replaces existing labels). + * AIDEV-NOTE: Integrates apply-single-label.js / apply-labels*.js into OMC CLI surface. + */ +export default class OmnivoreLabelSet extends BaseCommand { + static override description = 'Set labels on a page (replaces existing labels)'; + + static override examples = [ + '$ omc omnivore label set --label "ai"', + '$ omc omnivore label set --label "ai" --label "devops"', + '$ omc omnivore label set --label "ai" --source "omc" --json', + ]; + + static override args = { + pageId: Args.string({ description: 'Page/Article ID', required: true }), + }; + + static override flags = { + label: Flags.string({ + description: 'Label name (repeatable)', + multiple: true, + required: true, + }), + source: Flags.string({ + description: 'Optional source string (stored by Omnivore)', + default: 'omc', + }), + json: jsonFlag(), + }; + + protected async execute(flags: { pageId: string; label: string[]; source: string; json: boolean }): Promise { + const labels = flags.label.map((name) => ({ name })); + const result = await setLabels({ pageId: flags.pageId, labels, source: flags.source }); + checkGraphQLResult({ data: { setLabels: result }, errors: undefined }); + + if (flags.json) { + this.log(JSON.stringify(result, null, 2)); + return; + } + + const applied = (result.labels ?? []).map((l: any) => l.name).join(', '); + this.log(`Labels set: ${applied || '(none)'}`); + } +} + diff --git a/self-hosting/omc/src/commands/omnivore/list.ts b/self-hosting/omc/src/commands/omnivore/list.ts new file mode 100644 index 000000000..376ff7d08 --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/list.ts @@ -0,0 +1,44 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { checkGraphQLResult } from '@lib/cli/graphql.js'; +import { getRecentArticles } from '@lib/omnivore/client.js'; +import type { SearchResult } from '@omc-types/omnivore.js'; + +/** + * List recent articles. + * AIDEV-NOTE: Time-filtered article listing + */ +export default class OmnivoreList extends BaseCommand { + static override description = 'List recent articles'; + + static override examples = [ + '$ omc omnivore list', + '$ omc omnivore list --hours 48', + '$ omc omnivore list --hours 24 --limit 20 --json', + ]; + + static override flags = { + hours: Flags.integer({ char: 'h', description: 'Hours to look back', default: 24 }), + limit: Flags.integer({ char: 'l', description: 'Max results', default: 10 }), + json: jsonFlag(), + }; + + protected async execute(flags: Record): Promise { + const result = await getRecentArticles(flags.hours, flags.limit); + checkGraphQLResult({ data: { search: result }, errors: undefined }); + + this.outputResults(result.edges, flags.json); + } + + private outputResults(edges: SearchResult['edges'], json: boolean): void { + if (json) { + this.log(JSON.stringify(edges.map((e) => e.node), null, 2)); + } else { + edges.forEach(({ node }) => { + this.log(`- ${node.title}`); + this.log(` Saved: ${node.savedAt}`); + }); + } + } +} diff --git a/self-hosting/omc/src/commands/omnivore/mapping/download.ts b/self-hosting/omc/src/commands/omnivore/mapping/download.ts new file mode 100644 index 000000000..d71710346 --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/mapping/download.ts @@ -0,0 +1,92 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { searchArticles } from '@lib/omnivore/client.js'; +import Database from 'better-sqlite3'; +import { mkdirSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; + +/** + * Download a URL -> pageId mapping into a local SQLite DB. + * AIDEV-NOTE: Integrates scripts/download-items-mapping.js functionality into OMC CLI. + */ +export default class OmnivoreMappingDownload extends BaseCommand { + static override description = 'Download Omnivore URL-to-ID mapping into SQLite'; + + static override examples = [ + '$ omc omnivore mapping download', + '$ omc omnivore mapping download --destination temp/url-id-mapping.sqlite', + '$ omc omnivore mapping download --limit 1000 --json', + ]; + + static override flags = { + destination: Flags.string({ + description: 'Output SQLite path', + default: 'temp/url-id-mapping.sqlite', + }), + limit: Flags.integer({ + description: 'Optional max items (for testing)', + required: false, + }), + json: jsonFlag(), + }; + + protected async execute(flags: { destination: string; limit?: number; json: boolean }): Promise { + const destination = resolve(flags.destination); + mkdirSync(dirname(destination), { recursive: true }); + + const db = new Database(destination); + try { + this.initSchema(db); + const downloaded = await this.downloadInto(db, flags.limit); + const result = { destination, downloaded }; + + if (flags.json) this.log(JSON.stringify(result, null, 2)); + else this.log(`Saved ${downloaded} items to ${destination}`); + } finally { + db.close(); + } + } + + private initSchema(db: Database.Database): void { + db.exec(` + CREATE TABLE IF NOT EXISTS item_mapping ( + id TEXT PRIMARY KEY, + url TEXT NOT NULL, + title TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_item_mapping_url ON item_mapping(url); + DELETE FROM item_mapping; + `); + } + + private async downloadInto(db: Database.Database, limit?: number): Promise { + const insert = db.prepare('INSERT OR REPLACE INTO item_mapping (id, url, title) VALUES (?, ?, ?)'); + const insertMany = db.transaction((rows: Array<{ id: string; url: string; title?: string }>) => { + for (const r of rows) insert.run(r.id, r.url, r.title ?? null); + }); + + let after: string | undefined; + let total = 0; + + while (true) { + const first = 100; + const result: any = await searchArticles({ query: '', first, after: after ?? '', includeContent: false }); + + const edges = result.edges ?? []; + let rows = edges.map((e: any) => ({ id: e.node.id, url: e.node.url, title: e.node.title })); + if (limit && total + rows.length > limit) { + rows = rows.slice(0, Math.max(0, limit - total)); + } + insertMany(rows); + + total += rows.length; + if (limit && total >= limit) return total; + + const pageInfo = result.pageInfo; + if (!pageInfo?.hasNextPage) return total; + after = pageInfo.endCursor; + } + } +} diff --git a/self-hosting/omc/src/commands/omnivore/note/add.ts b/self-hosting/omc/src/commands/omnivore/note/add.ts new file mode 100644 index 000000000..cac7854df --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/note/add.ts @@ -0,0 +1,48 @@ +import { Args } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { checkGraphQLResult } from '@lib/cli/graphql.js'; +import { createHighlight } from '@lib/omnivore/client.js'; + +/** + * Add note to article. + * AIDEV-NOTE: Creates NOTE-type highlight (no quote required) + */ +export default class NoteAdd extends BaseCommand { + static override description = 'Add note to article'; + + static override examples = [ + '$ omc omnivore note add article-id "My note content"', + '$ omc omnivore note add article-id "Note" --json', + ]; + + static override args = { + articleId: Args.string({ description: 'Article ID', required: true }), + note: Args.string({ description: 'Note content', required: true }), + }; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: { articleId: string; note: string; json: boolean }): Promise { + const result = await createHighlight({ + id: crypto.randomUUID(), + shortId: crypto.randomUUID().substring(0, 8), + articleId: flags.articleId, + annotation: flags.note, + type: 'NOTE', + }); + checkGraphQLResult({ data: { createHighlight: result }, errors: undefined }); + + this.outputResult(result, flags.json); + } + + private outputResult(result: any, json: boolean): void { + if (json) { + this.log(JSON.stringify(result.highlight, null, 2)); + } else { + this.log('Note added successfully'); + } + } +} diff --git a/self-hosting/omc/src/commands/omnivore/note/get.ts b/self-hosting/omc/src/commands/omnivore/note/get.ts new file mode 100644 index 000000000..1726beca8 --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/note/get.ts @@ -0,0 +1,57 @@ +import { Args } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { fetchUsername } from '@lib/cli/graphql.js'; +import { getHighlights } from '@lib/omnivore/client.js'; + +type NoteRecord = { annotation?: string | null; createdAt?: string; quote?: string | null; type?: string }; + +/** + * Get notes for article. + * AIDEV-NOTE: Filters highlights to show only NOTE type + */ +export default class NoteGet extends BaseCommand { + static override description = 'Get notes for article'; + + static override examples = [ + '$ omc omnivore note get article-id', + '$ omc omnivore note get article-id --json', + ]; + + static override args = { + articleId: Args.string({ description: 'Article ID', required: true }), + }; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: { articleId: string; json: boolean }): Promise { + await withDatabase(async ({ repo }) => { + const job = repo.getByArticleId(flags.articleId); + if (!job) { + this.error(`Article not found in database: ${flags.articleId}`); + } + const username = await fetchUsername(); + const highlights = (await getHighlights(job.articleSlug, username)) as NoteRecord[]; + const notes = this.filterNotes(highlights); + this.outputNotes(notes, flags.json); + }); + } + + private filterNotes(highlights: NoteRecord[]): NoteRecord[] { + return highlights.filter((h) => h.type === 'NOTE' || (!h.quote && h.annotation)); + } + + private outputNotes(notes: NoteRecord[], json: boolean): void { + if (json) { + this.log(JSON.stringify(notes, null, 2)); + } else { + notes.forEach((note) => { + this.log(`- ${note.annotation ?? ''}`); + if (note.createdAt) this.log(` Created: ${note.createdAt}`); + }); + } + } +} diff --git a/self-hosting/omc/src/commands/omnivore/note/update.ts b/self-hosting/omc/src/commands/omnivore/note/update.ts new file mode 100644 index 000000000..76a83508a --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/note/update.ts @@ -0,0 +1,45 @@ +import { Args } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { checkGraphQLResult } from '@lib/cli/graphql.js'; +import { updateHighlight } from '@lib/omnivore/client.js'; + +/** + * Update note content. + * AIDEV-NOTE: Updates annotation field of highlight + */ +export default class NoteUpdate extends BaseCommand { + static override description = 'Update note content'; + + static override examples = [ + '$ omc omnivore note update highlight-id "Updated note"', + '$ omc omnivore note update highlight-id "New content" --json', + ]; + + static override args = { + highlightId: Args.string({ description: 'Highlight ID', required: true }), + note: Args.string({ description: 'New note content', required: true }), + }; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: { highlightId: string; note: string; json: boolean }): Promise { + const result = await updateHighlight({ + highlightId: flags.highlightId, + annotation: flags.note, + }); + checkGraphQLResult({ data: { updateHighlight: result }, errors: undefined }); + + this.outputResult(result, flags.json); + } + + private outputResult(result: any, json: boolean): void { + if (json) { + this.log(JSON.stringify(result.highlight, null, 2)); + } else { + this.log('Note updated successfully'); + } + } +} diff --git a/self-hosting/omc/src/commands/omnivore/search.ts b/self-hosting/omc/src/commands/omnivore/search.ts new file mode 100644 index 000000000..1d76def8c --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/search.ts @@ -0,0 +1,47 @@ +import { Args, Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { checkGraphQLResult } from '@lib/cli/graphql.js'; +import { searchArticles } from '@lib/omnivore/client.js'; +import type { SearchResult } from '@omc-types/omnivore.js'; + +/** + * Search articles. + * AIDEV-NOTE: Omnivore search with query DSL support + */ +export default class OmnivoreSearch extends BaseCommand { + static override description = 'Search articles'; + + static override examples = [ + '$ omc omnivore search "AI machine learning"', + '$ omc omnivore search "label:tech" --limit 20', + '$ omc omnivore search "in:inbox" --json', + ]; + + static override args = { + query: Args.string({ description: 'Search query', required: true }), + }; + + static override flags = { + limit: Flags.integer({ char: 'l', description: 'Max results', default: 10 }), + json: jsonFlag(), + }; + + protected async execute(flags: { query: string; limit: number; json: boolean }): Promise { + const result = await searchArticles({ query: flags.query, first: flags.limit }); + checkGraphQLResult({ data: { search: result }, errors: undefined }); + + this.outputResults(result.edges, flags.json); + } + + private outputResults(edges: SearchResult['edges'], json: boolean): void { + if (json) { + this.log(JSON.stringify(edges.map((e) => e.node), null, 2)); + } else { + edges.forEach(({ node }) => { + this.log(`- ${node.title}`); + this.log(` URL: ${node.url}`); + }); + } + } +} diff --git a/self-hosting/omc/src/commands/omnivore/update.ts b/self-hosting/omc/src/commands/omnivore/update.ts new file mode 100644 index 000000000..0ddf1960c --- /dev/null +++ b/self-hosting/omc/src/commands/omnivore/update.ts @@ -0,0 +1,56 @@ +import { Args, Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { checkGraphQLResult } from '@lib/cli/graphql.js'; +import { updatePage } from '@lib/omnivore/client.js'; + +/** + * Update article metadata. + * AIDEV-NOTE: Modify article title/description via Omnivore API + */ +export default class OmnivoreUpdate extends BaseCommand { + static override description = 'Update article metadata'; + + static override examples = [ + '$ omc omnivore update article-id --title "New Title"', + '$ omc omnivore update article-id --description "Summary"', + '$ omc omnivore update article-id --title "Title" --description "Desc" --json', + ]; + + static override args = { + articleId: Args.string({ description: 'Article ID', required: true }), + }; + + static override flags = { + title: Flags.string({ char: 't', description: 'New title' }), + description: Flags.string({ char: 'd', description: 'New description' }), + json: jsonFlag(), + }; + + protected async execute(flags: { articleId: string; title?: string; description?: string; json: boolean }): Promise { + this.validateFlags(flags); + + const result = await updatePage({ + pageId: flags.articleId, + title: flags.title, + description: flags.description, + }); + checkGraphQLResult({ data: { updatePage: result }, errors: undefined }); + + this.outputResult(result, flags.json); + } + + private validateFlags(flags: { title?: string; description?: string }): void { + if (!flags.title && !flags.description) { + throw new Error('At least one of --title or --description is required'); + } + } + + private outputResult(result: any, json: boolean): void { + if (json) { + this.log(JSON.stringify(result.updatedPage, null, 2)); + } else { + this.log('Article updated successfully'); + } + } +} diff --git a/self-hosting/omc/src/commands/queue/add.ts b/self-hosting/omc/src/commands/queue/add.ts new file mode 100644 index 000000000..57b8dcfe6 --- /dev/null +++ b/self-hosting/omc/src/commands/queue/add.ts @@ -0,0 +1,193 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatSuccess } from '@lib/cli/formatters.js'; +import { fetchUsername, checkGraphQLResult } from '@lib/cli/graphql.js'; +import { searchArticles, getArticle, getArticlesByLabel } from '@lib/omnivore/client.js'; +import type { OmnivoreArticle, SearchResult } from '@omc-types/omnivore.js'; + +interface QueueAddFlags { + hours?: number; + url?: string; + label?: string; + slug?: string; + json: boolean; +} + +type QueueSourceArticle = Pick & { slug: string }; + +/** + * Add articles to the analysis queue. + * AIDEV-NOTE: Entry point for queue population from Omnivore API + */ +export default class QueueAdd extends BaseCommand { + static override description = 'Add articles to the analysis queue'; + + static override examples = [ + '$ omc queue add --hours 24', + '$ omc queue add --hours 168', + '$ omc queue add --label AI', + '$ omc queue add --slug my-article-slug', + '$ omc queue add --url https://omnivore.app/username/article-slug', + '$ omc queue add --hours 24 --json', + ]; + + static override flags = { + hours: Flags.integer({ + char: 'h', + description: 'Add articles from last N hours', + exclusive: ['url', 'label', 'slug'], + }), + url: Flags.string({ + char: 'u', + description: 'Add single article by Omnivore URL', + exclusive: ['hours', 'label', 'slug'], + }), + label: Flags.string({ + char: 'l', + description: 'Add articles with specific label', + exclusive: ['hours', 'url', 'slug'], + }), + slug: Flags.string({ + char: 's', + description: 'Add single article by slug', + exclusive: ['hours', 'url', 'label'], + }), + json: jsonFlag(), + }; + + protected async execute(flags: QueueAddFlags): Promise { + this.validateFlags(flags); + + await withDatabase(async ({ repo }) => { + let articles: QueueSourceArticle[] = []; + + if (flags.label) { + articles = await this.fetchByLabel(flags.label); + } else if (flags.url) { + articles = await this.fetchByUrl(flags.url); + } else if (flags.slug) { + articles = await this.fetchBySlug(flags.slug); + } else { + articles = await this.fetchByHours(this.requireHours(flags.hours)); + } + + const queueArticles = this.formatArticlesForQueue(articles); + const inserted = repo.initializeQueue(queueArticles); + this.outputResults(inserted, articles.length, flags.json); + }); + } + + private validateFlags(flags: QueueAddFlags): void { + if (!flags.hours && !flags.url && !flags.label && !flags.slug) { + throw new Error('One of --hours, --url, --label, or --slug is required'); + } + } + + private requireHours(hours: number | undefined): number { + if (hours === undefined) throw new Error('--hours is required'); + return hours; + } + + private async fetchByHours(hours: number): Promise { + const hoursAgo = new Date(Date.now() - hours * 60 * 60 * 1000); + const collected: QueueSourceArticle[] = []; + let after: string | undefined; + let shouldContinue = true; + + while (shouldContinue) { + const result = await this.fetchSearchPage(after); + const { items, reachedCutoff, nextCursor } = this.collectWithinCutoff(result, hoursAgo); + collected.push(...items); + shouldContinue = !reachedCutoff && Boolean(nextCursor); + after = nextCursor ?? undefined; + } + + return collected; + } + + private async fetchSearchPage(after?: string): Promise { + const result = await searchArticles({ + query: 'in:all sort:saved-desc', + first: 100, + after: after ?? '', + }); + + if (!isSearchResult(result)) throw new Error('Unexpected searchArticles response shape'); + return result; + } + + private collectWithinCutoff(result: SearchResult, cutoff: Date): { items: QueueSourceArticle[]; reachedCutoff: boolean; nextCursor?: string } { + const items: QueueSourceArticle[] = []; + for (const edge of result.edges) { + const node = edge.node; + if (new Date(node.savedAt) < cutoff) return { items, reachedCutoff: true }; + if (isQueueSourceArticle(node)) items.push(node); + } + return { items, reachedCutoff: false, nextCursor: result.pageInfo.endCursor }; + } + + private async fetchByLabel(labelName: string): Promise { + const result = await getArticlesByLabel(labelName, 50); + if (!isSearchResult(result)) throw new Error('Unexpected getArticlesByLabel response shape'); + checkGraphQLResult({ data: { search: result }, errors: undefined }); + return result.edges.map((edge) => this.toQueueSourceArticle(edge.node)); + } + + private async fetchBySlug(slug: string): Promise { + const username = await fetchUsername(); + const result = await getArticle(slug, username); + checkGraphQLResult({ data: { article: result }, errors: undefined }); + if (!result.article) { + throw new Error('Article not found'); + } + return [this.toQueueSourceArticle({ ...result.article, slug })]; + } + + private async fetchByUrl(url: string): Promise { + // AIDEV-NOTE: URL format is https://omnivore.app/{username}/{slug} + const urlPattern = /https:\/\/omnivore\.app\/([^/]+)\/(.+)/; + const match = url.match(urlPattern); + + if (!match) { + throw new Error('Invalid Omnivore URL. Expected format: https://omnivore.app/{username}/{slug}'); + } + + const [, username, slug] = match; + const result = await getArticle(slug, username); + checkGraphQLResult({ data: { article: result }, errors: undefined }); + if (!result.article) { + throw new Error('Article not found'); + } + return [this.toQueueSourceArticle({ ...result.article, slug })]; + } + + private toQueueSourceArticle(article: OmnivoreArticle): QueueSourceArticle { + if (!isQueueSourceArticle(article)) throw new Error('Article missing required queue fields'); + return { id: article.id, slug: article.slug, url: article.url, title: article.title, savedAt: article.savedAt }; + } + + private formatArticlesForQueue(articles: QueueSourceArticle[]): Array<{ id: string; slug: string; url: string; title: string; savedAt: string }> { + return articles.map(({ id, slug, url, title, savedAt }) => ({ id, slug, url, title, savedAt })); + } + + private outputResults(inserted: number, total: number, json: boolean): void { + if (json) { + this.log(JSON.stringify({ added: inserted, total })); + } else { + this.log(formatSuccess(`Added ${inserted} articles to queue (${total} total found)`)); + } + } + +} + +function isSearchResult(value: unknown): value is SearchResult { + return typeof value === 'object' && value !== null && 'edges' in value && 'pageInfo' in value; +} + +function isQueueSourceArticle(value: unknown): value is QueueSourceArticle { + if (typeof value !== 'object' || value === null) return false; + const v = value as { id?: unknown; slug?: unknown; url?: unknown; title?: unknown; savedAt?: unknown }; + return typeof v.id === 'string' && typeof v.slug === 'string' && typeof v.url === 'string' && typeof v.title === 'string' && typeof v.savedAt === 'string'; +} diff --git a/self-hosting/omc/src/commands/queue/clear.ts b/self-hosting/omc/src/commands/queue/clear.ts new file mode 100644 index 000000000..58b2e7ce7 --- /dev/null +++ b/self-hosting/omc/src/commands/queue/clear.ts @@ -0,0 +1,61 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatSuccess } from '@lib/cli/formatters.js'; + +/** + * Clear articles from queue by status or all. + * AIDEV-NOTE: CLI command for bulk queue clearing + */ +export default class QueueClear extends BaseCommand { + static override description = 'Clear articles from queue'; + + static override examples = [ + '$ omc queue clear --status failed', + '$ omc queue clear --status completed', + '$ omc queue clear --all', + '$ omc queue clear --all --json', + ]; + + static override flags = { + status: Flags.string({ + description: 'Clear articles with specific status (pending|in_progress|completed|failed)', + exclusive: ['all'], + }), + all: Flags.boolean({ + description: 'Clear all articles from queue', + exclusive: ['status'], + }), + json: jsonFlag(), + }; + + private validateFlags(flags: { status?: string; all: boolean; json: boolean }): void { + if (!flags.status && !flags.all) { + throw new Error('Either --status or --all flag is required'); + } + + if (flags.status) { + const validStatuses = ['pending', 'in_progress', 'completed', 'failed']; + if (!validStatuses.includes(flags.status)) { + throw new Error(`Invalid status: ${flags.status}. Must be one of: ${validStatuses.join(', ')}`); + } + } + } + + protected async execute(flags: { status?: string; all: boolean; json: boolean }): Promise { + this.validateFlags(flags); + + await withDatabase(async ({ repo }) => { + const clearedCount = flags.all + ? repo.clearAll() + : repo.clearByStatus(flags.status as string); + + if (flags.json) { + this.log(JSON.stringify({ cleared: clearedCount })); + } else { + this.log(formatSuccess(`Cleared ${clearedCount} articles from queue`)); + } + }); + } +} diff --git a/self-hosting/omc/src/commands/queue/export.ts b/self-hosting/omc/src/commands/queue/export.ts new file mode 100644 index 000000000..094fcdd2b --- /dev/null +++ b/self-hosting/omc/src/commands/queue/export.ts @@ -0,0 +1,59 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag, statusFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import type { AnalysisJob } from '@storage/AnalysisQueueRepository.js'; + +/** + * Export queue to JSONL format. + * AIDEV-NOTE: Exports queue jobs for backup/transfer (one JSON object per line) + */ +export default class QueueExport extends BaseCommand { + static override description = 'Export queue to JSONL format'; + + static override examples = [ + '$ omc queue export', + '$ omc queue export --status pending', + '$ omc queue export --json', + ]; + + static override flags = { + status: statusFlag(), + json: jsonFlag(), + }; + + async execute(flags: any): Promise { + await withDatabase(async ({ repo }) => { + const jobs = await this.fetchJobs(repo, flags.status); + + if (flags.json) { + this.outputJsonArray(jobs); + } else { + this.outputJsonl(jobs); + } + }); + } + + private async fetchJobs(repo: any, status?: string): Promise { + if (status) { + return repo.getByStatus(status); + } + + const allJobs: AnalysisJob[] = []; + + for (const s of ['pending', 'in_progress', 'completed', 'failed']) { + allJobs.push(...repo.getByStatus(s)); + } + + return allJobs; + } + + private outputJsonArray(jobs: AnalysisJob[]): void { + this.log(JSON.stringify(jobs, null, 2)); + } + + private outputJsonl(jobs: AnalysisJob[]): void { + for (const job of jobs) { + this.log(JSON.stringify(job)); + } + } +} diff --git a/self-hosting/omc/src/commands/queue/import.ts b/self-hosting/omc/src/commands/queue/import.ts new file mode 100644 index 000000000..5d977a35b --- /dev/null +++ b/self-hosting/omc/src/commands/queue/import.ts @@ -0,0 +1,70 @@ +import { Args } from '@oclif/core'; +import { readFileSync } from 'fs'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { outputResult, parseJsonSafely } from '@lib/cli/command-utils.js'; + +/** + * Import queue from JSONL file. + * AIDEV-NOTE: Imports queue jobs from backup/transfer file (one JSON object per line) + */ +export default class QueueImport extends BaseCommand { + static override description = 'Import queue from JSONL file'; + + static override examples = [ + '$ omc queue import queue-backup.jsonl', + '$ omc queue import queue-backup.jsonl --json', + ]; + + static override args = { + file: Args.string({ + description: 'Path to JSONL file', + required: true, + }), + }; + + static override flags = { + json: jsonFlag(), + }; + + async execute(flags: any): Promise { + const file = await this.parse(QueueImport).then((p) => p.args.file); + const articles = this.readJsonl(file); + + await withDatabase(async ({ repo }) => { + const count = repo.initializeQueue(articles); + outputResult(this, { imported: count }, `Imported ${count} articles`, flags.json); + }); + } + + private readJsonl(filePath: string): Array<{ id: string; slug: string; url: string; title: string; savedAt: string }> { + const content = readFileSync(filePath, 'utf-8'); + const lines = content.trim().split('\n'); + return lines.map(line => this.parseJobLine(line)); + } + + private parseJobLine(line: string): { id: string; slug: string; url: string; title: string; savedAt: string } { + const job = parseJsonSafely(line); + if (!job) throw new Error('Failed to parse JSONL line'); + this.validateJob(job); + + return { + id: job.articleId, + slug: job.articleSlug, + url: job.articleUrl, + title: job.articleTitle, + savedAt: job.savedAt, + }; + } + + private validateJob(job: any): void { + const required = ['articleId', 'articleSlug', 'articleUrl', 'articleTitle', 'savedAt']; + + for (const field of required) { + if (!job[field]) { + throw new Error(`Missing required field: ${field}`); + } + } + } +} diff --git a/self-hosting/omc/src/commands/queue/list.ts b/self-hosting/omc/src/commands/queue/list.ts new file mode 100644 index 000000000..9c95e92c1 --- /dev/null +++ b/self-hosting/omc/src/commands/queue/list.ts @@ -0,0 +1,49 @@ +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader } from '@lib/cli/formatters.js'; +import { displayJobs } from '@lib/cli/queue-display.js'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag, statusFlag } from '@lib/cli/shared-flags.js'; + +/** + * List articles in the analysis queue. + * AIDEV-NOTE: Displays queue contents with optional status filtering + */ +export default class QueueList extends BaseCommand { + static override description = 'List articles in the analysis queue'; + + static override examples = [ + '$ omc queue list', + '$ omc queue list --status pending', + '$ omc queue list --status completed', + '$ omc queue list --json', + ]; + + static override flags = { + status: statusFlag(), + json: jsonFlag(), + }; + + async execute(flags: any): Promise { + await withDatabase(async ({ repo }) => { + let jobs; + + if (flags.status) { + jobs = repo.getByStatus(flags.status); + } else { + // Get all jobs from all statuses + const pending = repo.getByStatus('pending'); + const inProgress = repo.getByStatus('in_progress'); + const completed = repo.getByStatus('completed'); + const failed = repo.getByStatus('failed'); + jobs = [...pending, ...inProgress, ...completed, ...failed]; + } + + if (flags.json) { + this.log(JSON.stringify(jobs, null, 2)); + } else { + this.log(formatHeader('Queue Listing')); + displayJobs(jobs); + } + }); + } +} diff --git a/self-hosting/omc/src/commands/queue/remove.ts b/self-hosting/omc/src/commands/queue/remove.ts new file mode 100644 index 000000000..5be3522d0 --- /dev/null +++ b/self-hosting/omc/src/commands/queue/remove.ts @@ -0,0 +1,47 @@ +import { Args } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatSuccess } from '@lib/cli/formatters.js'; + +/** + * Remove article from analysis queue. + * AIDEV-NOTE: CLI command for queue article removal + */ +export default class QueueRemove extends BaseCommand { + static override description = 'Remove article from queue'; + + static override examples = [ + '$ omc queue remove ', + ]; + + static override args = { + articleId: Args.string({ + description: 'Article ID to remove', + required: true, + }), + }; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: Record): Promise { + const { args } = await this.parse(QueueRemove); + + await withDatabase(async ({ repo }) => { + const removed = repo.removeArticle(args.articleId); + + if (removed === 0) { + this.warn(`Article ${args.articleId} not found in queue`); + return; + } + + if (flags.json) { + this.log(JSON.stringify({ removed, articleId: args.articleId })); + } else { + this.log(formatSuccess(`Removed article ${args.articleId} from queue`)); + } + }); + } +} diff --git a/self-hosting/omc/src/commands/queue/reset.ts b/self-hosting/omc/src/commands/queue/reset.ts new file mode 100644 index 000000000..d5f74f86b --- /dev/null +++ b/self-hosting/omc/src/commands/queue/reset.ts @@ -0,0 +1,74 @@ +import { Args, Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatSuccess } from '@lib/cli/formatters.js'; + +/** + * Reset article(s) to pending status for reprocessing. + * AIDEV-NOTE: Useful for retrying stuck in_progress jobs + */ +export default class QueueReset extends BaseCommand { + static override description = 'Reset article to pending status'; + + static override examples = [ + '$ omc queue reset ', + '$ omc queue reset --all-in-progress', + ]; + + static override args = { + articleId: Args.string({ + description: 'Article ID to reset', + required: false, + }), + }; + + static override flags = { + 'all-in-progress': Flags.boolean({ + description: 'Reset all in_progress articles to pending', + default: false, + }), + json: jsonFlag(), + }; + + + // AIDEV-NOTE: Extracted helper to keep execute() under 20 lines + private resetAllInProgress(repo: any, flags: any): void { + const jobs = repo.getByStatus('in_progress'); + for (const job of jobs) { + repo.resetToPending(job.articleId); + } + + if (flags.json) { + this.log(JSON.stringify({ reset: jobs.length })); + } else { + this.log(formatSuccess(`Reset ${jobs.length} in_progress articles to pending`)); + } + } + + // AIDEV-NOTE: Extracted helper to keep execute() under 20 lines + private resetSingleArticle(repo: any, articleId: string, flags: any): void { + repo.resetToPending(articleId); + + if (flags.json) { + this.log(JSON.stringify({ reset: 1, articleId })); + } else { + this.log(formatSuccess(`Reset article ${articleId} to pending`)); + } + } + + async execute(flags: any): Promise { + await withDatabase(async ({ repo }) => { + if (flags['all-in-progress']) { + this.resetAllInProgress(repo, flags); + return; + } + + if (!flags.articleId) { + throw new Error('Either provide article-id or use --all-in-progress flag'); + } + + this.resetSingleArticle(repo, flags.articleId, flags); + }); + } +} diff --git a/self-hosting/omc/src/commands/queue/stats.ts b/self-hosting/omc/src/commands/queue/stats.ts new file mode 100644 index 000000000..cae2688b0 --- /dev/null +++ b/self-hosting/omc/src/commands/queue/stats.ts @@ -0,0 +1,95 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader, formatDivider } from '@lib/cli/formatters.js'; +import { displayQueueStats } from '@lib/cli/queue-display.js'; + +/** + * Show queue statistics. + * AIDEV-NOTE: Displays aggregate queue metrics for monitoring + */ +export default class QueueStats extends BaseCommand { + static override description = 'Show queue statistics'; + + static override examples = [ + '$ omc queue stats', + '$ omc queue stats --detailed', + '$ omc queue stats --json', + ]; + + static override flags = { + detailed: Flags.boolean({ + description: 'Show detailed per-status breakdown', + default: false, + }), + json: jsonFlag(), + }; + + async execute(flags: any): Promise { + await withDatabase(async ({ repo }) => { + const stats = repo.getStats(); + + if (flags.json) { + const output = flags.detailed ? this.buildDetailedStats(repo) : stats; + this.log(JSON.stringify(output, null, 2)); + } else { + this.log(formatHeader('Queue Statistics')); + displayQueueStats(stats); + + if (flags.detailed) { + this.displayDetailedBreakdown(repo); + } + } + }); + } + + private buildDetailedStats(repo: any): any { + const stats = repo.getStats(); + const detailed: any = { ...stats, breakdown: {} }; + + for (const status of ['pending', 'in_progress', 'completed', 'failed']) { + const jobs = repo.getByStatus(status); + detailed.breakdown[status] = this.analyzeJobs(jobs); + } + + return detailed; + } + + private displayDetailedBreakdown(repo: any): void { + this.log('\n' + formatDivider()); + this.log('Detailed Breakdown\n'); + + for (const status of ['pending', 'in_progress', 'completed', 'failed']) { + const jobs = repo.getByStatus(status); + this.displayStatusAnalysis(status, jobs); + } + } + + private displayStatusAnalysis(status: string, jobs: any[]): void { + const analysis = this.analyzeJobs(jobs); + + this.log(`${status.toUpperCase()}:`); + this.log(` Count: ${analysis.count}`); + if (analysis.oldestDate) this.log(` Oldest: ${analysis.oldestDate}`); + if (analysis.newestDate) this.log(` Newest: ${analysis.newestDate}`); + if (status === 'failed') this.log(` Total Retries: ${analysis.totalRetries}`); + this.log(''); + } + + private analyzeJobs(jobs: any[]): any { + if (jobs.length === 0) { + return { count: 0 }; + } + + const dates = jobs.map(j => new Date(j.createdAt).getTime()).filter(d => !isNaN(d)); + const totalRetries = jobs.reduce((sum, j) => sum + (j.retryCount || 0), 0); + + return { + count: jobs.length, + oldestDate: dates.length > 0 ? new Date(Math.min(...dates)).toISOString() : null, + newestDate: dates.length > 0 ? new Date(Math.max(...dates)).toISOString() : null, + totalRetries, + }; + } +} diff --git a/self-hosting/omc/src/commands/report/corpus.ts b/self-hosting/omc/src/commands/report/corpus.ts new file mode 100644 index 000000000..6e430910b --- /dev/null +++ b/self-hosting/omc/src/commands/report/corpus.ts @@ -0,0 +1,103 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader } from '@lib/cli/formatters.js'; +import { parseJsonSafely } from '@lib/cli/command-utils.js'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; + +/** + * Full corpus analysis report. + * AIDEV-NOTE: Aggregates topic frequencies, sentiment distribution, content types + */ +export default class ReportCorpus extends BaseCommand { + static override description = 'Generate full corpus analysis report'; + + static override examples = [ + '$ omc report corpus', + '$ omc report corpus --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: any): Promise { + await withDatabase(async ({ repo }) => { + const jobs = repo.getCompletedWithAnalysis(); + const analyses = this.parseAnalyses(jobs); + + if (flags.json) { + this.log(JSON.stringify(this.buildCorpusStats(analyses), null, 2)); + } else { + this.displayCorpusReport(analyses); + } + }); + } + + private parseAnalyses(jobs: any[]): ContentAnalysis[] { + return jobs.map(job => parseJsonSafely(job.analysisJson)) + .filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A'); + } + + private buildCorpusStats(analyses: ContentAnalysis[]): any { + const topicFreq = this.countTopics(analyses); + const sentimentDist = this.countSentiment(analyses); + const contentTypeDist = this.countContentTypes(analyses); + + return { + totalAnalyses: analyses.length, + topicDistribution: topicFreq, + sentimentDistribution: sentimentDist, + contentTypeDistribution: contentTypeDist, + }; + } + + private displayCorpusReport(analyses: ContentAnalysis[]): void { + this.log(formatHeader('Corpus Analysis Report')); + this.log(`\nTotal Analyses: ${analyses.length}\n`); + + this.log('Topic Distribution:'); + const topics = this.countTopics(analyses); + for (const [topic, count] of Object.entries(topics).sort((a, b) => b[1] - a[1])) { + this.log(` ${topic.padEnd(30)} ${count}`); + } + + this.log('\nSentiment Distribution:'); + const sentiment = this.countSentiment(analyses); + for (const [sent, count] of Object.entries(sentiment)) { + this.log(` ${sent}: ${count}`); + } + + this.log('\nContent Types:'); + const types = this.countContentTypes(analyses); + for (const [type, count] of Object.entries(types).sort((a, b) => b[1] - a[1])) { + this.log(` ${type.padEnd(30)} ${count}`); + } + } + + private countTopics(analyses: ContentAnalysis[]): Record { + const counts: Record = {}; + for (const a of analyses) { + for (const topic of a.topics) { + counts[topic] = (counts[topic] || 0) + 1; + } + } + return counts; + } + + private countSentiment(analyses: ContentAnalysis[]): Record { + const counts: Record = {}; + for (const a of analyses) { + counts[a.sentiment] = (counts[a.sentiment] || 0) + 1; + } + return counts; + } + + private countContentTypes(analyses: ContentAnalysis[]): Record { + const counts: Record = {}; + for (const a of analyses) { + counts[a.contentType] = (counts[a.contentType] || 0) + 1; + } + return counts; + } +} diff --git a/self-hosting/omc/src/commands/report/custom.ts b/self-hosting/omc/src/commands/report/custom.ts new file mode 100644 index 000000000..077d7b2a5 --- /dev/null +++ b/self-hosting/omc/src/commands/report/custom.ts @@ -0,0 +1,84 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader } from '@lib/cli/formatters.js'; +import { parseJsonSafely } from '@lib/cli/command-utils.js'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; + +/** + * Custom SQL-based report. + * AIDEV-NOTE: Allows filtering analyses with custom SQL WHERE clause + */ +export default class ReportCustom extends BaseCommand { + static override description = 'Run custom SQL query against analyses'; + + static override examples = [ + '$ omc report custom --query "article_title LIKE \'%AI%\'"', + '$ omc report custom --query "completed_at > \'2024-01-01\'" --json', + ]; + + static override flags = { + query: Flags.string({ + description: 'SQL WHERE clause for filtering', + required: true, + }), + json: jsonFlag(), + }; + + protected async execute(flags: any): Promise { + await withDatabase(async ({ db }) => { + const results = this.executeCustomQuery(db, flags.query); + + if (flags.json) { + this.log(JSON.stringify(results, null, 2)); + } else { + this.displayResults(results); + } + }); + } + + private executeCustomQuery(db: any, whereClause: string): any[] { + try { + const sql = this.buildQuerySQL(whereClause); + const rows = db.prepare(sql).all(); + return this.mapQueryResults(rows); + } catch (error) { + throw new Error(`SQL query failed: ${error instanceof Error ? error.message : String(error)}`); + } + } + + private buildQuerySQL(whereClause: string): string { + return ` + SELECT article_id as articleId, article_title as articleTitle, + analysis_json as analysisJson, completed_at as completedAt + FROM analysis_queue + WHERE status = 'completed' AND analysis_json IS NOT NULL AND (${whereClause}) + ORDER BY completed_at DESC + `; + } + + private mapQueryResults(rows: any[]): any[] { + return rows.map((row: any) => { + const analysis = parseJsonSafely(row.analysisJson); + return { + articleId: row.articleId, + articleTitle: row.articleTitle, + completedAt: row.completedAt, + analysis, + }; + }).filter(r => r.analysis); + } + + private displayResults(results: any[]): void { + this.log(formatHeader('Custom Query Results')); + this.log(`\nFound ${results.length} matching analyses\n`); + + for (const { articleTitle, completedAt, analysis } of results) { + this.log(`${articleTitle}`); + this.log(` Completed: ${completedAt}`); + this.log(` Topics: ${analysis.topics.join(', ')}`); + this.log(` Sentiment: ${analysis.sentiment}\n`); + } + } +} diff --git a/self-hosting/omc/src/commands/report/export.ts b/self-hosting/omc/src/commands/report/export.ts new file mode 100644 index 000000000..20fde0746 --- /dev/null +++ b/self-hosting/omc/src/commands/report/export.ts @@ -0,0 +1,127 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { parseJsonSafely } from '@lib/cli/command-utils.js'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; + +/** + * Export report data. + * AIDEV-NOTE: Supports json|csv|markdown output formats for different use cases + */ +export default class ReportExport extends BaseCommand { + static override description = 'Export report data to file or stdout'; + + static override examples = [ + '$ omc report export --format json', + '$ omc report export --format csv --report-type topics', + '$ omc report export --format markdown', + ]; + + static override flags = { + format: Flags.string({ + description: 'Output format', + required: true, + options: ['json', 'csv', 'markdown'], + }), + 'report-type': Flags.string({ + description: 'Type of report to export', + default: 'corpus', + options: ['corpus', 'topics', 'sentiment', 'monetization'], + }), + }; + + protected async execute(flags: any): Promise { + await withDatabase(async ({ repo }) => { + const jobs = repo.getCompletedWithAnalysis(); + const analyses = this.parseAnalyses(jobs); + + const output = this.formatOutput(analyses, flags.format, flags['report-type']); + this.log(output); + }); + } + + private parseAnalyses(jobs: any[]): ContentAnalysis[] { + return jobs.map(job => parseJsonSafely(job.analysisJson)) + .filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A'); + } + + private formatOutput(analyses: ContentAnalysis[], format: string, reportType: string): string { + if (format === 'json') return this.exportJSON(analyses, reportType); + if (format === 'csv') return this.exportCSV(analyses, reportType); + return this.exportMarkdown(analyses, reportType); + } + + private exportJSON(analyses: ContentAnalysis[], reportType: string): string { + const data = reportType === 'topics' ? this.buildTopicsData(analyses) : + reportType === 'sentiment' ? this.buildSentimentData(analyses) : + reportType === 'monetization' ? this.buildMonetizationData(analyses) : + { analyses }; + return JSON.stringify(data, null, 2); + } + + private exportCSV(analyses: ContentAnalysis[], reportType: string): string { + if (reportType === 'topics') { + const topics = this.buildTopicsData(analyses); + return 'Topic,Frequency,AvgScore\n' + + topics.map((t: any) => `${t.topic},${t.frequency},${t.avgScore}`).join('\n'); + } + return 'ArticleId,Topics,Sentiment,ContentType\n' + + analyses.map(a => `${a.articleId},"${a.topics.join(';')}",${a.sentiment},${a.contentType}`).join('\n'); + } + + private exportMarkdown(analyses: ContentAnalysis[], reportType: string): string { + let md = `# ${reportType.charAt(0).toUpperCase() + reportType.slice(1)} Report\n\n`; + md += `Total Analyses: ${analyses.length}\n\n`; + + if (reportType === 'topics') { + md += '## Topic Distribution\n\n'; + const topics = this.buildTopicsData(analyses); + for (const t of topics) { + md += `- **${t.topic}**: ${t.frequency} articles (avg score: ${t.avgScore})\n`; + } + } else { + md += '## Articles\n\n'; + for (const a of analyses.slice(0, 10)) { + md += `### ${a.articleId}\n`; + md += `- Topics: ${a.topics.join(', ')}\n`; + md += `- Sentiment: ${a.sentiment}\n\n`; + } + } + return md; + } + + private buildTopicsData(analyses: ContentAnalysis[]): any[] { + const topicData: Record = {}; + for (const a of analyses) { + for (const topic of a.topics) { + if (!topicData[topic]) topicData[topic] = { count: 0, scoreSum: 0 }; + topicData[topic].count++; + topicData[topic].scoreSum += a.topicScores[topic] || 0; + } + } + return Object.entries(topicData) + .map(([topic, data]) => ({ + topic, + frequency: data.count, + avgScore: (data.scoreSum / data.count).toFixed(2), + })) + .sort((a, b) => b.frequency - a.frequency); + } + + private buildSentimentData(analyses: ContentAnalysis[]): any { + const counts: Record = {}; + for (const a of analyses) { + counts[a.sentiment] = (counts[a.sentiment] || 0) + 1; + } + return { distribution: counts }; + } + + private buildMonetizationData(analyses: ContentAnalysis[]): any[] { + return analyses.map(a => ({ + articleId: a.articleId, + monetizationAngle: a.monetizationAngle, + topics: a.topics, + sentiment: a.sentiment, + })); + } +} diff --git a/self-hosting/omc/src/commands/report/monetization.ts b/self-hosting/omc/src/commands/report/monetization.ts new file mode 100644 index 000000000..b1993cac2 --- /dev/null +++ b/self-hosting/omc/src/commands/report/monetization.ts @@ -0,0 +1,95 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader } from '@lib/cli/formatters.js'; +import { parseJsonSafely } from '@lib/cli/command-utils.js'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; + +/** + * Monetization opportunities report. + * AIDEV-NOTE: Extracts monetizationAngle from analyses, ranks by potential + */ +export default class ReportMonetization extends BaseCommand { + static override description = 'Show monetization opportunities across analyses'; + + static override examples = [ + '$ omc report monetization', + '$ omc report monetization --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: any): Promise { + await withDatabase(async ({ repo }) => { + const jobs = repo.getCompletedWithAnalysis(); + const analyses = this.parseAnalyses(jobs); + + const opportunities = this.extractOpportunities(analyses); + + if (flags.json) { + this.log(JSON.stringify(opportunities, null, 2)); + } else { + this.displayOpportunities(opportunities); + } + }); + } + + private parseAnalyses(jobs: any[]): ContentAnalysis[] { + return jobs.map(job => parseJsonSafely(job.analysisJson)) + .filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A'); + } + + private extractOpportunities(analyses: ContentAnalysis[]): any[] { + const themeGroups: Record = {}; + + for (const a of analyses) { + const theme = this.extractTheme(a.monetizationAngle); + if (!themeGroups[theme]) themeGroups[theme] = []; + themeGroups[theme].push({ + articleId: a.articleId, + angle: a.monetizationAngle, + sentiment: a.sentiment, + topics: a.topics, + }); + } + + return Object.entries(themeGroups) + .map(([theme, items]) => ({ + theme, + count: items.length, + items, + potential: this.calculatePotential(items), + })) + .sort((a, b) => b.potential - a.potential); + } + + private extractTheme(angle: string): string { + const lower = angle.toLowerCase(); + if (lower.includes('comparison') || lower.includes('compare')) return 'comparison'; + if (lower.includes('roundup') || lower.includes('weekly')) return 'roundup'; + if (lower.includes('tutorial') || lower.includes('guide')) return 'tutorial'; + if (lower.includes('review')) return 'review'; + return 'other'; + } + + private calculatePotential(items: any[]): number { + const positiveCount = items.filter(i => i.sentiment === 'positive').length; + return items.length * 10 + positiveCount * 5; + } + + private displayOpportunities(opportunities: any[]): void { + this.log(formatHeader('Monetization Opportunities Report')); + this.log(''); + + for (const { theme, count, items, potential } of opportunities) { + this.log(`${theme.toUpperCase()} (${count} articles, potential: ${potential})`); + for (const item of items.slice(0, 3)) { + this.log(` • ${item.angle}`); + } + if (items.length > 3) this.log(` ... and ${items.length - 3} more\n`); + else this.log(''); + } + } +} diff --git a/self-hosting/omc/src/commands/report/sentiment.ts b/self-hosting/omc/src/commands/report/sentiment.ts new file mode 100644 index 000000000..b50d73d3b --- /dev/null +++ b/self-hosting/omc/src/commands/report/sentiment.ts @@ -0,0 +1,94 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader } from '@lib/cli/formatters.js'; +import { parseJsonSafely } from '@lib/cli/command-utils.js'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; + +/** + * Sentiment analysis report. + * AIDEV-NOTE: Sentiment distribution correlated with topics, identifies outliers + */ +export default class ReportSentiment extends BaseCommand { + static override description = 'Show sentiment analysis across corpus'; + + static override examples = [ + '$ omc report sentiment', + '$ omc report sentiment --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: any): Promise { + await withDatabase(async ({ repo }) => { + const jobs = repo.getCompletedWithAnalysis(); + const analyses = this.parseAnalyses(jobs); + + const sentimentData = this.analyzeSentiment(analyses); + + if (flags.json) { + this.log(JSON.stringify(sentimentData, null, 2)); + } else { + this.displaySentimentReport(sentimentData); + } + }); + } + + private parseAnalyses(jobs: any[]): ContentAnalysis[] { + return jobs.map(job => parseJsonSafely(job.analysisJson)) + .filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A'); + } + + private analyzeSentiment(analyses: ContentAnalysis[]): any { + const distribution = this.countSentiment(analyses); + const topicCorrelation = this.correlateSentimentWithTopics(analyses); + + return { distribution, topicCorrelation }; + } + + private countSentiment(analyses: ContentAnalysis[]): Record { + const counts: Record = {}; + for (const a of analyses) { + counts[a.sentiment] = (counts[a.sentiment] || 0) + 1; + } + return counts; + } + + private correlateSentimentWithTopics(analyses: ContentAnalysis[]): any[] { + const topicSentiment: Record> = {}; + + for (const a of analyses) { + for (const topic of a.topics) { + if (!topicSentiment[topic]) { + topicSentiment[topic] = { positive: 0, neutral: 0, negative: 0 }; + } + topicSentiment[topic][a.sentiment]++; + } + } + + return Object.entries(topicSentiment) + .map(([topic, sentiments]) => ({ + topic, + positive: sentiments.positive || 0, + neutral: sentiments.neutral || 0, + negative: sentiments.negative || 0, + total: (sentiments.positive || 0) + (sentiments.neutral || 0) + (sentiments.negative || 0), + })) + .sort((a, b) => b.total - a.total); + } + + private displaySentimentReport(data: any): void { + this.log(formatHeader('Sentiment Analysis Report')); + this.log('\nOverall Distribution:'); + for (const [sentiment, count] of Object.entries(data.distribution)) { + this.log(` ${sentiment}: ${count}`); + } + + this.log('\nSentiment by Topic:'); + for (const { topic, positive, neutral, negative, total } of data.topicCorrelation) { + this.log(` ${topic.padEnd(30)} +${positive} ~${neutral} -${negative} (total: ${total})`); + } + } +} diff --git a/self-hosting/omc/src/commands/report/topics.ts b/self-hosting/omc/src/commands/report/topics.ts new file mode 100644 index 000000000..debd36c8f --- /dev/null +++ b/self-hosting/omc/src/commands/report/topics.ts @@ -0,0 +1,94 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader } from '@lib/cli/formatters.js'; +import { parseJsonSafely } from '@lib/cli/command-utils.js'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; + +/** + * Topic distribution report. + * AIDEV-NOTE: Groups analyses by topics with frequency and average scores + */ +export default class ReportTopics extends BaseCommand { + static override description = 'Show topic distribution across all analyses'; + + static override examples = [ + '$ omc report topics', + '$ omc report topics --min-score 0.8', + '$ omc report topics --json', + ]; + + static override flags = { + 'min-score': Flags.string({ + description: 'Minimum topic score threshold (0-1)', + default: '0.0', + }), + json: jsonFlag(), + }; + + protected async execute(flags: any): Promise { + await withDatabase(async ({ repo }) => { + const jobs = repo.getCompletedWithAnalysis(); + const analyses = this.parseAnalyses(jobs); + const minScore = parseFloat(flags['min-score']); + + const topicStats = this.buildTopicStats(analyses, minScore); + + if (flags.json) { + this.log(JSON.stringify(topicStats, null, 2)); + } else { + this.displayTopicReport(topicStats); + } + }); + } + + private parseAnalyses(jobs: any[]): ContentAnalysis[] { + return jobs.map(job => parseJsonSafely(job.analysisJson)) + .filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A'); + } + + private buildTopicStats(analyses: ContentAnalysis[], minScore: number): any[] { + const topicData = this.aggregateTopicData(analyses, minScore); + return this.formatTopicStats(topicData); + } + + private aggregateTopicData(analyses: ContentAnalysis[], minScore: number): Record { + const topicData: Record = {}; + + for (const a of analyses) { + for (const topic of a.topics) { + const score = a.topicScores[topic] || 0; + if (score < minScore) continue; + + if (!topicData[topic]) { + topicData[topic] = { count: 0, scoreSum: 0, articles: [] }; + } + topicData[topic].count++; + topicData[topic].scoreSum += score; + topicData[topic].articles.push(a.articleId); + } + } + return topicData; + } + + private formatTopicStats(topicData: Record): any[] { + return Object.entries(topicData) + .map(([topic, data]) => ({ + topic, + frequency: data.count, + avgScore: data.count > 0 ? (data.scoreSum / data.count).toFixed(2) : '0', + articles: data.articles, + })) + .sort((a, b) => b.frequency - a.frequency); + } + + private displayTopicReport(stats: any[]): void { + this.log(formatHeader('Topic Distribution Report')); + this.log(''); + + for (const { topic, frequency, avgScore } of stats) { + this.log(`${topic.padEnd(30)} ${frequency} articles (avg score: ${avgScore})`); + } + } +} diff --git a/self-hosting/omc/src/commands/report/trends.ts b/self-hosting/omc/src/commands/report/trends.ts new file mode 100644 index 000000000..6775034d7 --- /dev/null +++ b/self-hosting/omc/src/commands/report/trends.ts @@ -0,0 +1,109 @@ +import { Flags } from '@oclif/core'; +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { withDatabase } from '@lib/cli/database.js'; +import { formatHeader } from '@lib/cli/formatters.js'; +import { parseJsonSafely } from '@lib/cli/command-utils.js'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; + +/** + * Trending topics over time. + * AIDEV-NOTE: Tracks topic frequency changes, identifies rising/falling trends + */ +export default class ReportTrends extends BaseCommand { + static override description = 'Show trending topics over time'; + + static override examples = [ + '$ omc report trends', + '$ omc report trends --period week', + '$ omc report trends --json', + ]; + + static override flags = { + period: Flags.string({ + description: 'Time period for grouping (day|week|month)', + default: 'week', + options: ['day', 'week', 'month'], + }), + json: jsonFlag(), + }; + + protected async execute(flags: any): Promise { + await withDatabase(async ({ repo }) => { + const jobs = repo.getCompletedWithAnalysis(); + const trends = this.buildTrendAnalysis(jobs, flags.period); + + if (flags.json) { + this.log(JSON.stringify(trends, null, 2)); + } else { + this.displayTrends(trends); + } + }); + } + + private buildTrendAnalysis(jobs: any[], period: string): any { + const periodData: Record> = {}; + + for (const job of jobs) { + const analysis = parseJsonSafely(job.analysisJson); + if (!analysis || !analysis.topics || analysis.topics[0] === 'N/A') continue; + + const periodKey = this.getPeriodKey(job.completedAt, period); + if (!periodData[periodKey]) periodData[periodKey] = {}; + + for (const topic of analysis.topics) { + periodData[periodKey][topic] = (periodData[periodKey][topic] || 0) + 1; + } + } + + return this.calculateTrends(periodData); + } + + private getPeriodKey(date: string, period: string): string { + const d = new Date(date); + if (period === 'day') return d.toISOString().split('T')[0]; + if (period === 'week') { + const weekNum = Math.floor(d.getTime() / (7 * 24 * 60 * 60 * 1000)); + return `Week-${weekNum}`; + } + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}`; + } + + private calculateTrends(periodData: Record>): any { + const periods = Object.keys(periodData).sort(); + const allTopics = this.extractAllTopics(periodData); + const trends = Array.from(allTopics).map(topic => this.calculateTopicTrend(topic, periods, periodData)); + + return { periods, trends: trends.sort((a, b) => parseFloat(b.avgSecond) - parseFloat(a.avgSecond)) }; + } + + private extractAllTopics(periodData: Record>): Set { + const topics = new Set(); + for (const topicCounts of Object.values(periodData)) { + for (const topic of Object.keys(topicCounts)) topics.add(topic); + } + return topics; + } + + private calculateTopicTrend(topic: string, periods: string[], periodData: Record>): any { + const counts = periods.map(p => periodData[p][topic] || 0); + const firstHalf = counts.slice(0, Math.floor(counts.length / 2)); + const secondHalf = counts.slice(Math.floor(counts.length / 2)); + + const avgFirst = firstHalf.reduce((a, b) => a + b, 0) / (firstHalf.length || 1); + const avgSecond = secondHalf.reduce((a, b) => a + b, 0) / (secondHalf.length || 1); + const trend = avgSecond > avgFirst ? 'rising' : avgSecond < avgFirst ? 'falling' : 'stable'; + + return { topic, trend, avgFirst: avgFirst.toFixed(1), avgSecond: avgSecond.toFixed(1) }; + } + + private displayTrends(data: any): void { + this.log(formatHeader('Topic Trends Report')); + this.log(`\nPeriods analyzed: ${data.periods.join(', ')}\n`); + + for (const { topic, trend, avgFirst, avgSecond } of data.trends) { + const arrow = trend === 'rising' ? '↑' : trend === 'falling' ? '↓' : '→'; + this.log(`${arrow} ${topic.padEnd(30)} ${avgFirst} → ${avgSecond} (${trend})`); + } + } +} diff --git a/self-hosting/omc/src/commands/version.ts b/self-hosting/omc/src/commands/version.ts new file mode 100644 index 000000000..073183712 --- /dev/null +++ b/self-hosting/omc/src/commands/version.ts @@ -0,0 +1,65 @@ +import { BaseCommand } from '@lib/cli/base-command.js'; +import { jsonFlag } from '@lib/cli/shared-flags.js'; +import { outputResult } from '@lib/cli/command-utils.js'; +import { readFileSync } from 'fs'; +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +/** + * Display version and system information + * AIDEV-NOTE: version-command - shows package version and runtime environment + */ +export default class Version extends BaseCommand { + static override description = 'Show version and system information'; + + static override examples = [ + '<%= config.bin %> <%= command.id %>', + '<%= config.bin %> <%= command.id %> --json', + ]; + + static override flags = { + json: jsonFlag(), + }; + + protected async execute(flags: { json: boolean }): Promise { + const info = { + version: this.getPackageVersion(), + node: process.version, + platform: process.platform, + arch: process.arch, + }; + + if (flags.json) { + outputResult(this, info, '', true); + } else { + this.displayVersionTable(info); + } + } + + private getPackageVersion(): string { + try { + const __filename = fileURLToPath(import.meta.url); + const __dirname = dirname(__filename); + const pkgPath = join(__dirname, '../../../package.json'); + const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8')); + return pkg.version || 'unknown'; + } catch { + return 'unknown'; + } + } + + private displayVersionTable(info: SystemInfo): void { + this.log('\nOmnivore Content System\n'); + this.log(`Version: ${info.version}`); + this.log(`Node: ${info.node}`); + this.log(`Platform: ${info.platform}`); + this.log(`Architecture: ${info.arch}`); + } +} + +interface SystemInfo { + version: string; + node: string; + platform: string; + arch: string; +} diff --git a/self-hosting/omc/src/lib/ai/anthropic-client.ts b/self-hosting/omc/src/lib/ai/anthropic-client.ts new file mode 100644 index 000000000..8c82835f4 --- /dev/null +++ b/self-hosting/omc/src/lib/ai/anthropic-client.ts @@ -0,0 +1,183 @@ +/** + * Anthropic API Client + * Wrapper for Claude API for content analysis + */ + +import Anthropic from '@anthropic-ai/sdk'; +import { config } from 'dotenv'; +import type { ContentAnalysis, AnalysisRequest } from '@omc-types/analysis.js'; + +// Load environment variables +config(); + +const API_KEY = process.env.ANTHROPIC_API_KEY; +const MODEL = 'claude-sonnet-4-5-20250929'; +const MAX_TOKENS = 8000; + +if (!API_KEY) { + console.error('❌ ANTHROPIC_API_KEY not set in environment'); + process.exit(1); +} + +// Initialize client +const client = new Anthropic({ apiKey: API_KEY }); + +/** + * Token and cost tracking + */ +interface UsageMetrics { + inputTokens: number; + outputTokens: number; + totalCost: number; // In USD +} + +const COST_PER_INPUT_TOKEN = 0.000003; // $3 per 1M tokens +const COST_PER_OUTPUT_TOKEN = 0.000015; // $15 per 1M tokens + +/** + * Calculate cost from token usage + */ +function calculateCost(inputTokens: number, outputTokens: number): number { + return (inputTokens * COST_PER_INPUT_TOKEN) + (outputTokens * COST_PER_OUTPUT_TOKEN); +} + +/** + * Sleep for exponential backoff + */ +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +/** + * Retry logic with exponential backoff + */ +async function withRetry( + operation: () => Promise, + maxAttempts: number = 3 +): Promise { + let lastError: Error; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await operation(); + } catch (error) { + lastError = error as Error; + if (attempt === maxAttempts) break; + + const backoffMs = Math.pow(2, attempt) * 1000; + console.error(`Attempt ${attempt} failed, retrying in ${backoffMs}ms...`); + await sleep(backoffMs); + } + } + + throw lastError!; +} + +/** + * Parse JSON response from Claude + */ +function parseAnalysisResponse(text: string): ContentAnalysis { + const jsonMatch = text.match(/\{[\s\S]*\}/); + if (!jsonMatch) { + throw new Error('No JSON found in response'); + } + + return JSON.parse(jsonMatch[0]); +} + +/** + * Build usage metrics from Anthropic response + */ +function buildUsageMetrics(usage: Anthropic.Usage): UsageMetrics { + return { + inputTokens: usage.input_tokens, + outputTokens: usage.output_tokens, + totalCost: calculateCost(usage.input_tokens, usage.output_tokens), + }; +} + +/** + * Analyze article content using Claude + * + * @param request Article data to analyze + * @param systemPrompt System prompt for analysis + * @returns Analysis result with usage metrics + */ +export async function analyzeArticle( + request: AnalysisRequest, + systemPrompt: string +): Promise<{ analysis: ContentAnalysis; usage: UsageMetrics }> { + const userMessage = buildUserMessage(request); + + return withRetry(async () => { + const response = await client.messages.create({ + model: MODEL, + max_tokens: MAX_TOKENS, + system: systemPrompt, + messages: [{ role: 'user', content: userMessage }], + }); + + const text = extractTextContent(response); + const analysis = parseAnalysisResponse(text); + const usage = buildUsageMetrics(response.usage); + + return { analysis, usage }; + }); +} + +/** + * Build user message from request data + */ +function buildUserMessage(request: AnalysisRequest): string { + let message = `Title: ${request.title}\n`; + message += `URL: ${request.url}\n`; + if (request.author) message += `Author: ${request.author}\n`; + if (request.publishedAt) message += `Published: ${request.publishedAt}\n`; + message += `Word Count: ${request.wordCount}\n\n`; + + if (request.highlights.length > 0) { + message += `User Highlights:\n`; + request.highlights.forEach(h => { + message += `- "${h.quote}"\n`; + if (h.annotation) message += ` Note: ${h.annotation}\n`; + }); + message += `\n`; + } + + message += `Content:\n${request.content}`; + return message; +} + +/** + * Extract text from Claude response content blocks + */ +function extractTextContent(response: Anthropic.Message): string { + return response.content + .filter(block => block.type === 'text') + .map(block => (block as Anthropic.TextBlock).text) + .join(''); +} + +/** + * Test API connection + */ +export async function testConnection(): Promise { + try { + const response = await client.messages.create({ + model: MODEL, + max_tokens: 100, + messages: [{ role: 'user', content: 'Hello' }], + }); + + console.log('✅ Connected to Anthropic API'); + console.log(` Model: ${MODEL}`); + console.log(` Response tokens: ${response.usage.output_tokens}`); + return true; + } catch (error) { + console.error('❌ Failed to connect to Anthropic API'); + console.error(` ${(error as Error).message}`); + return false; + } +} + +export { MODEL, MAX_TOKENS }; diff --git a/self-hosting/omc/src/lib/ai/codex-cli-client.ts b/self-hosting/omc/src/lib/ai/codex-cli-client.ts new file mode 100644 index 000000000..c0ac2e54d --- /dev/null +++ b/self-hosting/omc/src/lib/ai/codex-cli-client.ts @@ -0,0 +1,96 @@ +import { mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import type { AnalysisRequest, ContentAnalysis } from '@omc-types/analysis.js'; +import { withRetry } from '@lib/ai/retry.js'; +import { spawnToString } from '@lib/ai/spawn-to-string.js'; + +export interface CodexCliUsageMetrics { + note: string; +} + +export interface CodexCliOptions { + model?: string; + codexHomeDir?: string; + timeoutMs?: number; + maxAttempts?: number; +} + +export async function analyzeArticleWithCodexCli( + request: AnalysisRequest, + systemPrompt: string, + options: CodexCliOptions = {} +): Promise<{ analysis: ContentAnalysis; usage: CodexCliUsageMetrics; raw: string }> { + const prompt = buildPrompt(systemPrompt, request); + const raw = await withRetry(() => runCodexExec(prompt, options), { + maxAttempts: options.maxAttempts ?? 3, + }); + const analysis = parseJsonObject(raw); + return { analysis, usage: { note: 'Token/cost metrics not available via codex CLI' }, raw }; +} + +function buildPrompt(systemPrompt: string, request: AnalysisRequest): string { + const header = [ + 'You are a strict JSON generator.', + 'Do not run tools, commands, or read/write files.', + 'Return ONLY a single valid JSON object (no markdown, no prose).', + systemPrompt.trim(), + '', + `Title: ${request.title}`, + `URL: ${request.url}`, + request.author ? `Author: ${request.author}` : null, + request.publishedAt ? `Published: ${request.publishedAt}` : null, + `Word Count: ${request.wordCount}`, + '', + request.highlights?.length ? formatHighlights(request.highlights) : null, + 'Content:', + request.content, + ] + .filter(Boolean) + .join('\n'); + + return header + '\n'; +} + +function formatHighlights(highlights: AnalysisRequest['highlights']): string { + const lines = ['User Highlights:']; + for (const h of highlights) { + lines.push(`- "${h.quote}"`); + if (h.annotation) lines.push(` Note: ${h.annotation}`); + } + return lines.join('\n') + '\n'; +} + +async function runCodexExec(prompt: string, options: CodexCliOptions): Promise { + const args = buildCodexArgs(options); + const env = buildCodexEnv(options); + const { code, stdout, stderr } = await spawnToString('codex', args, prompt, env, options.timeoutMs); + if (code !== 0) throw new Error(`codex exec failed (exit ${code}): ${trimStderr(stderr)}`); + return stdout.trim() || stderr.trim(); +} + +function buildCodexArgs(options: CodexCliOptions): string[] { + const args = ['exec', '-s', 'read-only']; + if (options.model) args.push('-m', options.model); + args.push('-'); + return args; +} + +function buildCodexEnv(options: CodexCliOptions): NodeJS.ProcessEnv { + const codexHomeDir = options.codexHomeDir ?? join(process.cwd(), 'temp', 'codex-home'); + mkdirSync(codexHomeDir, { recursive: true }); + return { ...process.env, CODEX_HOME: codexHomeDir }; +} + +function parseJsonObject(text: string): T { + const start = text.indexOf('{'); + const end = text.lastIndexOf('}'); + if (start < 0 || end <= start) throw new Error('No JSON object found in codex output'); + const json = text.slice(start, end + 1); + return JSON.parse(json) as T; +} + +function trimStderr(stderr: string): string { + const value = stderr.trim(); + if (!value) return '(no stderr)'; + return value.length > 1200 ? value.slice(0, 1200) + '…' : value; +} diff --git a/self-hosting/omc/src/lib/ai/retry.ts b/self-hosting/omc/src/lib/ai/retry.ts new file mode 100644 index 000000000..1257c33df --- /dev/null +++ b/self-hosting/omc/src/lib/ai/retry.ts @@ -0,0 +1,27 @@ +export interface RetryOptions { + maxAttempts: number; + baseDelayMs?: number; +} + +export async function withRetry(fn: () => Promise, options: RetryOptions): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= options.maxAttempts; attempt++) { + try { + return await fn(); + } catch (err) { + lastError = err; + if (attempt === options.maxAttempts) break; + await sleepMs(backoffDelayMs(attempt, options.baseDelayMs ?? 500)); + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)); +} + +function backoffDelayMs(attempt: number, baseDelayMs: number): number { + return 2 ** attempt * baseDelayMs; +} + +function sleepMs(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + diff --git a/self-hosting/omc/src/lib/ai/spawn-to-string.ts b/self-hosting/omc/src/lib/ai/spawn-to-string.ts new file mode 100644 index 000000000..421fa4e2b --- /dev/null +++ b/self-hosting/omc/src/lib/ai/spawn-to-string.ts @@ -0,0 +1,43 @@ +import { spawn } from 'node:child_process'; + +export interface SpawnToStringResult { + code: number | null; + stdout: string; + stderr: string; +} + +export function spawnToString( + cmd: string, + args: string[], + stdin: string, + env: NodeJS.ProcessEnv, + timeoutMs?: number +): Promise { + return new Promise((resolve, reject) => { + const child = spawn(cmd, args, { env, stdio: 'pipe' }); + const out = attachCollectors(child); + child.on('error', reject); + applyTimeout(child, timeoutMs); + child.stdin.end(stdin); + child.on('close', code => resolve({ code, stdout: out.stdout, stderr: out.stderr })); + }); +} + +function attachCollectors(child: ReturnType): { stdout: string; stderr: string } { + let stdout = ''; + let stderr = ''; + if (!child.stdout || !child.stderr) { + throw new Error('spawnToString requires stdout/stderr pipes'); + } + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', chunk => (stdout += chunk)); + child.stderr.on('data', chunk => (stderr += chunk)); + return { get stdout() { return stdout; }, get stderr() { return stderr; } }; +} + +function applyTimeout(child: ReturnType, timeoutMs?: number): void { + if (!timeoutMs) return; + const timer = setTimeout(() => child.kill('SIGKILL'), timeoutMs); + timer.unref(); +} diff --git a/self-hosting/omc/src/lib/cli/__tests__/graphql.test.ts b/self-hosting/omc/src/lib/cli/__tests__/graphql.test.ts new file mode 100644 index 000000000..9192a09bd --- /dev/null +++ b/self-hosting/omc/src/lib/cli/__tests__/graphql.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; +import { checkGraphQLResult } from '../graphql.js'; + +describe('checkGraphQLResult', () => { + it('throws on GraphQL-level errors', () => { + expect(() => checkGraphQLResult({ errors: [{ message: 'Boom' }] })).toThrow(/GraphQL errors: Boom/); + }); + + it('throws on domain-level errorCodes under data', () => { + expect(() => checkGraphQLResult({ data: { article: { errorCodes: ['NOT_FOUND'] } } })).toThrow(/article error: NOT_FOUND/); + }); + + it('does not throw when there are no errors', () => { + expect(() => checkGraphQLResult({ data: { updatePage: { updatedPage: { id: '1' } } } })).not.toThrow(); + }); +}); + diff --git a/self-hosting/omc/src/lib/cli/base-command.ts b/self-hosting/omc/src/lib/cli/base-command.ts new file mode 100644 index 000000000..b6c2a49e8 --- /dev/null +++ b/self-hosting/omc/src/lib/cli/base-command.ts @@ -0,0 +1,40 @@ +import { Command } from '@oclif/core'; +import { handleCommandError } from './command-utils.js'; + +/** + * Abstract base command class for all CLI commands. + * AIDEV-NOTE: DRY utility - eliminates duplicate run() method across 10 commands + * + * @example + * class MyCommand extends BaseCommand { + * protected async execute(): Promise { + * // command logic here + * } + * } + */ +export abstract class BaseCommand extends Command { + // Note: We handle --json manually via shared-flags, not oclif's built-in JSON flag + static enableJsonFlag = false; + + /** + * Standard run method with error handling. + * Pattern: try { parse → execute } catch { handleError } + * Note: oclif commands should return normally on success, not call this.exit(0) + */ + async run(): Promise { + try { + const { args, flags } = await this.parse(this.constructor as typeof BaseCommand); + await this.execute({ ...args, ...flags }); + // Success: return normally (oclif handles exit code 0) + } catch (error) { + handleCommandError(this, error); + } + } + + /** + * Abstract method to be implemented by each command. + * Contains command-specific logic. + * @param flags - Parsed command flags from oclif + */ + protected abstract execute(flags: Record): Promise; +} diff --git a/self-hosting/omc/src/lib/cli/command-utils.ts b/self-hosting/omc/src/lib/cli/command-utils.ts new file mode 100644 index 000000000..2d7a94b62 --- /dev/null +++ b/self-hosting/omc/src/lib/cli/command-utils.ts @@ -0,0 +1,75 @@ +import { Command } from '@oclif/core'; +import { readFileSync, existsSync } from 'fs'; +import { formatSuccess } from './formatters.js'; +import { EXIT_CODES } from './constants.js'; + +/** + * Shared error handling for all commands. + * AIDEV-NOTE: DRY utility - eliminates duplicate handleError across 10+ commands + * + * @example + * try { + * await doWork(); + * } catch (error) { + * handleCommandError(this, error); + * } + */ +export function handleCommandError(command: Command, error: unknown): void { + const message = error instanceof Error ? error.message : String(error); + command.error(message, { exit: false }); + process.exit(EXIT_CODES.ERROR); +} + +/** + * Shared output formatter for JSON vs text modes. + * AIDEV-NOTE: DRY utility - eliminates duplicate output logic across commands + * + * @example + * outputResult(this, { count: 5 }, 'Processed 5 items', flags.json); + */ +export function outputResult( + command: Command, + data: any, + successMessage: string, + jsonMode: boolean +): void { + if (jsonMode) { + command.log(JSON.stringify(data, null, 2)); + } else { + command.log(formatSuccess(successMessage)); + } +} + +/** + * Safe JSON parsing with error handling. + * AIDEV-NOTE: DRY utility - eliminates unsafe JSON.parse across 15+ locations + * + * @example + * const analysis = parseJsonSafely(job.analysisJson); + */ +export function parseJsonSafely(jsonString: string | undefined, fallback?: T): T | undefined { + if (!jsonString) return fallback; + try { + return JSON.parse(jsonString) as T; + } catch { + return fallback; + } +} + +/** + * Load .env file as key-value pairs. + * AIDEV-NOTE: DRY utility - eliminates duplicate .env parsing logic + * + * @example + * const config = loadEnvFile('.env'); + */ +export function loadEnvFile(envPath: string = '.env'): Record { + if (!existsSync(envPath)) return {}; + const content = readFileSync(envPath, 'utf-8'); + const env: Record = {}; + content.split('\n').forEach(line => { + const match = line.match(/^([^#][^=]+)=(.*)$/); + if (match) env[match[1].trim()] = match[2].trim(); + }); + return env; +} diff --git a/self-hosting/omc/src/lib/cli/constants.ts b/self-hosting/omc/src/lib/cli/constants.ts new file mode 100644 index 000000000..6cece2e20 --- /dev/null +++ b/self-hosting/omc/src/lib/cli/constants.ts @@ -0,0 +1,39 @@ +/** + * Shared constants for CLI commands + * AIDEV-NOTE: DRY utility - centralizes exit codes and status values used across all commands + */ + +/** + * Standard exit codes for CLI commands + * Used consistently across all command implementations + */ +export const EXIT_CODES = { + SUCCESS: 0, + ERROR: 1, + NOT_FOUND: 3, +} as const; + +export type ExitCode = typeof EXIT_CODES[keyof typeof EXIT_CODES]; + +/** + * Queue status values matching database schema + * Source: src/storage/AnalysisQueueRepository.ts:17 + */ +export const QUEUE_STATUS = { + PENDING: 'pending', + IN_PROGRESS: 'in_progress', + COMPLETED: 'completed', + FAILED: 'failed', +} as const; + +export type QueueStatus = typeof QUEUE_STATUS[keyof typeof QUEUE_STATUS]; + +/** + * Array of valid status values for validation + */ +export const VALID_STATUSES: readonly QueueStatus[] = [ + QUEUE_STATUS.PENDING, + QUEUE_STATUS.IN_PROGRESS, + QUEUE_STATUS.COMPLETED, + QUEUE_STATUS.FAILED, +] as const; diff --git a/self-hosting/omc/src/lib/cli/database.ts b/self-hosting/omc/src/lib/cli/database.ts new file mode 100644 index 000000000..5c86bec2e --- /dev/null +++ b/self-hosting/omc/src/lib/cli/database.ts @@ -0,0 +1,32 @@ +import Database from 'better-sqlite3'; +import { initDatabase } from '@storage/database.js'; +import { AnalysisQueueRepository } from '@storage/AnalysisQueueRepository.js'; + +export interface DatabaseContext { + db: Database.Database; + repo: AnalysisQueueRepository; +} + +/** + * Execute callback with initialized database and repository. + * Ensures proper cleanup via try/finally pattern. + * + * AIDEV-NOTE: Eliminates repeated db init/close pattern across 6+ CLI scripts + * + * @example + * await withDatabase(async ({ db, repo }) => { + * const stats = repo.getQueueStats(); + * return stats; + * }); + */ +export async function withDatabase( + callback: (ctx: DatabaseContext) => Promise +): Promise { + const db = initDatabase(); + const repo = new AnalysisQueueRepository(db); + try { + return await callback({ db, repo }); + } finally { + db.close(); + } +} diff --git a/self-hosting/omc/src/lib/cli/formatters.ts b/self-hosting/omc/src/lib/cli/formatters.ts new file mode 100644 index 000000000..999098dbe --- /dev/null +++ b/self-hosting/omc/src/lib/cli/formatters.ts @@ -0,0 +1,85 @@ +/** + * Console output formatting utilities for CLI scripts. + * Provides consistent visual styling across all commands. + */ + +const HEADER_WIDTH = 80; +const COLUMN_WIDTH = 30; + +/** + * Format section header with double-line border. + * + * @example + * console.log(formatHeader('Queue Statistics')); + * // ═══════════════════════════════════... + * // Queue Statistics + * // ═══════════════════════════════════... + */ +export function formatHeader(title: string): string { + const border = '═'.repeat(HEADER_WIDTH); + return `${border}\n${title}\n${border}`; +} + +/** + * Format section divider with single-line border. + */ +export function formatDivider(): string { + return '─'.repeat(HEADER_WIDTH); +} + +/** + * Format success message with checkmark prefix. + * + * @example + * console.log(formatSuccess('Analysis complete')); + * // ✓ Analysis complete + */ +export function formatSuccess(message: string): string { + return `✓ ${message}`; +} + +/** + * Format error message with cross prefix. + * + * @example + * console.log(formatError('Failed to fetch article')); + * // ✗ Failed to fetch article + */ +export function formatError(message: string): string { + return `✗ ${message}`; +} + +/** + * Format data as simple aligned table. + * + * AIDEV-NOTE: Simple column formatting - not full table library + * Columns are padded to COLUMN_WIDTH for alignment + * + * @example + * formatTable( + * [{ name: 'foo', status: 'done' }], + * ['name', 'status'] + * ); + */ +export function formatTable(data: any[], columns: string[]): string { + if (data.length === 0) { + return '(no data)'; + } + + const rows: string[] = []; + + // Header row + const header = columns.map(col => col.padEnd(COLUMN_WIDTH)).join(''); + rows.push(header); + rows.push('─'.repeat(header.length)); + + // Data rows + for (const item of data) { + const row = columns + .map(col => String(item[col] ?? '').padEnd(COLUMN_WIDTH)) + .join(''); + rows.push(row); + } + + return rows.join('\n'); +} diff --git a/self-hosting/omc/src/lib/cli/graphql.ts b/self-hosting/omc/src/lib/cli/graphql.ts new file mode 100644 index 000000000..7b54d8b04 --- /dev/null +++ b/self-hosting/omc/src/lib/cli/graphql.ts @@ -0,0 +1,79 @@ +import { getMe } from '@lib/omnivore/client.js'; +import type { OmnivoreUser } from '@omc-types/omnivore.js'; + +/** + * GraphQL client utilities with error handling. + * Standardizes error checking and user authentication patterns. + */ + +/** + * Check GraphQL result for errors and throw if found. + * Handles both GraphQL-level errors and domain-level error codes. + * + * AIDEV-NOTE: Pattern from fetch-articles.ts - checks errors at both levels + * GraphQL errors = network/parse issues, errorCodes = domain errors + * + * @throws Error with GraphQL error details + * @example + * const result = await client.query(...); + * checkGraphQLResult(result); + */ +export function checkGraphQLResult(result: unknown): void { + const errors = getGraphQLErrorMessages(result); + if (errors.length) throw new Error(`GraphQL errors: ${errors.join(', ')}`); + + const domainError = getDomainError(result); + if (domainError) throw new Error(domainError); +} + +/** + * Fetch authenticated user's username. + * + * @throws Error if authentication fails or username not found + * @example + * const username = await fetchUsername(); + * console.log(`Authenticated as: ${username}`); + */ +export async function fetchUsername(): Promise { + const me = await getMe(); + + if (!isOmnivoreUser(me)) throw new Error('Failed to fetch username - authentication may have failed'); + return me.profile.username; +} + +function isOmnivoreUser(value: unknown): value is OmnivoreUser { + if (typeof value !== 'object' || value === null) return false; + const profile = (value as { profile?: unknown }).profile; + if (typeof profile !== 'object' || profile === null) return false; + return typeof (profile as { username?: unknown }).username === 'string'; +} + +function getGraphQLErrorMessages(result: unknown): string[] { + if (typeof result !== 'object' || result === null) return []; + const errors = (result as { errors?: unknown }).errors; + if (!Array.isArray(errors)) return []; + return errors.map((e) => (typeof e === 'object' && e !== null ? String((e as { message?: unknown }).message ?? '') : '')).filter(Boolean); +} + +function getDomainError(result: unknown): string | null { + const directCodes = extractErrorCodes(result); + if (directCodes?.length) return `Error: ${directCodes.join(', ')}`; + + if (typeof result !== 'object' || result === null) return null; + const data = (result as { data?: unknown }).data; + if (typeof data !== 'object' || data === null) return null; + + for (const [op, value] of Object.entries(data as Record)) { + const codes = extractErrorCodes(value); + if (codes?.length) return `${op} error: ${codes.join(', ')}`; + } + + return null; +} + +function extractErrorCodes(value: unknown): string[] | null { + if (typeof value !== 'object' || value === null) return null; + const codes = (value as { errorCodes?: unknown }).errorCodes; + if (!Array.isArray(codes)) return null; + return codes.map(String); +} diff --git a/self-hosting/omc/src/lib/cli/queue-display.ts b/self-hosting/omc/src/lib/cli/queue-display.ts new file mode 100644 index 000000000..06ac84aa5 --- /dev/null +++ b/self-hosting/omc/src/lib/cli/queue-display.ts @@ -0,0 +1,98 @@ +import type { AnalysisJob } from '@storage/AnalysisQueueRepository.js'; + +/** + * Queue statistics display type. + * Matches AnalysisQueueRepository.getQueueStats() return type. + */ +export interface QueueStats { + total: number; + pending: number; + inProgress: number; + completed: number; + failed: number; +} + +/** + * Display queue statistics in consistent format. + * Pattern from cli/analysis-status.ts + * + * @example + * const stats = repo.getQueueStats(); + * displayQueueStats(stats); + * // Total: 42 + * // Pending: 10 + * // In Progress: 2 + * // Completed: 28 + * // Failed: 2 + */ +export function displayQueueStats(stats: QueueStats): void { + console.log(`Total: ${stats.total}`); + console.log(`Pending: ${stats.pending}`); + console.log(`In Progress: ${stats.inProgress}`); + console.log(`Completed: ${stats.completed}`); + console.log(`Failed: ${stats.failed}`); +} + +/** + * Format table header with column names and divider. + * AIDEV-NOTE: Column widths match formatJobRow for alignment + */ +function formatJobHeader(): string { + const COL_ID = 10; + const COL_SLUG = 50; + const COL_STATUS = 15; + const COL_DATE = 25; + + const header = 'ID'.padEnd(COL_ID) + + 'Slug'.padEnd(COL_SLUG) + + 'Status'.padEnd(COL_STATUS) + + 'Created'.padEnd(COL_DATE); + const divider = '─'.repeat(COL_ID + COL_SLUG + COL_STATUS + COL_DATE); + + return header + '\n' + divider; +} + +/** + * Format single job row with truncated slug and ISO date. + * AIDEV-NOTE: Truncates slug at 47 chars + '...' to fit column width + */ +function formatJobRow(job: AnalysisJob): string { + const COL_ID = 10; + const COL_SLUG = 50; + const COL_STATUS = 15; + const COL_DATE = 25; + + const slug = job.articleSlug.length > COL_SLUG - 3 + ? job.articleSlug.substring(0, COL_SLUG - 3) + '...' + : job.articleSlug; + + const created = job.createdAt + ? new Date(job.createdAt).toISOString() + : 'N/A'; + + return String(job.id).padEnd(COL_ID) + + slug.padEnd(COL_SLUG) + + job.status.padEnd(COL_STATUS) + + created.padEnd(COL_DATE); +} + +/** + * Display analysis jobs in tabular format. + * Shows key job details: ID, slug, status, timestamps. + * + * @example + * const jobs = repo.getRecentJobs(10); + * displayJobs(jobs); + */ +export function displayJobs(jobs: AnalysisJob[]): void { + if (jobs.length === 0) { + console.log('(no jobs found)'); + return; + } + + console.log(formatJobHeader()); + + for (const job of jobs) { + console.log(formatJobRow(job)); + } +} diff --git a/self-hosting/omc/src/lib/cli/shared-flags.ts b/self-hosting/omc/src/lib/cli/shared-flags.ts new file mode 100644 index 000000000..9a7bd5c9d --- /dev/null +++ b/self-hosting/omc/src/lib/cli/shared-flags.ts @@ -0,0 +1,28 @@ +import { Flags } from '@oclif/core'; +import { VALID_STATUSES } from './constants.js'; + +/** + * Shared flag definitions for CLI commands + * AIDEV-NOTE: DRY utility - eliminates duplicate flag definitions across 10 commands + * Reference: OCLIF docs recommend baseFlags pattern for shared flags + */ + +/** + * JSON output flag - used by all 10 commands + * Appears in every command with identical definition + */ +export const jsonFlag = () => + Flags.boolean({ + description: 'Output as JSON', + default: false, + }); + +/** + * Status filter flag - used by queue list and content list + * Options match QUEUE_STATUS constants + */ +export const statusFlag = () => + Flags.string({ + description: 'Filter by status', + options: [...VALID_STATUSES], + }); diff --git a/self-hosting/omc/src/lib/omnivore/client.ts b/self-hosting/omc/src/lib/omnivore/client.ts new file mode 100644 index 000000000..119d1028e --- /dev/null +++ b/self-hosting/omc/src/lib/omnivore/client.ts @@ -0,0 +1,24 @@ +/** + * Re-export Omnivore client functions for TypeScript imports + * AIDEV-NOTE: Thin wrapper - actual implementation in /lib/omnivore/client.js + */ +export { + getMe, + searchArticles, + getArticle, + getArticlesByDate, + getArticlesByLabel, + getRecentArticles, + searchByTopic, + getUnreadArticles, + getLabels, + getHighlights, + updatePage, + createLabel, + setLabels, + saveUrl, + createHighlight, + updateHighlight, + deleteHighlight, + testConnection, +} from '../../../lib/omnivore/client.js'; diff --git a/self-hosting/omc/src/storage/AnalysisQueueRepository.ts b/self-hosting/omc/src/storage/AnalysisQueueRepository.ts new file mode 100644 index 000000000..5d11548e0 --- /dev/null +++ b/self-hosting/omc/src/storage/AnalysisQueueRepository.ts @@ -0,0 +1,405 @@ +// AIDEV-NOTE: tracking + immutable analysis storage - coordination and AI snapshots +// AIDEV-NOTE: analysisJson stores original AI output; git Markdown files are human-editable + +import Database from 'better-sqlite3'; + +export interface AnalysisJob { + id: number; + articleId: string; + articleSlug: string; // AIDEV: REQUIRED for article(slug, username) GQL query + articleUrl: string; + articleTitle: string; + savedAt: string; + publishedAt?: string; + updatedAtArticle?: string; + analysisJson?: string; // Full ContentAnalysis as JSON + markdownPath?: string; // Path to git-tracked .md file + status: 'pending' | 'in_progress' | 'completed' | 'failed'; + assignedAt?: string; + completedAt?: string; + errorMessage?: string; + retryCount: number; + createdAt: string; + updatedAt: string; +} + +export interface QueueStats { + total: number; + pending: number; + inProgress: number; + completed: number; + failed: number; +} + +/** + * Repository for analysis job queue + * + * BOUNDARY: This is TRACKING ONLY + * - Coordinates parallel analysis execution + * - Tracks which articles need analysis + * - Prevents duplicate work + * - Analysis RESULTS stored in Markdown/JSONL, not here + */ +export class AnalysisQueueRepository { + constructor(private db: Database.Database) {} + + private readonly PENDING_QUERY = ` + SELECT + id, + article_id as articleId, + article_slug as articleSlug, + article_url as articleUrl, + article_title as articleTitle, + saved_at as savedAt, + published_at as publishedAt, + updated_at_article as updatedAtArticle, + analysis_json as analysisJson, + markdown_path as markdownPath, + status, + assigned_at as assignedAt, + completed_at as completedAt, + error_message as errorMessage, + retry_count as retryCount, + created_at as createdAt, + updated_at as updatedAt + FROM analysis_queue + WHERE status = 'pending' + ORDER BY created_at ASC + LIMIT ? + `; + + /** + * Initialize queue from list of article metadata + * AIDEV-NOTE: tracking-initialization - sets up job queue, not analysis storage + * AIDEV-NOTE: gql-article-query-fields - slug required for fetching article content + */ + initializeQueue(articles: Array<{ id: string; slug: string; url: string; title: string; savedAt: string }>): number { + const insert = this.db.prepare(` + INSERT OR IGNORE INTO analysis_queue + (article_id, article_slug, article_url, article_title, saved_at, status, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, 'pending', datetime('now'), datetime('now')) + `); + + const insertMany = this.db.transaction((articles: any[]) => { + let inserted = 0; + for (const article of articles) { + const result = insert.run(article.id, article.slug, article.url, article.title, article.savedAt); + inserted += result.changes; + } + return inserted; + }); + + return insertMany(articles); + } + + /** + * Get next batch of pending jobs + * AIDEV-NOTE: tracking-coordination - fetches jobs for parallel processing + */ + getPending(limit: number = 5): AnalysisJob[] { + return this.db.prepare(this.PENDING_QUERY).all(limit) as AnalysisJob[]; + } + + /** + * Mark job as in_progress (coordination lock) + * AIDEV-NOTE: tracking-lock - prevents duplicate analysis by parallel workers + */ + markInProgress(articleId: string): void { + this.db.prepare(` + UPDATE analysis_queue + SET status = 'in_progress', + assigned_at = datetime('now'), + updated_at = datetime('now') + WHERE article_id = ? + `).run(articleId); + } + + /** + * Mark job as completed without storing analysis (use storeAnalysis instead) + * AIDEV-NOTE: tracking-completion - deprecated, use storeAnalysis() to save results + */ + markCompleted(articleId: string): void { + this.db.prepare(` + UPDATE analysis_queue + SET status = 'completed', + completed_at = datetime('now'), + updated_at = datetime('now') + WHERE article_id = ? + `).run(articleId); + } + + /** + * Store complete analysis result (immutable AI snapshot) + * AIDEV-NOTE: analysis-storage - saves original AI output to database + */ + storeAnalysis( + articleId: string, + publishedAt: string | null, + updatedAtArticle: string | null, + analysisJson: string, + markdownPath: string + ): void { + this.db.prepare(` + UPDATE analysis_queue + SET status = 'completed', + published_at = ?, + updated_at_article = ?, + analysis_json = ?, + markdown_path = ?, + completed_at = datetime('now'), + updated_at = datetime('now') + WHERE article_id = ? + `).run(publishedAt, updatedAtArticle, analysisJson, markdownPath, articleId); + } + + /** + * Mark job as failed for retry + * AIDEV-NOTE: tracking-error - increments retry counter, job stays in queue + */ + markFailed(articleId: string, errorMessage: string): void { + this.db.prepare(` + UPDATE analysis_queue + SET status = 'failed', + error_message = ?, + retry_count = retry_count + 1, + updated_at = datetime('now') + WHERE article_id = ? + `).run(errorMessage, articleId); + } + + /** + * Reset failed job to pending for retry + * AIDEV-NOTE: tracking-retry - gives failed job another chance + */ + resetToPending(articleId: string): void { + this.db.prepare(` + UPDATE analysis_queue + SET status = 'pending', + error_message = NULL, + assigned_at = NULL, + updated_at = datetime('now') + WHERE article_id = ? + `).run(articleId); + } + + /** + * Get queue statistics + * AIDEV-NOTE: tracking-stats - shows progress, not analysis content + */ + getStats(): QueueStats { + const stats = this.db.prepare(` + SELECT + COUNT(*) as total, + SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) as pending, + SUM(CASE WHEN status = 'in_progress' THEN 1 ELSE 0 END) as inProgress, + SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) as completed, + SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed + FROM analysis_queue + `).get() as any; + + return { + total: stats.total || 0, + pending: stats.pending || 0, + inProgress: stats.inProgress || 0, + completed: stats.completed || 0, + failed: stats.failed || 0 + }; + } + + /** + * Check if article already in queue + * AIDEV-NOTE: tracking-deduplication - prevents duplicate queue entries + */ + hasArticle(articleId: string): boolean { + const result = this.db.prepare(` + SELECT COUNT(*) as count FROM analysis_queue + WHERE article_id = ? + `).get(articleId) as { count: number }; + + return result.count > 0; + } + + /** + * Get all failed jobs for review + * AIDEV-NOTE: tracking-failures - lists jobs that need retry or investigation + */ + getFailed(): AnalysisJob[] { + return this.db.prepare(` + SELECT + id, + article_id as articleId, + article_slug as articleSlug, + article_url as articleUrl, + article_title as articleTitle, + status, + assigned_at as assignedAt, + completed_at as completedAt, + error_message as errorMessage, + retry_count as retryCount, + created_at as createdAt, + updated_at as updatedAt + FROM analysis_queue + WHERE status = 'failed' + ORDER BY retry_count DESC, updated_at DESC + `).all() as AnalysisJob[]; + } + + /** + * Get jobs by status + */ + getByStatus(status: string): AnalysisJob[] { + return this.db.prepare(` + SELECT + id, + article_id as articleId, + article_slug as articleSlug, + article_url as articleUrl, + article_title as articleTitle, + saved_at as savedAt, + published_at as publishedAt, + updated_at_article as updatedAtArticle, + status, + analysis_json as analysisJson, + markdown_path as markdownPath, + assigned_at as assignedAt, + completed_at as completedAt, + error_message as errorMessage, + retry_count as retryCount, + created_at as createdAt, + updated_at as updatedAt + FROM analysis_queue + WHERE status = ? + ORDER BY created_at DESC + `).all(status) as AnalysisJob[]; + } + + /** + * Get all jobs regardless of status + * AIDEV-NOTE: tracking-all-jobs - fetches all articles for --all flag + */ + getAll(limit?: number): AnalysisJob[] { + const limitClause = limit ? `LIMIT ${limit}` : ''; + return this.db.prepare(` + SELECT id, article_id as articleId, article_slug as articleSlug, + article_url as articleUrl, article_title as articleTitle, + saved_at as savedAt, published_at as publishedAt, + updated_at_article as updatedAtArticle, status, + assigned_at as assignedAt, completed_at as completedAt, + error_message as errorMessage, retry_count as retryCount, + created_at as createdAt, updated_at as updatedAt + FROM analysis_queue + ORDER BY created_at ASC ${limitClause} + `).all() as AnalysisJob[]; + } + + /** + * Clear completed jobs (cleanup after export) + * AIDEV-NOTE: tracking-cleanup - removes completed jobs after export to Markdown + */ + clearCompleted(): number { + const result = this.db.prepare(` + DELETE FROM analysis_queue + WHERE status = 'completed' + `).run(); + + return result.changes; + } + + /** + * Remove specific article from queue + * AIDEV-NOTE: tracking-removal - deletes single article by ID + */ + removeArticle(articleId: string): number { + const result = this.db.prepare(` + DELETE FROM analysis_queue + WHERE article_id = ? + `).run(articleId); + + return result.changes; + } + + /** + * Clear all articles with specific status + * AIDEV-NOTE: tracking-bulk-clear - removes articles by status filter + */ + clearByStatus(status: string): number { + const result = this.db.prepare(` + DELETE FROM analysis_queue + WHERE status = ? + `).run(status); + + return result.changes; + } + + /** + * Clear entire queue (all articles) + * AIDEV-NOTE: tracking-reset - nuclear option, removes all queue entries + */ + clearAll(): number { + const result = this.db.prepare(` + DELETE FROM analysis_queue + `).run(); + + return result.changes; + } + + /** + * Get specific job by article ID + */ + getByArticleId(articleId: string): AnalysisJob | null { + const result = this.db.prepare(` + SELECT + id, + article_id as articleId, + article_slug as articleSlug, + article_url as articleUrl, + article_title as articleTitle, + saved_at as savedAt, + published_at as publishedAt, + updated_at_article as updatedAtArticle, + status, + analysis_json as analysisJson, + markdown_path as markdownPath, + assigned_at as assignedAt, + completed_at as completedAt, + error_message as errorMessage, + retry_count as retryCount, + created_at as createdAt, + updated_at as updatedAt + FROM analysis_queue + WHERE article_id = ? + `).get(articleId) as AnalysisJob | undefined; + + return result || null; + } + + /** + * Get completed jobs with analysis JSON + * AIDEV-NOTE: report-helper - returns only completed analyses with parsed JSON + */ + getCompletedWithAnalysis(): AnalysisJob[] { + return this.db.prepare(` + SELECT + id, + article_id as articleId, + article_slug as articleSlug, + article_url as articleUrl, + article_title as articleTitle, + saved_at as savedAt, + published_at as publishedAt, + updated_at_article as updatedAtArticle, + analysis_json as analysisJson, + markdown_path as markdownPath, + status, + assigned_at as assignedAt, + completed_at as completedAt, + error_message as errorMessage, + retry_count as retryCount, + created_at as createdAt, + updated_at as updatedAt + FROM analysis_queue + WHERE status = 'completed' AND analysis_json IS NOT NULL + ORDER BY completed_at DESC + `).all() as AnalysisJob[]; + } +} diff --git a/self-hosting/omc/src/storage/AnalysisWriter.ts b/self-hosting/omc/src/storage/AnalysisWriter.ts new file mode 100644 index 000000000..e54d1e8cb --- /dev/null +++ b/self-hosting/omc/src/storage/AnalysisWriter.ts @@ -0,0 +1,180 @@ +// AIDEV-NOTE: analysis-output-boundary - writes to git-tracked Markdown/JSONL, NOT database +// AIDEV-NOTE: git-tracked-output - permanent storage for analysis results + +import { appendFileSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import matter from 'gray-matter'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; + +export interface AnalysisWriterConfig { + outputDir: string; // e.g., 'content/analysis' +} + +export interface AnalysisJsonlRecord { + articleId: string; + articleSlug?: string; + articleUrl: string; + articleTitle: string; + savedAt: string; + publishedAt?: string | null; + updatedAt?: string | null; + markdownPath?: string; + analyzedAt: string; + topics: string[]; + topicScores: Record; + sentiment: string; + summary: string; + keyPoints: string[]; + monetizationAngle: string; + contentType: string; + problemStatement: string; + audienceLevel: string; + technologiesMentioned: string[]; + companiesMentioned: string[]; + peopleMentioned: string[]; + conceptsExplained: string[]; + relatedTechnologies: string[]; + useCases: string[]; + targetKeywords: string[]; + searchQuestions: string[]; + githubRepo: string; + releaseInfo: string; +} + +export class AnalysisWriter { + private outputDir: string; + + constructor(config: AnalysisWriterConfig) { + this.outputDir = config.outputDir; + + // Ensure output directory exists + mkdirSync(this.outputDir, { recursive: true }); + } + + /** + * Write ContentAnalysis to Markdown file with YAML front-matter + * @param articleId - Omnivore article ID + * @param articleUrl - Source article URL + * @param articleTitle - Source article title + * @param savedAt - When article was saved to Omnivore + * @param analysis - ContentAnalysis from Claude + * @param articleSlug - Optional Omnivore slug (preferred for filename stability) + * @returns File path where analysis was written + */ + async write( + articleId: string, + articleUrl: string, + articleTitle: string, + savedAt: string, + analysis: ContentAnalysis, + articleSlug?: string + ): Promise { + const slug = this.generateSlug(articleSlug ?? articleTitle); + const date = new Date(savedAt).toISOString().split('T')[0]; // YYYY-MM-DD + const filename = `${date}-${slug}-analysis.md`; + const filePath = join(this.outputDir, filename); + + const content = this.formatMarkdown( + articleId, + articleUrl, + articleTitle, + savedAt, + analysis, + articleSlug + ); + + writeFileSync(filePath, content, 'utf-8'); + + return filePath; + } + + /** + * Generate URL-friendly slug from article title + */ + private generateSlug(title: string): string { + return title + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') // Replace non-alphanumeric with hyphens + .replace(/^-+|-+$/g, '') // Remove leading/trailing hyphens + .substring(0, 50); // Limit length + } + + /** + * Format ContentAnalysis as Markdown with YAML front-matter + */ + private formatMarkdown( + articleId: string, + articleUrl: string, + articleTitle: string, + savedAt: string, + analysis: ContentAnalysis, + articleSlug?: string + ): string { + const frontMatter = { + articleId, + articleSlug: articleSlug ?? undefined, + articleUrl, + articleTitle, + savedAt, + analyzedAt: analysis.analyzedAt, + topics: analysis.topics, + topicScores: analysis.topicScores, + sentiment: analysis.sentiment, + contentType: analysis.contentType, + problemStatement: analysis.problemStatement, + audienceLevel: analysis.audienceLevel, + technologiesMentioned: analysis.technologiesMentioned, + companiesMentioned: analysis.companiesMentioned, + peopleMentioned: analysis.peopleMentioned, + conceptsExplained: analysis.conceptsExplained, + relatedTechnologies: analysis.relatedTechnologies, + useCases: analysis.useCases, + targetKeywords: analysis.targetKeywords, + searchQuestions: analysis.searchQuestions, + githubRepo: analysis.githubRepo, + releaseInfo: analysis.releaseInfo, + }; + + const bodyLines = [ + '## Summary', + '', + analysis.summary, + '', + '## Key Points', + '', + ...analysis.keyPoints.map((point) => `- ${point}`), + '', + '## Monetization Angle', + '', + analysis.monetizationAngle, + '', + ]; + + return matter.stringify(bodyLines.join('\n'), frontMatter); + } + + /** + * Append analysis to JSONL file for machine-readable storage + * AIDEV-NOTE: git-tracked-output - JSONL format for batch processing + * + * @param jsonlPath - Path to JSONL file (e.g., 'content/analysis/analyses.jsonl') + * @param data - Complete analysis data with metadata + */ + async appendToJsonl( + jsonlPath: string, + data: AnalysisJsonlRecord + ): Promise { + // Ensure directory exists + const dir = dirname(jsonlPath); + mkdirSync(dir, { recursive: true }); + + // Create file if it doesn't exist + if (!existsSync(jsonlPath)) { + writeFileSync(jsonlPath, '', 'utf-8'); + } + + // Append JSON line + const jsonLine = JSON.stringify(data) + '\n'; + appendFileSync(jsonlPath, jsonLine, 'utf-8'); + } +} diff --git a/self-hosting/omc/src/storage/ContentReader.ts b/self-hosting/omc/src/storage/ContentReader.ts new file mode 100644 index 000000000..55bbf6f78 --- /dev/null +++ b/self-hosting/omc/src/storage/ContentReader.ts @@ -0,0 +1,129 @@ +// AIDEV-NOTE: storage-reader; reads and parses Markdown files with YAML front-matter + +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import matter from 'gray-matter'; +import type { AnalysisFrontMatter, StoredAnalysis } from '@omc-types/content.js'; + +export interface ContentReaderConfig { + directory: string; // e.g., 'content/analysis' +} + +export class ContentReader { + private directory: string; + + constructor(config: ContentReaderConfig) { + this.directory = config.directory; + } + + /** + * List all Markdown files in directory, sorted by date (newest first) + * @param _pattern - Optional glob pattern (not implemented yet, just lists all) + * @returns Array of file paths + */ + async list(_pattern?: string): Promise { + try { + const files = readdirSync(this.directory) + .filter(f => f.endsWith('.md')) + .sort() + .reverse(); // Newest first (assumes YYYY-MM-DD prefix) + + return files.map(f => join(this.directory, f)); + } catch (error) { + // Directory doesn't exist yet + return []; + } + } + + /** + * Read and parse a Markdown file with front-matter + * @param filePath - Absolute path to file + * @returns StoredAnalysis with parsed front-matter and content sections + */ + async read(filePath: string): Promise { + const fileContent = readFileSync(filePath, 'utf-8'); + const parsed = matter(fileContent); + + // Extract sections from markdown body + const sections = this.parseMarkdownSections(parsed.content); + + return { + frontMatter: parsed.data as AnalysisFrontMatter, + summary: sections.summary || '', + keyPoints: sections.keyPoints || [], + monetizationAngle: sections.monetizationAngle || '' + }; + } + + /** + * Find analysis by article ID + * @param articleId - Omnivore article ID + * @returns StoredAnalysis or null if not found + */ + async findByArticleId(articleId: string): Promise { + const files = await this.list(); + + for (const filePath of files) { + const analysis = await this.read(filePath); + if (analysis.frontMatter.articleId === articleId) { + return analysis; + } + } + + return null; + } + + /** + * Search analyses by topic + * @param topic - Topic to search for + * @returns Array of StoredAnalysis matching topic + */ + async searchByTopic(topic: string): Promise { + const files = await this.list(); + const results: StoredAnalysis[] = []; + + for (const filePath of files) { + const analysis = await this.read(filePath); + if (analysis.frontMatter.topics.includes(topic)) { + results.push(analysis); + } + } + + return results; + } + + /** + * Parse markdown sections from body + */ + private parseMarkdownSections(content: string): { + summary?: string; + keyPoints?: string[]; + monetizationAngle?: string; + } { + const sections: { summary?: string; keyPoints?: string[]; monetizationAngle?: string } = {}; + + // Extract Summary + const summaryMatch = content.match(/## Summary\s+([\s\S]*?)(?=\n##|$)/); + if (summaryMatch) { + sections.summary = summaryMatch[1].trim(); + } + + // Extract Key Points + const keyPointsMatch = content.match(/## Key Points\s+([\s\S]*?)(?=\n##|$)/); + if (keyPointsMatch) { + const pointsText = keyPointsMatch[1].trim(); + sections.keyPoints = pointsText + .split('\n') + .filter(line => line.trim().startsWith('-')) + .map(line => line.replace(/^-\s*/, '').trim()); + } + + // Extract Monetization Angle + const monetizationMatch = content.match(/## Monetization Angle\s+([\s\S]*?)(?=\n##|$)/); + if (monetizationMatch) { + sections.monetizationAngle = monetizationMatch[1].trim(); + } + + return sections; + } +} diff --git a/self-hosting/omc/src/storage/__tests__/analysis-writer-frontmatter.test.ts b/self-hosting/omc/src/storage/__tests__/analysis-writer-frontmatter.test.ts new file mode 100644 index 000000000..c0d853162 --- /dev/null +++ b/self-hosting/omc/src/storage/__tests__/analysis-writer-frontmatter.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import matter from 'gray-matter'; +import { AnalysisWriter } from '../AnalysisWriter.js'; +import type { ContentAnalysis } from '@omc-types/analysis.js'; + +describe('AnalysisWriter.write front-matter', () => { + it('produces YAML that round-trips through gray-matter for common edge cases', async () => { + const dir = mkdtempSync(join(tmpdir(), 'omc-frontmatter-')); + try { + const writer = new AnalysisWriter({ outputDir: join(dir, 'out') }); + const analysis = buildAnalysis('id-1'); + const path = await writer.write( + 'id-1', + 'https://example.com/a?x=1&y=two:three', + 'Title: "quotes" and colon: test', + '2025-01-01T00:00:00.000Z', + analysis, + 'omnivore-slug:with:colons' + ); + + const parsed = matter(readFileSync(path, 'utf-8')); + expect(parsed.data.articleId).toBe('id-1'); + expect(parsed.data.articleSlug).toBe('omnivore-slug:with:colons'); + expect(parsed.data.articleUrl).toBe('https://example.com/a?x=1&y=two:three'); + expect(parsed.data.articleTitle).toBe('Title: "quotes" and colon: test'); + expect(parsed.data.topics).toEqual(['ai tooling', 'foo:bar', 'quoted "topic"']); + expect(parsed.data.topicScores).toEqual({ 'ai tooling': 0.9, 'foo:bar': 0.8 }); + expect(parsed.data.problemStatement).toBe('N/A: "no problem"'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +function buildAnalysis(articleId: string): ContentAnalysis { + return { + articleId, + topics: ['ai tooling', 'foo:bar', 'quoted "topic"'], + topicScores: { 'ai tooling': 0.9, 'foo:bar': 0.8 }, + sentiment: 'neutral', + summary: 'Summary with a colon: and "quotes".', + keyPoints: ['One: two', 'Quote "x"'], + monetizationAngle: 'Angle', + contentType: 'tutorial:advanced', + problemStatement: 'N/A: "no problem"', + audienceLevel: 'intermediate', + technologiesMentioned: ['node.js', 'yaml:1.2'], + companiesMentioned: ['ACME, Inc.'], + peopleMentioned: [], + conceptsExplained: ['front-matter'], + relatedTechnologies: [], + useCases: [], + targetKeywords: ['a:b', 'c d'], + searchQuestions: ['What is YAML?'], + githubRepo: 'N/A', + releaseInfo: 'N/A', + analyzedAt: '2025-01-02T00:00:00.000Z', + }; +} + diff --git a/self-hosting/omc/src/storage/__tests__/analysis-writer-jsonl.test.ts b/self-hosting/omc/src/storage/__tests__/analysis-writer-jsonl.test.ts new file mode 100644 index 000000000..baf8175b7 --- /dev/null +++ b/self-hosting/omc/src/storage/__tests__/analysis-writer-jsonl.test.ts @@ -0,0 +1,62 @@ +import { describe, it, expect } from 'vitest'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AnalysisWriter } from '../AnalysisWriter.js'; +import type { AnalysisJsonlRecord } from '../AnalysisWriter.js'; + +describe('AnalysisWriter.appendToJsonl', () => { + it('appends one JSON object per line', async () => { + const dir = mkdtempSync(join(tmpdir(), 'omc-jsonl-')); + try { + const writer = new AnalysisWriter({ outputDir: join(dir, 'out') }); + const jsonlPath = join(dir, 'analyses.jsonl'); + const record1 = buildRecord('a1'); + const record2 = buildRecord('a2'); + + await writer.appendToJsonl(jsonlPath, record1); + await writer.appendToJsonl(jsonlPath, record2); + + const lines = readFileSync(jsonlPath, 'utf-8').trim().split('\n'); + expect(lines).toHaveLength(2); + expect(JSON.parse(lines[0])).toEqual(record1); + expect(JSON.parse(lines[1])).toEqual(record2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +function buildRecord(articleId: string): AnalysisJsonlRecord { + return { + articleId, + articleSlug: 'test-slug', + articleUrl: 'https://example.com', + articleTitle: 'Test', + savedAt: new Date('2025-01-01T00:00:00Z').toISOString(), + publishedAt: null, + updatedAt: null, + markdownPath: 'content/analysis/2025-01-01-test-analysis.md', + analyzedAt: new Date('2025-01-02T00:00:00Z').toISOString(), + topics: ['ai'], + topicScores: { ai: 0.9 }, + sentiment: 'neutral', + summary: 'Summary', + keyPoints: ['One'], + monetizationAngle: 'Angle', + contentType: 'N/A', + problemStatement: 'N/A', + audienceLevel: 'N/A', + technologiesMentioned: [], + companiesMentioned: [], + peopleMentioned: [], + conceptsExplained: [], + relatedTechnologies: [], + useCases: [], + targetKeywords: [], + searchQuestions: [], + githubRepo: 'N/A', + releaseInfo: 'N/A', + }; +} + diff --git a/self-hosting/omc/src/storage/database.ts b/self-hosting/omc/src/storage/database.ts new file mode 100644 index 000000000..69c0d6617 --- /dev/null +++ b/self-hosting/omc/src/storage/database.ts @@ -0,0 +1,138 @@ +// AIDEV-NOTE: tracking-db-boundary - SQLite stores immutable AI snapshots + job tracking +// AIDEV-NOTE: git-tracked Markdown files are user-editable, SQLite has original AI output +// AIDEV-NOTE: omnivore-boundary - existing Omnivore tables are READ-ONLY, never modify + +import Database from 'better-sqlite3'; +import { readFileSync, existsSync } from 'fs'; +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +/** + * Initialize database connection with boundary enforcement + * + * CRITICAL BOUNDARIES: + * 1. Existing Omnivore tables (if present) are READ-ONLY - never modify + * 2. New tracking tables are READ-WRITE - store immutable AI snapshots + coordination + * 3. Markdown files are user-editable; SQLite has original immutable AI output + * + * @param dbPath - Path to SQLite database file + * @returns Database connection + */ +export function initDatabase(dbPath: string = 'data/omnivore-content.db'): Database.Database { + // AIDEV-NOTE: tracking-db - stores immutable AI snapshots + coordination + const db = new Database(dbPath); + + // Enable WAL mode for better concurrent access + db.pragma('journal_mode = WAL'); + + // AIDEV-NOTE: Only create tracking tables, never modify existing Omnivore tables + const trackingSchema = readFileSync(resolveTrackingSchemaPath(), 'utf-8'); + + db.exec(trackingSchema); + + return db; +} + +function resolveTrackingSchemaPath(): string { + const candidates = [ + process.env.OMC_TRACKING_SCHEMA_PATH, + join(__dirname, 'schema/tracking-schema.sql'), + join(process.cwd(), 'src/storage/schema/tracking-schema.sql'), + join(process.cwd(), 'dist/schema/tracking-schema.sql'), + ].filter(Boolean) as string[]; + + for (const p of candidates) { + if (existsSync(p)) return p; + } + + throw new Error( + `Could not find tracking-schema.sql. Tried: ${candidates.join(', ')}` + ); +} + +/** + * List all tables in database for inspection + * Used to identify Omnivore tables vs tracking tables + */ +export function listTables(db: Database.Database): string[] { + const tables = db.prepare(` + SELECT name FROM sqlite_master + WHERE type='table' + ORDER BY name + `).all() as { name: string }[]; + + return tables.map(t => t.name); +} + +/** + * Check if table belongs to Omnivore cache (READ-ONLY) + * Omnivore tables typically use Z-prefixed column names (Core Data convention) + * + * @param db - Database connection + * @param tableName - Table to check + * @returns true if likely an Omnivore table + */ +export function isOmnivoreTable(db: Database.Database, tableName: string): boolean { + // AIDEV-NOTE: boundary-check - identify Omnivore tables by schema pattern + try { + const columns = db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{ name: string }>; + + // Core Data (Omnivore) tables use Z-prefixed column names + const hasZPrefixedColumns = columns.some(col => col.name.startsWith('Z')); + + return hasZPrefixedColumns; + } catch (err) { + console.warn(`Could not inspect table ${tableName}:`, err); + return false; + } +} + +/** + * Validate that our code hasn't modified Omnivore tables + * This is a safety check to enforce the READ-ONLY boundary + * + * NOTE: This is not foolproof - it checks for suspicious patterns, + * but cannot detect all modifications + */ +export function validateOmnivoreTablesReadOnly(db: Database.Database): void { + // AIDEV-NOTE: boundary-check - ensure Omnivore tables never modified by our code + const tables = listTables(db); + + for (const table of tables) { + if (isOmnivoreTable(db, table)) { + console.log(`[BOUNDARY CHECK] Omnivore table detected: ${table} (READ-ONLY)`); + } + } + + // Log our tracking tables + const trackingTables = ['analysis_queue', 'analysis_sessions']; + for (const table of trackingTables) { + if (tables.includes(table)) { + console.log(`[BOUNDARY CHECK] Tracking table: ${table} (READ-WRITE)`); + } + } +} + +/** + * Get table row counts for monitoring + * Useful for debugging and understanding database state + */ +export function getTableCounts(db: Database.Database): Record { + const tables = listTables(db); + const counts: Record = {}; + + for (const table of tables) { + try { + const result = db.prepare(`SELECT COUNT(*) as count FROM ${table}`).get() as { count: number }; + counts[table] = result.count; + } catch (err) { + console.warn(`Could not count rows in ${table}:`, err); + counts[table] = -1; + } + } + + return counts; +} diff --git a/self-hosting/omc/src/storage/schema/tracking-schema.sql b/self-hosting/omc/src/storage/schema/tracking-schema.sql new file mode 100644 index 000000000..6e1bbaad5 --- /dev/null +++ b/self-hosting/omc/src/storage/schema/tracking-schema.sql @@ -0,0 +1,53 @@ +-- AIDEV-NOTE: tracking and original analysis storage for querying +-- AIDEV-NOTE: analysisJson stores AI-generated output (immutable snapshot) +-- AIDEV-NOTE: git-tracked Markdown files are editable by humans (mutable) +-- AIDEV-NOTE: this database also contains READ-ONLY Omnivore cache tables (if present) + +-- Analysis job queue for parallel execution coordination +-- AIDEV-NOTE: gql-article-query-requirements - slug required for article(slug, username) query +-- AIDEV-NOTE: gql-search-query - search(query) does NOT support id: filter +CREATE TABLE IF NOT EXISTS analysis_queue ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + article_id TEXT NOT NULL UNIQUE, -- AIDEV: from SearchItem.id field + article_slug TEXT NOT NULL, -- AIDEV: REQUIRED for article(slug, username) GQL query + article_url TEXT NOT NULL, + article_title TEXT NOT NULL, + + -- Article metadata from Omnivore + saved_at TEXT NOT NULL, -- AIDEV: when saved to Omnivore (ISO 8601) + published_at TEXT, -- AIDEV: article publication date (ISO 8601), null if unknown + updated_at_article TEXT, -- AIDEV: article last update (ISO 8601), null if unknown + + -- Analysis results (immutable AI snapshot) + analysis_json TEXT, -- AIDEV: full ContentAnalysis as JSON (null until analyzed) + markdown_path TEXT, -- AIDEV: path to git-tracked .md file (null until saved) + + -- Tracking status (coordination lock) + status TEXT NOT NULL CHECK(status IN ('pending', 'in_progress', 'completed', 'failed')), + assigned_at TEXT, -- When marked in_progress (ISO 8601) + completed_at TEXT, -- When marked completed (ISO 8601) + + -- Error handling + error_message TEXT, + retry_count INTEGER DEFAULT 0, + + -- Metadata + created_at TEXT NOT NULL, -- When added to queue (ISO 8601) + updated_at TEXT NOT NULL -- AIDEV: last tracking status change (ISO 8601) +); + +-- Indexes for efficient queries +CREATE INDEX IF NOT EXISTS idx_analysis_queue_status ON analysis_queue(status); +CREATE INDEX IF NOT EXISTS idx_analysis_queue_article_id ON analysis_queue(article_id); +CREATE INDEX IF NOT EXISTS idx_analysis_queue_created_at ON analysis_queue(created_at DESC); + +-- Analysis session metadata (optional - for tracking batches) +CREATE TABLE IF NOT EXISTS analysis_sessions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + started_at TEXT NOT NULL, + completed_at TEXT, + total_articles INTEGER, + completed_articles INTEGER DEFAULT 0, + failed_articles INTEGER DEFAULT 0, + notes TEXT +); diff --git a/self-hosting/omc/src/test-helpers/database.ts b/self-hosting/omc/src/test-helpers/database.ts new file mode 100644 index 000000000..e485da22e --- /dev/null +++ b/self-hosting/omc/src/test-helpers/database.ts @@ -0,0 +1,36 @@ +import Database from 'better-sqlite3'; +import { initDatabase } from '../storage/database.js'; +import { AnalysisQueueRepository } from '../storage/AnalysisQueueRepository.js'; + +/** + * Create an in-memory test database + * + * @example + * >>> const db = createTestDatabase() + * >>> db.prepare('SELECT 1').get() + * { '1': 1 } + */ +export function createTestDatabase(): Database.Database { + return initDatabase(':memory:'); +} + +/** + * Execute test with isolated database instance + * + * @example + * >>> await withTestDatabase(async (db, repo) => { + * >>> repo.addArticle({ articleId: 'test', ... }) + * >>> return repo.getJob('test') + * >>> }) + */ +export async function withTestDatabase( + callback: (db: Database.Database, repo: AnalysisQueueRepository) => Promise +): Promise { + const db = createTestDatabase(); + const repo = new AnalysisQueueRepository(db); + try { + return await callback(db, repo); + } finally { + db.close(); + } +} diff --git a/self-hosting/omc/src/test-helpers/setup.ts b/self-hosting/omc/src/test-helpers/setup.ts new file mode 100644 index 000000000..127b7976c --- /dev/null +++ b/self-hosting/omc/src/test-helpers/setup.ts @@ -0,0 +1,12 @@ +import { config } from 'dotenv'; +import { beforeAll } from 'vitest'; + +// Load test environment variables +config({ path: '.env.test' }); + +// Set test environment +process.env.NODE_ENV = 'test'; + +beforeAll(() => { + // Global test setup if needed +}); diff --git a/self-hosting/omc/src/types/analysis.ts b/self-hosting/omc/src/types/analysis.ts new file mode 100644 index 000000000..077cdddb4 --- /dev/null +++ b/self-hosting/omc/src/types/analysis.ts @@ -0,0 +1,99 @@ +/** + * Content Analysis Type Definitions + * + * Types for AI-powered content analysis using Claude. + * These represent the structured output from analysis prompts. + */ + +/** + * Complete content analysis result + * Returned by ContentAnalyzer after analyzing an article + */ +export interface ContentAnalysis { + articleId: string; // Omnivore article ID + topics: string[]; // 2-5 main topics (e.g., ["AI", "Machine Learning", "LLMs"]) + topicScores: Record; // Confidence scores 0-1 (e.g., { "AI": 0.95, "ML": 0.88 }) + summary: string; // 2-3 sentence summary capturing main points + keyPoints: string[]; // 3-5 key takeaways or actionable insights + sentiment: 'positive' | 'neutral' | 'negative'; // Overall article tone + monetizationAngle: string; // How to package for audience (e.g., "Compare with 2 other LLM papers") + + // Content Planning (extracted from article only) + contentType: string; // Open-ended description (e.g., "tutorial", "comparison review", "announcement") + problemStatement: string; // Problem article addresses or "N/A" + audienceLevel: string; // "beginner"|"intermediate"|"advanced" or "N/A" + + // Knowledge Graph (for future corpus linking - from article only) + technologiesMentioned: string[]; // Tools/frameworks/languages mentioned + companiesMentioned: string[]; // Companies/organizations mentioned + peopleMentioned: string[]; // Notable people mentioned (if relevant) + conceptsExplained: string[]; // Key concepts/techniques explained + relatedTechnologies: string[]; // Technologies compared to or built upon + useCases: string[]; // Specific use cases or scenarios described + + // SEO Signals (from article only) + targetKeywords: string[]; // Keywords emphasized in article or ["N/A"] + searchQuestions: string[]; // Questions the article answers or ["N/A"] + + // Trend Signals (ONLY if present in article) + githubRepo: string; // GitHub URL or "N/A" + releaseInfo: string; // Version/release information or "N/A" + + analyzedAt: string; // ISO timestamp of analysis +} + +/** + * Input to content analysis + * Data sent to Claude for analysis + */ +export interface AnalysisRequest { + title: string; + author?: string; + url: string; + content: string; // Full article text + wordCount: number; + highlights: Array<{ // User's highlights from Omnivore + quote: string; + annotation?: string; + }>; + publishedAt?: string; +} + +/** + * Topic with confidence score + * Used for ranking and filtering topics + */ +export interface TopicScore { + topic: string; // Topic name (e.g., "AI", "DevOps") + score: number; // Confidence 0-1 + keywords: string[]; // Associated keywords found in content +} + +/** + * Analysis configuration + * Options for customizing analysis behavior + */ +export interface AnalysisConfig { + focusTopics?: string[]; // Prioritize these topics (from CLAUDE.md strategy) + minTopicScore?: number; // Minimum confidence threshold (default 0.7) + maxTopics?: number; // Maximum topics to extract (default 5) + includeSentiment?: boolean; // Whether to analyze sentiment (default true) +} + +/** + * Batch analysis result + * Result of analyzing multiple articles together + */ +export interface BatchAnalysisResult { + articles: Array<{ + articleId: string; + analysis: ContentAnalysis; + }>; + commonTopics: string[]; // Topics appearing across multiple articles + trends: Array<{ // Emerging trends detected + topic: string; + frequency: number; // How many articles mention this + averageScore: number; // Average confidence across articles + }>; + analyzedAt: string; +} diff --git a/self-hosting/omc/src/types/content.ts b/self-hosting/omc/src/types/content.ts new file mode 100644 index 000000000..32b7ec70c --- /dev/null +++ b/self-hosting/omc/src/types/content.ts @@ -0,0 +1,119 @@ +/** + * Content Storage Type Definitions + * + * Types for Markdown file storage with YAML front-matter. + * The content system uses front-matter + Git instead of a database. + */ + +/** + * Front-matter metadata for stored articles + * Saved as YAML at the top of Markdown files in content/articles/ + */ +export interface ArticleFrontMatter { + id: string; // Omnivore article ID + url: string; // Original article URL + title: string; + author?: string; + savedAt: string; // ISO timestamp when saved to Omnivore + publishedAt?: string; // Original publication date + labels: string[]; // Label names (not IDs) + highlights: number; // Count of highlights + wordCount: number; + siteName?: string; // Source website name + topics?: string[]; // Added by analysis phase + sentiment?: string; // Added by analysis phase + analyzed?: boolean; // Whether analysis has been run +} + +/** + * Front-matter metadata for analysis results + * Saved in content/analysis/ with same filename as article + */ +export interface AnalysisFrontMatter { + articleId: string; // References original article + articleSlug?: string; // Omnivore slug (preferred filename key) + articleUrl: string; // Source article URL + articleTitle: string; // Source article title + savedAt: string; // When article was saved to Omnivore + analyzedAt: string; // ISO timestamp of analysis + + topics: string[]; // Extracted topics + topicScores: Record; // Topic confidence scores + sentiment: 'positive' | 'neutral' | 'negative'; + contentType: string; + problemStatement: string; + audienceLevel: string; + + technologiesMentioned: string[]; + companiesMentioned: string[]; + peopleMentioned: string[]; + conceptsExplained: string[]; + relatedTechnologies: string[]; + useCases: string[]; + targetKeywords: string[]; + searchQuestions: string[]; + githubRepo: string; + releaseInfo: string; +} + +/** + * Front-matter metadata for generated content + * Saved in content/generated/blog-posts/ or content/generated/newsletters/ + */ +export interface GeneratedContentFrontMatter { + title: string; // SEO-optimized title + metaDescription: string; // SEO meta description (155 chars) + generatedAt: string; // ISO timestamp of generation + type: 'blog-post' | 'newsletter'; + sources: string[]; // Source article URLs + topics: string[]; // Main topics covered + publishedAt?: string; // ISO timestamp if published + slug?: string; // URL-friendly slug +} + +/** + * Complete stored article with content + * Result of reading an article Markdown file + */ +export interface StoredArticle { + frontMatter: ArticleFrontMatter; + content: string; // Markdown content + highlights?: Array<{ + quote: string; + annotation?: string; + }>; +} + +/** + * Complete stored analysis with details + * Result of reading an analysis Markdown file + */ +export interface StoredAnalysis { + frontMatter: AnalysisFrontMatter; + summary: string; // 2-3 sentence summary + keyPoints: string[]; // 3-5 key takeaways + monetizationAngle: string; // How to turn into content +} + +/** + * Complete generated content + * Result of reading a generated Markdown file + */ +export interface StoredGeneratedContent { + frontMatter: GeneratedContentFrontMatter; + content: string; // Full Markdown content +} + +/** + * Search index entry + * Stored in content/.metadata/index.json + */ +export interface SearchIndexEntry { + id: string; + slug: string; + title: string; + topics: string[]; + labels: string[]; + savedAt: string; + analyzed: boolean; +} diff --git a/self-hosting/omc/src/types/index.ts b/self-hosting/omc/src/types/index.ts new file mode 100644 index 000000000..c94a795e8 --- /dev/null +++ b/self-hosting/omc/src/types/index.ts @@ -0,0 +1,62 @@ +/** + * Type definitions for Omnivore Content System + * + * Central export point for all type definitions. + * Import from this file to access any type in the system. + * + * @example + * ```typescript + * import { OmnivoreArticle, ContentAnalysis, ArticleFrontMatter } from '@omc-types'; + * ``` + */ + +// Re-export all Omnivore API types +export * from './omnivore'; + +// Re-export all content storage types +export * from './content'; + +// Re-export all analysis types +export * from './analysis'; + +// Common utility types +export type DateString = string; // ISO 8601 format (e.g., "2025-09-30T10:00:00Z") +export type UUID = string; // Unique identifier +export type Slug = string; // URL-friendly string (e.g., "ai-trends-2025") + +/** + * Configuration for blog post generation + */ +export interface BlogPostConfig { + type: 'single-article' | 'weekly-roundup' | 'deep-dive'; + title?: string; // Override auto-generated title + targetWordCount?: number; // Target length (default varies by type) + includeSources?: boolean; // Include source links (default true) + seoOptimize?: boolean; // Apply SEO optimization (default true) +} + +/** + * Configuration for newsletter generation + */ +export interface NewsletterConfig { + type: 'weekly' | 'monthly'; + includeSections?: string[]; // Sections to include (e.g., ["topStories", "quickHits"]) + maxArticles?: number; // Maximum articles to include + personalCommentary?: boolean; // Include personal notes (default true) +} + +/** + * Publishing platform options + */ +export type PublishingPlatform = 'markdown' | 'ghost' | 'wordpress' | 'medium'; + +/** + * Publishing result + */ +export interface PublishResult { + platform: PublishingPlatform; + success: boolean; + url?: string; // Published URL if successful + error?: string; // Error message if failed + publishedAt: string; // ISO timestamp +} diff --git a/self-hosting/omc/src/types/omnivore.ts b/self-hosting/omc/src/types/omnivore.ts new file mode 100644 index 000000000..2232e0695 --- /dev/null +++ b/self-hosting/omc/src/types/omnivore.ts @@ -0,0 +1,97 @@ +/** + * Omnivore API Type Definitions + * + * Types matching the GraphQL API responses from lib/omnivore/client.js + * These interfaces represent the data structures returned by the Omnivore API. + */ + +/** + * Label attached to an article + */ +export interface Label { + id: string; + name: string; + color: string; + description?: string; +} + +/** + * Highlight/annotation made on an article + */ +export interface Highlight { + id: string; + quote: string; + annotation?: string | null; + createdAt: string; +} + +/** + * Article from Omnivore API + * Matches the response structure from searchArticles() and getArticle() + */ +export interface OmnivoreArticle { + id: string; + title: string; + url: string; + originalArticleUrl?: string; + slug?: string; + content?: string | null; + description?: string | null; + author?: string | null; + image?: string; + siteName?: string; + pageType?: string; + wordCount?: number; + createdAt: string; + savedAt: string; + publishedAt?: string | null; + updatedAt: string; + readingProgressTopPercent?: number; + isArchived?: boolean; + folder?: string; + labels: Label[]; + highlights: Highlight[]; +} + +/** + * Parameters for article search queries + */ +export interface SearchParams { + query?: string; // Omnivore query syntax (e.g., "label:ai", "in:inbox") + first?: number; // Number of results per page + after?: string; // Pagination cursor + includeContent?: boolean; // Include full article content in response +} + +/** + * Pagination information + */ +export interface PageInfo { + hasNextPage: boolean; + hasPreviousPage?: boolean; + startCursor?: string; + endCursor?: string; + totalCount: number; +} + +/** + * Search result wrapper with pagination + */ +export interface SearchResult { + edges: Array<{ + node: OmnivoreArticle; + }>; + pageInfo: PageInfo; +} + +/** + * User profile information + */ +export interface OmnivoreUser { + id: string; + name: string; + email: string; + profile: { + username: string; + }; +} diff --git a/self-hosting/omc/test-api-fields.js b/self-hosting/omc/test-api-fields.js new file mode 100755 index 000000000..6f7306b54 --- /dev/null +++ b/self-hosting/omc/test-api-fields.js @@ -0,0 +1,221 @@ +#!/usr/bin/env node + +/** + * Test script to verify which Omnivore API fields work + * Tests baseline fields and extra fields individually + */ + +import fetch from 'node-fetch'; +import { config } from 'dotenv'; + +// Load environment variables +config(); + +const OMNIVORE_API_URL = 'https://api-prod.omnivore.app/api/graphql'; + +// Get API key from environment +const API_KEY = process.env.OMNIVORE_API_KEY; + +if (!API_KEY) { + console.error('Error: OMNIVORE_API_KEY environment variable not set'); + process.exit(1); +} + +// Baseline fields that work in client.js +const BASELINE_FIELDS = ` + id + title + url + originalArticleUrl + createdAt + updatedAt + publishedAt + savedAt + author + description + image + siteName + pageType + wordsCount + readingProgressTopPercent + isArchived + folder + content +`; + +// Extra fields to test individually +const EXTRA_FIELDS = [ + 'slug' +]; + +/** + * Execute a GraphQL query against Omnivore API + */ +async function executeQuery(fields) { + const query = ` + query Search($query: String!, $first: Int) { + search(query: $query, first: $first) { + ... on SearchSuccess { + pageInfo { + totalCount + hasNextPage + } + edges { + node { + ${fields} + } + } + } + ... on SearchError { + errorCodes + } + } + } + `; + + const variables = { + query: 'in:all', + first: 1 + }; + + const requestBody = { query, variables }; + + // Debug: Log what we're sending (only first time) + if (!executeQuery.logged) { + console.log('\n[DEBUG] Request body preview:'); + console.log('Query starts with:', query.substring(0, 100) + '...'); + console.log('Variables:', variables); + executeQuery.logged = true; + } + + const response = await fetch(OMNIVORE_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': API_KEY + }, + body: JSON.stringify(requestBody) + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const data = await response.json(); + + // Debug: Log what we're receiving (only first time) + if (!executeQuery.receivedLog) { + console.log('[DEBUG] Response preview:'); + console.log(JSON.stringify(data, null, 2).substring(0, 200) + '...\n'); + executeQuery.receivedLog = true; + } + + return data; +} + +/** + * Test a specific field set + */ +async function testFields(name, fields) { + console.log(`\n${'='.repeat(60)}`); + console.log(`Testing: ${name}`); + console.log(`${'='.repeat(60)}`); + + try { + const result = await executeQuery(fields); + + if (result.errors) { + console.log('❌ FAILED'); + console.log('\nErrors:'); + result.errors.forEach(err => { + console.log(` - ${err.message}`); + if (err.locations) { + console.log(` Location: line ${err.locations[0].line}, column ${err.locations[0].column}`); + } + }); + return false; + } else if (result.data?.search?.errorCodes) { + console.log('❌ SEARCH ERROR'); + console.log(` Error codes: ${result.data.search.errorCodes.join(', ')}`); + return false; + } else if (result.data?.search?.edges?.[0]?.node) { + console.log('✅ SUCCESS'); + console.log('\nSample data received:'); + const node = result.data.search.edges[0].node; + // Show first 3 fields as sample + const sampleKeys = Object.keys(node).slice(0, 3); + sampleKeys.forEach(key => { + const value = node[key]; + const display = typeof value === 'string' && value.length > 50 + ? value.substring(0, 50) + '...' + : value; + console.log(` ${key}: ${JSON.stringify(display)}`); + }); + console.log(` ... (${Object.keys(node).length} total fields)`); + return true; + } else { + console.log('⚠️ UNEXPECTED RESPONSE'); + console.log(JSON.stringify(result, null, 2)); + return false; + } + } catch (error) { + console.log('❌ EXCEPTION'); + console.log(` ${error.message}`); + return false; + } +} + +/** + * Main test execution + */ +async function runTests() { + console.log('Omnivore API Field Compatibility Test'); + console.log(`API Endpoint: ${OMNIVORE_API_URL}`); + console.log(`API Key: ${API_KEY.substring(0, 10)}...`); + + const results = { + baseline: null, + extra: {} + }; + + // Test baseline fields + results.baseline = await testFields('BASELINE FIELDS', BASELINE_FIELDS); + + // Test each extra field individually with baseline + for (const field of EXTRA_FIELDS) { + const combinedFields = BASELINE_FIELDS + '\n ' + field; + results.extra[field] = await testFields( + `BASELINE + ${field}`, + combinedFields + ); + } + + // Summary report + console.log('\n' + '='.repeat(60)); + console.log('SUMMARY REPORT'); + console.log('='.repeat(60)); + + console.log(`\nBaseline fields: ${results.baseline ? '✅ PASS' : '❌ FAIL'}`); + + console.log('\nExtra fields:'); + for (const field of EXTRA_FIELDS) { + const status = results.extra[field] ? '✅ PASS' : '❌ FAIL'; + console.log(` ${field}: ${status}`); + } + + // Final verdict + const allExtraPass = Object.values(results.extra).every(r => r === true); + console.log('\n' + '='.repeat(60)); + if (results.baseline && allExtraPass) { + console.log('✅ ALL TESTS PASSED - All fields are compatible'); + } else { + console.log('❌ SOME TESTS FAILED - See details above'); + } + console.log('='.repeat(60) + '\n'); +} + +// Run tests +runTests().catch(error => { + console.error('Fatal error:', error); + process.exit(1); +}); diff --git a/self-hosting/omc/tsconfig.build.json b/self-hosting/omc/tsconfig.build.json new file mode 100644 index 000000000..dd6790662 --- /dev/null +++ b/self-hosting/omc/tsconfig.build.json @@ -0,0 +1,32 @@ +{ + // Extends base tsconfig.json + "extends": "./tsconfig.json", + + // Production build overrides + "compilerOptions": { + // No source maps in production + "sourceMap": false, + + // No declaration files in production (smaller bundle) + "declaration": false, + "declarationMap": false, + + // Remove comments to reduce bundle size + "removeComments": true, + + // Optimize for production + "noEmitOnError": true + }, + + // Exclude test files from production build + "exclude": [ + "node_modules", + "dist", + "test-scripts", + "legacy-scripts", + "content", + "templates", + "**/*.test.ts", + "**/*.spec.ts" + ] +} diff --git a/self-hosting/omc/tsconfig.json b/self-hosting/omc/tsconfig.json new file mode 100644 index 000000000..6f9bae27a --- /dev/null +++ b/self-hosting/omc/tsconfig.json @@ -0,0 +1,64 @@ +{ + "compilerOptions": { + // Target modern Node.js + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022"], + + // Output configuration + "outDir": "./dist", + "rootDir": "./src", + "noEmit": true, + + // Strict type checking + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + + // Module interop + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "forceConsistentCasingInFileNames": true, + + // Additional checks + "skipLibCheck": true, + "resolveJsonModule": true, + + // Source maps and declarations + "declaration": true, + "declarationMap": true, + "sourceMap": true, + + // Path aliases for clean imports + "paths": { + "@lib/*": ["./src/lib/*"], + "@storage/*": ["./src/storage/*"], + "@analysis/*": ["./src/analysis/*"], + "@generation/*": ["./src/generation/*"], + "@publishing/*": ["./src/publishing/*"], + "@workflows/*": ["./src/workflows/*"], + "@utils/*": ["./src/utils/*"], + "@omc-types/*": ["./src/types/*"] + } + }, + + // Include TypeScript source and JavaScript library + "include": [ + "src/**/*", + "lib/**/*" + ], + + // Exclude compiled output and dependencies + "exclude": [ + "node_modules", + "dist", + "test-scripts", + "legacy-scripts", + "content", + "templates" + ] +} diff --git a/self-hosting/omc/vitest.config.ts b/self-hosting/omc/vitest.config.ts new file mode 100644 index 000000000..52585f237 --- /dev/null +++ b/self-hosting/omc/vitest.config.ts @@ -0,0 +1,29 @@ +import { defineConfig } from 'vitest/config'; +import { resolve } from 'path'; + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + setupFiles: ['./src/test-helpers/setup.ts'], + include: ['src/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'lcov', 'html'], + exclude: ['dist/**', 'node_modules/**', '**/*.test.ts'] + } + }, + resolve: { + alias: { + '@lib': resolve(__dirname, './src/lib'), + '@storage': resolve(__dirname, './src/storage'), + '@analysis': resolve(__dirname, './src/analysis'), + '@generation': resolve(__dirname, './src/generation'), + '@publishing': resolve(__dirname, './src/publishing'), + '@workflows': resolve(__dirname, './src/workflows'), + '@utils': resolve(__dirname, './src/utils'), + '@omc-types': resolve(__dirname, './src/types'), + '@commands': resolve(__dirname, './src/commands') + } + } +});