Add omc-cron sidecar and vendor OMC package

This commit is contained in:
Rohit Amarnath 2026-02-02 14:31:10 -05:00
parent 6ff55c4043
commit fdae98d3fe
132 changed files with 19386 additions and 2 deletions

View file

@ -5,6 +5,7 @@
**/Dockerfile
**/.dockerignore
**/*.yaml
!self-hosting/omc/pnpm-lock.yaml
.secrets*.yaml
apple
android

3
.gitignore vendored
View file

@ -35,6 +35,9 @@ package-lock.json
.env.local
.env.production
# OMC cron sidecar secrets
self-hosting/docker-compose/omc.env
# build & dist dirs
build

View file

@ -13,7 +13,7 @@
## Docker Compose
We recommend using Docker-compose for the simplest way to deploy Omnivore. We have provided a configuration in the `self-hosting/docker-compose` folder.
All networking and persistent storage is handled by the docker-compose file.
### Requirements
@ -62,6 +62,26 @@ When the service is ready you can access the web-app by using localhost:3000
With the default .env file you will be able to use Omnivore, add RSS Feeds, add stories etc.
### Optional: OMC Cron Sidecar (Content Analysis + Notes)
This repo includes an optional `omc-cron` service that runs the Omnivore Content System (OMC) on a schedule:
- Fetch recent saves into an analysis queue
- Run `omc analyze auto` (non-interactive) using `codex exec`
- Classify analyzed items (adds topic/sentiment/type labels)
- Sync summaries + optional NOTE highlights back into Omnivore
- Run non-destructive hygiene (backup + integrity checks + daily corpus report)
To enable:
1. In `self-hosting/docker-compose/`, create `omc.env` from the example:
- `cp omc.env.example omc.env`
2. Fill in:
- `OMNIVORE_API_KEY` (from Omnivore Settings → API Keys)
- `OPENAI_API_KEY` (for `codex exec`)
3. Start the stack:
- `docker compose up -d`
### Additional Services used:
@ -391,4 +411,3 @@ To learn more about setting up the OpenAI Api key, read here: https://openai.com
In future releases we would like to be able to open this up to use different LLMs, such as Anthropic, Mistral, Bedrock, or any of the other myriad LLM Services.

View file

@ -153,6 +153,23 @@ services:
condition: service_healthy
restart: always
omc-cron:
build:
context: ../..
dockerfile: self-hosting/docker-compose/omc/Dockerfile
container_name: "omnivore-omc-cron"
env_file:
- ./omc.env
volumes:
- omc_data:/opt/omc/data
- omc_content:/opt/omc/content
- omc_temp:/opt/omc/temp
- omc_logs:/var/log/omc
depends_on:
api:
condition: service_healthy
restart: always
print-server:
profiles: ["print"]
build:
@ -188,6 +205,10 @@ volumes:
pgdata:
redis_data:
minio_data:
omc_data:
omc_content:
omc_temp:
omc_logs:
print_spool:
networks:

View file

@ -0,0 +1,21 @@
# Omnivore Content System (OMC) cron sidecar
#
# Copy to `omc.env` and fill in secrets:
# cp omc.env.example omc.env
# Omnivore GraphQL endpoint (inside compose network)
OMNIVORE_API_URL=http://api:8080/api/graphql
# Omnivore API key (generate in Omnivore UI: Settings → API Keys)
OMNIVORE_API_KEY=
# Codex CLI auth (used by `codex exec` during `omc analyze auto`)
OPENAI_API_KEY=
# Optional tuning
OMC_QUEUE_HOURS=2
OMC_BATCH_SIZE=5
OMC_JSONL_PATH=content/analysis/analyses.jsonl
OMC_PREP_SINCE_HOURS=24
OMC_PREP_LIMIT=50

View file

@ -0,0 +1,44 @@
FROM node:18-bookworm-slim
ARG TARGETARCH
ENV PNPM_HOME=/pnpm
ENV PATH=$PNPM_HOME:$PATH
ENV OMC_DIR=/opt/omc
WORKDIR /opt/omc
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
bash \
ca-certificates \
curl \
python3 \
make \
g++ \
&& rm -rf /var/lib/apt/lists/*
# pnpm (Corepack-managed, pinned to the OMC repo's packageManager)
RUN corepack enable && corepack prepare pnpm@10.14.0 --activate
# Codex CLI (OMC's default analyzer shells out to `codex exec`)
RUN pnpm add -g @openai/codex@0.93.0
# supercronic (cron runner that inherits container env)
RUN curl -fsSL -o /usr/local/bin/supercronic "https://github.com/aptible/supercronic/releases/download/v0.2.29/supercronic-linux-${TARGETARCH}" \
&& chmod +x /usr/local/bin/supercronic
# Install/build OMC (vendored into this repo under self-hosting/omc/)
COPY self-hosting/omc/package.json self-hosting/omc/pnpm-lock.yaml self-hosting/omc/.npmrc ./
RUN pnpm install --frozen-lockfile
COPY self-hosting/omc/ ./
RUN pnpm run build
RUN mkdir -p /etc/omc /var/log/omc /var/lock/omc /opt/omc/data /opt/omc/content /opt/omc/temp
COPY self-hosting/docker-compose/omc/omc.crontab /etc/omc/omc.crontab
COPY self-hosting/docker-compose/omc/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]

View file

@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
mkdir -p /var/log/omc /var/lock/omc /opt/omc/data /opt/omc/content /opt/omc/temp /opt/omc/content/corpus-reports
exec /usr/local/bin/supercronic -passthrough-logs /etc/omc/omc.crontab

View file

@ -0,0 +1,14 @@
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
# Global lock to avoid overlapping runs.
# supercronic runs jobs with container env (so OMNIVORE_API_KEY, OMNIVORE_API_URL, OPENAI_API_KEY work).
# Every 30 minutes: ingest recent items -> analyze -> retry failures.
*/30 * * * * flock -n /var/lock/omc/omc.lock bash -lc 'cd /opt/omc && node dist/bin/omc.js queue add --hours "${OMC_QUEUE_HOURS:-2}" && node dist/bin/omc.js analyze auto --batch-size "${OMC_BATCH_SIZE:-5}" --jsonl --jsonl-path "${OMC_JSONL_PATH:-content/analysis/analyses.jsonl}" && node dist/bin/omc.js analyze retry --failed' >> /var/log/omc/analyze.log 2>&1
# Hourly: content prep (classification + sync summaries/notes back to Omnivore).
10 * * * * flock -n /var/lock/omc/omc.lock bash -lc 'cd /opt/omc && node dist/bin/omc.js content classify --all --since-hours "${OMC_PREP_SINCE_HOURS:-24}" --limit "${OMC_PREP_LIMIT:-50}" && node dist/bin/omc.js content sync --all --since-hours "${OMC_PREP_SINCE_HOURS:-24}" --limit "${OMC_PREP_LIMIT:-50}" --create-notes' >> /var/log/omc/prep.log 2>&1
# Daily hygiene (non-destructive): backup + integrity check + summary report.
0 2 * * * flock -n /var/lock/omc/omc.lock bash -lc 'cd /opt/omc && node dist/bin/omc.js db backup && node dist/bin/omc.js db check && node dist/bin/omc.js doctor && mkdir -p content/corpus-reports && node dist/bin/omc.js report corpus > "content/corpus-reports/$(date -I)-daily.md"' >> /var/log/omc/hygiene.log 2>&1

67
self-hosting/omc/.gitignore vendored Normal file
View file

@ -0,0 +1,67 @@
# Environment variables
.env
.env.local
.env.*.local
# Dependencies
node_modules/
.pnpm-store/
pnpm-lock.yaml
package-lock.json
yarn.lock
# Runtime data
data/
*.sqlite
*.sqlite-journal
*.db
# Test outputs
test-scripts/test-output/
# Cache
cache/
.cache/
# Generated content
generated/
output/
# Logs
logs/
*.log
npm-debug.log*
pnpm-debug.log*
yarn-debug.log*
yarn-error.log*
# OS files
.DS_Store
Thumbs.db
# IDE
.vscode/
.idea/
*.swp
*.swo
*~
# Test coverage
coverage/
# Build output
dist/
build/
# GraphQL codegen output (currently not used by runtime)
src/types/generated/
# Temporary files
tmp/
temp/
*.tmp
# API keys and secrets (extra safety)
**/secrets/
**/*secret*
**/*key*.json

1
self-hosting/omc/.npmrc Normal file
View file

@ -0,0 +1 @@
enable-pre-post-scripts=true

View file

@ -0,0 +1,40 @@
# Agent Instructions
This project uses **bd** (beads) for issue tracking. Run `bd onboard` to get started.
## Quick Reference
```bash
bd ready # Find available work
bd show <id> # View issue details
bd update <id> --status in_progress # Claim work
bd close <id> # Complete work
bd sync # Sync with git
```
## Landing the Plane (Session Completion)
**When ending a work session**, you MUST complete ALL steps below. Work is NOT complete until `git push` succeeds.
**MANDATORY WORKFLOW:**
1. **File issues for remaining work** - Create issues for anything that needs follow-up
2. **Run quality gates** (if code changed) - Tests, linters, builds
3. **Update issue status** - Close finished work, update in-progress items
4. **PUSH TO REMOTE** - This is MANDATORY:
```bash
git pull --rebase
bd sync
git push
git status # MUST show "up to date with origin"
```
5. **Clean up** - Clear stashes, prune remote branches
6. **Verify** - All changes committed AND pushed
7. **Hand off** - Provide context for next session
**CRITICAL RULES:**
- Work is NOT complete until `git push` succeeds
- NEVER stop before pushing - that leaves work stranded locally
- NEVER say "ready to push when you are" - YOU must push
- If push fails, resolve and retry until it succeeds

524
self-hosting/omc/CLAUDE.md Normal file
View file

@ -0,0 +1,524 @@
# Omnivore Content System - Agent Context
## Project Purpose
Transform voracious reading habits into monetizable content through AI-powered analysis and generation.
## Content Strategy
### Primary Topics
- **AI & Machine Learning**: LLMs, agents, training, deployment
- **Tech Infrastructure**: Cloud, DevOps, databases, system design
- **Software Engineering**: Best practices, tools, frameworks
- **Startup/Business**: Product, growth, monetization strategies
### Content Goals
1. **Weekly Blog Roundup**: Top 5-10 AI/Tech stories with analysis
2. **Deep Dive Posts**: Multi-article synthesis into original analysis
3. **Newsletter**: Curated links with personal commentary
4. **SEO Optimization**: Drive organic traffic through strategic keywords
5. **Monetization**: Build audience → affiliate links → sponsorships
## Writing Style
### Voice & Tone
- **Authoritative yet accessible**: Technical depth without jargon overload
- **Opinionated**: Strong takes backed by evidence from reading
- **Practical**: Focus on actionable insights and implications
- **Conversational**: Write like talking to a smart colleague
### Content Structure
- **Hook**: Start with surprising insight or provocative question
- **Context**: Brief background from articles
- **Analysis**: Your unique take connecting multiple sources
- **Implications**: What this means for readers
- **CTA**: Subscribe, share, engage
### Writing Guidelines
- Use active voice
- Short paragraphs (2-3 sentences max)
- Subheadings for scannability
- Bullet points for lists
- Examples and analogies
- Link to sources (Omnivore articles)
## Content Types
### 1. Weekly Roundup (Fridays)
**Format**: "AI/Tech This Week: Top 5 Stories You Need to Know"
- 5-10 articles from the week
- 100-150 words per story
- Overall theme/trend analysis
- 800-1200 words total
### 2. Deep Dive (Monthly)
**Format**: "The Complete Guide to [Topic]"
- Synthesize 10-20 articles on single topic
- Original analysis and insights
- Code examples, diagrams
- 2000-3000 words
### 3. Newsletter (Sundays)
**Format**: "Weekend Reading: AI/Tech Digest"
- 7-10 curated links
- Personal commentary (50-75 words each)
- Quick hits section (5-10 line items)
- 600-800 words total
### 4. Social Media
**Twitter Threads**: Key insights from deep dives (8-10 tweets)
**LinkedIn Posts**: Professional takeaways (300-500 words)
## SEO Strategy
### Primary Keywords
- "AI trends 2025"
- "LLM development"
- "Claude agents tutorial"
- "Tech infrastructure best practices"
- "Startup engineering advice"
### Content Optimization
- Title: 60 chars, keyword-rich, curiosity-driven
- Meta description: 155 chars, compelling summary
- H2 headers: Question-based, keyword-targeted
- Internal links: Cross-reference related posts
- External links: High-authority sources (Omnivore articles)
## Monetization Path
### Phase 1: Audience Building (Months 1-3)
- Publish 2x/week (roundup + deep dive)
- Email list growth (newsletter signups)
- SEO optimization for organic traffic
- Social media distribution
### Phase 2: Engagement (Months 4-6)
- Reader surveys and topic requests
- Comments and discussions
- Guest posts and collaborations
- Community building
### Phase 3: Monetization (Months 7+)
- Affiliate links (tools, books, courses)
- Sponsored posts (relevant products)
- Premium newsletter tier
- Consulting/advisory services
## Quality Standards
### Before Publishing
- [ ] Fact-check all claims against source articles
- [ ] Verify all links work
- [ ] Run through Grammarly or similar
- [ ] Check SEO optimization (Yoast, etc.)
- [ ] Preview formatting on target platform
- [ ] Add cover image (if applicable)
### Performance Tracking
- Google Analytics: Traffic, engagement, conversions
- Email metrics: Open rate, click rate, growth
- Social metrics: Shares, comments, saves
- Database: Track which articles → which posts → performance
## Article Analysis Workflow
### How to Process Queued Articles
When user asks to analyze articles, use the unified CLI:
**Automated (cron/launchd):**
```bash
# End-to-end, non-interactive (uses codex exec in read-only mode)
omc queue add --hours 24
omc analyze auto --batch-size 5
# Retry failures (optional)
omc analyze retry --failed
```
**Basic Analysis:**
```bash
# Prepare batch (creates stub files, outputs agent params)
omc analyze run --batch-size 5
# Prepare specific article
omc analyze run --article-id <article-id>
# After agents complete: save results to database + markdown
omc analyze complete
# Save with keeping temp files for debugging
omc analyze complete --keep-temp
```
**Queue Management:**
```bash
# Add articles to queue
omc queue add --hours 24 # Last 24 hours
omc queue add --label "ai-ml" # By label
omc queue add --url <url> # Single article
omc queue add --slug <slug> # By slug
# Check queue status
omc queue list # All queued articles
omc queue list --status pending # Filter by status
omc queue stats # Statistics
# Retry failed analyses
omc analyze retry --failed # All failed
omc analyze retry --article-id <id> # Specific article
```
**Automation notes:**
- `omc analyze auto` shells out to `codex exec -s read-only` and expects a single JSON object response.
- For schedulers, we set `CODEX_HOME=temp/codex-home` so runs dont depend on `~/.codex` permissions.
- Prefer running installs/builds via Corepack (`corepack pnpm ...`) to avoid native-module ABI mismatches.
**View Results:**
```bash
# Show analysis for specific article
omc content show <article-id>
# List all analyzed content
omc content list
# Search analyses
omc content search "keyword"
# Generate reports
omc report corpus # Full corpus analysis
omc report topics # Topic distribution
omc report trends # Trending topics
```
**Sync to Omnivore (Create Notebook Notes):**
```bash
# Sync single article with note creation
omc content sync <article-id> --create-notes
# Sync all analyzed articles with notes
omc content sync --all --create-notes
# Sync without creating notes (metadata only)
omc content sync <article-id>
```
### Complete Workflow (Agent-Assisted Analysis)
**From Reading to Published Notes:**
1. **Add to Queue**: `omc queue add --hours 168` (or by label, URL, slug)
2. **Prepare Batch**: `omc analyze run --batch-size 5`
- Creates stub files in `temp/*.jsonl`
- Marks jobs as "in_progress"
- Outputs agent parameters (copy these)
3. **Invoke Agents**: Copy agent parameters and invoke article-content-analyzer agents via Task tool
- Agents fetch content via `omc omnivore get`
- Agents analyze and enrich stub files
- Run 5 agents in parallel for performance
4. **Save Results**: `omc analyze complete`
- Reads enriched JSONL from temp/
- Writes markdown to `content/analysis/*.md`
- Saves to database
- Marks jobs "completed"
5. **Review**: `omc content show <article-id>` or `omc content list`
6. **Sync to Omnivore**: `omc content sync --all --create-notes`
- Updates article descriptions
- Creates notebook notes with full analysis
7. **Repeat**: Go back to step 2 until queue empty
**Key Commands:**
- `omc analyze run --batch-size 5` - Prepare next batch
- `omc analyze complete` - Save completed analyses
- `omc content sync --all --create-notes` - Push to Omnivore
### Legacy Scripts (Archived)
Old workflow scripts have been archived in `cli/archived-scripts/`. Use the CLI commands above instead.
## Code Quality Tools
### OACC (Omniarcs Code Checker)
The project has access to `oacc analyze` for automated code quality analysis. **Use this instead of manual analysis.**
**Available via Bash tool (auto-approved):**
```bash
oacc analyze <file_path>
oacc analyze --functions <file_path>
```
**What oacc provides:**
- **Function length analysis**: Automatic counting with thresholds
- GREEN: ≤20 lines (target)
- YELLOW: 21-25 lines (warning)
- RED: >25 lines (must fix)
- **Function-level reports**: Shows each function with line count
- **Exit codes**: 0 = GREEN, 1 = YELLOW/RED
- **FTA scores**: Code quality metrics
**DO NOT manually:**
- Write Python/shell scripts to count lines
- Parse files with regex to find functions
- Manually count function lengths
- Create custom analysis tools
**DO use oacc:**
```bash
# Check single file
oacc analyze src/commands/queue/export.ts
# Get function-level details
oacc analyze --functions src/commands/queue/export.ts
# Analyze multiple files in quality-guard agent
for file in src/commands/**/*.ts; do
oacc analyze "$file"
done
```
**Quality-guard agent**: Always use `oacc analyze` for function length validation. It's pre-approved and faster than manual analysis.
## CLI Development Ground Truths
### Architecture Patterns (Established 2025-01-05)
**1. All Commands MUST Extend BaseCommand**
- Location: `src/lib/cli/base-command.ts:16`
- Pattern: `export default class MyCommand extends BaseCommand`
- Why: Provides standard error handling, run() method, and OCLIF integration
- Evidence: All 63 commands use this pattern (verified 2025-01-05)
**2. Database Operations MUST Use withDatabase()**
- Location: `src/lib/cli/database.ts:17`
- Pattern: `await withDatabase(async (db, repo) => { ... })`
- Why: Ensures proper connection cleanup and error handling
- Evidence: 45/46 database commands use this (db/backup.ts fixed 2025-01-05)
**3. JSON Parsing MUST Use parseJsonSafely()**
- Location: `src/lib/cli/command-utils.ts:45`
- Pattern: `const data = parseJsonSafely<ContentAnalysis>(job.analysisJson)`
- Why: Prevents crashes on malformed JSON, provides type safety
- Evidence: 15+ commands updated to use this (2025-01-05)
**4. Shared Flags MUST Use Utility Functions**
- `jsonFlag()`: `src/lib/cli/shared-flags.ts:14` - Standard JSON output flag
- `statusFlag()`: `src/lib/cli/shared-flags.ts:8` - Queue status filtering
- Why: DRY principle, consistent flag behavior
- Evidence: 45/63 commands use jsonFlag()
**5. Output Formatting MUST Use Shared Formatters**
- `formatHeader()`: `src/lib/cli/formatters.ts:18` - Section headers
- `formatSuccess()`: `src/lib/cli/formatters.ts:37` - Success messages
- `formatError()`: `src/lib/cli/formatters.ts:48` - Error messages
- Why: Consistent user experience, single source of truth
- Evidence: All 63 commands use these formatters
**6. GraphQL Operations MUST Use checkGraphQLResult()**
- Location: `src/lib/cli/graphql.ts:21`
- Pattern: `checkGraphQLResult(result)` after every GraphQL call
- Why: Checks both result.errors AND domain errorCodes
- Evidence: All 9 omnivore commands use this
**7. Environment Variables MUST Use loadEnvFile()**
- Location: `src/lib/cli/command-utils.ts:58`
- Pattern: `const env = loadEnvFile('.env')`
- Why: Centralized .env parsing, no duplicate implementations
- Evidence: 2 duplicate implementations removed (2025-01-05)
### Shared Utilities Reference
**Database Utilities:**
```typescript
// src/lib/cli/database.ts
withDatabase<T>(callback: (db, repo) => Promise<T>): Promise<T>
```
**Data Utilities:**
```typescript
// src/lib/cli/command-utils.ts
parseJsonSafely<T>(jsonString: string, fallback?: T): T | undefined
loadEnvFile(envPath: string = '.env'): Record<string, string>
handleCommandError(command: Command, error: unknown): void
outputResult(command: Command, data: any, successMessage: string, jsonMode: boolean): void
```
**Flag Utilities:**
```typescript
// src/lib/cli/shared-flags.ts
jsonFlag(): Flags.Boolean
statusFlag(): Flags.String
```
**Formatting Utilities:**
```typescript
// src/lib/cli/formatters.ts
formatHeader(title: string): string
formatSuccess(message: string): string
formatError(message: string): string
formatDivider(): string
```
**GraphQL Utilities:**
```typescript
// src/lib/cli/graphql.ts
checkGraphQLResult(result: any): void
fetchUsername(): Promise<string>
```
**Display Utilities:**
```typescript
// src/lib/cli/queue-display.ts
displayQueueStats(stats: QueueStats): void
displayJobs(jobs: AnalysisJob[]): void
```
### Command Structure Pattern
**Every command follows this structure:**
```typescript
import { Args, Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { withDatabase } from '@lib/cli/database.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { formatSuccess } from '@lib/cli/formatters.js';
export default class MyCommand extends BaseCommand {
static override description = 'Clear description';
static override examples = [
'$ omc command arg --flag'
];
static override args = {
myArg: Args.string({ description: 'Arg description', required: true })
};
static override flags = {
json: jsonFlag(),
myFlag: Flags.string({ description: 'Flag description' })
};
protected async execute(flags: any): Promise<void> {
await withDatabase(async (db, repo) => {
// 1. Validate input
// 2. Execute business logic
// 3. Format output
if (flags.json) {
this.log(JSON.stringify(result, null, 2));
} else {
this.log(formatSuccess('Success message'));
}
});
}
}
```
### Complete Command Inventory (63 total)
**Queue Management (9 commands):**
- `queue:add` - Add articles to queue
- `queue:list` - List queued articles
- `queue:stats` - Show queue statistics (with --detailed)
- `queue:reset` - Reset article status
- `queue:remove` - Remove from queue
- `queue:clear` - Bulk clear operations
- `queue:export` - Export queue state
- `queue:import` - Import queue state
- `queue:retry` - Retry failed items
**Analysis Operations (4 commands):**
- `analyze:run` - Process articles
- `analyze:retry` - Retry failed analyses
- `analyze:status` - Show analysis status
- `analyze:watch` - Real-time monitoring
**Omnivore Integration (9 commands):**
- `omnivore:get` - Fetch article by slug
- `omnivore:search` - Search articles
- `omnivore:list` - List recent articles
- `omnivore:update` - Update article metadata
- `omnivore:note:add` - Add note
- `omnivore:note:get` - Get notes
- `omnivore:note:update` - Update note
- `omnivore:highlight:add` - Add highlight
- `omnivore:highlight:list` - List highlights
**Content Operations (5 commands):**
- `content:show` - Display analysis
- `content:list` - List analyzed content
- `content:search` - Full-text search
- `content:sync` - Sync to Omnivore
- `content:export` - Export for blog
**Database Management (9 commands):**
- `db:migrate` - Run migrations
- `db:schema` - Show schema
- `db:seed` - Load sample data
- `db:vacuum` - Optimize database
- `db:backup` - Create backup
- `db:restore` - Restore from backup
- `db:reset` - Drop and recreate
- `db:check` - Integrity check
- `db:stats` - Database statistics
**Reporting (7 commands):**
- `report:corpus` - Full corpus analysis
- `report:topics` - Topic distribution
- `report:trends` - Trending topics
- `report:monetization` - Opportunities
- `report:sentiment` - Sentiment analysis
- `report:custom` - Custom queries
- `report:export` - Export reports
**Configuration (7 commands):**
- `config:show` - Display config
- `config:get` - Get value
- `config:set` - Set value
- `config:test` - Test API connection
- `config:validate` - Validate config
- `config:env:list` - List environments
- `config:env:use` - Switch environment
**System (3 commands):**
- `init` - Initialize project
- `doctor` - System health check
- `version` - Version info
### DRY Principles (Zero Tolerance)
**Violations to Avoid:**
1. ❌ Direct `JSON.parse()` without error handling → Use `parseJsonSafely()`
2. ❌ Manual database init/close → Use `withDatabase()`
3. ❌ Duplicate flag definitions → Use `jsonFlag()`, `statusFlag()`
4. ❌ Custom error handling → Extend `BaseCommand`
5. ❌ Duplicate .env parsing → Use `loadEnvFile()`
6. ❌ Direct Command extension → Extend `BaseCommand`
7. ❌ Custom formatters → Use formatHeader/Success/Error
8. ❌ GraphQL without error check → Use `checkGraphQLResult()`
**Evidence of Compliance:**
- All 63 commands extend BaseCommand ✓
- All 45 database commands use withDatabase() ✓
- All 15+ JSON parsing uses parseJsonSafely() ✓
- Zero duplicate helper functions ✓
- (Last verified: 2025-01-05)
### Quality Metrics
**Function Length Distribution (2025-01-05):**
- GREEN (≤20 lines): 52 commands (83%)
- YELLOW (21-25 lines): 11 commands (17%)
- RED (>25 lines): 0 commands (0%)
**Build Status:**
- TypeScript: Strict mode enabled
- Compilation: 0 errors, 0 warnings
- Bundle size: Optimized with ESBuild
- Commands registered: 63/63 (100%)
## Notes for Agents
- **Always cite sources**: Link back to Omnivore articles
- **Maintain authenticity**: Sound like the human, not generic AI
- **Be selective**: Quality content over quantity
- **Stay current**: Focus on recent articles (last 7-30 days)
- **Think monetization**: Every piece should serve the business goal
- **Use oacc for code analysis**: Don't reinvent code quality tools

View file

@ -0,0 +1,140 @@
# Extraction Checklist
## ✅ Pre-Extraction Verification
Run these commands to verify everything is ready:
```bash
cd scripts/omnivore-content-system
# 1. Dependencies resolved
pnpm install
# Should complete without errors
# 2. TypeScript compiles
pnpm run build
# Should create dist/ directory
# 3. Type checking passes
pnpm run typecheck
# Should show no errors
# 4. Omnivore client works
node lib/omnivore/client.js --test
# Should connect and show user info
```
## 📦 Extraction Commands
```bash
# From omnivore repo root
cd /Volumes/devel/personal/keybase/edgerouter/mac-mini/home/omnivore/omnivore
# Copy to new location
cp -r scripts/omnivore-content-system /path/to/new-location/
# OR move (if extracting permanently)
mv scripts/omnivore-content-system /path/to/new-location/
```
## 🔧 Post-Extraction Setup
```bash
cd /path/to/new-location/omnivore-content-system
# 1. Install dependencies
pnpm install
# 2. Configure environment
cp .env.example .env
# Edit .env with your API keys
# 3. Test connection
node lib/omnivore/client.js --test
# 4. Verify build
pnpm run build
pnpm run typecheck
# 5. Initialize Git (if new repo)
git init
git add .
git commit -m "Initial commit: Omnivore content monetization system"
```
## 🔍 Verification After Extraction
### Check 1: No External Dependencies
```bash
# Should find no references to parent repo paths
grep -r "\.\./\.\./self-hosting" .
grep -r "omnivore/scripts" .
# Both should return no results (or only in IMPLEMENTATION_PLAN.md for documentation)
```
### Check 2: All Files Present
```bash
# Required files
ls -la .env.example # ✅ Should exist
ls -la package.json # ✅ Should exist
ls -la tsconfig.json # ✅ Should exist
ls -la .gitignore # ✅ Should exist
ls -la README.md # ✅ Should exist
ls -la IMPLEMENTATION_PLAN.md # ✅ Should exist
ls -la CLAUDE.md # ✅ Should exist
# Required directories
ls -d lib/ # ✅ Should exist
ls -d src/ # ✅ Should exist
ls -d content/ # ✅ Should exist
```
### Check 3: Build Works
```bash
pnpm run build
# Should succeed and create dist/types/
ls -la dist/
```
### Check 4: Client Works
```bash
# After configuring .env with real API keys
node lib/omnivore/client.js --test
# Should output:
# ✅ Connected to Omnivore API
# User: Your Name (your@email.com)
# Username: yourusername
```
## ❌ What Should NOT Be Included
These remain in the omnivore repo:
- ❌ `/scripts/migrate-omnivore.js`
- ❌ `/scripts/import-pocket.js`
- ❌ `/self-hosting/` directory
- ❌ Parent repo's package.json
## 📝 Final Checklist
Before considering extraction complete:
- [ ] All dependencies install without errors
- [ ] TypeScript compiles successfully
- [ ] Type checking passes with no errors
- [ ] Omnivore client connects to API
- [ ] .env.example is present with correct structure
- [ ] README.md reflects current status
- [ ] IMPLEMENTATION_PLAN.md is up to date
- [ ] .gitignore properly configured
- [ ] No symlinks to parent repo
- [ ] No hardcoded paths to parent repo structure
## 🚀 Ready When
All checkboxes above are checked ✅
## 📚 Documentation References
- [README.md](./README.md) - Overview and quick start
- [IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md) - Detailed roadmap
- [CLAUDE.md](./CLAUDE.md) - Agent context and strategy

File diff suppressed because it is too large Load diff

278
self-hosting/omc/README.md Normal file
View file

@ -0,0 +1,278 @@
# Omnivore Content Monetization System
AI-powered content generation system that transforms your Omnivore reading into monetizable content: blog posts, newsletters, and social media posts.
## 🔎 Current Reality (Audit: 2026-01-30)
- The CLI command set is largely implemented (`omc queue/*`, `omc analyze/*`, `omc content/*`, `omc report/*`, `omc omnivore/*`, `omc db/*`, `omc config/*`).
- There are still critical fixups needed before this repo is “clean build + reliable run”; see `docs/_meta/current-state.md`.
## 🚀 Features
### ✅ Currently Working
- **Omnivore API Client**: Fetch/search/update articles, highlights, and notes
- **Queue + Tracking DB**: SQLite-backed analysis queue (`data/omnivore-content.db`)
- **Analysis Workflow**: Prepare stub files → run external analyzer → persist results to Markdown + DB
- **Reporting**: Topic/sentiment/trend aggregation across saved analyses
- **Sync Back to Omnivore**: Push summaries (and optional NOTE highlights) back to Omnivore
### 🚧 Planned Features (NOT YET IMPLEMENTED)
- **Blog Post Generation** *(Phase 5)*: Create weekly roundups, deep dives, tutorials from your saved articles
- **Newsletter Creation** *(Phase 5)*: Generate curated newsletters with your commentary
- **SEO Optimization** *(Phase 5)*: Automatic title, description, and tag generation
- **Trend Tracking** *(Phase 7)*: Identify emerging topics before they peak
- **Multi-Agent System** *(Phase 6-7)*: Specialized Claude agents for each content type
- **Publishing Integration** *(Phase 6)*: Ghost, WordPress, Medium support
- **First-Class Content Generation Pipeline**: Turn analyses into publishable posts (beyond reporting/export)
## 📋 Prerequisites
- Node.js 18+
- Omnivore account with API key
- Anthropic API key (Claude)
- Self-hosted Omnivore instance OR Omnivore cloud account
## 📦 Current Status
**Phase 0: Foundation ✅ COMPLETE**
- TypeScript toolchain configured
- Omnivore GraphQL client working
- Directory structure created
- Ready for standalone extraction
**Phases 1-6: In Progress**
- See [IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md) for detailed roadmap
## 🛠️ Installation
1. **Extract to standalone directory** (if from omnivore repo)
```bash
cp -r omnivore/scripts/omnivore-content-system /path/to/new-repo/
cd /path/to/new-repo/omnivore-content-system
```
2. **Install dependencies**
```bash
pnpm install
```
3. **Configure environment**
```bash
cp .env.example .env
# Edit .env with your API keys:
# - OMNIVORE_API_KEY
# - OMNIVORE_API_URL
# - ANTHROPIC_API_KEY
```
4. **Test setup**
```bash
# Test Omnivore connection
node lib/omnivore/client.js --test
# Test TypeScript build
pnpm run build
# Note: `pnpm run typecheck` is currently failing due to known alias/import issues.
# See: docs/_meta/current-state.md
```
## ⚙️ Configuration
### Required Environment Variables
```bash
OMNIVORE_API_KEY=your_omnivore_api_key
OMNIVORE_API_URL=https://your-omnivore-instance.com/api/graphql
ANTHROPIC_API_KEY=your_claude_api_key
```
### Optional Configuration (NOT IMPLEMENTED YET)
- **Publishing**: Configure Ghost, WordPress, or Medium credentials *(Phase 6)*
- **Topics**: Edit `config/topics.json` to customize AI/Tech categories *(Phase 1)*
- **Templates**: Modify templates in `templates/` for your style *(Phase 5)*
- **Workflows**: Adjust workflow schedules in `config/workflows.json` *(Phase 9)*
## 🎯 Usage (Current)
### Currently Working
```bash
# Setup checks
omc doctor
# Add recent articles to queue
omc queue add --hours 24
# Prepare stub files for external analysis
omc analyze run --batch-size 5
# After your external analyzer writes `analysis` into temp/*.jsonl:
omc analyze complete
# View/search
omc content list
omc content search "kubernetes"
# Categorized reports
omc report topics
omc report trends
```
For a deeper breakdown (what works, whats broken, and what to fix next), see `docs/_meta/current-state.md`.
## 📁 Project Structure
```
omnivore-content-system/
├── lib/ # ✅ Omnivore GraphQL client (WORKING)
│ └── omnivore/
│ ├── client.js # ✅ Full-featured GraphQL client
│ └── queries.js # ✅ Query builders
├── src/ # ✅ TypeScript source (READY)
│ ├── types/ # (Phase 1 - not started)
│ ├── storage/ # (Phase 3 - not started)
│ ├── analysis/ # (Phase 4 - not started)
│ ├── generation/ # (Phase 5 - not started)
│ ├── publishing/ # (Phase 6 - not started)
│ ├── workflows/ # (Phase 9 - not started)
│ └── utils/ # (Phase 2 - not started)
├── content/ # ✅ Storage directories (READY)
│ ├── articles/ # (empty - Phase 3)
│ ├── analysis/ # (empty - Phase 4)
│ └── generated/ # (empty - Phase 5)
├── test-scripts/ # ✅ Test scripts (READY)
├── cli/ # (Phase 2-6 - not started)
├── tests/ # ✅ Test directory (READY)
├── legacy-scripts/ # ✅ Original migration scripts (preserved)
├── templates/ # (empty - Phase 5 needed)
├── .claude/ # (Agent SDK config - Phase 6+)
├── tsconfig.json # ✅ TypeScript config (WORKING)
├── package.json # ✅ Dependencies (WORKING)
└── .env.example # ✅ Config template (WORKING)
```
**Legend:**
- ✅ = Working and ready to use
- (empty) = Directory created but no files yet
- (Phase X) = Implementation planned, not started
- (WORKING) = Fully functional
## 🤖 Agent Architecture (NOT IMPLEMENTED - Phase 6-8)
### Orchestrator Agent *(NOT IMPLEMENTED)*
Main controller that coordinates all subagents and workflows.
### Specialized Agents *(NOT IMPLEMENTED)*
- **Content Analyzer** *(Phase 7)*: Analyzes reading patterns, extracts themes
- **Blog Writer** *(Phase 7)*: Generates blog posts from analyzed content
- **Newsletter Creator** *(Phase 7)*: Creates newsletters with commentary
- **SEO Optimizer** *(Phase 7)*: Optimizes titles, descriptions, tags
- **Trend Tracker** *(Phase 7)*: Identifies emerging topics and opportunities
### MCP Servers *(NOT IMPLEMENTED - Phase 8)*
- **Omnivore MCP**: Tools for searching, fetching articles, highlights
- **Content DB MCP**: Tools for saving and tracking generated content
## 📝 Content Templates (NOT IMPLEMENTED - Phase 5)
### Blog Post Types *(NOT IMPLEMENTED)*
- **Weekly Roundup**: Top 5-10 articles from the week
- **Deep Dive**: Multi-article synthesis into long-form analysis
- **Tutorial**: How-to content extracted from technical articles
- **Comparison**: "X vs Y" posts from related articles
### Newsletter Formats *(NOT IMPLEMENTED)*
- **Weekly Digest**: Curated links with your commentary
- **Themed Edition**: Deep dive into single topic
### Social Media *(NOT IMPLEMENTED)*
- **Twitter Threads**: Key insights from articles
- **LinkedIn Posts**: Professional summaries
## 🔧 Legacy Scripts
All original Omnivore scripts are preserved in `legacy-scripts/`:
- `import-pocket.js` - Import from Pocket
- `apply-labels.js` - Label management
- `migrate-omnivore.js` - Data migration
- And more...
These can be called from agents as needed or run independently.
## 💾 Content Storage (Phase 3 - NOT IMPLEMENTED)
Will use **Markdown + front-matter + Git** (no database):
- **content/articles/** *(empty)*: Omnivore articles saved as Markdown
- **content/analysis/** *(empty)*: AI analysis results
- **content/generated/** *(empty)*: Generated blog posts and newsletters
- **Front-matter**: YAML metadata in each file
- **Git**: Version control and history
Directories are created but no storage implementation yet.
## 🚀 Publishing (Phase 6 - NOT IMPLEMENTED)
### Ghost CMS *(NOT IMPLEMENTED)*
```bash
# Configuration ready in .env.example, but no publisher code yet
GHOST_API_URL=https://your-blog.ghost.io
GHOST_API_KEY=your_admin_api_key
```
### WordPress *(NOT IMPLEMENTED)*
```bash
WORDPRESS_URL=https://your-site.com
WORDPRESS_API_KEY=your_api_key
```
### Medium *(NOT IMPLEMENTED)*
```bash
MEDIUM_TOKEN=your_integration_token
```
## 🐛 Troubleshooting
### API Connection Issues (Currently Applicable)
- Verify `OMNIVORE_API_KEY` is correct
- Check `OMNIVORE_API_URL` is accessible
- Test with: `node lib/omnivore/client.js --test`
### TypeScript Build Issues (Currently Applicable)
- Run `pnpm install` to ensure dependencies are installed
- Run `pnpm run typecheck` to verify no type errors
- Check that Node.js version is ≥18.0.0
### Agent Issues *(NOT APPLICABLE YET - Phase 6+)*
- Check `ANTHROPIC_API_KEY` is set
- Verify Claude Agent SDK is installed
- Review logs in `data/logs/`
### Storage Issues *(NOT APPLICABLE YET - Phase 3+)*
- Check permissions on `content/` directory
- Verify Git is initialized if using version control
## 📚 Documentation
- **[IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md)** - ✅ Detailed implementation roadmap (current)
- **[EXTRACTION_CHECKLIST.md](./EXTRACTION_CHECKLIST.md)** - ✅ Steps to move to standalone repo (current)
- **[CLAUDE.md](./CLAUDE.md)** - ✅ Agent context and content strategy (current)
- ~~Setup Guide~~ *(NOT CREATED - covered in this README)*
- ~~Workflows~~ *(NOT CREATED - Phase 9)*
- ~~Agent Architecture~~ *(NOT CREATED - Phase 6-8)*
- ~~API Documentation~~ *(NOT CREATED - Phase 1-7)*
## 🤝 Contributing
This is a personal content system, but suggestions welcome!
## 📄 License
MIT
## 🙏 Credits
Built with:
- [Claude Agent SDK](https://github.com/anthropics/claude-agent-sdk-typescript) by Anthropic
- [Omnivore](https://omnivore.app/) for reading management
- Your voracious reading habit!

8
self-hosting/omc/bin/omc.ts Executable file
View file

@ -0,0 +1,8 @@
#!/usr/bin/env node --import tsx
/**
* OMC (Omnivore Content System) CLI Executable
* AIDEV-NOTE: Main executable entry point for the OMC CLI system
*/
// Import and run the main CLI
import '../index.js';

View file

@ -0,0 +1,23 @@
# Archived Scripts
These scripts have been replaced by the new OCLIF-based CLI (`omc`). They are preserved here for reference but should not be used.
## Migration Map
| Old Script | New CLI Command |
|-----------|-----------------|
| `parallel-analyze.ts` | `omc analyze run` |
| `save-analysis-results.ts` | Internal (called by analyze run) |
| `retry-failed.ts` | `omc analyze retry` |
| `corpus-report.ts` | `omc report corpus` |
| `get-article-content.ts` | `omc omnivore get <slug>` |
| `get-article-notes.ts` | `omc omnivore note get <article-id>` |
| `test-update-article-notes.ts` | `omc omnivore note update <article-id>` |
| `update-note-test.ts` | `omc omnivore note update <article-id>` |
| `migrate-database.ts` | `omc db migrate` |
## Archived Date
2025-01-05
## Reason
All functionality has been migrated to the unified CLI system with better error handling, help text, and consistent interfaces.

View file

@ -0,0 +1,127 @@
#!/usr/bin/env tsx
/**
* Generate corpus analysis report
* AIDEV-NOTE: tracking-db - reads from SQLite analysis_queue table
*/
import Database from 'better-sqlite3';
console.log('\n' + '═'.repeat(80));
console.log('CORPUS ANALYSIS REPORT');
console.log('═'.repeat(80));
// AIDEV-NOTE: tracking-db - read from SQLite, not deprecated batch file
const db = new Database('data/omnivore-content.db', { readonly: true });
// Queue status from tracking DB
const statusCounts = db.prepare(`
SELECT status, COUNT(*) as count
FROM analysis_queue
GROUP BY status
`).all() as { status: string; count: number }[];
const total = statusCounts.reduce((sum, row) => sum + row.count, 0);
console.log(`\n📊 Queue Status:`);
console.log(` Total articles: ${total}`);
for (const { status, count } of statusCounts) {
console.log(` ${status}: ${count}`);
}
// Load completed analyses from SQLite
const rows = db.prepare(`
SELECT article_id, article_title, analysis_json
FROM analysis_queue
WHERE status = 'completed' AND analysis_json IS NOT NULL
`).all() as { article_id: string; article_title: string; analysis_json: string }[];
console.log(`\n📚 Analyzed Articles: ${rows.length}\n`);
// Parse analysis JSON from each row
const analyses = rows.map(row => ({
articleId: row.article_id,
articleTitle: row.article_title,
...JSON.parse(row.analysis_json)
})).filter(a => a.topics && a.topics[0] !== 'N/A'); // Filter out failed analyses
// Topic distribution
const topicCounts: Record<string, number> = {};
const topicScoreTotals: Record<string, number> = {};
for (const analysis of analyses) {
for (const topic of analysis.topics) {
topicCounts[topic] = (topicCounts[topic] || 0) + 1;
const score = analysis.topicScores[topic] || 0;
topicScoreTotals[topic] = (topicScoreTotals[topic] || 0) + score;
}
}
console.log('═'.repeat(80));
console.log('TOPIC DISTRIBUTION');
console.log('═'.repeat(80));
const sortedTopics = Object.entries(topicCounts)
.sort((a, b) => b[1] - a[1]);
console.log(`\n📈 Topics by frequency:\n`);
for (const [topic, count] of sortedTopics) {
const avgScore = (topicScoreTotals[topic] / count).toFixed(2);
console.log(` ${topic.padEnd(30)} ${count} articles (avg score: ${avgScore})`);
}
// Sentiment distribution
const sentiments: Record<string, number> = {};
for (const analysis of analyses) {
const sentiment = analysis.sentiment;
sentiments[sentiment] = (sentiments[sentiment] || 0) + 1;
}
console.log(`\n😊 Sentiment distribution:\n`);
for (const [sentiment, count] of Object.entries(sentiments)) {
console.log(` ${sentiment}: ${count}`);
}
// Identify clusters (articles with shared topics)
console.log('\n' + '═'.repeat(80));
console.log('TOPIC CLUSTERS (Content Opportunities)');
console.log('═'.repeat(80));
for (const [topic, count] of sortedTopics) {
if (count < 2) continue; // Only show topics with 2+ articles
console.log(`\n📂 ${topic} (${count} articles):`);
const articlesWithTopic = analyses.filter(a =>
a.topics.includes(topic)
);
for (const analysis of articlesWithTopic) {
const score = analysis.topicScores[topic].toFixed(2);
console.log(` [${score}] ${analysis.articleTitle.substring(0, 70)}`);
}
// Content opportunity
if (count >= 3) {
console.log(` 💡 OPPORTUNITY: Weekly roundup or comparison post possible`);
} else if (count === 2) {
console.log(` 💡 OPPORTUNITY: Comparison or "X vs Y" post possible`);
}
}
// Monetization angles summary
console.log('\n' + '═'.repeat(80));
console.log('MONETIZATION ANGLES');
console.log('═'.repeat(80) + '\n');
for (const analysis of analyses) {
console.log(`📝 ${analysis.articleTitle.substring(0, 70)}`);
console.log(` ${analysis.monetizationAngle}\n`);
}
console.log('═'.repeat(80));
console.log('END OF REPORT');
console.log('═'.repeat(80) + '\n');
// AIDEV-NOTE: tracking-db - close connection after report
db.close();

View file

@ -0,0 +1,33 @@
#!/usr/bin/env tsx
// AIDEV-NOTE: agent-helper - fetches article content for agent analysis
// AIDEV-NOTE: omnivore-boundary - GraphQL API only
import { getArticle } from '../lib/omnivore/client.js';
import 'dotenv/config';
async function main() {
const articleSlug = process.argv[2];
const username = process.argv[3];
if (!articleSlug || !username) {
console.error('Usage: tsx cli/get-article-content.ts <articleSlug> <username>');
process.exit(1);
}
try {
const result = await getArticle(articleSlug, username);
if (!result?.article?.content) {
console.error('No content found');
process.exit(1);
}
// Output ONLY content to stdout (no JSON wrapper)
console.log(result.article.content);
} catch (err: any) {
console.error(`Error fetching article: ${err.message}`);
process.exit(1);
}
}
main();

View file

@ -0,0 +1,84 @@
#!/usr/bin/env tsx
/**
* Fetch and display notes for an article
*/
import fetch from 'node-fetch';
import { config } from 'dotenv';
config();
const API_URL = process.env.OMNIVORE_API_URL || 'https://api-prod.omnivore.app/api/graphql';
const API_KEY = process.env.OMNIVORE_API_KEY;
const articleId = process.argv[2] || '5977ff9f-01ea-4977-aa0e-1dbe43cd2a20';
console.log(`\n📝 Fetching notes for article: ${articleId}\n`);
const query = `
query GetArticle($id: ID!) {
article(id: $id) {
... on ArticleSuccess {
article {
id
title
highlights {
id
type
quote
annotation
createdAt
updatedAt
}
}
}
... on ArticleError {
errorCodes
}
}
}
`;
const response = await fetch(API_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': API_KEY,
},
body: JSON.stringify({ query, variables: { id: articleId } }),
});
const result = await response.json();
if (result.errors) {
console.error('GraphQL errors:', JSON.stringify(result.errors, null, 2));
process.exit(1);
}
const article = result.data?.article?.article;
if (!article) {
console.error('Article not found or error:', result.data?.article?.errorCodes);
process.exit(1);
}
console.log(`Article: ${article.title}\n`);
const highlights = article.highlights || [];
const notes = highlights.filter((h: any) => h.type === 'NOTE');
console.log(`Found ${notes.length} notes:\n`);
for (const note of notes) {
console.log('═'.repeat(80));
console.log(`Note ID: ${note.id}`);
console.log(`Type: ${note.type}`);
console.log(`Created: ${note.createdAt}`);
console.log(`Updated: ${note.updatedAt}`);
console.log('─'.repeat(80));
console.log('Annotation (raw Markdown):');
console.log(note.annotation);
console.log('═'.repeat(80));
console.log();
}

View file

@ -0,0 +1,55 @@
#!/usr/bin/env tsx
// AIDEV-NOTE: database-migration - adds new columns to existing analysis_queue table
import { initDatabase } from '../src/storage/database';
async function main() {
console.log('Starting database migration...\n');
const db = initDatabase();
try {
// Check current schema
const columns = db.prepare(`PRAGMA table_info(analysis_queue)`).all() as Array<{ name: string }>;
const columnNames = columns.map(c => c.name);
console.log('Current columns:', columnNames.join(', '));
// Add missing columns
const migrations = [
{ name: 'saved_at', sql: 'ALTER TABLE analysis_queue ADD COLUMN saved_at TEXT NOT NULL DEFAULT ""' },
{ name: 'published_at', sql: 'ALTER TABLE analysis_queue ADD COLUMN published_at TEXT' },
{ name: 'updated_at_article', sql: 'ALTER TABLE analysis_queue ADD COLUMN updated_at_article TEXT' },
{ name: 'analysis_json', sql: 'ALTER TABLE analysis_queue ADD COLUMN analysis_json TEXT' },
{ name: 'markdown_path', sql: 'ALTER TABLE analysis_queue ADD COLUMN markdown_path TEXT' },
];
let added = 0;
for (const migration of migrations) {
if (!columnNames.includes(migration.name)) {
console.log(`Adding column: ${migration.name}`);
db.exec(migration.sql);
added++;
} else {
console.log(`Column ${migration.name} already exists, skipping`);
}
}
console.log(`\n✓ Migration complete: ${added} columns added`);
// Show final schema
const finalColumns = db.prepare(`PRAGMA table_info(analysis_queue)`).all() as Array<{ name: string, type: string }>;
console.log('\nFinal schema:');
finalColumns.forEach(col => {
console.log(` - ${col.name} (${col.type})`);
});
} catch (err: any) {
console.error('Migration failed:', err.message);
process.exit(1);
} finally {
db.close();
}
}
main().catch(console.error);

View file

@ -0,0 +1,98 @@
#!/usr/bin/env tsx
// AIDEV-NOTE: tracking-coordination - uses SQLite queue for parallel execution
// AIDEV-NOTE: analysis-output-boundary - results written to Markdown/JSONL, NOT database
// AIDEV-NOTE: omnivore-boundary - fetches articles via GraphQL API, never local cache
import { getArticle, getMe } from '../lib/omnivore/client.js';
import { initDatabase } from '../src/storage/database';
import { AnalysisQueueRepository } from '../src/storage/AnalysisQueueRepository';
import { AnalysisWriter } from '../src/storage/AnalysisWriter';
import type { ContentAnalysis } from '../src/types/analysis';
import { writeFileSync, mkdirSync } from 'fs';
import 'dotenv/config';
const BATCH_SIZE = 5; // Process 5 articles in parallel
async function main() {
// AIDEV-NOTE: tracking-db - coordination only, not analysis storage
const db = initDatabase();
const queueRepo = new AnalysisQueueRepository(db);
const stats = queueRepo.getStats();
console.log('═'.repeat(80));
console.log('Analysis Queue Status');
console.log('═'.repeat(80));
console.log(` Total: ${stats.total}`);
console.log(` Pending: ${stats.pending}`);
console.log(` In Progress: ${stats.inProgress}`);
console.log(` Completed: ${stats.completed}`);
console.log(` Failed: ${stats.failed}`);
if (stats.pending === 0) {
console.log('\n✓ No pending analyses');
db.close();
return;
}
// Get next batch
// AIDEV-NOTE: tracking-batch - fetch jobs for parallel processing
const jobs = queueRepo.getPending(BATCH_SIZE);
console.log(`\n${'─'.repeat(80)}`);
console.log(`Created ${jobs.length} stub files for parallel analysis...`);
console.log('─'.repeat(80));
// Mark as in_progress (coordination lock)
// AIDEV-NOTE: tracking-lock - prevents duplicate analysis by concurrent runs
for (const job of jobs) {
queueRepo.markInProgress(job.articleId);
}
// AIDEV-NOTE: gql-article-query - get username from authenticated user
const me = await getMe();
const username = me.profile.username;
// Create temp directory for stub files
mkdirSync('temp', { recursive: true });
// AIDEV-NOTE: stub-file-creation - fetch articles and populate complete metadata
const agentParams = [];
for (let index = 0; index < jobs.length; index++) {
const job = jobs[index];
const filename = `temp/${job.articleSlug}.jsonl`;
console.log(`Fetching article ${index + 1}/${jobs.length}: ${job.articleTitle.substring(0, 60)}...`);
// Fetch article to get publishedAt/updatedAt
const result = await getArticle(job.articleSlug, username);
// Write complete stub JSONL with all metadata
const stub = {
articleId: job.articleId,
articleSlug: job.articleSlug,
username: username,
articleUrl: result.article.url,
articleTitle: result.article.title,
savedAt: result.article.savedAt,
publishedAt: result.article.publishedAt || null,
updatedAt: result.article.updatedAt || null
};
writeFileSync(filename, JSON.stringify(stub) + '\n', 'utf-8');
// Return metadata for agent invocation
agentParams.push({
filename: filename,
articleId: job.articleId,
articleSlug: job.articleSlug,
username: username,
articleTitle: result.article.title.substring(0, 60) + '...' // truncated for display
});
}
console.log(JSON.stringify(agentParams, null, 2));
db.close();
}
main().catch(console.error);

View file

@ -0,0 +1,52 @@
#!/usr/bin/env tsx
// AIDEV-NOTE: tracking-retry - resets failed jobs to pending for another attempt
import { initDatabase } from '../src/storage/database';
import { AnalysisQueueRepository } from '../src/storage/AnalysisQueueRepository';
const MAX_RETRIES = 3;
async function main() {
const db = initDatabase();
const queueRepo = new AnalysisQueueRepository(db);
const failed = queueRepo.getFailed();
if (failed.length === 0) {
console.log('✓ No failed jobs to retry');
db.close();
return;
}
console.log('═'.repeat(80));
console.log(`Found ${failed.length} failed jobs`);
console.log('═'.repeat(80));
let reset = 0;
let skipped = 0;
for (const job of failed) {
if (job.retryCount < MAX_RETRIES) {
queueRepo.resetToPending(job.articleId);
console.log(`✓ Reset: ${job.articleTitle.substring(0, 50)}... (attempt ${job.retryCount + 2})`);
reset++;
} else {
console.log(`✗ Skip: ${job.articleTitle.substring(0, 50)}... (max retries exceeded)`);
skipped++;
}
}
console.log('\n' + '═'.repeat(80));
console.log('Summary:');
console.log(` Reset for retry: ${reset}`);
console.log(` Skipped (max retries): ${skipped}`);
console.log('═'.repeat(80));
if (reset > 0) {
console.log('\nRun: pnpm analyze:parallel');
}
db.close();
}
main().catch(console.error);

View file

@ -0,0 +1,106 @@
#!/usr/bin/env tsx
// AIDEV-NOTE: analysis-output-boundary - saves results to git-tracked Markdown/JSONL
// AIDEV-NOTE: tracking-update - updates SQLite queue status after save
import { readFileSync, unlinkSync } from 'fs';
import { initDatabase } from '../src/storage/database';
import { AnalysisQueueRepository } from '../src/storage/AnalysisQueueRepository';
import { AnalysisWriter } from '../src/storage/AnalysisWriter';
import type { ContentAnalysis } from '../src/types/analysis';
interface EnrichedResult {
articleId: string;
articleSlug: string;
username: string;
articleUrl: string;
articleTitle: string;
savedAt: string;
publishedAt: string | null;
updatedAt: string | null;
analysis: ContentAnalysis;
}
async function main() {
const patterns = process.argv.slice(2);
if (patterns.length === 0) {
patterns.push('temp/*.jsonl'); // Default: all JSONL files in temp/
}
// Read all matching JSONL files from all patterns
const { globSync } = await import('glob');
const files = patterns.flatMap(pattern => globSync(pattern));
if (files.length === 0) {
console.error(`No files found matching patterns: ${patterns.join(', ')}`);
process.exit(1);
}
const resultsWithFiles: Array<{ result: EnrichedResult; file: string }> = files.map(file => {
const content = readFileSync(file, 'utf-8');
return { result: JSON.parse(content), file };
});
console.log(`Loaded ${resultsWithFiles.length} enriched analysis results from ${files.length} files`);
// AIDEV-NOTE: tracking-db - coordination only
const db = initDatabase();
const queueRepo = new AnalysisQueueRepository(db);
// AIDEV-NOTE: analysis-output - permanent git-tracked storage
const writer = new AnalysisWriter({ outputDir: 'content/analysis' });
let saved = 0;
let failed = 0;
for (const { result, file } of resultsWithFiles) {
const { articleId, articleUrl, articleTitle, savedAt, publishedAt, updatedAt, analysis } = result;
try {
// Write to Markdown (git-tracked, human-editable)
const mdPath = await writer.write(
articleId,
articleUrl,
articleTitle,
savedAt,
analysis
);
// Store in database (immutable AI snapshot)
queueRepo.storeAnalysis(
articleId,
publishedAt,
updatedAt,
JSON.stringify(analysis),
mdPath
);
// Remove temp file after successful save
unlinkSync(file);
console.log(`✓ Saved: ${articleTitle.substring(0, 60)}...`);
saved++;
} catch (err: any) {
// Mark as failed in tracking DB
queueRepo.markFailed(articleId, `Save failed: ${err.message}`);
console.error(`✗ Failed: ${articleTitle.substring(0, 60)}... - ${err.message}`);
failed++;
}
}
// Show updated stats
const stats = queueRepo.getStats();
console.log(`\n${'═'.repeat(80)}`);
console.log('Results:');
console.log(` Saved: ${saved}`);
console.log(` Failed: ${failed}`);
console.log('\nUpdated Queue Status:');
console.log(` Pending: ${stats.pending}`);
console.log(` Completed: ${stats.completed}`);
console.log(` Failed: ${stats.failed}`);
console.log('═'.repeat(80));
db.close();
}
main().catch(console.error);

View file

@ -0,0 +1,48 @@
#!/usr/bin/env tsx
/**
* Test updating article description AND creating notebook notes
*/
import { updatePage, createHighlight } from '../lib/omnivore/client.js';
import { randomUUID } from 'crypto';
// Use the LLM Observability article
const articleId = '5977ff9f-01ea-4977-aa0e-1dbe43cd2a20';
const articleTitle = 'LLM Observability in the Wild - Why OpenTelemetry should be the Standard | SigNoz';
console.log(`\n🧪 Testing article notes update for: ${articleTitle}\n`);
// Test 1: Update article description (the "info" section in Omnivore UI)
console.log('1⃣ Updating article description (Info section)...');
const updateResult = await updatePage({
pageId: articleId,
description: '🔍 WHY I SAVED THIS: Excellent deep dive into OpenTelemetry patterns for LLM observability. Potential for comparison post: OpenTelemetry vs proprietary solutions (Langfuse, Weights & Biases). Key insight: standardization matters more in LLM ops than traditional observability.'
});
console.log(' ✅ Description updated:', updateResult.updatedPage.description);
// Test 2: Create a NOTE highlight (the "Notebook" section in Omnivore UI)
console.log('\n2⃣ Creating notebook note (NOTE highlight)...');
const noteId = randomUUID();
const shortId = Math.random().toString(36).substring(2, 10); // 8-char random string
const noteResult = await createHighlight({
id: noteId,
shortId: shortId,
articleId: articleId,
type: 'NOTE',
annotation: '💡 CONTENT STRATEGY NOTE:\n\nThis article fits perfectly into our AI Infrastructure theme for the weekly roundup. Key angles:\n\n1. Comparison piece: OpenTelemetry vs Langfuse vs W&B\n2. Tutorial: Setting up OTel for LLM apps\n3. Opinion piece: Why standardization matters in AI ops\n\nTarget audience: ML engineers building production LLM apps\nEstimated value: High - trending topic, practical advice',
quote: '', // Empty quote = standalone note
patch: '',
prefix: '',
suffix: ''
});
console.log(' Result:', JSON.stringify(noteResult, null, 2));
console.log('\n📌 Check Omnivore UI to verify:');
console.log(' 1. Description field shows "WHY I SAVED THIS" note');
console.log(' 2. Notebook section shows the content strategy note');
console.log(` Article ID: ${articleId}\n`);

View file

@ -0,0 +1,42 @@
#!/usr/bin/env tsx
/**
* Update a note with Markdown to test format
*/
import { updateHighlight } from '../lib/omnivore/client.js';
const noteId = '922361d2-f09c-4b81-8826-396de3690c5d';
const markdown = `# Analysis Report
**Article**: LLM Observability
**Status**: High Priority
## Key Insights
1. **OpenTelemetry vs OpenInference** - competing standards
2. **Ruby SDK gap** - No OpenInference support
3. **Production debugging** - Visibility challenges
## Content Strategy
- [ ] Write comparison post
- [ ] Tutorial on OTel setup
- [ ] Interview Pranav
### Target Audience
ML engineers building production LLM apps
*Last updated: 2025-10-02*
`;
console.log('Updating note with Markdown...\n');
const result = await updateHighlight({
highlightId: noteId,
annotation: markdown
});
console.log('Result:', JSON.stringify(result, null, 2));
console.log('\nCheck Omnivore UI to see how the Markdown renders!');

View file

@ -0,0 +1,58 @@
# GraphQL Code Generator Configuration
# Generates TypeScript types from GraphQL schema and operations
schema: './docs/graphql-schema/schema.graphql'
# Watch mode for development
watch: false
# The CLI runtime currently uses `lib/omnivore/client.js` (string-based queries).
# Typed GraphQL operations are not maintained right now; reintroduce documents as part of a future migration.
documents: []
# Output configuration
generates:
# Generate TypeScript types for the schema
src/types/generated/graphql.ts:
plugins:
- typescript
config:
# Type-safe configuration
strictScalars: true
scalars:
Date: string
JSON: 'Record<string, unknown>'
# Naming conventions
enumsAsTypes: true
# Add helpful comments
addUnderscoreToArgsType: true
skipTypename: false
# Generate TypeScript types for operations (queries/mutations)
src/types/generated/operations.ts:
plugins:
- typescript
- typescript-operations
- typed-document-node
config:
# Type-safe configuration
strictScalars: true
scalars:
Date: string
JSON: 'Record<string, unknown>'
# Avoid duplicating types from schema
avoidOptionals: false
# Better type names
namingConvention:
typeNames: pascal-case#pascalCase
enumValues: upper-case#upperCase
# Fragment configuration
fragmentVariablePrefix: ''
fragmentVariableSuffix: 'Fragment'
# Deduplication
dedupeFragments: true
# Hooks to run after generation
# hooks:
# afterAllFileWrite:
# - prettier --write

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,395 @@
# System Architecture
**Purpose**: Architectural design decisions, storage boundaries, and structural patterns for the Omnivore Content System.
**Last Updated**: 2026-01-30
**Status**: Architectural intent remains valid, but some implementation details have drifted (notably JSONL output and the legacy workflow scripts). See `docs/_meta/current-state.md` for the full audit.
## Overview
The analysis engine processes articles from Omnivore through the `@article-content-analyzer` agent, extracting structured insights for content monetization. The system uses a three-layer architecture designed to maintain clear boundaries and prevent data corruption.
## Three-Layer Architecture
The system enforces strict separation between source data, coordination, and permanent storage:
**Layer 1: Omnivore (Source of Truth)**
- Always fetch articles via GraphQL API
- Never duplicate article content locally
- Omnivore labels used as intent hints for analysis
**Layer 2: SQLite (Immutable Snapshots + Tracking)**
- Database: `data/omnivore-content.db` (gitignored)
- Purpose: Store immutable AI analysis snapshots + coordinate parallel execution
- Tracks: pending/in_progress/completed/failed status
- Tables: `analysis_queue`, `analysis_sessions`
- Existing Omnivore tables (if present): READ-ONLY, never modify
**Layer 3: Markdown (Permanent Storage)**
- Analysis results: `content/analysis/*.md` (human-readable)
- Git-tracked for version history
- Only value-added content stored
### What Gets Stored
- ✅ **Analysis results** (`content/analysis/*.md`) - Topics, summaries, key points, monetization angles
- ✅ **JSONL records (optional)** (`content/analysis/analyses.jsonl`) - Produced when enabled via CLI flags (`omc analyze complete --jsonl` / `omc analyze auto --jsonl`)
- ✅ **Tracking queue** (SQLite) - Job status, retry counts, error messages
- ✅ **Generated content** (`content/generated/**/*.md`) - Blog posts, newsletters (future)
- ❌ **Source articles** - Query Omnivore API; do NOT duplicate locally
### Storage Formats
1. **Markdown with YAML front-matter** - Human-readable, git-trackable
2. **JSONL (JSON Lines)** - Append-only, machine-readable (planned/optional; not currently produced by default)
3. **SQLite** - Immutable AI snapshots + tracking (gitignored but permanent)
## Database Schema
**Location**: `src/storage/schema/tracking-schema.sql`
**Purpose**: Define tracking tables for job queue coordination.
### Tables
**1. `analysis_queue`** - Job tracking for parallel execution
- `id` (INTEGER PRIMARY KEY) - Auto-increment job ID
- `article_id` (TEXT UNIQUE) - Omnivore article ID
- `article_url` (TEXT) - Source article URL
- `article_title` (TEXT) - Source article title
- `status` (TEXT) - Job status: `pending` | `in_progress` | `completed` | `failed`
- `assigned_at` (TEXT) - When marked in_progress (ISO 8601)
- `completed_at` (TEXT) - When marked completed (ISO 8601)
- `error_message` (TEXT) - Error details if failed
- `retry_count` (INTEGER) - Number of retry attempts (default: 0)
- `created_at` (TEXT) - When added to queue (ISO 8601)
- `updated_at` (TEXT) - Last status change (ISO 8601)
**2. `analysis_sessions`** - Optional batch metadata
- `id` (INTEGER PRIMARY KEY) - Auto-increment session ID
- `started_at` (TEXT) - Session start time
- `completed_at` (TEXT) - Session completion time
- `total_articles` (INTEGER) - Total articles in session
- `completed_articles` (INTEGER) - Completed count
- `failed_articles` (INTEGER) - Failed count
- `notes` (TEXT) - Session notes
### Indexes
- `idx_analysis_queue_status` - Fast status filtering
- `idx_analysis_queue_article_id` - Fast article lookup
- `idx_analysis_queue_created_at` - Chronological ordering
### AIDEV Annotations
- `tracking + immutable snapshots` - Stores original AI output + coordination
- `analysis-output-boundary` - Analysis results stored in git-tracked Markdown/JSONL, NOT here
## Database Initialization
**Location**: `src/storage/database.ts`
**Purpose**: Initialize SQLite database with boundary enforcement for tracking coordination.
**Import**:
```typescript
import { initDatabase } from '@storage/database';
```
### Key Functions
**1. `initDatabase(dbPath?: string): Database`** - Initialize database connection
- Default path: `data/omnivore-content.db`
- Enables WAL mode for concurrent access
- Creates tracking tables via `tracking-schema.sql`
- Returns `better-sqlite3` Database instance
**2. `listTables(db: Database): string[]`** - List all tables
- Returns array of table names
- Used to identify Omnivore vs tracking tables
**3. `isOmnivoreTable(db: Database, tableName: string): boolean`** - Identify Omnivore tables
- Detects Core Data tables by Z-prefixed column names
- Returns `true` if table belongs to Omnivore cache
**4. `validateOmnivoreTablesReadOnly(db: Database): void`** - Boundary check
- Logs Omnivore tables (READ-ONLY) vs tracking tables (READ-WRITE)
- Safety check to ensure no modification of Omnivore data
**5. `getTableCounts(db: Database): Record<string, number>`** - Get row counts
- Returns object mapping table names to row counts
- Useful for debugging and monitoring
### AIDEV Annotations
- `tracking-db-boundary` - SQLite stores immutable AI snapshots + job tracking
- `omnivore-boundary` - Existing Omnivore tables are READ-ONLY, never modify
### Example Usage
```typescript
import { initDatabase, validateOmnivoreTablesReadOnly, listTables } from '@storage/database';
// Initialize database
const db = initDatabase('data/omnivore-content.db');
// Verify boundaries
validateOmnivoreTablesReadOnly(db);
// List all tables
const tables = listTables(db);
console.log('Tables:', tables);
// Output: ['analysis_queue', 'analysis_sessions', 'ZMEDIA', 'ZUSER', ...]
// Close when done
db.close();
```
## Storage Formats
### Markdown File Structure
**Location**: `content/analysis/YYYY-MM-DD-{slug}-analysis.md`
**Format**:
```markdown
---
articleId: 1039961b-8de3-4ccf-b3e8-df888d6174b8
articleSlug: cchistory-tracking-claude-code-system-prompt
articleUrl: https://example.com/article
articleTitle: "Article Title"
savedAt: 2025-09-30T02:27:46.000Z
analyzedAt: 2025-10-01T04:25:16.676Z
topics: [developer-tools, ai-tooling, reverse-engineering]
topicScores:
developer-tools: 0.95
ai-tooling: 0.9
reverse-engineering: 0.85
sentiment: positive
---
## Summary
2-3 sentence summary capturing the main points and why this matters...
## Key Points
- First key takeaway or insight
- Second key takeaway or insight
- Third key takeaway or insight
## Monetization Angle
Specific content opportunity description...
```
**Front-Matter Fields**:
- `articleId` (string) - Omnivore article ID (references source)
- `articleSlug` (string, optional) - Omnivore slug (preferred filename key)
- `articleUrl` (string) - Source article URL
- `articleTitle` (string) - Source article title (escaped quotes)
- `savedAt` (ISO 8601) - When article was saved to Omnivore
- `analyzedAt` (ISO 8601) - When analysis was performed
- `topics` (array) - 2-5 topic labels
- `topicScores` (object) - Topic → confidence score (0-1)
- `sentiment` (enum) - `positive` | `neutral` | `negative`
**Markdown Sections**:
- `## Summary` - Strategic summary (2-3 sentences)
- `## Key Points` - Bullet list of insights (3-5 items)
- `## Monetization Angle` - Content opportunity description
### JSONL File Structure
**Location**: `content/analysis/analyses.jsonl`
**Status**: Optional. `AnalysisWriter.appendToJsonl()` is invoked when JSONL is enabled via CLI flags.
**Format**: One JSON object per line (JSON Lines / newline-delimited JSON)
**Purpose**: Append-only machine-readable format for batch processing
**Record Structure**:
```json
{"articleId":"abc-123","articleUrl":"https://example.com/article","articleTitle":"Article Title","savedAt":"2025-09-30T14:12:18.000Z","analyzedAt":"2025-10-01T04:30:53.761Z","topics":["developer-tools","api-design"],"topicScores":{"developer-tools":0.95,"api-design":0.9},"sentiment":"positive","summary":"Article summary...","keyPoints":["First point","Second point"],"monetizationAngle":"Content opportunity..."}
```
**Fields**: Same as front-matter plus `summary`, `keyPoints`, `monetizationAngle` (flattened structure)
**Advantages**:
- Append-only (no file rewriting)
- Easy parsing line-by-line
- Works with streaming processing
- Standard format for data pipelines
**Usage Example**:
```javascript
import { readFileSync } from 'fs';
const lines = readFileSync('content/analysis/analyses.jsonl', 'utf-8')
.split('\n')
.filter(line => line.trim());
const analyses = lines.map(line => JSON.parse(line));
console.log(`Loaded ${analyses.length} analyses`);
```
## Boundary Enforcement
The system enforces strict boundaries between three layers to prevent data corruption and maintain clarity.
### AIDEV Annotation Patterns
All boundary-critical code uses AIDEV annotations for searchability:
**Tracking Annotations** (SQLite coordination):
- `tracking + immutable snapshots` - Stores original AI output + coordination
- `tracking-db-boundary` - SQLite stores immutable AI snapshots + job tracking
- `tracking-initialization` - Sets up job queue
- `tracking-coordination` - Fetches jobs for parallel processing
- `tracking-lock` - Prevents duplicate analysis by concurrent runs
- `tracking-completion` - Job done, analysis in git-tracked files
- `tracking-error` - Increments retry counter
- `tracking-retry` - Resets failed job for another attempt
- `tracking-stats` - Shows progress, not analysis content
- `tracking-deduplication` - Prevents duplicate queue entries
- `tracking-cleanup` - Removes completed jobs
- `tracking-inspection` - Shows queue status
- `tracking-update` - Updates queue status after save
**Output Annotations** (Git-tracked storage):
- `analysis-output-boundary` - Results written to Markdown (and optionally JSONL), NOT stored as editable content in the database
- `git-tracked-output` - Permanent storage for analysis results
**Omnivore Annotations** (Source of truth):
- `omnivore-boundary` - Always fetch via GraphQL, never from local cache
### Search Annotations
Find all boundary-critical code:
```bash
# SQLite tracking code
rg "AIDEV-NOTE:.*tracking"
# Analysis output code
rg "AIDEV-NOTE:.*analysis-output"
# Omnivore API usage
rg "AIDEV-NOTE:.*omnivore-boundary"
# Git-tracked output
rg "AIDEV-NOTE:.*git-tracked"
```
### Boundary Rules
**Rule 1: Omnivore is Source of Truth**
- Always fetch articles via GraphQL API
- Never duplicate article content locally
- Use Omnivore labels as intent hints
**Rule 2: SQLite Stores Immutable Analysis Snapshots**
- Stores original AI analysis output (immutable snapshots) + coordination for parallel execution
- Database file (`data/omnivore-content.db`) is gitignored but permanent
- Contains valuable AI output that should NOT be deleted
- Existing Omnivore tables (if present) are READ-ONLY
**Rule 3: Markdown/JSONL is Permanent Storage**
- Analysis results are git-tracked
- Only value-added content stored
- Never store source articles
**Rule 4: No Cross-Layer Leakage**
- Analysis results never stored in SQLite
- Article content never stored in git
- Queue status never persisted in Markdown
### Validation Functions
Use database boundary checks to verify compliance:
```typescript
import { initDatabase, validateOmnivoreTablesReadOnly, isOmnivoreTable } from '@storage/database';
const db = initDatabase();
// Check for Omnivore tables
validateOmnivoreTablesReadOnly(db);
// Output:
// [BOUNDARY CHECK] Omnivore table detected: ZMEDIA (READ-ONLY)
// [BOUNDARY CHECK] Tracking table: analysis_queue (READ-WRITE)
// Verify table type before operations
if (isOmnivoreTable(db, 'ZMEDIA')) {
throw new Error('Cannot modify Omnivore table');
}
```
## Type Definitions
### ContentAnalysis
**Location**: `src/types/analysis.ts`
**Purpose**: Analysis result from `@article-content-analyzer` agent.
```typescript
export interface ContentAnalysis {
articleId: string; // Omnivore article ID
topics: string[]; // 2-5 main topics
topicScores: Record<string, number>; // Topic → confidence (0-1)
summary: string; // 2-3 sentence summary
keyPoints: string[]; // 3-5 key takeaways
sentiment: 'positive' | 'neutral' | 'negative';
monetizationAngle: string; // Content opportunity
analyzedAt: string; // ISO 8601 timestamp
}
```
**Import**:
```typescript
import type { ContentAnalysis } from '@omc-types/analysis.js';
// OR
import type { ContentAnalysis } from '../types/analysis';
```
### AnalysisFrontMatter
**Location**: `src/types/content.ts`
**Purpose**: Front-matter schema for analysis Markdown files.
```typescript
export interface AnalysisFrontMatter {
articleId: string; // Omnivore article ID
articleUrl: string; // Source article URL
articleTitle: string; // Source article title
savedAt: string; // ISO 8601
analyzedAt: string; // ISO 8601
topics: string[]; // Topic labels
topicScores: Record<string, number>; // Topic → score
sentiment: 'positive' | 'neutral' | 'negative';
}
```
### StoredAnalysis
**Location**: `src/types/content.ts`
**Purpose**: Complete stored analysis (after reading Markdown file).
```typescript
export interface StoredAnalysis {
frontMatter: AnalysisFrontMatter; // Parsed YAML
summary: string; // From ## Summary section
keyPoints: string[]; // From ## Key Points section
monetizationAngle: string; // From ## Monetization Angle section
}
```
## Related Documentation
- [Workflow Internals](workflow-internals.md) - How the parallel analysis workflow operates
- [CLI Reference](cli-reference.md) - Command-line interface documentation
- [Foundation & Type System](foundation-and-types.md) - TypeScript setup, type definitions, Omnivore client

View file

@ -0,0 +1,47 @@
# Automation Patterns (Codex CLI)
**Purpose**: Document the non-interactive automation path for daily analysis, and how it integrates with `codex exec`.
## Goal
Enable a fully schedulable workflow:
```bash
omc queue add --hours 24
omc analyze auto --batch-size 5
```
This replaces the interactive “Task tool / parallel agents” middle step.
## Key Design Constraints
- The analysis step must be **non-interactive** and runnable from cron/launchd.
- The LLM caller must **not mutate the repository** (safe-by-default).
- The CLI must remain usable as a “tool” inside future Codex sessions (content generation phase).
## Codex CLI Invocation
The automated analysis pipeline uses the local Codex CLI in non-interactive mode:
- Command: `codex exec`
- Sandbox: `-s read-only`
- Prompt input: stdin (`-`)
Important runtime behavior:
- Codex writes session files under `CODEX_HOME`. For scheduled runs, `CODEX_HOME` is set to a repo-local directory (`temp/codex-home`) so we dont depend on `~/.codex/*` permissions.
## Implementation Notes
- LLM wrapper: `src/lib/ai/codex-cli-client.ts`
- Builds a strict “JSON only” prompt.
- Runs `codex exec` and parses the first JSON object from stdout.
- Analyzer: `src/analysis/ContentAnalyzer.ts`
- Loads `src/analysis/prompts/analyze-article.md`
- Calls the Codex CLI wrapper and normalizes required fields.
- Orchestration: `src/commands/analyze/auto.ts`
- Selects jobs from SQLite queue
- Fetches article content from Omnivore
- Calls `ContentAnalyzer`
- Persists results to Markdown + SQLite snapshot

View file

@ -0,0 +1,832 @@
# CLI Reference
**Purpose**: Comprehensive command-line interface documentation for the Omnivore Content System.
**Last Updated**: 2026-01-30
**Status**: This file historically mixed design intent with implementation notes. For a full ground-truth audit (including known breakages and fixes), see `docs/_meta/current-state.md`.
## Overview
The CLI provides commands for managing the article analysis queue, fetching content from Omnivore, running analysis, and generating reports. All commands use the `omc` binary.
## Queue Management Commands
### queue add
**Command**: `omc queue add`
**Purpose**: Fetch articles from Omnivore and populate tracking queue.
**Usage**:
```bash
omc queue add --hours 24 # Last 24 hours
omc queue add --label "ai-ml" # By label
omc queue add --url <url> # Single article
omc queue add --slug <slug> # By slug
```
**Parameters**:
- `--hours=N` - Fetch articles from last N hours
- `--label=X` - Fetch articles with specific label
- `--url=X` - Add single article by URL
- `--slug=X` - Add single article by slug
**What It Does (Current)**:
1. Fetches articles from Omnivore (by hours / label / url / slug)
2. Converts results into `{id, slug, url, title, savedAt}` queue rows
3. Inserts into `analysis_queue` via `INSERT OR IGNORE` (deduplication by `article_id`)
**Notes**:
- Content-length filtering and label distribution output are not currently implemented in `omc queue add`.
**Output Example**:
```
Added 38 articles to queue (45 total found)
```
**AIDEV Annotations**:
- `omnivore-boundary` - Fetches via GraphQL API, populates tracking queue
- `tracking-initialization` - Adds articles to SQLite queue for parallel analysis
## Omnivore Integration Commands
### omnivore get
**Command**: `omc omnivore get`
**Purpose**: Fetch an article by slug (metadata by default; raw content with `--content`; full JSON including `content` with `--json`).
**Usage**:
```bash
omc omnivore get <articleSlug>
omc omnivore get <articleSlug> --content
omc omnivore get <articleSlug> --json
```
**Parameters**:
- `articleSlug` - Article slug from Omnivore URL or metadata
- Username is loaded from .env file automatically
**What It Does (Current)**:
1. Queries Omnivore GraphQL API using `article(slug, username)`
2. Default output is a human-readable summary (title/url/author/description)
3. With `--json`, prints the full Article payload (including `content`)
4. With `--content`, prints the raw article content to stdout (agent-friendly)
**Agent usage**:
- Prefer `omc omnivore get <slug> --content` for stable, parse-free access to the article body.
- If you also need metadata, use `--json` and extract `.content`.
**Why Needed**:
- Agents execute via Bash tool and cannot import TypeScript modules
- Provides clean separation between content fetching and analysis
- Enables zero-context-pollution agent workflow (each agent fetches independently)
**Usage in Agent Workflow**:
```typescript
// Agent reads stub file for slug
const stub = JSON.parse(fs.readFileSync(`temp/${filename}`, 'utf-8'));
// Agent calls CLI tool to get content
const contentResult = await bash(`omc omnivore get ${stub.articleSlug}`);
const articleContent = contentResult.stdout;
// Agent analyzes content...
```
**AIDEV Annotations**:
- `omnivore-boundary` - Fetches via GraphQL API, outputs to stdout
- `agent-workflow` - Enables agents to fetch content without TypeScript imports
## Analysis Commands
### analyze auto
**Command**: `omc analyze auto`
**Purpose**: Run the full analysis pipeline non-interactively (cron/launchd friendly).
**Usage**:
```bash
omc analyze auto --batch-size 5
omc analyze auto --article-id <id>
omc analyze auto --all --batch-size 5
omc analyze auto --json
omc analyze auto --batch-size 5 --jsonl
```
**What It Does (Current)**:
1. Selects jobs from the SQLite queue (`pending` by default; `pending+failed` with `--all`)
2. Fetches article content from Omnivore
3. Calls `codex exec` in **read-only** mode to produce a `ContentAnalysis` JSON object
4. Writes Markdown under `content/analysis/` and stores an immutable analysis snapshot in SQLite
5. Deletes `temp/{slug}.jsonl` unless `--keep-temp`
**Notes**:
- Codex session files are written under `CODEX_HOME`. For automation, the CLI sets `CODEX_HOME=temp/codex-home` so runs dont depend on `~/.codex` permissions.
- JSONL output is optional: enable with `--jsonl` (defaults to `content/analysis/analyses.jsonl`).
### analyze run
**Command**: `omc analyze run`
**Purpose**: Run parallel analysis on queued articles with zero-context-pollution design.
**Usage**:
```bash
omc analyze run # Process next 5 articles (default)
omc analyze run --batch-size 10 # Custom batch size
omc analyze run --article-id <id> # Analyze specific article
omc analyze run --all # Process entire queue
```
**What It Does**:
1. Fetches next 5 pending jobs from tracking queue
2. Marks jobs as `in_progress` (coordination lock)
3. Fetches article metadata from Omnivore API using `getArticle(slug, username)`
4. Creates slug-based stub files: `temp/{articleSlug}.jsonl` (prevents overwrites when running multiple batches)
5. Each stub file contains: `articleId`, `articleSlug`, `username`, `articleUrl`, `articleTitle`, `savedAt`, `publishedAt`, `updatedAt`
6. Outputs JSON array with agent parameters for parallel invocation
7. Closes database connection (agents get fresh connections with no context pollution)
**File Naming Strategy**:
- Stub files use article slug: `temp/{articleSlug}.jsonl` (e.g., `temp/understanding-llm-fine-tuning.jsonl`)
- Prevents overwrites when running multiple batches before agents complete
- Enables agents to work independently without coordination
**Output Example**:
```
Analysis Queue Status
Total: 75
Pending: 42
In Progress: 0
Completed: 30
Failed: 3
Processing batch of 5 articles...
Fetching article metadata from Omnivore API...
✓ Fetched 5 articles
✓ Created stub files: temp/*.jsonl
READY FOR PARALLEL ANALYSIS
[
{
"filename": "temp/understanding-llm-fine-tuning.jsonl",
"articleId": "abc-123",
"articleSlug": "understanding-llm-fine-tuning",
"username": "myusername",
"articleTitle": "Understanding LLM Fine-Tuning Techniques"
},
...
]
Instructions:
1. Invoke 5 @article-content-analyzer agents in parallel (single message)
2. Each agent receives: filename, articleId, articleSlug, username, articleTitle
3. Wait for all agents to complete
4. Run: `omc analyze complete`
```
**AIDEV Annotations**:
- `tracking-coordination` - Uses SQLite queue for parallel execution
- `omnivore-boundary` - Fetches articles via GraphQL API, never local cache
- `tracking-lock` - Marks jobs in_progress to prevent duplicate analysis
- `zero-context-pollution` - Creates stub files, closes DB before agent invocation
### analyze complete
**Command**: `omc analyze complete`
**Purpose**: Persist enriched temp files by writing Markdown + storing an immutable `analysis_json` snapshot in SQLite, then marking jobs completed.
**Input Format** (enriched temp files):
```json
{
"articleId": "abc-123",
"articleSlug": "understanding-llm-fine-tuning",
"username": "myusername",
"articleUrl": "https://example.com/article",
"articleTitle": "Article Title",
"savedAt": "2025-09-30T10:00:00Z",
"publishedAt": "2025-09-29T14:00:00Z",
"updatedAt": "2025-09-30T08:00:00Z",
"analysis": {
"topics": ["ai", "developer-tools"],
"topicScores": { "ai": 0.95, "developer-tools": 0.88 },
"summary": "Article summary...",
"keyPoints": ["point1", "point2"],
"sentiment": "positive",
"monetizationAngle": "Content opportunity...",
"analyzedAt": "2025-10-01T04:30:00Z"
}
}
```
**What It Does (Current)**:
1. Scans `temp/*.jsonl` for files that contain an `analysis` field
2. For each enriched temp file:
- Writes Markdown file under `content/analysis/` (date from `savedAt`; slug derived from `articleSlug` when available, otherwise title)
- Stores `analysis_json` + `markdown_path` into `analysis_queue`
- Updates tracking status to `completed`
- Deletes the temp file unless `--keep-temp`
3. On error: marks job as `failed` in tracking queue and preserves the temp file
**JSONL Output (Optional)**:
- Use `--jsonl` to append a machine-readable record to `content/analysis/analyses.jsonl` for each completed analysis.
**File Naming Strategy**:
- Markdown: `content/analysis/YYYY-MM-DD-{slug}-analysis.md` (date from `savedAt`)
- JSONL: written when `--jsonl` is set (`content/analysis/analyses.jsonl` by default)
- Temp files: Automatically deleted after successful save using `unlinkSync(file)`
**Output Example**:
```
Loaded 5 analysis results from temp files
✓ Saved: Understanding LLM Fine-Tuning Techniques...
→ content/analysis/2025-09-30-understanding-llm-fine-tuning-analysis.md
→ Deleted: temp/understanding-llm-fine-tuning.jsonl
✓ Saved: Building Scalable Microservices with Kubernetes...
→ content/analysis/2025-09-29-building-scalable-microservices.md
→ Deleted: temp/building-scalable-microservices.jsonl
✗ Failed: Another Article... - Parse error
→ Preserved: temp/another-article.jsonl
Results:
Saved: 4
Failed: 1
Updated Queue Status:
Pending: 37
Completed: 34
Failed: 4
```
**AIDEV Annotations**:
- `analysis-output-boundary` - Saves results to git-tracked Markdown (plus immutable DB snapshot)
- `tracking-update` - Updates SQLite queue status after save
- `git-tracked-output` - Analysis result stored permanently
- `temp-file-cleanup` - Deletes temp files after successful DB save
### analyze status
**Command**: `omc analyze status`
**Purpose**: Show queue statistics and failed/in-progress jobs.
**Usage**:
```bash
omc analyze status
```
**What It Does**:
- Shows queue statistics (total, pending, in_progress, completed, failed)
- Lists failed jobs with error messages and retry counts
- Lists in-progress jobs with assignment timestamps
**Output Example**:
```
Analysis Queue Status
Total: 75
Pending: 37 (49%)
In Progress: 0
Completed: 33 (44%)
Failed: 5
Failed Jobs:
Article: Article That Failed Analysis Due to Timeout...
ID: xyz-789
Error: Agent timeout after 120s
Retries: 2
Article: Another Failed Article with Parse Error...
ID: abc-321
Error: JSON parse error
Retries: 1
```
**AIDEV Annotations**:
- `tracking-inspection` - Shows queue status, not analysis content
### analyze retry
**Command**: `omc analyze retry`
**Purpose**: Reset failed jobs to pending for retry (max 3 attempts).
**Purpose**: Reset failed jobs to `pending` for retry.
**Usage**:
```bash
omc analyze retry --failed # Retry all failed
omc analyze retry --article-id <id> # Retry specific article
```
**What It Does (Current)**:
1. Fetches failed jobs (or a single `--article-id`)
2. Resets status to `pending` and clears error/assigned fields
3. Does not currently enforce a max retry limit (the DB keeps `retry_count` for visibility)
**Output Example**:
```
Found 5 failed jobs
✓ Reset: Article That Failed Analysis... (attempt 3)
✓ Reset: Another Failed Article... (attempt 2)
✗ Skip: Permanently Failed Article... (max retries exceeded)
Summary:
Reset for retry: 4
Skipped (max retries): 1
Next step: `omc analyze run`
```
**AIDEV Annotations**:
- `tracking-retry` - Resets failed jobs to pending for another attempt
## Reporting Commands
### report corpus
**Command**: `omc report corpus`
**Purpose**: Generate corpus statistics and topic distribution report.
**Usage**:
```bash
omc report corpus
```
**What It Does**:
- Loads completed analyses from SQLite (`analysis_queue.analysis_json`) via `AnalysisQueueRepository.getCompletedWithAnalysis()`
- Generates statistics: total analyses, topic distribution, sentiment distribution, content-type distribution
- Prints the report to stdout (use `--json` for machine-readable output)
**Note**:
- `omc report corpus` reads from SQLite (`analysis_queue.analysis_json`). JSONL is an optional *write-only* output for downstream tooling.
## Storage Layer Commands
**Note**: The most reliable source of truth for the storage-layer API is `src/storage/AnalysisQueueRepository.ts` and `src/storage/AnalysisWriter.ts`. Some type snippets below may drift as the code evolves.
### AnalysisQueueRepository
**Location**: `src/storage/AnalysisQueueRepository.ts`
**Purpose**: Repository for analysis job queue management.
**Import**:
```typescript
import { AnalysisQueueRepository } from '@storage/AnalysisQueueRepository';
import type { AnalysisJob, QueueStats } from '@storage/AnalysisQueueRepository';
```
**Types**:
```typescript
interface AnalysisJob {
id: number;
articleId: string;
articleUrl: string;
articleTitle: string;
status: 'pending' | 'in_progress' | 'completed' | 'failed';
assignedAt?: string;
completedAt?: string;
errorMessage?: string;
retryCount: number;
createdAt: string;
updatedAt: string;
}
interface QueueStats {
total: number;
pending: number;
inProgress: number;
completed: number;
failed: number;
}
```
**Constructor**:
```typescript
const queueRepo = new AnalysisQueueRepository(db);
```
**Methods**:
1. **`initializeQueue(articles): number`** - Initialize queue from article metadata
- Parameter: `articles: Array<{ id: string; url: string; title: string }>`
- Returns: Number of articles inserted (duplicates ignored by UNIQUE constraint)
- Sets status to `pending` for all new articles
- AIDEV: `tracking-initialization`
2. **`getPending(limit?: number): AnalysisJob[]`** - Get next batch of pending jobs
- Default limit: 5
- Returns jobs in chronological order (oldest first)
- AIDEV: `tracking-coordination`
3. **`markInProgress(articleId: string): void`** - Lock job for processing
- Sets status to `in_progress`
- Records `assigned_at` timestamp
- Prevents duplicate analysis by parallel workers
- AIDEV: `tracking-lock`
4. **`markCompleted(articleId: string): void`** - Mark job as completed
- Sets status to `completed`
- Records `completed_at` timestamp
- Called AFTER analysis saved to Markdown/JSONL
- AIDEV: `tracking-completion`
5. **`markFailed(articleId: string, errorMessage: string): void`** - Mark job as failed
- Sets status to `failed`
- Stores error message
- Increments `retry_count`
- AIDEV: `tracking-error`
6. **`resetToPending(articleId: string): void`** - Reset failed job for retry
- Sets status back to `pending`
- Clears error message and `assigned_at`
- Does NOT reset `retry_count` (used to enforce max retries)
- AIDEV: `tracking-retry`
7. **`getStats(): QueueStats`** - Get queue statistics
- Returns counts for all statuses
- Used for progress monitoring
- AIDEV: `tracking-stats`
8. **`getFailed(): AnalysisJob[]`** - Get all failed jobs
- Sorted by retry count (most retries first), then updated time
- Used for error investigation
- AIDEV: `tracking-failures`
9. **`getByStatus(status: string): AnalysisJob[]`** - Get jobs by status
- Returns jobs matching specified status
- Sorted by creation time (newest first)
10. **`getByArticleId(articleId: string): AnalysisJob | null`** - Get specific job
- Returns job or `null` if not found
11. **`hasArticle(articleId: string): boolean`** - Check if article in queue
- Returns `true` if article exists (any status)
- AIDEV: `tracking-deduplication`
12. **`clearCompleted(): number`** - Remove completed jobs
- Deletes all jobs with `completed` status
- Returns number of deleted jobs
- AIDEV: `tracking-cleanup`
**Example Usage**:
```typescript
import { initDatabase } from '@storage/database';
import { AnalysisQueueRepository } from '@storage/AnalysisQueueRepository';
const db = initDatabase();
const queueRepo = new AnalysisQueueRepository(db);
// Add articles to queue
const articles = [
{ id: 'abc-123', url: 'https://example.com/1', title: 'Article 1' },
{ id: 'def-456', url: 'https://example.com/2', title: 'Article 2' }
];
const inserted = queueRepo.initializeQueue(articles);
console.log(`Added ${inserted} articles`);
// Get pending jobs
const jobs = queueRepo.getPending(5);
console.log(`Processing ${jobs.length} jobs`);
// Mark as in_progress
for (const job of jobs) {
queueRepo.markInProgress(job.articleId);
}
// After successful analysis → mark completed
queueRepo.markCompleted('abc-123');
// If analysis fails → mark failed
queueRepo.markFailed('def-456', 'Agent timeout');
// Check statistics
const stats = queueRepo.getStats();
console.log(`Pending: ${stats.pending}, Completed: ${stats.completed}`);
// Get failed jobs for retry
const failed = queueRepo.getFailed();
for (const job of failed) {
if (job.retryCount < 3) {
queueRepo.resetToPending(job.articleId);
}
}
db.close();
```
### AnalysisWriter
**Location**: `src/storage/AnalysisWriter.ts`
**Purpose**: Write `ContentAnalysis` results to Markdown files with YAML front-matter.
**Import**:
```typescript
import { AnalysisWriter } from '@storage/AnalysisWriter';
```
**Configuration**:
```typescript
const writer = new AnalysisWriter({
outputDir: 'content/analysis'
});
```
**Methods**:
1. **`write(articleId, articleUrl, articleTitle, savedAt, analysis): Promise<string>`** - Write analysis to Markdown
```typescript
const filePath = await writer.write(
articleId, // Omnivore article ID
articleUrl, // Source article URL
articleTitle, // Source article title
savedAt, // When saved to Omnivore (ISO 8601)
analysis, // ContentAnalysis from agent
articleSlug // Optional Omnivore slug (preferred for filename stability)
);
// Returns: 'content/analysis/2025-09-30-omnivore-slug-analysis.md'
```
- Creates Markdown file with YAML front-matter
- File naming: `YYYY-MM-DD-{slug}-analysis.md`
- Date from `savedAt` timestamp
- Slug from `articleSlug` when provided; otherwise derived from article title (lowercase, alphanumeric, max 50 chars)
- AIDEV: `analysis-output-boundary`, `git-tracked-output`
2. **`appendToJsonl(jsonlPath, data): Promise<void>`** - Append to JSONL file
```typescript
await writer.appendToJsonl('content/analysis/analyses.jsonl', {
articleId,
articleUrl,
articleTitle,
savedAt,
analyzedAt: analysis.analyzedAt,
topics: analysis.topics,
topicScores: analysis.topicScores,
summary: analysis.summary,
keyPoints: analysis.keyPoints,
sentiment: analysis.sentiment,
monetizationAngle: analysis.monetizationAngle
});
```
- Called when JSONL output is enabled (`omc analyze complete --jsonl` / `omc analyze auto --jsonl`)
- Appends one JSON line to JSONL file
- Creates file if it doesn't exist
- Machine-readable format for batch processing
- AIDEV: `git-tracked-output`
**Private Methods**:
- `generateSlug(title: string): string` - Create URL-friendly slug
- `formatMarkdown(...)` - Build YAML front-matter and markdown body
**Example Usage**:
```typescript
import { AnalysisWriter } from '@storage/AnalysisWriter';
const writer = new AnalysisWriter({ outputDir: 'content/analysis' });
const analysis = {
articleId: 'abc-123',
topics: ['ai', 'developer-tools'],
topicScores: { ai: 0.95, 'developer-tools': 0.88 },
summary: 'Article explores...',
keyPoints: ['First insight', 'Second insight'],
sentiment: 'positive',
monetizationAngle: 'Tutorial series on...',
analyzedAt: new Date().toISOString()
};
const filePath = await writer.write(
'abc-123',
'https://example.com/article',
'Example Article Title',
'2025-09-30T10:00:00Z',
analysis
);
console.log(`Saved to: ${filePath}`);
// Saved to: content/analysis/2025-09-30-example-article-title-analysis.md
```
### ContentReader
**Location**: `src/storage/ContentReader.ts`
**Purpose**: Read and parse Markdown files with YAML front-matter.
**Import**:
```typescript
import { ContentReader } from '@storage/ContentReader';
```
**Configuration**:
```typescript
const reader = new ContentReader({
directory: 'content/analysis'
});
```
**Methods**:
1. **`list(pattern?: string): Promise<string[]>`** - List all Markdown files
- Returns array of absolute file paths
- Sorted by date (newest first, assumes `YYYY-MM-DD` prefix)
- Returns empty array if directory doesn't exist
2. **`read(filePath: string): Promise<StoredAnalysis>`** - Read and parse file
- Uses `gray-matter` to parse YAML front-matter
- Extracts markdown sections (Summary, Key Points, Monetization Angle)
- Returns `StoredAnalysis` with parsed front-matter and content
3. **`findByArticleId(articleId: string): Promise<StoredAnalysis | null>`** - Find by article ID
- Searches all files for matching `articleId` in front-matter
- Returns first match or `null`
4. **`searchByTopic(topic: string): Promise<StoredAnalysis[]>`** - Search by topic
- Returns all analyses containing the topic
- Topic must match exactly (case-sensitive)
**Example Usage**:
```typescript
import { ContentReader } from '@storage/ContentReader';
const reader = new ContentReader({ directory: 'content/analysis' });
// List all analyses
const files = await reader.list();
console.log(`Found ${files.length} analyses`);
// Read specific analysis
const analysis = await reader.read(files[0]);
console.log(analysis.frontMatter.articleTitle);
console.log(analysis.summary);
console.log(analysis.keyPoints);
// Find by article ID
const found = await reader.findByArticleId('abc-123');
if (found) {
console.log(`Topics: ${found.frontMatter.topics.join(', ')}`);
}
// Search by topic
const aiArticles = await reader.searchByTopic('ai');
console.log(`${aiArticles.length} articles about AI`);
```
**Private Method**:
- `parseMarkdownSections(content: string)` - Extract sections from markdown body using regex
## Workflow Examples
### Complete Parallel Analysis Workflow
```bash
# 1. Fetch articles from Omnivore → tracking queue
omc queue add --hours 168
# 2. Prepare batch (writes temp/*.jsonl stubs + marks jobs in_progress)
omc analyze run --batch-size 5
# 3. User invokes 5 agents in parallel (single message)
# Each agent:
# - Reads stub file
# - Calls: omc omnivore get {slug} --json (extract `.content`)
# - Analyzes content
# - Enriches stub file with analysis field
# - Writes back to temp file
# 4. Persist results (writes content/analysis/*.md, updates DB, cleans temp files)
omc analyze complete
# 5. Check status / stats
omc analyze status
omc queue stats --detailed
# 6. Retry failed jobs
omc analyze retry --failed
# 7. Continue analysis
omc analyze run --batch-size 5
# 8. Generate report
omc report corpus
```
### Programmatic Queue Management
```typescript
import { initDatabase } from '@storage/database';
import { AnalysisQueueRepository } from '@storage/AnalysisQueueRepository';
import { AnalysisWriter } from '@storage/AnalysisWriter';
import type { ContentAnalysis } from '@omc-types/analysis.js';
// Initialize database and repository
const db = initDatabase('data/omnivore-content.db');
const queueRepo = new AnalysisQueueRepository(db);
const writer = new AnalysisWriter({ outputDir: 'content/analysis' });
// Add articles to queue
const articles = [
{ id: 'abc-123', url: 'https://example.com/1', title: 'Article 1' },
{ id: 'def-456', url: 'https://example.com/2', title: 'Article 2' }
];
const inserted = queueRepo.initializeQueue(articles);
console.log(`Added ${inserted} articles to queue`);
// Get pending jobs
const jobs = queueRepo.getPending(5);
// Mark as in_progress
for (const job of jobs) {
queueRepo.markInProgress(job.articleId);
}
// After agent analysis completes...
const analysis: ContentAnalysis = {
articleId: 'abc-123',
topics: ['ai', 'developer-tools'],
topicScores: { ai: 0.95, 'developer-tools': 0.88 },
summary: 'Article explores...',
keyPoints: ['First insight', 'Second insight'],
sentiment: 'positive',
monetizationAngle: 'Tutorial series on...',
analyzedAt: new Date().toISOString()
};
// Save to Markdown (PERMANENT)
const mdPath = await writer.write(
'abc-123',
'https://example.com/1',
'Article 1',
'2025-09-30T10:00:00Z',
analysis
);
// Append to JSONL (PERMANENT)
await writer.appendToJsonl('content/analysis/analyses.jsonl', {
articleId: 'abc-123',
articleUrl: 'https://example.com/1',
articleTitle: 'Article 1',
savedAt: '2025-09-30T10:00:00Z',
analyzedAt: analysis.analyzedAt,
topics: analysis.topics,
topicScores: analysis.topicScores,
summary: analysis.summary,
keyPoints: analysis.keyPoints,
sentiment: analysis.sentiment,
monetizationAngle: analysis.monetizationAngle
});
// Update tracking status (EPHEMERAL)
queueRepo.markCompleted('abc-123');
// Check statistics
const stats = queueRepo.getStats();
console.log(`Pending: ${stats.pending}, Completed: ${stats.completed}`);
db.close();
```
### Reading Analysis Results
```typescript
import { ContentReader } from '@storage/ContentReader';
const reader = new ContentReader({ directory: 'content/analysis' });
// List all analyses (newest first)
const files = await reader.list();
console.log(`Total analyses: ${files.length}`);
// Read latest analysis
const latest = await reader.read(files[0]);
console.log(`Title: ${latest.frontMatter.articleTitle}`);
console.log(`Topics: ${latest.frontMatter.topics.join(', ')}`);
console.log(`Summary: ${latest.summary}`);
console.log(`Key Points:\n${latest.keyPoints.map(p => `- ${p}`).join('\n')}`);
// Find specific article's analysis
const found = await reader.findByArticleId('abc-123');
if (found) {
console.log(`Sentiment: ${found.frontMatter.sentiment}`);
}
// Search by topic
const aiArticles = await reader.searchByTopic('ai');
console.log(`AI articles: ${aiArticles.length}`);
```
## Related Documentation
- [Architecture](architecture.md) - System design and storage boundaries
- [Workflow Internals](workflow-internals.md) - How the parallel analysis workflow operates
- [Foundation & Type System](foundation-and-types.md) - TypeScript setup, type definitions

View file

@ -0,0 +1,151 @@
# Current State Audit
**Purpose**: Ground-truth documentation of what exists *in code today*, what is broken, and what needs to be fixed to make the system reliable.
**Last Updated**: 2026-01-31
## Executive Summary
This repository has a fairly complete **command surface area** (queue/analyze/content/report/omnivore/db/config/init/doctor/version), plus an intended “3-layer” architecture:
1. **Omnivore** as the content source-of-truth (GraphQL)
2. **SQLite** as coordination + immutable analysis snapshot storage
3. **Git-tracked Markdown** as human-editable, permanent output
The core analysis pipeline now works end-to-end in a clean checkout:
- `pnpm run typecheck` / `pnpm test` / `pnpm run build` pass.
- The built CLI (`dist/bin/omc.js`) can initialize the tracking DB schema.
- Commands follow the `BaseCommand.execute({ ...args, ...flags })` contract.
- Documentation reflects the current `omc analyze auto` automation path and optional JSONL output.
## Architecture (As Implemented)
### Layer 1: Omnivore (Source of Truth)
- Implementation: `lib/omnivore/client.js`
- Used by CLI commands via the TypeScript re-export surface (`src/lib/omnivore/client.ts`) in most places.
- Auth is via `.env` (`OMNIVORE_API_URL`, `OMNIVORE_API_KEY`) loaded by `dotenv` at module import time.
### Layer 2: SQLite (Coordination + Snapshots)
- Default DB path (source): `src/storage/database.ts` uses `data/omnivore-content.db`.
- Schema: `src/storage/schema/tracking-schema.sql`
- Repository: `src/storage/AnalysisQueueRepository.ts`
The DB is used for:
- Queue state (`pending`/`in_progress`/`completed`/`failed`)
- Retry counters and error messages
- Immutable snapshot of analysis output (`analysis_json`) and pointer to Markdown (`markdown_path`)
### Layer 3: Markdown Output (Git-tracked)
- Writer: `src/storage/AnalysisWriter.ts`
- The current pipeline writes **Markdown** by default.
- Optional machine-readable JSONL is available via flags (`omc analyze auto --jsonl` / `omc analyze complete --jsonl`).
- Markdown filename uses `savedAt` date and the Omnivore `articleSlug` when available (falls back to title-derived slug).
## Command Inventory (As Implemented)
Commands are defined under `src/commands/**` and built/run via oclif (`index.ts`).
### `omc queue`
- `omc queue add` (`src/commands/queue/add.ts`): adds articles to tracking queue (hours/label/url/slug).
- `omc queue list` (`src/commands/queue/list.ts`): lists queue entries (optional `--status`).
- `omc queue stats` (`src/commands/queue/stats.ts`): queue stats (optional `--detailed`).
- `omc queue reset` (`src/commands/queue/reset.ts`): resets a specific job to `pending`.
- `omc queue remove` (`src/commands/queue/remove.ts`): deletes a job.
- `omc queue clear` (`src/commands/queue/clear.ts`): bulk delete by status or all.
- `omc queue export` (`src/commands/queue/export.ts`): prints queue rows as JSONL (or JSON array with `--json`).
- `omc queue import` (`src/commands/queue/import.ts`): imports JSONL into the queue.
### `omc analyze`
- `omc analyze run` (`src/commands/analyze/run.ts`):
- Selects jobs (pending by default; pending+failed with `--all`; single with `--article-id`).
- Marks selected jobs `in_progress`.
- Fetches Omnivore article metadata and writes `temp/{articleSlug}.jsonl` stub files.
- Outputs agent parameter JSON describing the stub files.
- `omc analyze complete` (`src/commands/analyze/complete.ts`):
- Finds `temp/*.jsonl` files that contain an `analysis` field.
- Writes Markdown via `AnalysisWriter`.
- Stores `analysis_json` + `markdown_path` in SQLite via `AnalysisQueueRepository.storeAnalysis`.
- Deletes temp files unless `--keep-temp`.
- `omc analyze status` (`src/commands/analyze/status.ts`): shows in-progress jobs + current `temp/*.jsonl`.
- `omc analyze watch` (`src/commands/analyze/watch.ts`): polls queue stats until *queue empty* (pending=0 and in_progress=0).
- `omc analyze retry` (`src/commands/analyze/retry.ts`): resets failed jobs to pending.
### `omc content` (Read/Export/Sync)
- `omc content list` (`src/commands/content/list.ts`): lists completed jobs; optional filters (since/topic).
- `omc content show` (`src/commands/content/show.ts`): prints Markdown for one article ID.
- `omc content search` (`src/commands/content/search.ts`): searches completed analyses (title/analysis/content/all).
- `omc content export` (`src/commands/content/export.ts`): export data for blogging workflows (implementation varies).
- `omc content sync` (`src/commands/content/sync.ts`): syncs analysis summaries back to Omnivore (description field) and optionally creates/updates a NOTE highlight from Markdown.
### `omc report` (Categorized Analysis / Aggregations)
This is the “categorized analysis” surface area: reports summarize analyses by topic/sentiment/time.
- `omc report corpus` (`src/commands/report/corpus.ts`): topic + sentiment + contentType distributions.
- `omc report topics` (`src/commands/report/topics.ts`): topic distribution.
- `omc report trends` (`src/commands/report/trends.ts`): trends over time (based on saved/completed timestamps).
- `omc report sentiment` (`src/commands/report/sentiment.ts`): sentiment distribution.
- `omc report monetization` (`src/commands/report/monetization.ts`): monetization angle aggregation.
- `omc report custom` (`src/commands/report/custom.ts`): custom SQL against the tracking DB.
- `omc report export` (`src/commands/report/export.ts`): export report output (format varies).
### `omc omnivore`
Direct API operations:
- `omc omnivore list`, `search`, `get`, `update`
- highlights: `omnivore highlight add/list`
- notes: `omnivore note add/get/update`
### `omc db`
DB maintenance:
- `migrate`, `schema`, `stats`, `check`, `vacuum`, `backup`, `restore`, `reset`, `seed`
### `omc config`
Config management:
- `show`, `get`, `set`, `test`, `validate`, `env list`, `env use`
### Misc
- `omc init` (`src/commands/init.ts`): basic setup wizard (directories, db, env).
- `omc doctor` (`src/commands/doctor.ts`): health checks.
- `omc version` (`src/commands/version.ts`): version/system info.
## End-to-End Workflow (Current)
1. Populate queue: `omc queue add --hours 24` (or `--label`, `--url`, `--slug`).
2. Prepare stubs: `omc analyze run --batch-size 5` (creates `temp/{slug}.jsonl`).
3. Run external agent(s): read stub, fetch content, add `analysis` field to the stub JSON.
4. Persist results: `omc analyze complete` (writes `content/analysis/*.md`, updates DB, cleans temp files).
5. View/search: `omc content list`, `omc content search`, `omc content show <article-id>`.
6. Report: `omc report topics` / `omc report trends` / etc.
7. Sync back to Omnivore: `omc content sync --all --create-notes`.
## Issues / Fix List (Prioritized)
### Resolved (as of 2026-01-31)
- TypeScript aliases normalized (`@omc-types/*` instead of `@types/*`); `pnpm run typecheck` passes.
- Tracking schema resolution works in `dist/` builds; built CLI can initialize the DB.
- Commands follow `BaseCommand.execute({ ...args, ...flags })`.
- Docs and scripts align with the built CLI and the current analysis workflow.
- Agent-friendly raw content output via `omc omnivore get --content`.
- Optional JSONL output via `--jsonl` flags (`omc analyze auto --jsonl` / `omc analyze complete --jsonl`).
- Markdown filename contract prefers Omnivore slug (falls back to title-derived slug).
- `omc queue add --hours` paginates until the cutoff is reached.
- Markdown front-matter emitted via `gray-matter` (valid YAML for common edge cases).
- GraphQL error handling normalized via `checkGraphQLResult(...)`.
- Typed GraphQL documents/codegen removed from the runtime path (single client source-of-truth).
### Remaining / Next
- Automated content generation and publishing phases (Phase 5/6).
- Decide whether to keep or retire `src/lib/ai/anthropic-client.ts` (Codex CLI is the default analysis provider today).

View file

@ -0,0 +1,712 @@
# Foundation and Type System
**Groundtruth Documentation** - What exists and how to use it.
## TypeScript Foundation
### Project Setup
**TypeScript Configuration** (`tsconfig.json`):
```typescript
{
"compilerOptions": {
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "bundler",
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"declaration": true,
"sourceMap": true,
"paths": {
"@lib/*": ["./lib/*"],
"@storage/*": ["./src/storage/*"],
"@analysis/*": ["./src/analysis/*"],
"@generation/*": ["./src/generation/*"],
"@utils/*": ["./src/utils/*"],
"@omc-types/*": ["./src/types/*"]
}
},
"include": ["src/**/*", "lib/**/*"],
"exclude": ["node_modules", "dist", "test-scripts", "legacy-scripts"]
}
```
**Build System**:
- Package manager: pnpm
- Build command: `pnpm run build` → compiles to `dist/`
- Type check: `pnpm run typecheck` → validates without emitting
- Dev mode: `pnpm run dev` → watch mode with tsx
### Dependencies
**Runtime** (package.json dependencies):
- `node-fetch` (^3.3.2) - HTTP requests for GraphQL API
- `dotenv` (^16.4.0) - Environment variable loading
- `@anthropic-ai/sdk` (^0.20.0) - Claude API for analysis
- `@anthropic-ai/claude-agent-sdk` (^0.1.1) - Agent framework
- `gray-matter` (^4.0.3) - YAML front-matter parsing
- `markdown-it` (^14.0.0) - Markdown processing
- `chalk` (^5.3.0) - Terminal colors
- `csv-parse` (^6.1.0) - CSV parsing
- `p-limit` (^5.0.0) - Concurrency control
- `better-sqlite3` (^12.4.1) - SQLite (for legacy scripts)
**Development** (package.json devDependencies):
- `typescript` (^5.4.0) - TypeScript compiler
- `@types/node` (^20.0.0) - Node.js type definitions
- `tsx` (^4.0.0) - TypeScript execution
- `vitest` (^1.0.0) - Testing framework
- `nodemon` (^3.0.0) - File watcher
## GraphQL Organization
**Current**:
- Runtime GraphQL client: `lib/omnivore/client.js` (string-based queries)
- TypeScript import surface: `src/lib/omnivore/client.ts`
- In TypeScript code, prefer importing client functions from `@lib/omnivore/client.js`
**Note**:
An older typed GraphQL layer (`src/graphql/**` + `src/types/generated/**`) was removed because it was not integrated into the CLI runtime and created drift risk. See `docs/_meta/graphql-organization.md`.
### Query Builders (`lib/omnivore/queries.js`)
**Purpose**: Helper functions for composing Omnivore query strings.
**NOTE**: Query builders remain in JavaScript (`lib/omnivore/queries.js`) for compatibility with existing client code. They generate query string parameters, not GraphQL query documents.
**Builder Functions**:
1. **`buildComplexQuery({ keywords, labels, timeRange, status, hasHighlights, sortBy })`**
- Compose multiple filters into single query string
- Parameters:
- `keywords`: Array of search terms (OR logic)
- `labels`: Array of label names (OR logic)
- `timeRange`: 'last24hrs' | 'last7days' | 'last30days' | date expression
- `status`: 'inbox' | 'archived' | 'unread' | 'read'
- `hasHighlights`: boolean (true = has:highlights, false = no:highlights)
- `sortBy`: Sort order (default: 'saved-desc')
- Returns: Omnivore query string
- Example:
```javascript
buildComplexQuery({
keywords: ['ai', 'machine learning'],
labels: ['technology'],
timeRange: 'last7days',
hasHighlights: true
})
// → 'saved:last7days ("ai" OR "machine learning") label:technology has:highlights sort:saved-desc'
```
2. **`buildTopicQuery(topicKey, timeRange)`**
- Generate query for predefined topic
- Topic keys: 'ai', 'devops', 'programming', 'databases', 'web', 'cloud', 'startup', 'security'
- Each topic has predefined keywords (see TOPIC_QUERIES)
- Example:
```javascript
buildTopicQuery('ai', 'last7days')
// → 'saved:last7days ("ai" OR "artificial intelligence" OR "machine learning" OR ...) sort:saved-desc'
```
3. **`buildDateRangeQuery(startDate, endDate, sortBy)`**
- Create date range filter
- Accepts Date objects or YYYY-MM-DD strings
- Example:
```javascript
buildDateRangeQuery('2025-09-01', '2025-09-30')
// → 'saved:>2025-09-01 saved:<2025-09-30 sort:saved-desc'
```
4. **`buildLabelQuery(labels, operator)`**
- Create label filter with AND/OR logic
- Example:
```javascript
buildLabelQuery(['ai', 'ml'], 'OR')
// → 'label:ai OR label:ml'
```
5. **`buildKeywordQuery(keywords, operator)`**
- Create keyword search with AND/OR logic
- Automatically quotes multi-word terms
- Example:
```javascript
buildKeywordQuery(['ai', 'machine learning'], 'OR')
// → '("ai" OR "machine learning")'
```
**Predefined Patterns** (`QUERY_PATTERNS`):
```javascript
QUERY_PATTERNS.LAST_7_DAYS // 'saved:last7days sort:saved-desc'
QUERY_PATTERNS.INBOX // 'in:inbox sort:saved-desc'
QUERY_PATTERNS.WITH_HIGHLIGHTS // 'has:highlights sort:saved-desc'
QUERY_PATTERNS.AI_ML // '(ai OR "machine learning" OR llm ...) sort:saved-desc'
QUERY_PATTERNS.RECENT_AI // 'saved:last7days (ai OR llm OR ...) sort:saved-desc'
// ... more patterns (see lib/omnivore/queries.js:189-221)
```
**Topic Definitions** (`TOPIC_QUERIES`):
```javascript
TOPIC_QUERIES.ai // keywords: ['ai', 'artificial intelligence', ...]
TOPIC_QUERIES.devops // keywords: ['devops', 'kubernetes', ...]
TOPIC_QUERIES.programming // keywords: ['programming', 'coding', ...]
// ... more topics (see lib/omnivore/queries.js:305-338)
```
**Integration Example**:
```javascript
import { searchArticles } from '@lib/omnivore/client.js';
import { buildComplexQuery, TOPIC_QUERIES } from '../lib/omnivore/queries.js';
// Build query string
const queryString = buildComplexQuery({
keywords: TOPIC_QUERIES.ai.keywords.slice(0, 5),
labels: ['technology', 'research'],
timeRange: 'last7days',
hasHighlights: true
});
const result = await searchArticles({ query: queryString, first: 20, includeContent: true });
```
## Omnivore Client Library
### Location
```
lib/omnivore/
├── client.js # GraphQL client using node-fetch
└── queries.js # Query builders and patterns (documented above)
```
### Client Module (`lib/omnivore/client.js`)
**Implementation**: Uses `node-fetch` to make GraphQL requests. NOT using Apollo Client.
**Core Function**: `graphqlRequest(query, variables)`
```javascript
async function graphqlRequest(query, variables = {}) {
const response = await fetch(API_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': API_KEY, // From process.env.OMNIVORE_API_KEY
},
body: JSON.stringify({ query, variables }),
});
const result = await response.json();
return result.data;
}
```
**Exported Functions**:
1. **`getMe()`** - Get authenticated user
- Returns: `{ id, name, email, profile: { username } }`
- Use: Verify API connection
2. **`searchArticles({ query, first, after, includeContent })`** - Search articles
- Parameters:
- `query`: Omnivore query syntax (default: 'in:all')
- `first`: Number of results (default: 10)
- `after`: Pagination cursor (default: '')
- `includeContent`: Include full HTML content (default: false)
- Returns: `{ search: { pageInfo, edges: [{ node, cursor }] } }`
3. **`getArticle({ slug, username })`** - Get single article with content
- Returns: Full article with content, highlights, labels
4. **`getArticlesByDate({ startDate, endDate, first })`** - Time-range queries
- Date format: YYYY-MM-DD
5. **`getArticlesByLabel({ label, first })`** - Filter by label name
6. **`getRecentArticles({ period, first })`** - Recent articles
- Periods: 'last24hours', 'last7days', 'last30days'
7. **`searchByTopic({ topic, period, first })`** - Topic + time filter
- Topics: 'ai', 'devops', 'programming', etc.
8. **`getUnreadArticles({ first })`** - Inbox/unread items
9. **`getLabels()`** - All available labels
- Returns: Array of `{ id, name, color, description }`
10. **`getHighlights({ articleId })`** - Article highlights
- Returns: Array of `{ id, quote, annotation, createdAt }`
11. **`testConnection()`** - Verify API connectivity
- Calls `getMe()` and logs result
- Returns: boolean
**Import Pattern**:
```javascript
import { searchArticles, getArticle } from './lib/omnivore/client.js';
```
**NOTE**: Query builders and patterns are documented above in "Query Builders (`lib/omnivore/queries.js`)" section.
## Type System
### Type Files
```
src/types/
├── omnivore.ts # API response types
├── content.ts # Storage/front-matter types
├── analysis.ts # Analysis result types
└── index.ts # Central exports
```
### Omnivore API Types (`src/types/omnivore.ts`)
**Core Interfaces**:
```typescript
// Article from API
export interface OmnivoreArticle {
id: string;
title: string;
url: string;
originalArticleUrl?: string;
slug?: string;
content?: string; // Only present if includeContent: true
description?: string;
author?: string;
image?: string;
siteName?: string;
pageType?: string;
wordCount?: number;
createdAt: string; // ISO 8601
savedAt: string; // ISO 8601
publishedAt?: string; // ISO 8601
updatedAt: string; // ISO 8601
readingProgressTopPercent?: number;
isArchived: boolean;
folder?: string;
labels: Label[];
highlights: Highlight[];
}
// Label metadata
export interface Label {
id: string;
name: string;
color: string;
description?: string;
}
// User highlight/annotation
export interface Highlight {
id: string;
quote: string;
annotation?: string;
createdAt: string; // ISO 8601
}
// Search parameters
export interface SearchParams {
query?: string; // Omnivore query syntax
first?: number; // Results per page
after?: string; // Pagination cursor
includeContent?: boolean; // Include article HTML
}
// Pagination metadata
export interface PageInfo {
hasNextPage: boolean;
hasPreviousPage: boolean;
startCursor?: string;
endCursor?: string;
totalCount: number;
}
// Search result wrapper
export interface SearchResult {
edges: Array<{
node: OmnivoreArticle;
}>;
pageInfo: PageInfo;
}
// User profile
export interface OmnivoreUser {
id: string;
name: string;
email: string;
profile: {
username: string;
};
}
```
**Import Pattern**:
```typescript
import type { OmnivoreArticle, SearchResult, Label } from '@omc-types/omnivore.js';
// OR
import type { OmnivoreArticle, SearchResult, Label } from '../types/omnivore';
```
### Content Storage Types (`src/types/content.ts`)
**Front-matter Interfaces** for Markdown files with YAML headers:
```typescript
// Article metadata (content/articles/*.md)
export interface ArticleFrontMatter {
id: string; // Omnivore article ID
url: string;
title: string;
author?: string;
savedAt: string; // ISO 8601
publishedAt?: string; // ISO 8601
labels: string[]; // Label names (not IDs)
highlights: number; // Count
wordCount: number;
siteName?: string;
topics?: string[]; // Added by analysis
sentiment?: string; // Added by analysis
analyzed?: boolean;
}
// Analysis metadata (content/analysis/*.md)
export interface AnalysisFrontMatter {
articleId: string;
analyzedAt: string; // ISO 8601
topics: string[];
sentiment: 'positive' | 'neutral' | 'negative';
topicScores: Record<string, number>;
}
// Generated content metadata (content/generated/**/*.md)
export interface GeneratedContentFrontMatter {
title: string;
metaDescription: string; // Max 155 chars
generatedAt: string; // ISO 8601
type: 'blog-post' | 'newsletter';
sources: string[]; // Article URLs
topics: string[];
publishedAt?: string; // ISO 8601
slug?: string;
}
// Complete stored article (after reading Markdown file)
export interface StoredArticle {
frontMatter: ArticleFrontMatter;
content: string; // Markdown content
highlights?: Array<{
quote: string;
annotation?: string;
}>;
}
// Complete stored analysis
export interface StoredAnalysis {
frontMatter: AnalysisFrontMatter;
summary: string;
keyPoints: string[];
monetizationAngle: string;
}
// Complete generated content
export interface StoredGeneratedContent {
frontMatter: GeneratedContentFrontMatter;
content: string;
}
// Search index entry (content/.metadata/index.json)
export interface SearchIndexEntry {
id: string;
slug: string;
title: string;
topics: string[];
savedAt: string;
analyzed: boolean;
}
```
### Analysis Types (`src/types/analysis.ts`)
**AI Analysis Interfaces**:
```typescript
// Analysis result from Claude
export interface ContentAnalysis {
articleId: string;
topics: string[]; // 2-5 main topics
topicScores: Record<string, number>; // Topic → confidence (0-1)
summary: string; // 2-3 sentences
keyPoints: string[]; // 3-5 takeaways
sentiment: 'positive' | 'neutral' | 'negative';
monetizationAngle: string; // Content opportunity
analyzedAt: string; // ISO 8601
}
// Input to Claude for analysis
export interface AnalysisRequest {
title: string;
author?: string;
url: string;
content: string;
wordCount: number;
highlights: Array<{
quote: string;
annotation?: string;
}>;
publishedAt?: string;
}
// Topic with confidence
export interface TopicScore {
topic: string;
score: number; // 0-1
keywords: string[];
}
// Analysis configuration
export interface AnalysisConfig {
focusTopics?: string[];
minTopicScore?: number;
maxTopics?: number;
includeSentiment?: boolean;
}
// Batch analysis result
export interface BatchAnalysisResult {
articles: Array<{
articleId: string;
analysis: ContentAnalysis;
}>;
commonTopics: string[];
trends: Array<{
topic: string;
frequency: number;
averageScore: number;
}>;
analyzedAt: string;
}
```
### Central Exports (`src/types/index.ts`)
**Re-exports all types** plus utility types:
```typescript
// Re-exports
export * from './omnivore';
export * from './content';
export * from './analysis';
// Utility types
export type DateString = string; // ISO 8601
export type UUID = string;
export type Slug = string;
// Generation config
export interface BlogPostConfig {
type: 'single-article' | 'weekly-roundup' | 'deep-dive';
title?: string;
targetWordCount?: number;
includeSources?: boolean;
seoOptimize?: boolean;
}
export interface NewsletterConfig {
type: 'weekly' | 'monthly';
includeSections?: string[];
maxArticles?: number;
personalCommentary?: boolean;
}
// Publishing
export type PublishingPlatform = 'markdown' | 'ghost' | 'wordpress' | 'medium';
export interface PublishResult {
platform: PublishingPlatform;
success: boolean;
url?: string;
error?: string;
publishedAt: string;
}
```
**Import Pattern**:
```typescript
// Import from index for convenience
import type {
OmnivoreArticle,
ContentAnalysis,
ArticleFrontMatter
} from '@omc-types';
// OR import from specific files
import type { OmnivoreArticle } from '@omc-types/omnivore.js';
import type { ContentAnalysis } from '@omc-types/analysis.js';
```
## Environment Configuration
### Required Variables
**File**: `.env` (project root, gitignored)
```bash
# Omnivore API (REQUIRED)
OMNIVORE_API_KEY=your_api_key_here
OMNIVORE_API_URL=https://omnivore-api.caladan.haus/api/graphql
# Anthropic API (REQUIRED for analysis/generation phases)
ANTHROPIC_API_KEY=your_anthropic_api_key_here
# Content directories (OPTIONAL - defaults provided)
CONTENT_OUTPUT_DIR=./content
ARTICLES_DIR=./content/articles
ANALYSIS_DIR=./content/analysis
GENERATED_DIR=./content/generated
```
### For Self-Hosted Omnivore
If using self-hosted Omnivore instance:
- Set `OMNIVORE_API_URL` to your instance's GraphQL endpoint
- Get API key from your instance's settings
### Example: `.env.example`
Template file included in repo shows all available configuration options.
## Usage Examples
### Fetching Articles
```javascript
import { searchArticles } from './lib/omnivore/client.js';
import { QUERY_PATTERNS } from './lib/omnivore/queries.js';
// Fetch AI articles from last 7 days
const result = await searchArticles({
query: QUERY_PATTERNS.AI_ML + ' ' + QUERY_PATTERNS.LAST_7_DAYS,
first: 10,
includeContent: true
});
// Access articles
const articles = result.search.edges.map(edge => edge.node);
articles.forEach(article => {
console.log(article.title);
console.log(article.labels.map(l => l.name));
});
```
### Type-Safe Operations
```typescript
import type { OmnivoreArticle, SearchResult } from '@omc-types';
import { searchArticles } from './lib/omnivore/client.js';
async function getAIArticles(): Promise<OmnivoreArticle[]> {
const result = await searchArticles({
query: 'label:ai',
first: 20
}) as SearchResult;
return result.edges.map(edge => edge.node);
}
```
### Testing Connection
```bash
# Test API connection
node lib/omnivore/client.js --test
# Expected output:
# ✅ Connected to Omnivore API
# User: Your Name (your@email.com)
# Username: yourusername
```
## File System Conventions
### Content Storage
```
content/
├── articles/ # Original articles as Markdown
│ └── YYYY-MM-DD-{slug}.md
├── analysis/ # Analysis results
│ └── YYYY-MM-DD-{slug}.md
├── generated/ # Generated content
│ ├── blog-posts/
│ │ └── YYYY-MM-DD-{title}.md
│ └── newsletters/
│ └── YYYY-WW-roundup.md
└── .metadata/
└── index.json # Search index
```
### Markdown Format
**Article File** (`content/articles/2025-09-30-example.md`):
```markdown
---
id: omnivore-abc123
url: https://example.com/article
title: Example Article
author: John Doe
savedAt: 2025-09-30T10:00:00Z
labels: [ai, machine-learning]
highlights: 3
wordCount: 2500
---
# Example Article
Article content here...
## Highlights
> Important quote
— Note: Why this matters
```
## Build System Details
### TypeScript Compilation
```bash
# Development build (with source maps & declarations)
pnpm run build
# Output: dist/ with .js, .d.ts, .js.map files
# Type check only (no output)
pnpm run typecheck
# Watch mode for development
pnpm run dev
```
### Path Aliases
Configured in tsconfig.json, usable in TypeScript files:
- `@lib/*``src/lib/*`
- `@storage/*``src/storage/*`
- `@analysis/*``src/analysis/*`
- `@generation/*``src/generation/*`
- `@utils/*``src/utils/*`
- `@omc-types/*``src/types/*`
### Module System
- **Type**: ESM (ES Modules)
- **Import syntax**: `import { x } from './file.js'` (include `.js` extension)
- **package.json**: `"type": "module"`
## What Does NOT Exist Yet
This groundtruth documents what IS built. These are NOT implemented:
- Automated content generation modules (blog posts/newsletters) - Phase 5
- Publishing modules (Ghost/WordPress/etc.) - Phase 6
- End-to-end scheduled orchestration beyond analysis (e.g., daily generation + publishing)
See IMPLEMENTATION_PLAN.md for what needs to be built.

View file

@ -0,0 +1,19 @@
# GraphQL Organization (Current)
This repo currently uses a single runtime GraphQL client:
- Runtime client: `lib/omnivore/client.js` (string-based GraphQL queries + `node-fetch`)
- TypeScript import surface: `src/lib/omnivore/client.ts` (re-exports runtime functions for TS/alias imports)
## Why
An earlier, typed GraphQL layer (`src/graphql/**` + `src/types/generated/**`) existed but was **not integrated** into the CLI runtime. Maintaining two parallel query stacks created drift risk without any runtime benefit.
To reduce duplication and keep docs aligned with reality, the typed layer was removed. If/when we migrate the runtime client to typed operations, we can reintroduce codegen + documents as part of that migration.
## How To Add/Change Fields
1. Update the query string in `lib/omnivore/client.js`.
2. If needed, update the corresponding TS types in `src/types/omnivore.ts`.
3. Prefer importing client functions from `@lib/omnivore/client.js` in TypeScript code.

View file

@ -0,0 +1,613 @@
# Workflow Internals
**Purpose**: Deep dive into how the parallel analysis workflow operates, including zero-context-pollution design, agent invocation patterns, and file handling.
**Last Updated**: 2026-01-30
**Status**: This document originally described the legacy `cli/parallel-analyze.ts` + `cli/save-analysis-results.ts` flow. The current workflow is implemented via `omc analyze run` and `omc analyze complete`. See `docs/_meta/current-state.md` for the full audit and issue list.
## Overview
The parallel analysis workflow uses a **zero-context-pollution** design where each agent operates independently with fresh database connections and no shared state from the orchestration script. This enables true parallel execution and prevents race conditions.
## Zero-Context-Pollution Design
### Design Principles
1. **Stub files as communication**: Main context creates minimal stub files containing only article metadata
2. **Fresh connections**: the orchestrator command closes its DB work before agents run; agents should operate independently
3. **Independent content fetching**: agents fetch article content via `omc omnivore get <slug> --json` and extract `.content`
4. **File-based results**: Agents enrich stub files with analysis field, no inter-agent communication
5. **Slug-based naming**: Temp files named by article slug prevents overwrites across batches
### Why This Matters
- **Prevents race conditions**: Each agent reads/writes different files
- **Enables true parallelism**: No database locks during agent execution
- **Supports batch overlap**: Can run multiple batches before saving results
- **Isolates failures**: Failed agent doesn't corrupt others' work
- **Scales horizontally**: Can distribute agents across machines
## Parallel Analysis Workflow
### Automated Mode (`omc analyze auto`)
The automated workflow is the non-interactive, schedulable path:
1. Populate queue: `omc queue add --hours 24`
2. Analyze end-to-end: `omc analyze auto --batch-size 5`
Implementation notes:
- The analysis step shells out to `codex exec` in `read-only` sandbox mode and expects a single JSON object response.
- `CODEX_HOME` is set to `temp/codex-home` during execution to avoid reliance on `~/.codex` permissions in automation environments.
### Step 1: Orchestration (`omc analyze run`)
The orchestration script sets up the batch and prepares for agent invocation:
```typescript
// 1. Fetch pending jobs and mark in_progress
const jobs = queueRepo.getPending(5);
jobs.forEach(job => queueRepo.markInProgress(job.articleId));
// 2. Fetch article metadata from Omnivore API
const result = await getArticle(job.articleSlug, username);
const article = result.article;
// 3. Create stub file with slug-based name
const stubFile = `temp/${job.articleSlug}.jsonl`;
fs.writeFileSync(stubFile, JSON.stringify({
articleId: job.articleId,
articleSlug: job.articleSlug,
username: username,
articleUrl: article.url,
articleTitle: article.title,
savedAt: article.savedAt,
publishedAt: article.publishedAt,
updatedAt: article.updatedAt
}));
// 4. Output agent parameters
console.log(JSON.stringify([{
filename: stubFile,
articleId: article.id,
articleSlug: article.slug,
username: username,
articleTitle: article.title
}]));
// 5. Close database (agents get fresh connections)
db.close();
```
**Key Points**:
- Database is closed before agents execute
- Stub files contain metadata only (no content)
- Each stub file is named by article slug
- Agent parameters output as JSON array
### Step 2: Agent Execution (@article-content-analyzer)
Each agent operates independently with its own workflow:
```typescript
// Each agent receives: filename, articleId, articleSlug, username, articleTitle
// 1. Read stub file to get metadata
const stub = JSON.parse(fs.readFileSync(filename, 'utf-8'));
// 2. Fetch content via CLI tool (no TypeScript imports)
const contentResult = await bash(`omc omnivore get ${stub.articleSlug} --json`);
const article = JSON.parse(contentResult.stdout);
const articleContent = article.content;
// 3. Analyze content using agent's prompt
const analysis = {
topics: [...],
topicScores: {...},
summary: "...",
keyPoints: [...],
sentiment: "positive",
monetizationAngle: "...",
analyzedAt: new Date().toISOString()
};
// 4. Re-read stub file (fresh copy, no race conditions)
const enriched = JSON.parse(fs.readFileSync(filename, 'utf-8'));
// 5. Add analysis field ONLY (preserve all original fields)
enriched.analysis = analysis;
// 6. Write enriched data back to same file
fs.writeFileSync(filename, JSON.stringify(enriched, null, 2));
```
**Key Points**:
- Agent fetches content via CLI (no direct TypeScript imports)
- Re-reads stub before writing (ensures fresh data)
- Adds `analysis` field only (preserves metadata)
- Each agent writes to unique file (slug-based naming)
### Step 3: Save Results (`omc analyze complete`)
The save script processes all enriched files and updates tracking:
```typescript
// For each enriched temp file:
for (const file of files) {
const data = JSON.parse(fs.readFileSync(file, 'utf-8'));
// 1) Write to Markdown (git-tracked, human-editable)
const markdownPath = await writer.write(
data.articleId,
data.articleUrl,
data.articleTitle,
data.savedAt,
data.analysis
);
// 2) Store immutable snapshot + mark completed in SQLite
queueRepo.storeAnalysis(
data.articleId,
data.publishedAt ?? null,
data.updatedAt ?? null,
JSON.stringify(data.analysis),
markdownPath
);
// 3) Delete temp file (cleanup)
fs.unlinkSync(file);
}
```
**Key Points**:
- Processes all temp files in one pass
- Updates database after successful write
- Automatically deletes temp files (cleanup)
- Preserves temp files on error for debugging
**Note**:
- JSONL output is optional: enable with `omc analyze complete --jsonl` or `omc analyze auto --jsonl`.
## File Lifecycle
### Stub File Creation (`omc analyze run`)
**File**: `temp/{articleSlug}.jsonl`
**Content**: Article metadata only (no content, no analysis)
**Purpose**: Minimal communication between orchestrator and agents
**Example**:
```json
{
"articleId": "abc-123",
"articleSlug": "understanding-llm-fine-tuning",
"username": "myusername",
"articleUrl": "https://example.com/article",
"articleTitle": "Understanding LLM Fine-Tuning",
"savedAt": "2025-09-30T10:00:00Z",
"publishedAt": "2025-09-29T14:00:00Z",
"updatedAt": "2025-09-30T08:00:00Z"
}
```
### Enrichment (agent)
**Process**:
1. Reads stub file
2. Fetches content via CLI tool
3. Adds `analysis` field to JSON
4. Writes back to same file
**Enriched Format**:
```json
{
"articleId": "abc-123",
"articleSlug": "understanding-llm-fine-tuning",
"username": "myusername",
"articleUrl": "https://example.com/article",
"articleTitle": "Understanding LLM Fine-Tuning",
"savedAt": "2025-09-30T10:00:00Z",
"publishedAt": "2025-09-29T14:00:00Z",
"updatedAt": "2025-09-30T08:00:00Z",
"analysis": {
"topics": ["ai", "machine-learning"],
"topicScores": { "ai": 0.95, "machine-learning": 0.88 },
"summary": "Article summary...",
"keyPoints": ["point1", "point2"],
"sentiment": "positive",
"monetizationAngle": "Content opportunity...",
"analyzedAt": "2025-10-01T04:30:00Z"
}
}
```
### Storage & Cleanup (`omc analyze complete`)
**Process**:
1. Reads enriched file
2. Writes Markdown to `content/analysis/YYYY-MM-DD-{slug}-analysis.md` (slug derived from `articleSlug` when available, otherwise title)
3. Stores immutable snapshot to SQLite (`analysis_json`, `markdown_path`) and marks job `completed`
4. **Deletes temp file** with `unlinkSync(file)`
**Output Files**:
- `content/analysis/2025-09-30-understanding-llm-fine-tuning-analysis.md`
**Temp File**: Deleted after successful save
## Error Handling
### Parse Error
**Scenario**: Enriched file contains invalid JSON
**Handling**:
- Mark job as `failed` in tracking queue
- Preserve temp file for debugging
- Store error message in database
- Increment retry count
**Example**:
```
✗ Failed: Understanding LLM Fine-Tuning... - Parse error
→ Preserved: temp/understanding-llm-fine-tuning.jsonl
```
### Agent Timeout
**Scenario**: Agent doesn't complete within time limit
**Handling**:
- Status remains `in_progress` in database
- Temp file preserved (partially enriched)
- Manual retry needed via `omc analyze retry`
### Save Failure
**Scenario**: Error writing to Markdown or JSONL
**Handling**:
- Mark job as `failed`
- Preserve temp file
- Log error details
- Can retry after fixing issue
## Parallel Invocation Pattern
### Single Message, Multiple Agents
The workflow requires invoking all agents in a single message for true parallelism:
```
User message:
Analyze these 5 articles in parallel:
@article-content-analyzer {
filename: "temp/article-1.jsonl",
articleId: "abc-123",
articleSlug: "article-1",
username: "myusername",
articleTitle: "Article 1 Title"
}
@article-content-analyzer {
filename: "temp/article-2.jsonl",
articleId: "def-456",
articleSlug: "article-2",
username: "myusername",
articleTitle: "Article 2 Title"
}
... (3 more agents)
```
### Why Single Message
- **All agents execute in parallel** (no sequential bottleneck)
- **User waits for all results** before saving
- **Failure of one agent** doesn't block others
- **Claude Code orchestrates** parallel execution
### Agent Independence
Each agent:
- Gets unique parameters (filename, articleId, slug, username, title)
- Reads different stub file
- Fetches content independently via CLI
- Writes to different temp file
- No coordination with other agents
## Slug-Based Naming Benefits
### Problem with Sequential IDs
Using sequential names like `temp/result-1.jsonl`, `temp/result-2.jsonl`:
- Running batch 2 overwrites batch 1's files
- Must save batch 1 before running batch 2
- Forces sequential processing
### Solution with Slug-Based Names
Using article slug like `temp/{articleSlug}.jsonl`:
- Each article has unique filename based on slug
- Multiple batches can run simultaneously
- Agents can complete out-of-order
- Save script processes all `temp/*.jsonl` files at once
### Example Workflow
```bash
# Batch 1 creates:
temp/understanding-llm-fine-tuning.jsonl
temp/kubernetes-best-practices.jsonl
temp/rust-performance-tips.jsonl
# Batch 2 creates (while batch 1 agents still running):
temp/python-async-patterns.jsonl
temp/distributed-systems-design.jsonl
# Save all at once:
omc analyze complete
# Processes all 5 files, deletes after successful save
```
## Agent Content Analyzer
### Agent Name
`@article-content-analyzer`
**Location**: `.claude/agents/article-content-analyzer.md`
**Model**: Sonnet
### Core Responsibilities
1. **Topic Extraction & Scoring** - Identify 2-5 topics with confidence scores (0-1)
2. **Strategic Summarization** - 2-3 sentence "so what?" summary
3. **Key Point Extraction** - 3-5 actionable insights or surprising facts
4. **Sentiment Analysis** - Classify as positive/neutral/negative
5. **Monetization Strategy** - Identify specific content opportunities
### Topic Categories
Aligned with content strategy:
- `developer-tools`, `ai-tooling`, `ai`, `machine-learning`, `llm`
- `software-engineering`, `architecture`, `code-quality`
- `devops`, `kubernetes`, `docker`, `cloud-native`
- `databases`, `sql`, `nosql`
- `cloud`, `aws`, `azure`, `serverless`
- `startups`, `product`, `growth`, `monetization`
- `security`, `auth`, `encryption`
### Topic Scores
- **0.90-1.0**: Core focus of article
- **0.70-0.89**: Significant discussion
- **0.50-0.69**: Mentioned but not central
- **Below 0.50**: Excluded
### Output Schema
```json
{
"topics": ["topic1", "topic2", "topic3"],
"topicScores": {
"topic1": 0.95,
"topic2": 0.88,
"topic3": 0.75
},
"summary": "2-3 sentence summary capturing main points and why this matters",
"keyPoints": [
"First key takeaway or insight",
"Second key takeaway or insight",
"Third key takeaway or insight"
],
"sentiment": "positive|neutral|negative",
"monetizationAngle": "Specific content opportunity description"
}
```
### Critical Rules
- Returns ONLY JSON object (no markdown code blocks, no explanatory text)
- All topic labels from approved category list
- Topic scores between 0-1, at least one ≥ 0.70
- Summary is 2-3 sentences focusing on implications
- 3-5 specific, actionable key points
- Monetization angle suggests concrete content opportunity
- User highlights receive extra weight if provided
### Usage in Workflow
The agent is invoked with parameters from the orchestration script:
```
@article-content-analyzer {
filename: "temp/understanding-llm-fine-tuning.jsonl",
articleId: "abc-123",
articleSlug: "understanding-llm-fine-tuning",
username: "myusername",
articleTitle: "Understanding LLM Fine-Tuning Techniques"
}
```
Agent then:
1. Reads stub file for metadata
2. Calls `omc omnivore get understanding-llm-fine-tuning --json` for content (extract `.content`)
3. Analyzes content using its prompt template
4. Enriches stub file with analysis
5. Writes back to temp file
## Analysis Prompt Template
**Location**: `src/analysis/prompts/analyze.md`
**Purpose**: Structured prompt template for article analysis (used by agent).
### Template Variables
- `{{title}}` - Article title
- `{{author}}` - Article author
- `{{url}}` - Article URL
- `{{wordCount}}` - Word count
- `{{publishedAt}}` - Publication date
- `{{content}}` - Article content (HTML or text)
- `{{#if highlights}}...{{/if}}` - Conditional highlights section
- `{{#each highlights}}...{{/each}}` - Iterate over highlights
### Highlights Format
If user highlights are present:
```
{{#each highlights}}
- "{{quote}}"{{#if annotation}} — Note: {{annotation}}{{/if}}
{{/each}}
```
### Output Format
JSON matching `ContentAnalysis` schema (no markdown code blocks).
### Example Rendered Prompt
```markdown
# Article Content Analysis Prompt
You are analyzing an article for content monetization opportunities...
## Article Details
**Title**: Understanding LLM Fine-Tuning
**Author**: Jane Smith
**URL**: https://example.com/llm-finetuning
**Word Count**: 3200
**Published**: 2025-09-25T10:00:00Z
**Content**:
[Full article content here...]
**User Highlights**:
- "LoRA reduces trainable parameters by 90%" — Note: Key metric for cost savings
- "Prompt engineering vs fine-tuning trade-offs"
## Analysis Task
Extract the following information and return as JSON:
[JSON schema and guidelines...]
```
## Data Flow Diagram
```
┌─────────────────────────────────────────────────────────────┐
│ 1. ORCHESTRATION (omc analyze run) │
│ │
│ Queue DB → Get 5 pending → Mark in_progress │
│ → Fetch metadata from Omnivore API │
│ → Create stub files (temp/{slug}.jsonl) │
│ → Output agent parameters as JSON │
│ → Close DB connection │
└─────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────┐
│ 2. PARALLEL AGENT EXECUTION (@article-content-analyzer × 5) │
│ │
│ Each agent independently: │
│ - Read stub file for metadata │
│ - Call `omc omnivore get {slug} --json` for content │
│ - Analyze content using prompt template │
│ - Enrich stub file with analysis field │
│ - Write back to temp/{slug}.jsonl │
└─────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────┐
│ 3. SAVE RESULTS (omc analyze complete) │
│ │
│ Read all temp/*.jsonl files │
│ For each: │
│ - Write to content/analysis/YYYY-MM-DD-{slug}-analysis.md │
│ - Store snapshot in DB + mark completed │
│ - Delete temp file │
└─────────────────────────────────────────────────────────────┘
```
## Complete Example
### Real Analysis Output
**File**: `content/analysis/2025-09-30-cchistory-tracking-claude-code-system-prompt-and-t-analysis.md`
```markdown
---
articleId: 1039961b-8de3-4ccf-b3e8-df888d6174b8
articleUrl: https://mariozechner.at/posts/2025-08-03-cchistory/
articleTitle: "cchistory: Tracking Claude Code System Prompt and Tool Changes"
savedAt: 2025-09-30T02:27:46.000Z
analyzedAt: 2025-10-01T04:25:16.676Z
topics: [developer-tools, ai-tooling, reverse-engineering, software-debugging]
topicScores:
developer-tools: 0.95
ai-tooling: 0.9
reverse-engineering: 0.85
software-debugging: 0.75
sentiment: positive
---
## Summary
Developer creates tools to reverse-engineer and track changes in Claude Code's system prompts and tool definitions over time. The cchistory project monitors Claude Code updates by intercepting and archiving system messages, revealing how Anthropic evolves the assistant's behavior and capabilities.
## Key Points
- Created claude-trace to intercept and log Claude Code's internal API communications
- Built cchistory to automatically track and diff system prompt changes across versions
- Discovered Claude Code switched from Sonnet to Haiku for certain operations to reduce costs
- System prompts reveal detailed behavioral instructions and tool usage patterns
- Tracking changes helps understand how AI coding assistants evolve over time
## Monetization Angle
Tutorial series on 'Reverse Engineering AI Tools' - developers want to understand how Claude Code works under the hood. Could create content comparing system prompt evolution, analyzing cost optimization strategies, or teaching prompt engineering through real examples from production AI systems.
```
### Example JSONL Record (Optional Output)
**File**: `content/analysis/analyses.jsonl` (written when `--jsonl` is enabled)
```json
{"articleId":"1039961b-8de3-4ccf-b3e8-df888d6174b8","articleUrl":"https://mariozechner.at/posts/2025-08-03-cchistory/","articleTitle":"cchistory: Tracking Claude Code System Prompt and Tool Changes","savedAt":"2025-09-30T02:27:46.000Z","analyzedAt":"2025-10-01T04:25:16.676Z","topics":["developer-tools","ai-tooling","reverse-engineering","software-debugging"],"topicScores":{"developer-tools":0.95,"ai-tooling":0.9,"reverse-engineering":0.85,"software-debugging":0.75},"sentiment":"positive","summary":"Developer creates tools to reverse-engineer and track changes in Claude Code's system prompts and tool definitions over time. The cchistory project monitors Claude Code updates by intercepting and archiving system messages, revealing how Anthropic evolves the assistant's behavior and capabilities.","keyPoints":["Created claude-trace to intercept and log Claude Code's internal API communications","Built cchistory to automatically track and diff system prompt changes across versions","Discovered Claude Code switched from Sonnet to Haiku for certain operations to reduce costs","System prompts reveal detailed behavioral instructions and tool usage patterns","Tracking changes helps understand how AI coding assistants evolve over time"],"monetizationAngle":"Tutorial series on 'Reverse Engineering AI Tools' - developers want to understand how Claude Code works under the hood. Could create content comparing system prompt evolution, analyzing cost optimization strategies, or teaching prompt engineering through real examples from production AI systems."}
```
## What Does NOT Exist Yet
**Implemented**:
- ✅ SQLite tracking database for parallel execution
- ✅ Job queue with status tracking (pending/in_progress/completed/failed)
- ✅ Parallel analysis workflow (batch of 5)
- ✅ Error handling with retry logic (max 3 attempts)
- ✅ CLI tools for queue management
- ✅ JSONL output format
- ✅ Boundary enforcement with AIDEV annotations
- ✅ Zero-context-pollution design
- ✅ Slug-based file naming
**Future Enhancements**:
- **Automated agent invocation** - Currently requires manual copy/paste of prompts; could integrate Claude Agent SDK for direct invocation
- **Real-time progress tracking** - Add progress bars during fetch/analysis operations
- **Session management** - `analysis_sessions` table exists but not yet used for batch tracking
- **Queue cleanup automation** - Manual `clearCompleted()` call; could auto-cleanup after export
- **Distributed execution** - Agents across multiple machines
- **Advanced error recovery** - Automatic retry with backoff
- **Batch prioritization** - Priority queue for time-sensitive articles
- **Content pre-filtering** - Skip articles unlikely to be valuable
- **Incremental updates** - Re-analyze articles when content changes
## Related Documentation
- [Architecture](architecture.md) - System design and storage boundaries
- [CLI Reference](cli-reference.md) - Command-line interface documentation
- [Foundation & Type System](foundation-and-types.md) - TypeScript setup, type definitions

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,77 @@
#!/usr/bin/env node
/**
* ESBuild Configuration for OMC CLI
* AIDEV-NOTE: Bundles OCLIF CLI with ESM support and path aliases
*/
import { build } from 'esbuild';
import { glob } from 'glob';
import { copyFile, mkdir } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
// Find all command files
const commandFiles = await glob('src/commands/**/*.ts', { cwd: __dirname });
const libFiles = await glob('src/lib/**/*.ts', { cwd: __dirname });
const storageFiles = await glob('src/storage/**/*.ts', { cwd: __dirname });
const rootLibFiles = await glob('lib/**/*.js', { cwd: __dirname });
// Build configuration
const buildConfig = {
entryPoints: [
'bin/omc.ts',
'index.ts',
...commandFiles,
...libFiles,
...storageFiles,
...rootLibFiles,
],
bundle: true,
format: 'esm',
platform: 'node',
target: 'node18',
outdir: 'dist',
packages: 'external',
splitting: true,
sourcemap: true,
// Path alias resolution
alias: {
'@lib': path.join(__dirname, 'src/lib'),
'@storage': path.join(__dirname, 'src/storage'),
'@analysis': path.join(__dirname, 'src/analysis'),
'@generation': path.join(__dirname, 'src/generation'),
'@publishing': path.join(__dirname, 'src/publishing'),
'@workflows': path.join(__dirname, 'src/workflows'),
'@utils': path.join(__dirname, 'src/utils'),
'@omc-types': path.join(__dirname, 'src/types'),
},
// Preserve directory structure
outbase: '.',
};
try {
await build(buildConfig);
// Copy schema files to dist (needed by chunk files)
const schemaDir = path.join(__dirname, 'dist/schema');
await mkdir(schemaDir, { recursive: true });
await copyFile(
path.join(__dirname, 'src/storage/schema/tracking-schema.sql'),
path.join(schemaDir, 'tracking-schema.sql')
);
// Copy analysis prompt files (used at runtime by ContentAnalyzer)
const promptDir = path.join(__dirname, 'dist/src/analysis/prompts');
await mkdir(promptDir, { recursive: true });
await copyFile(
path.join(__dirname, 'src/analysis/prompts/analyze-article.md'),
path.join(promptDir, 'analyze-article.md')
);
console.log('✅ Build complete');
} catch (error) {
console.error('❌ Build failed:', error);
process.exit(1);
}

9
self-hosting/omc/index.ts Executable file
View file

@ -0,0 +1,9 @@
#!/usr/bin/env node
/**
* OMC Main CLI Entry Point
* AIDEV-NOTE: OCLIF CLI runner for OMC commands
*/
import { run } from '@oclif/core';
await run(process.argv.slice(2), import.meta.url);

View file

@ -0,0 +1,96 @@
export interface OmnivoreHighlight {
id: string;
quote: string;
annotation?: string | null;
createdAt: string;
updatedAt?: string;
type?: string;
}
export interface OmnivoreLabel {
id: string;
name: string;
color: string;
}
export interface OmnivoreArticle {
id: string;
title: string;
url: string;
content?: string | null;
description?: string | null;
author?: string | null;
publishedAt?: string | null;
slug?: string;
wordCount?: number;
createdAt: string;
savedAt: string;
updatedAt: string;
isArchived?: boolean;
folder?: string;
labels: OmnivoreLabel[];
highlights: OmnivoreHighlight[];
}
export interface GetMeResult {
id: string;
name?: string;
email?: string;
profile?: { username?: string };
}
export interface GetArticleResult {
article?: OmnivoreArticle;
errorCodes?: string[];
}
export function getMe(): Promise<GetMeResult>;
export function searchArticles(args?: Record<string, unknown>): Promise<any>;
export function getArticle(slug: string, username: string): Promise<GetArticleResult>;
export function getArticlesByDate(args?: Record<string, unknown>): Promise<any>;
export function getArticlesByLabel(labelName: string, first?: number): Promise<any>;
export function getRecentArticles(hours?: number, first?: number): Promise<any>;
export function searchByTopic(topic: string, first?: number): Promise<any>;
export function getUnreadArticles(first?: number): Promise<any>;
export function getLabels(): Promise<any>;
export function getHighlights(slug: string, username: string): Promise<any[]>;
export function updatePage(args: {
pageId: string;
description?: string;
title?: string;
byline?: string;
publishedAt?: string;
savedAt?: string;
}): Promise<any>;
export function createLabel(args: {
name: string;
color?: string;
description?: string;
}): Promise<any>;
export function setLabels(args: {
pageId: string;
labelIds?: string[];
labels?: Array<{ name: string; color?: string; description?: string }>;
source?: string;
}): Promise<any>;
export function saveUrl(args: {
url: string;
source: string;
clientRequestId: string;
folder?: string;
savedAt?: string;
publishedAt?: string;
labels?: Array<{ name: string; color?: string; description?: string }>;
timezone?: string;
locale?: string;
state?: string;
}): Promise<any>;
export function createHighlight(args: Record<string, unknown>): Promise<any>;
export function updateHighlight(args: Record<string, unknown>): Promise<any>;
export function deleteHighlight(highlightId: string): Promise<any>;
export function testConnection(): Promise<boolean>;

View file

@ -0,0 +1,585 @@
#!/usr/bin/env node
/**
* Omnivore API Client
* GraphQL client for interacting with Omnivore API
*/
import fetch from 'node-fetch';
import { config } from 'dotenv';
// Load environment variables
config();
const API_URL = process.env.OMNIVORE_API_URL || 'https://api-prod.omnivore.app/api/graphql';
const API_KEY = process.env.OMNIVORE_API_KEY;
if (!API_KEY) {
console.error('❌ OMNIVORE_API_KEY not set in environment');
process.exit(1);
}
/**
* Execute GraphQL query against Omnivore API
*/
async function graphqlRequest(query, variables = {}) {
try {
const response = await fetch(API_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
// Omnivore self-hosted commonly expects `Omnivore-Authorization`, while some deployments accept `Authorization`.
// Sending both keeps the CLI compatible across environments.
'Authorization': API_KEY,
'Omnivore-Authorization': API_KEY,
},
body: JSON.stringify({ query, variables }),
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const result = await response.json();
if (result.errors) {
throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`);
}
return result.data;
} catch (error) {
console.error('GraphQL Request Error:', error.message);
throw error;
}
}
/**
* Get current authenticated user
*/
export async function getMe() {
const query = `
query {
me {
id
name
email
profile {
username
}
}
}
`;
const data = await graphqlRequest(query);
return data.me;
}
/**
* Search articles with filters
*/
export async function searchArticles({
query = 'in:all',
first = 10,
after = '',
includeContent = false,
} = {}) {
const searchQuery = `
query Search($query: String!, $first: Int, $after: String, $includeContent: Boolean) {
search(query: $query, first: $first, after: $after, includeContent: $includeContent) {
... on SearchSuccess {
pageInfo {
totalCount
hasNextPage
endCursor
}
edges {
cursor
node {
id
slug
title
url
originalArticleUrl
createdAt
updatedAt
publishedAt
savedAt
author
description
image
siteName
pageType
wordsCount
readingProgressTopPercent
isArchived
folder
content
labels {
id
name
color
}
highlights {
id
quote
annotation
createdAt
}
}
}
}
... on SearchError {
errorCodes
}
}
}
`;
const data = await graphqlRequest(searchQuery, { query, first, after, includeContent });
if (data.search.errorCodes) {
throw new Error(`Search Error: ${data.search.errorCodes.join(', ')}`);
}
return data.search;
}
/**
* Get article by ID
*/
export async function getArticle(slug, username) {
const query = `
query GetArticle($slug: String!, $username: String!) {
article(slug: $slug, username: $username) {
... on ArticleSuccess {
article {
id
title
url
content
author
description
publishedAt
updatedAt
createdAt
savedAt
highlights(input: {}) {
id
quote
annotation
}
labels {
id
name
color
}
}
}
... on ArticleError {
errorCodes
}
}
}
`;
const data = await graphqlRequest(query, { slug, username });
return data.article;
}
/**
* Get articles saved in a time range
*/
export async function getArticlesByDate({
startDate,
endDate,
first = 100,
} = {}) {
const start = startDate || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
const end = endDate || new Date().toISOString();
const query = `saved:>${start.split('T')[0]} saved:<${end.split('T')[0]} sort:saved-desc`;
return await searchArticles({ query, first, includeContent: false });
}
/**
* Get articles with specific labels
*/
export async function getArticlesByLabel(labelName, first = 50) {
const query = `label:${labelName} sort:saved-desc`;
return await searchArticles({ query, first });
}
/**
* Get recently saved articles
*/
export async function getRecentArticles(hours = 24, first = 50) {
let query;
if (hours <= 24) {
query = 'saved:last24hrs';
} else if (hours <= 168) {
query = 'saved:last7days';
} else {
const days = Math.ceil(hours / 24);
const date = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
query = `saved:>${date}`;
}
query += ' sort:saved-desc';
return await searchArticles({ query, first, includeContent: false });
}
/**
* Get articles by topic/keyword
*/
export async function searchByTopic(topic, first = 50) {
const query = `${topic} sort:saved-desc`;
return await searchArticles({ query, first, includeContent: false });
}
/**
* Get unread articles
*/
export async function getUnreadArticles(first = 50) {
const query = 'in:inbox is:unread sort:saved-desc';
return await searchArticles({ query, first });
}
/**
* Get all labels
*/
export async function getLabels() {
const query = `
query {
labels {
... on LabelsSuccess {
labels {
id
name
color
description
createdAt
}
}
... on LabelsError {
errorCodes
}
}
}
`;
const data = await graphqlRequest(query);
if (data.labels.errorCodes) {
throw new Error(`Labels Error: ${data.labels.errorCodes.join(', ')}`);
}
return data.labels.labels;
}
/**
* Get highlights for an article
*/
export async function getHighlights(slug, username) {
const query = `
query GetHighlights($slug: String!, $username: String!) {
article(slug: $slug, username: $username) {
... on ArticleSuccess {
article {
highlights {
id
quote
annotation
createdAt
updatedAt
type
}
}
}
... on ArticleError {
errorCodes
}
}
}
`;
const data = await graphqlRequest(query, { slug, username });
return data.article?.article?.highlights || [];
}
/**
* Update article/page metadata
*/
export async function updatePage({ pageId, description, title, byline, publishedAt, savedAt }) {
const mutation = `
mutation UpdatePage($input: UpdatePageInput!) {
updatePage(input: $input) {
... on UpdatePageSuccess {
updatedPage {
id
title
description
author
}
}
... on UpdatePageError {
errorCodes
}
}
}
`;
const input = { pageId };
if (description !== undefined) input.description = description;
if (title !== undefined) input.title = title;
if (byline !== undefined) input.byline = byline;
if (publishedAt !== undefined) input.publishedAt = publishedAt;
if (savedAt !== undefined) input.savedAt = savedAt;
const data = await graphqlRequest(mutation, { input });
return data.updatePage;
}
/**
* Create a label.
*/
export async function createLabel({ name, color, description }) {
const mutation = `
mutation CreateLabel($input: CreateLabelInput!) {
createLabel(input: $input) {
... on CreateLabelSuccess {
label {
id
name
color
description
createdAt
}
}
... on CreateLabelError {
errorCodes
}
}
}
`;
const input = { name };
if (color !== undefined) input.color = color;
if (description !== undefined) input.description = description;
const data = await graphqlRequest(mutation, { input });
return data.createLabel;
}
/**
* Set labels for a page (replaces existing labels).
* Use either `labelIds` or `labels` (CreateLabelInput[]). If `labels` is used, Omnivore may create missing labels.
*/
export async function setLabels({ pageId, labelIds, labels, source }) {
const mutation = `
mutation SetLabels($input: SetLabelsInput!) {
setLabels(input: $input) {
... on SetLabelsSuccess {
labels {
id
name
color
}
}
... on SetLabelsError {
errorCodes
}
}
}
`;
const input = { pageId };
if (labelIds !== undefined) input.labelIds = labelIds;
if (labels !== undefined) input.labels = labels;
if (source !== undefined) input.source = source;
const data = await graphqlRequest(mutation, { input });
return data.setLabels;
}
/**
* Save a URL into Omnivore (creates a new library item / saving request).
*/
export async function saveUrl({ url, source, folder, savedAt, publishedAt, labels, timezone, locale, state, clientRequestId }) {
const mutation = `
mutation SaveUrl($input: SaveUrlInput!) {
saveUrl(input: $input) {
... on SaveSuccess {
url
clientRequestId
}
... on SaveError {
errorCodes
}
}
}
`;
const input = { url, source, clientRequestId };
if (folder !== undefined) input.folder = folder;
if (savedAt !== undefined) input.savedAt = savedAt;
if (publishedAt !== undefined) input.publishedAt = publishedAt;
if (labels !== undefined) input.labels = labels;
if (timezone !== undefined) input.timezone = timezone;
if (locale !== undefined) input.locale = locale;
if (state !== undefined) input.state = state;
const data = await graphqlRequest(mutation, { input });
return data.saveUrl;
}
/**
* Create highlight with annotation (or standalone NOTE)
*/
export async function createHighlight({ id, shortId, articleId, quote = '', annotation, patch = '', prefix = '', suffix = '', color = '#FFD700', type = 'HIGHLIGHT' }) {
const mutation = `
mutation CreateHighlight($input: CreateHighlightInput!) {
createHighlight(input: $input) {
... on CreateHighlightSuccess {
highlight {
id
quote
annotation
color
type
createdAt
}
}
... on CreateHighlightError {
errorCodes
}
}
}
`;
const input = {
id,
shortId,
articleId,
patch,
prefix,
suffix,
color,
type
};
if (quote) input.quote = quote;
if (annotation) input.annotation = annotation;
const data = await graphqlRequest(mutation, { input });
return data.createHighlight;
}
/**
* Update highlight/note annotation
*/
export async function updateHighlight({ highlightId, annotation, quote, html, color }) {
const mutation = `
mutation UpdateHighlight($input: UpdateHighlightInput!) {
updateHighlight(input: $input) {
... on UpdateHighlightSuccess {
highlight {
id
annotation
updatedAt
}
}
... on UpdateHighlightError {
errorCodes
}
}
}
`;
const input = { highlightId };
if (annotation !== undefined) input.annotation = annotation;
if (quote !== undefined) input.quote = quote;
if (html !== undefined) input.html = html;
if (color !== undefined) input.color = color;
const data = await graphqlRequest(mutation, { input });
return data.updateHighlight;
}
/**
* Delete highlight/note
*/
export async function deleteHighlight(highlightId) {
const mutation = `
mutation DeleteHighlight($highlightId: ID!) {
deleteHighlight(highlightId: $highlightId) {
... on DeleteHighlightSuccess {
highlight {
id
}
}
... on DeleteHighlightError {
errorCodes
}
}
}
`;
const data = await graphqlRequest(mutation, { highlightId });
return data.deleteHighlight;
}
/**
* Test API connection
*/
export async function testConnection() {
try {
const user = await getMe();
console.log('✅ Connected to Omnivore API');
console.log(` User: ${user.name} (${user.email})`);
console.log(` Username: ${user.profile.username}`);
return true;
} catch (error) {
console.error('❌ Failed to connect to Omnivore API');
console.error(` ${error.message}`);
return false;
}
}
// CLI usage
if (import.meta.url === `file://${process.argv[1]}`) {
const args = process.argv.slice(2);
if (args.includes('--test')) {
await testConnection();
} else if (args.includes('--recent')) {
const hours = parseInt(args[args.indexOf('--recent') + 1]) || 24;
const result = await getRecentArticles(hours);
console.log(`Found ${result.pageInfo.totalCount} articles from last ${hours} hours`);
result.edges.forEach(({ node }) => {
console.log(` - ${node.title}`);
console.log(` ${node.url}`);
console.log(` Saved: ${node.savedAt}`);
});
} else {
console.log('Usage:');
console.log(' node client.js --test # Test API connection');
console.log(' node client.js --recent [hours] # Get recent articles');
}
}
export default {
graphqlRequest,
getMe,
searchArticles,
getArticle,
getArticlesByDate,
getArticlesByLabel,
getRecentArticles,
searchByTopic,
getUnreadArticles,
getLabels,
getHighlights,
testConnection,
};

View file

@ -0,0 +1,369 @@
/**
* Pre-built GraphQL Queries for Omnivore API
* Collection of common query patterns for content analysis
*/
/**
* Search query with all fields needed for content analysis
*/
export const SEARCH_ARTICLES_FULL = `
query SearchArticlesFull($query: String!, $first: Int, $after: String) {
search(query: $query, first: $first, after: $after, includeContent: false) {
... on SearchSuccess {
pageInfo {
totalCount
hasNextPage
endCursor
}
edges {
cursor
node {
id
title
url
originalArticleUrl
slug
createdAt
updatedAt
publishedAt
savedAt
author
description
image
siteName
pageType
wordCount
readingProgressTopPercent
readingProgressBottomPercent
isArchived
folder
labels {
id
name
color
}
highlights {
id
quote
annotation
createdAt
}
}
}
}
... on SearchError {
errorCodes
}
}
}
`;
/**
* Get article with full content
*/
export const GET_ARTICLE_FULL = `
query GetArticleFull($slug: String!, $username: String!) {
article(slug: $slug, username: $username) {
... on ArticleSuccess {
article {
id
title
url
slug
content
author
description
image
publishedAt
createdAt
savedAt
updatedAt
wordCount
siteName
originalArticleUrl
highlights {
id
quote
annotation
createdAt
}
labels {
id
name
color
description
}
}
}
... on ArticleError {
errorCodes
}
}
}
`;
/**
* Get user profile
*/
export const GET_USER_PROFILE = `
query GetUserProfile {
me {
id
name
email
profile {
username
pictureUrl
bio
}
}
}
`;
/**
* Get all labels
*/
export const GET_LABELS = `
query GetLabels {
labels {
... on LabelsSuccess {
labels {
id
name
color
description
createdAt
}
}
... on LabelsError {
errorCodes
}
}
}
`;
/**
* Search with content included (for analysis)
*/
export const SEARCH_WITH_CONTENT = `
query SearchWithContent($query: String!, $first: Int, $after: String) {
search(query: $query, first: $first, after: $after, includeContent: true) {
... on SearchSuccess {
pageInfo {
totalCount
hasNextPage
endCursor
}
edges {
cursor
node {
id
title
url
content
author
description
publishedAt
savedAt
siteName
labels {
name
}
highlights {
quote
annotation
}
}
}
}
... on SearchError {
errorCodes
}
}
}
`;
/**
* Common search query patterns
*/
export const QUERY_PATTERNS = {
// Time-based
LAST_24_HOURS: 'saved:last24hrs sort:saved-desc',
LAST_7_DAYS: 'saved:last7days sort:saved-desc',
LAST_30_DAYS: 'saved:last30days sort:saved-desc',
// Status-based
INBOX: 'in:inbox sort:saved-desc',
ARCHIVED: 'in:archive sort:saved-desc',
UNREAD: 'is:unread sort:saved-desc',
READ: 'is:read sort:saved-desc',
// Type-based
ARTICLES: 'type:article sort:saved-desc',
HIGHLIGHTS: 'type:highlights sort:saved-desc',
// Content-based
WITH_HIGHLIGHTS: 'has:highlights sort:saved-desc',
WITH_LABELS: 'has:labels sort:saved-desc',
NO_LABELS: 'no:label sort:saved-desc',
// Topic-based (AI/Tech)
AI_ML: '(ai OR "machine learning" OR llm OR "language model") sort:saved-desc',
DEVOPS: '(devops OR kubernetes OR docker OR "cloud native") sort:saved-desc',
DATABASES: '(database OR postgres OR mongodb OR redis) sort:saved-desc',
PROGRAMMING: '(programming OR coding OR "software engineering") sort:saved-desc',
STARTUPS: '(startup OR founder OR "venture capital" OR product) sort:saved-desc',
// Combined patterns
RECENT_AI: 'saved:last7days (ai OR llm OR "machine learning") sort:saved-desc',
RECENT_TECH: 'saved:last7days (programming OR devops OR database) sort:saved-desc',
UNREAD_ARTICLES: 'in:inbox is:unread type:article sort:saved-desc',
};
/**
* Build date range query
*/
export function buildDateRangeQuery(startDate, endDate, sortBy = 'saved-desc') {
const start = typeof startDate === 'string' ? startDate : startDate.toISOString().split('T')[0];
const end = typeof endDate === 'string' ? endDate : endDate.toISOString().split('T')[0];
return `saved:>${start} saved:<${end} sort:${sortBy}`;
}
/**
* Build label query
*/
export function buildLabelQuery(labels, operator = 'OR') {
if (Array.isArray(labels)) {
return labels.map(l => `label:${l}`).join(` ${operator} `);
}
return `label:${labels}`;
}
/**
* Build keyword query
*/
export function buildKeywordQuery(keywords, operator = 'OR') {
if (Array.isArray(keywords)) {
const quoted = keywords.map(k => `"${k}"`);
return `(${quoted.join(` ${operator} `)})`;
}
return `"${keywords}"`;
}
/**
* Build complex query
*/
export function buildComplexQuery({
keywords = [],
labels = [],
timeRange = 'last7days',
status = null,
hasHighlights = null,
sortBy = 'saved-desc',
} = {}) {
const parts = [];
// Time range
if (timeRange === 'last24hrs' || timeRange === 'last7days' || timeRange === 'last30days') {
parts.push(`saved:${timeRange}`);
} else if (timeRange.includes('>') || timeRange.includes('<')) {
parts.push(timeRange);
}
// Keywords
if (keywords.length > 0) {
parts.push(buildKeywordQuery(keywords));
}
// Labels
if (labels.length > 0) {
parts.push(buildLabelQuery(labels));
}
// Status
if (status) {
if (status === 'inbox') parts.push('in:inbox');
else if (status === 'archived') parts.push('in:archive');
else if (status === 'unread') parts.push('is:unread');
else if (status === 'read') parts.push('is:read');
}
// Highlights
if (hasHighlights === true) parts.push('has:highlights');
else if (hasHighlights === false) parts.push('no:highlights');
// Sort
if (sortBy) parts.push(`sort:${sortBy}`);
return parts.join(' ');
}
/**
* Topic-based query builders
*/
export const TOPIC_QUERIES = {
ai: {
keywords: ['ai', 'artificial intelligence', 'machine learning', 'llm', 'language model', 'gpt', 'claude', 'neural network'],
description: 'AI and Machine Learning',
},
devops: {
keywords: ['devops', 'kubernetes', 'docker', 'ci/cd', 'cloud native', 'infrastructure', 'terraform'],
description: 'DevOps and Infrastructure',
},
programming: {
keywords: ['programming', 'coding', 'software engineering', 'algorithm', 'data structure', 'clean code'],
description: 'Programming and Software Engineering',
},
databases: {
keywords: ['database', 'sql', 'nosql', 'postgres', 'mongodb', 'redis', 'query optimization'],
description: 'Databases and Data Engineering',
},
web: {
keywords: ['web development', 'javascript', 'typescript', 'react', 'vue', 'frontend', 'backend'],
description: 'Web Development',
},
cloud: {
keywords: ['aws', 'azure', 'gcp', 'cloud computing', 'serverless', 'lambda'],
description: 'Cloud Computing',
},
startup: {
keywords: ['startup', 'founder', 'entrepreneurship', 'venture capital', 'product market fit', 'growth'],
description: 'Startups and Business',
},
security: {
keywords: ['security', 'cybersecurity', 'authentication', 'encryption', 'vulnerability'],
description: 'Security',
},
};
/**
* Build query for specific topic
*/
export function buildTopicQuery(topicKey, timeRange = 'last7days') {
const topic = TOPIC_QUERIES[topicKey];
if (!topic) {
throw new Error(`Unknown topic: ${topicKey}`);
}
return buildComplexQuery({
keywords: topic.keywords,
timeRange,
sortBy: 'saved-desc',
});
}
export default {
SEARCH_ARTICLES_FULL,
GET_ARTICLE_FULL,
GET_USER_PROFILE,
GET_LABELS,
SEARCH_WITH_CONTENT,
QUERY_PATTERNS,
buildDateRangeQuery,
buildLabelQuery,
buildKeywordQuery,
buildComplexQuery,
buildTopicQuery,
TOPIC_QUERIES,
};

View file

@ -0,0 +1,92 @@
{
"name": "omnivore-content-system",
"version": "1.0.0",
"description": "AI-powered content monetization system for Omnivore reading - generates blog posts, newsletters, and tracks trends",
"packageManager": "pnpm@10.14.0",
"type": "module",
"bin": {
"omc": "./dist/bin/omc.js"
},
"scripts": {
"build": "node esbuild.config.mjs",
"prebuild": "rm -rf dist",
"build:watch": "tsc -p tsconfig.json --watch",
"codegen": "graphql-codegen --config codegen.yml",
"codegen:watch": "graphql-codegen --config codegen.yml --watch",
"dev": "tsx watch index.ts",
"clean": "rm -rf dist",
"typecheck": "tsc --noEmit",
"test": "vitest run",
"test:watch": "vitest --watch",
"fetch": "corepack pnpm -s run build && node dist/bin/omc.js queue add --hours 24",
"analyze:auto": "corepack pnpm -s run build && node dist/bin/omc.js analyze auto --batch-size 5",
"analyze:retry": "corepack pnpm -s run build && node dist/bin/omc.js analyze retry --failed",
"report:corpus": "corepack pnpm -s run build && node dist/bin/omc.js report corpus"
},
"keywords": [
"omnivore",
"content",
"blog",
"newsletter",
"ai",
"claude",
"agent"
],
"author": "",
"license": "MIT",
"dependencies": {
"@anthropic-ai/claude-agent-sdk": "^0.1.1",
"@anthropic-ai/sdk": "^0.20.0",
"@graphql-typed-document-node/core": "^3.2.0",
"@oclif/core": "^3.26.6",
"@oclif/plugin-help": "^6.0.21",
"better-sqlite3": "^12.4.1",
"chalk": "^5.3.0",
"csv-parse": "^6.1.0",
"dotenv": "^16.4.0",
"glob": "^11.0.3",
"graphql": "^16.11.0",
"graphql-tag": "^2.12.6",
"gray-matter": "^4.0.3",
"markdown-it": "^14.0.0",
"node-fetch": "^3.3.2",
"p-limit": "^5.0.0"
},
"devDependencies": {
"@graphql-codegen/cli": "^6.0.0",
"@graphql-codegen/typed-document-node": "^6.0.1",
"@graphql-codegen/typescript": "^5.0.1",
"@graphql-codegen/typescript-operations": "^5.0.1",
"@types/better-sqlite3": "^7.6.13",
"@types/node": "^20.0.0",
"esbuild": "^0.19.0",
"nodemon": "^3.0.0",
"tsx": "^4.0.0",
"typescript": "^5.4.0",
"vitest": "^1.0.0"
},
"engines": {
"node": ">=18.0.0"
},
"oclif": {
"commands": "./dist/src/commands",
"bin": "omc",
"dirname": "omc",
"plugins": [
"@oclif/plugin-help"
],
"topicSeparator": " ",
"additionalHelpFlags": [
"-h"
],
"additionalVersionFlags": [
"-v"
]
},
"pnpm": {
"onlyBuiltDependencies": [
"better-sqlite3",
"esbuild"
]
}
}

View file

@ -0,0 +1,47 @@
#!/usr/bin/env node
import { execSync } from 'child_process';
import { getHighlights, deleteHighlight, getMe } from '../lib/omnivore/client.js';
import Database from 'better-sqlite3';
const articleId = process.argv[2];
if (!articleId) {
console.error('Usage: node scripts/cleanup-duplicate-notes.mjs <article-id>');
process.exit(1);
}
// Use CLI to get article slug
const db = Database(process.env.DATABASE_PATH || 'data/omnivore-content.db');
const job = db.prepare('SELECT article_slug FROM analysis_queue WHERE article_id = ?').get(articleId);
db.close();
if (!job) {
console.error(`Article not found: ${articleId}`);
process.exit(1);
}
// Get username from Omnivore API
const user = await getMe();
const username = user.profile.username;
const highlights = await getHighlights(job.article_slug, username);
const notes = highlights.filter(h => h.type === 'NOTE');
console.log(`Found ${notes.length} notes total`);
if (notes.length > 0) {
console.log(`\nFirst note content (first 500 chars):`);
console.log(notes[0].annotation?.substring(0, 500));
}
const analysisNotes = notes.filter(h => h.annotation?.includes('_Analysis generated by Omnivore Content System_'));
const duplicates = notes.filter(h => !h.annotation?.includes('_Analysis generated by Omnivore Content System_'));
console.log(`\nAnalysis notes with marker: ${analysisNotes.length}`);
console.log(`Duplicate notes to delete: ${duplicates.length}`);
for (const note of duplicates) {
console.log(`Deleting note ${note.id}...`);
await deleteHighlight(note.id);
}
console.log('✓ Cleanup complete');

View file

@ -0,0 +1,25 @@
#!/usr/bin/env bash
set -euo pipefail
REPO_DIR="${REPO_DIR:-/Volumes/devel/personal/omnivore-content-system}"
BATCH_SIZE="${BATCH_SIZE:-5}"
HOURS="${HOURS:-24}"
cd "$REPO_DIR"
mkdir -p content/corpus-reports
# Ensure dependencies/build are available. Prefer Corepack-managed pnpm so native modules
# (better-sqlite3) are built for the active Node version.
if command -v corepack >/dev/null 2>&1; then
corepack install >/dev/null 2>&1 || true
fi
corepack pnpm -s run build
node dist/bin/omc.js queue add --hours "$HOURS"
node dist/bin/omc.js analyze auto --batch-size "$BATCH_SIZE"
node dist/bin/omc.js analyze retry --failed
node dist/bin/omc.js report corpus > "content/corpus-reports/$(date +%Y-%m-%d)-daily.md"

View file

@ -0,0 +1,85 @@
import { readFileSync, existsSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import type { AnalysisRequest, ContentAnalysis } from '@omc-types/analysis.js';
import { analyzeArticleWithCodexCli } from '@lib/ai/codex-cli-client.js';
export interface AnalyzeArticleInput extends AnalysisRequest {
articleId: string;
}
export interface ContentAnalyzerOptions {
promptPath?: string;
model?: string;
timeoutMs?: number;
maxAttempts?: number;
}
export class ContentAnalyzer {
private readonly prompt: string;
private readonly options: ContentAnalyzerOptions;
constructor(options: ContentAnalyzerOptions = {}) {
this.options = options;
this.prompt = loadPromptText(options.promptPath);
}
async analyzeArticle(input: AnalyzeArticleInput): Promise<ContentAnalysis> {
const { articleId, ...request } = input;
const { analysis } = await analyzeArticleWithCodexCli(request, this.prompt, {
model: this.options.model,
timeoutMs: this.options.timeoutMs,
maxAttempts: this.options.maxAttempts,
});
return normalizeAnalysis(articleId, analysis);
}
}
function loadPromptText(explicitPath?: string): string {
const candidates = [
explicitPath,
process.env.OMC_ANALYZE_PROMPT_PATH,
promptPathNearThisFile(),
join(process.cwd(), 'src/analysis/prompts/analyze-article.md'),
]
.filter(Boolean) as string[];
for (const p of candidates) {
if (existsSync(p)) return readFileSync(p, 'utf-8');
}
throw new Error('Could not locate analyze-article.md prompt (set OMC_ANALYZE_PROMPT_PATH)');
}
function promptPathNearThisFile(): string {
const here = dirname(fileURLToPath(import.meta.url));
return join(here, 'prompts', 'analyze-article.md');
}
function normalizeAnalysis(articleId: string, analysis: ContentAnalysis): ContentAnalysis {
return {
...fillDefaults(analysis),
articleId,
analyzedAt: analysis.analyzedAt || new Date().toISOString(),
};
}
function fillDefaults(analysis: ContentAnalysis): ContentAnalysis {
return {
...analysis,
topics: analysis.topics ?? [],
topicScores: analysis.topicScores ?? {},
keyPoints: analysis.keyPoints ?? [],
contentType: analysis.contentType || 'N/A',
problemStatement: analysis.problemStatement || 'N/A',
audienceLevel: analysis.audienceLevel || 'N/A',
technologiesMentioned: analysis.technologiesMentioned ?? [],
companiesMentioned: analysis.companiesMentioned ?? [],
peopleMentioned: analysis.peopleMentioned ?? [],
conceptsExplained: analysis.conceptsExplained ?? [],
relatedTechnologies: analysis.relatedTechnologies ?? [],
useCases: analysis.useCases ?? [],
targetKeywords: analysis.targetKeywords ?? [],
searchQuestions: analysis.searchQuestions ?? [],
githubRepo: analysis.githubRepo || 'N/A',
releaseInfo: analysis.releaseInfo || 'N/A',
};
}

View file

@ -0,0 +1,160 @@
import { existsSync, mkdirSync, writeFileSync, unlinkSync } from 'node:fs';
import { join } from 'node:path';
import type { AnalysisJob, AnalysisQueueRepository } from '@storage/AnalysisQueueRepository.js';
import type { ContentAnalyzer, AnalyzeArticleInput } from '@analysis/ContentAnalyzer.js';
import type { AnalysisWriter } from '@storage/AnalysisWriter.js';
import type { ContentAnalysis } from '@omc-types/analysis.js';
import type { OmnivoreArticle } from '@omc-types/omnivore.js';
import { getArticle } from '@lib/omnivore/client.js';
import type { AnalysisJsonlRecord } from '@storage/AnalysisWriter.js';
export interface AnalyzeAutoRunnerDeps {
jobs: AnalysisJob[];
username: string;
analyzer: ContentAnalyzer;
writer: AnalysisWriter;
repo: AnalysisQueueRepository;
keepTemp: boolean;
jsonlPath?: string;
}
export interface AnalyzeAutoRunnerResult {
saved: number;
failed: number;
skipped: number;
}
export async function runAnalyzeAuto(deps: AnalyzeAutoRunnerDeps): Promise<AnalyzeAutoRunnerResult> {
ensureTempDir();
const totals = { saved: 0, failed: 0, skipped: 0 };
for (const job of deps.jobs) {
const outcome = await runOne(job, deps);
totals[outcome]++;
}
return totals;
}
async function runOne(job: AnalysisJob, deps: AnalyzeAutoRunnerDeps): Promise<'saved' | 'failed' | 'skipped'> {
const article = await fetchArticle(job.articleSlug, deps.username);
if (!article) return markFailure(deps.repo, job.articleId, 'Failed to fetch article from Omnivore');
if (!article.content) return markFailure(deps.repo, job.articleId, 'Omnivore returned no content for article');
const input = buildAnalyzeInput(job.articleId, article);
const analysis = await deps.analyzer.analyzeArticle(input);
const tempPath = writeTempRecord(job, article, deps.username, analysis);
return await saveAndMarkComplete(job, article, analysis, tempPath, deps);
}
async function fetchArticle(slug: string, username: string): Promise<OmnivoreArticle | null> {
try {
const result = await getArticle(slug, username);
if (result.errorCodes?.length) return null;
return result.article ?? null;
} catch {
return null;
}
}
function buildAnalyzeInput(articleId: string, article: OmnivoreArticle): AnalyzeArticleInput {
const content = String(article.content ?? '');
return {
articleId,
title: article.title,
author: article.author ?? undefined,
url: article.url,
content,
wordCount: estimateWordCount(content),
highlights: (article.highlights ?? []).map((h) => ({
quote: h.quote,
annotation: h.annotation ?? undefined,
})),
publishedAt: article.publishedAt ?? undefined,
};
}
function writeTempRecord(job: AnalysisJob, article: OmnivoreArticle, username: string, analysis: ContentAnalysis): string {
const path = join('temp', `${job.articleSlug}.jsonl`);
const record = {
articleId: job.articleId,
articleSlug: job.articleSlug,
username,
articleUrl: article.url,
articleTitle: article.title,
savedAt: article.savedAt,
publishedAt: article.publishedAt ?? null,
updatedAt: article.updatedAt ?? null,
analysis,
};
writeFileSync(path, JSON.stringify(record) + '\n', 'utf-8');
return path;
}
async function saveAndMarkComplete(
job: AnalysisJob,
article: OmnivoreArticle,
analysis: ContentAnalysis,
tempPath: string,
deps: AnalyzeAutoRunnerDeps
): Promise<'saved' | 'failed'> {
try {
const mdPath = await deps.writer.write(job.articleId, article.url, article.title, article.savedAt, analysis, job.articleSlug);
deps.repo.storeAnalysis(job.articleId, article.publishedAt ?? null, article.updatedAt ?? null, JSON.stringify(analysis), mdPath);
if (deps.jsonlPath) {
await deps.writer.appendToJsonl(deps.jsonlPath, buildJsonlRecord(job, article, analysis, mdPath));
}
if (!deps.keepTemp) unlinkSync(tempPath);
return 'saved';
} catch (err: unknown) {
const message = err instanceof Error ? err.message : String(err);
deps.repo.markFailed(job.articleId, `Save failed: ${message}`);
return 'failed';
}
}
function buildJsonlRecord(job: AnalysisJob, article: OmnivoreArticle, analysis: ContentAnalysis, markdownPath: string): AnalysisJsonlRecord {
return {
articleId: job.articleId,
articleSlug: job.articleSlug,
articleUrl: article.url,
articleTitle: article.title,
savedAt: article.savedAt,
publishedAt: article.publishedAt ?? null,
updatedAt: article.updatedAt ?? null,
markdownPath,
analyzedAt: analysis.analyzedAt,
topics: analysis.topics,
topicScores: analysis.topicScores,
sentiment: analysis.sentiment,
summary: analysis.summary,
keyPoints: analysis.keyPoints,
monetizationAngle: analysis.monetizationAngle,
contentType: analysis.contentType,
problemStatement: analysis.problemStatement,
audienceLevel: analysis.audienceLevel,
technologiesMentioned: analysis.technologiesMentioned,
companiesMentioned: analysis.companiesMentioned,
peopleMentioned: analysis.peopleMentioned,
conceptsExplained: analysis.conceptsExplained,
relatedTechnologies: analysis.relatedTechnologies,
useCases: analysis.useCases,
targetKeywords: analysis.targetKeywords,
searchQuestions: analysis.searchQuestions,
githubRepo: analysis.githubRepo,
releaseInfo: analysis.releaseInfo,
};
}
function markFailure(repo: AnalysisQueueRepository, articleId: string, message: string): 'failed' {
repo.markFailed(articleId, message);
return 'failed';
}
function ensureTempDir(): void {
if (!existsSync('temp')) mkdirSync('temp', { recursive: true });
}
function estimateWordCount(text: string): number {
const trimmed = text.trim();
if (!trimmed) return 0;
return trimmed.split(/\s+/).length;
}

View file

@ -0,0 +1,129 @@
# Article Content Analysis Prompt
You are an elite content strategist and knowledge graph architect specializing in evidence-based article analysis for content monetization. Your dual expertise lies in extracting monetizable insights while building structured metadata for corpus-wide knowledge linking.
## Primary Directive: Anti-Hallucination Rule
This rule applies to EVERY field you extract - it is non-negotiable:
- If information is NOT explicitly present in the article, use "N/A" (strings) or ["N/A"] (arrays)
- DO NOT infer, guess, estimate, or use your training data
- Only extract what is directly stated in the article text
- When uncertain, always use "N/A" - data integrity trumps completeness
**Why this matters**: Hallucinated data corrupts the knowledge graph and misleads content strategy. "N/A" enables accurate corpus analysis; invented data causes systematic failures.
## Task
Analyze the provided article content and extract structured insights in JSON format. You are PURELY ADDITIVE - you will receive article metadata and ONLY add the `analysis` field.
## Analysis Fields
### Core Analysis (Always Required)
**topics** (array, 2-5 items): Main topics from approved categories:
- AI & Machine Learning, Developer Tools, Software Engineering, DevOps & Infrastructure
- Databases, Cloud, Startups & Business, Security
**topicScores** (object): Confidence 0-1 for each topic
- 0.90-1.0: Core focus
- 0.70-0.89: Significant discussion
- 0.50-0.69: Mentioned but not central
- <0.50: Exclude
**summary** (string, 2-3 sentences): Focus on "so what?" - why this matters to developers/tech professionals. **Chain-of-thought**: (1) What's the main point? (2) Why does it matter? (3) What's the implication?
**keyPoints** (array, 3-5 items): Actionable insights, surprising facts, or important takeaways readers can learn or apply. Not generic observations.
**sentiment** (string): "positive" (optimistic, solutions-focused), "neutral" (informative, balanced), "negative" (critical, problems-focused)
**monetizationAngle** (string): Specific content opportunity - comparison post, tutorial, weekly roundup, deep dive, contrarian take
### Content Planning (Evidence-Based)
**contentType** (string): Open-ended description of article type - "getting started guide", "comparison review", "release announcement", "case study", "technical deep dive", "news article", "tool documentation", etc. Use "N/A" if unclear.
**problemStatement** (string): What specific problem does article address? Use "N/A" if not explicitly stated.
**audienceLevel** (string): Based on technical depth - "beginner", "intermediate", "advanced", or "N/A" if unclear
### Knowledge Graph (Corpus Linking)
Extract ONLY what's explicitly mentioned - these fields enable future corpus-wide analysis and RAG:
**technologiesMentioned** (array): Specific tools, frameworks, languages named (e.g., ["Ray", "Python", "Dask"])
**companiesMentioned** (array): Organizations, companies mentioned (e.g., ["Anyscale", "OpenAI"])
**peopleMentioned** (array): Notable people if relevant (founders, researchers, etc.), else ["N/A"]
**conceptsExplained** (array): Technical concepts or techniques explained (e.g., ["distributed computing", "actor model"])
**relatedTechnologies** (array): Technologies this compares to or builds upon (e.g., ["Spark", "Dask"])
**useCases** (array): Specific scenarios described (e.g., ["ML training", "data processing"])
### SEO Signals (Article-Based Only)
**targetKeywords** (array): Keywords that appear emphasized/repeated in article, or ["N/A"] if none obvious
**searchQuestions** (array): If article answers specific questions, list them, else ["N/A"]
### Trend Signals (If Present)
**githubRepo** (string): GitHub URL if article is about/links to repo, else "N/A"
**releaseInfo** (string): Version/release info if article announces one, else "N/A"
## Output Format
**Input stub (what you receive):**
```json
{"articleId":"...","articleSlug":"...","username":"...","articleUrl":"...","articleTitle":"...","savedAt":"...","publishedAt":"...","updatedAt":"..."}
```
**Output (what you return):**
```json
{"articleId":"...","articleSlug":"...","username":"...","articleUrl":"...","articleTitle":"...","savedAt":"...","publishedAt":"...","updatedAt":"...","analysis":{"topics":[],"topicScores":{},"summary":"","keyPoints":[],"sentiment":"","monetizationAngle":"","contentType":"","problemStatement":"","audienceLevel":"","technologiesMentioned":[],"companiesMentioned":[],"peopleMentioned":[],"conceptsExplained":[],"relatedTechnologies":[],"useCases":[],"targetKeywords":[],"searchQuestions":[],"githubRepo":"","releaseInfo":"","analyzedAt":"ISO-timestamp"}}
```
**What you do**: Add the `analysis` field ONLY. All other fields are pass-through from stub.
## Decision-Making Framework
**For each field, ask**:
1. Is this information explicitly stated in the article?
2. If yes: Extract verbatim or in standardized form
3. If no: Use "N/A" or ["N/A"]
4. If uncertain: Default to "N/A"
**Quality checks**:
- All topic labels from approved categories?
- Summary captures "so what?", not just "what?"
- Key points actionable and specific?
- Knowledge graph fields evidence-based?
- No hallucinated dates, companies, or technologies?
You are precise, evidence-driven, and committed to data integrity over completeness.
## Instructions for LLM Integration
When using this prompt:
1. **Load article content**: Fetch the full article text from your content source
2. **Provide article metadata**: Pass in the stub JSON with article metadata
3. **Request analysis**: Ask the LLM to analyze the content and return the complete JSON with added `analysis` field
4. **Validate output**: Ensure the returned JSON is valid and contains all required fields
5. **Save result**: Write the enriched JSON to your output destination
**Example request format**:
```
Using the prompt from analyze-article.md, analyze this article:
[Article content here]
Article metadata:
[Stub JSON here]
Return the complete JSON with analysis field added.
```

View file

@ -0,0 +1,147 @@
# Article Content Analysis Prompt
You are analyzing an article for content monetization opportunities. Extract structured insights that can be used to create valuable blog posts, newsletters, and social media content.
⚠️ **CRITICAL ANTI-HALLUCINATION RULE**
This rule applies to EVERY field in your analysis:
- If information is NOT explicitly present in the article content, use "N/A" (for strings) or ["N/A"] (for arrays)
- DO NOT infer, guess, estimate, or use knowledge from your training data
- Only extract information that is directly stated in the article text
- When in doubt, use "N/A" - it's better to have missing data than invented data
## Article Details
**Title**: {{title}}
**Author**: {{author}}
**URL**: {{url}}
**Word Count**: {{wordCount}}
**Published**: {{publishedAt}}
**Content**:
{{content}}
{{#if highlights}}
**User Highlights**:
{{#each highlights}}
- "{{quote}}"{{#if annotation}} — Note: {{annotation}}{{/if}}
{{/each}}
{{/if}}
## Analysis Task
Extract the following information and return as JSON:
```json
{
"topics": ["topic1", "topic2"],
"topicScores": {
"topic1": 0.95,
"topic2": 0.88
},
"summary": "2-3 sentence summary capturing main points and why this matters",
"keyPoints": [
"First key takeaway",
"Second key takeaway",
"Third key takeaway"
],
"sentiment": "positive|neutral|negative",
"monetizationAngle": "How to turn this into valuable content",
"contentType": "tutorial|comparison|announcement|case-study|deep-dive|news|tool-review|other",
"publishedDate": "2024-03-15 OR N/A",
"updatedDate": "2024-03-20 OR N/A",
"problemStatement": "What problem does this article address OR N/A",
"audienceLevel": "beginner|intermediate|advanced OR N/A",
"technologiesMentioned": ["Ray", "Python", "Dask"],
"companiesMentioned": ["Anyscale", "OpenAI"],
"peopleMentioned": ["author name if notable"],
"conceptsExplained": ["distributed computing", "actor model"],
"relatedTechnologies": ["Spark", "Dask"],
"useCases": ["ML training", "data processing"],
"targetKeywords": ["distributed python", "ray framework"],
"searchQuestions": ["how to scale python workloads"],
"githubRepo": "https://github.com/org/repo OR N/A",
"releaseInfo": "v2.0 released OR N/A"
}
```
## Guidelines
### Topics (2-5 main topics)
Prioritize these categories that align with the content strategy:
- **AI & Machine Learning**: ai, machine-learning, llm, neural-networks, training, inference
- **Developer Tools**: developer-tools, ide, debugging, testing, ci-cd
- **Software Engineering**: software-engineering, architecture, design-patterns, code-quality
- **DevOps & Infrastructure**: devops, kubernetes, docker, cloud-native, infrastructure
- **Databases**: databases, sql, nosql, data-modeling, performance
- **Cloud**: cloud, aws, azure, gcp, serverless
- **Startups & Business**: startups, product, growth, monetization, business-strategy
- **Security**: security, auth, encryption, vulnerabilities
### Topic Scores (0-1 confidence)
- 0.90-1.0: Core focus of article
- 0.70-0.89: Significant discussion
- 0.50-0.69: Mentioned/relevant but not central
- Below 0.50: Don't include
### Summary
Focus on the "so what?" - why does this matter to developers/tech professionals?
### Key Points
Extract actionable insights, surprising facts, or important takeaways. Focus on what readers can learn or apply.
### Sentiment
- **positive**: Optimistic, solutions-focused, celebrates innovation
- **neutral**: Informative, balanced, educational
- **negative**: Critical, identifies problems, warnings
### Monetization Angle
Identify content opportunities:
- "Compare with 2-3 similar tools for comparison post"
- "Tutorial on implementing this technique"
- "Weekly roundup: combine with X other articles on topic Y"
- "Deep dive: expand on concept Z with 10+ articles"
- "Contrarian take: why this approach has limitations"
### Content Type
Describe what kind of article this is using open-ended language:
- "getting started guide", "comparison review", "release announcement", "case study", "technical deep dive", "news article", "tool documentation", etc.
- Use N/A if unclear from content
### Dates
- Extract publishedDate and updatedDate if shown in article (use ISO format YYYY-MM-DD)
- Use "N/A" if dates not present
### Problem Statement
- What specific problem does the article address?
- Use N/A if not explicitly stated
### Audience Level
- Based on technical depth: "beginner", "intermediate", "advanced"
- Use N/A if unclear
### Knowledge Graph Fields
Extract ONLY what's explicitly mentioned:
- **technologiesMentioned**: Specific tools, frameworks, languages named in article
- **companiesMentioned**: Organizations, companies mentioned
- **peopleMentioned**: Notable people mentioned (founders, researchers, etc.) - use N/A if none
- **conceptsExplained**: Technical concepts or techniques explained
- **relatedTechnologies**: Technologies this compares to or builds upon
- **useCases**: Specific use cases or scenarios described
### SEO Signals
- **targetKeywords**: Keywords that appear emphasized/repeated in article (N/A if none obvious)
- **searchQuestions**: If article answers specific questions, list them (N/A if not question-format)
### Trend Signals
- **githubRepo**: If article is about or links to a GitHub repository
- **releaseInfo**: If article announces a new version or release
- Use N/A for both if not applicable
## Output Format
Return ONLY valid JSON matching the schema above. No markdown code blocks, no additional text.

View file

@ -0,0 +1,128 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { withDatabase } from '@lib/cli/database.js';
import { fetchUsername } from '@lib/cli/graphql.js';
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { displayQueueStats } from '@lib/cli/queue-display.js';
import { AnalysisWriter } from '@storage/AnalysisWriter.js';
import type { AnalysisJob, QueueStats, AnalysisQueueRepository } from '@storage/AnalysisQueueRepository.js';
import { ContentAnalyzer } from '@analysis/ContentAnalyzer.js';
import { runAnalyzeAuto } from '@analysis/analyze-auto-runner.js';
interface AnalyzeAutoFlags {
'batch-size': number;
'article-id'?: string;
all: boolean;
model?: string;
'timeout-ms': number;
'keep-temp': boolean;
jsonl: boolean;
'jsonl-path': string;
json: boolean;
}
export default class AnalyzeAuto extends BaseCommand {
static override description = 'Analyze queued articles end-to-end (non-interactive)';
static override examples = [
'$ omc analyze auto --batch-size 5',
'$ omc analyze auto --article-id abc123',
'$ omc analyze auto --json',
];
static override flags = {
'batch-size': Flags.integer({
char: 'b',
description: 'Number of articles to analyze',
default: 5,
}),
'article-id': Flags.string({
description: 'Analyze specific article by ID',
exclusive: ['all'],
}),
all: Flags.boolean({
description: 'Analyze pending + failed jobs (up to batch size)',
exclusive: ['article-id'],
}),
model: Flags.string({
description: 'Codex model override (passed to codex exec -m)',
}),
'timeout-ms': Flags.integer({
description: 'Timeout per article analysis (milliseconds)',
default: 10 * 60 * 1000,
}),
'keep-temp': Flags.boolean({
description: 'Keep temp/*.jsonl files after successful save',
default: false,
}),
jsonl: Flags.boolean({
description: 'Append each completed analysis to content/analysis/analyses.jsonl',
default: false,
}),
'jsonl-path': Flags.string({
description: 'Path to JSONL output (requires --jsonl)',
default: 'content/analysis/analyses.jsonl',
}),
json: jsonFlag(),
};
protected async execute(flags: AnalyzeAutoFlags): Promise<void> {
return await withDatabase(async ({ repo }) => {
const jobs = this.selectJobs(repo, flags);
if (jobs.length === 0) return this.outputEmpty(flags.json);
if (!flags.json) {
this.log(formatHeader('Analysis Queue Status'));
displayQueueStats(repo.getStats());
}
this.markJobsInProgress(repo, jobs);
const username = await fetchUsername();
const analyzer = new ContentAnalyzer({ model: flags.model, timeoutMs: flags['timeout-ms'] });
const writer = new AnalysisWriter({ outputDir: 'content/analysis' });
const jsonlPath = flags.jsonl ? flags['jsonl-path'] : undefined;
const results = await runAnalyzeAuto({ jobs, username, analyzer, writer, repo, keepTemp: flags['keep-temp'], jsonlPath });
this.outputResults(results, repo.getStats(), flags.json);
});
}
private selectJobs(repo: AnalysisQueueRepository, flags: AnalyzeAutoFlags): AnalysisJob[] {
if (flags['article-id']) {
const job = repo.getByArticleId(flags['article-id']);
if (!job) throw new Error(`Article with ID ${flags['article-id']} not found`);
return [job];
}
if (flags.all) {
const combined = [...repo.getByStatus('pending'), ...repo.getByStatus('failed')];
return combined.slice(0, flags['batch-size'] ?? combined.length);
}
return repo.getPending(flags['batch-size']);
}
private outputEmpty(jsonMode: boolean): void {
if (jsonMode) this.log(JSON.stringify({ saved: 0, failed: 0, skipped: 0 }, null, 2));
else this.log('No jobs to process');
}
private markJobsInProgress(repo: AnalysisQueueRepository, jobs: AnalysisJob[]): void {
for (const job of jobs) repo.markInProgress(job.articleId);
}
private outputResults(results: { saved: number; failed: number; skipped: number }, stats: QueueStats, jsonMode: boolean): void {
if (jsonMode) {
this.log(JSON.stringify({ ...results, stats }, null, 2));
return;
}
this.log('');
this.log(formatSuccess(`Saved ${results.saved} article(s)`));
if (results.failed) this.log(`Failed: ${results.failed}`);
if (results.skipped) this.log(`Skipped: ${results.skipped}`);
this.log('');
this.log(formatHeader('Updated Queue Status'));
displayQueueStats(stats);
}
}

View file

@ -0,0 +1,212 @@
import { Flags } from '@oclif/core';
import { readFileSync, unlinkSync, existsSync } from 'node:fs';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
import { parseJsonSafely } from '@lib/cli/command-utils.js';
import { displayQueueStats } from '@lib/cli/queue-display.js';
import { AnalysisWriter } from '@storage/AnalysisWriter.js';
import { glob } from 'glob';
import type { ContentAnalysis } from '@omc-types/analysis.js';
import type { AnalysisJsonlRecord } from '@storage/AnalysisWriter.js';
import type { AnalysisQueueRepository, QueueStats } from '@storage/AnalysisQueueRepository.js';
interface EnrichedResult {
articleId: string;
articleSlug: string;
username: string;
articleUrl: string;
articleTitle: string;
savedAt: string;
publishedAt: string | null;
updatedAt: string | null;
analysis?: ContentAnalysis;
}
interface AnalyzeCompleteFlags {
'keep-temp': boolean;
jsonl: boolean;
'jsonl-path': string;
json: boolean;
}
interface SaveResults {
saved: number;
failed: number;
stats: QueueStats;
}
function buildJsonlRecord(result: EnrichedResult, analysis: ContentAnalysis, markdownPath: string): AnalysisJsonlRecord {
return {
articleId: result.articleId,
articleSlug: result.articleSlug,
articleUrl: result.articleUrl,
articleTitle: result.articleTitle,
savedAt: result.savedAt,
publishedAt: result.publishedAt,
updatedAt: result.updatedAt,
markdownPath,
analyzedAt: analysis.analyzedAt,
topics: analysis.topics,
topicScores: analysis.topicScores,
sentiment: analysis.sentiment,
summary: analysis.summary,
keyPoints: analysis.keyPoints,
monetizationAngle: analysis.monetizationAngle,
contentType: analysis.contentType,
problemStatement: analysis.problemStatement,
audienceLevel: analysis.audienceLevel,
technologiesMentioned: analysis.technologiesMentioned,
companiesMentioned: analysis.companiesMentioned,
peopleMentioned: analysis.peopleMentioned,
conceptsExplained: analysis.conceptsExplained,
relatedTechnologies: analysis.relatedTechnologies,
useCases: analysis.useCases,
targetKeywords: analysis.targetKeywords,
searchQuestions: analysis.searchQuestions,
githubRepo: analysis.githubRepo,
releaseInfo: analysis.releaseInfo,
};
}
/**
* OCLIF command: omc analyze complete
* Saves agent-analyzed results from temp/ to database and marks jobs complete.
* AIDEV-NOTE: analysis-complete - final step in analyze workflow after agents finish
*/
export default class AnalyzeComplete extends BaseCommand {
static override description = 'Save analyzed results and mark jobs complete';
static override examples = [
'$ omc analyze complete',
'$ omc analyze complete --keep-temp',
];
static override flags = {
'keep-temp': Flags.boolean({
description: 'Keep temp files after saving',
default: false,
}),
jsonl: Flags.boolean({
description: 'Append each completed analysis to content/analysis/analyses.jsonl',
default: false,
}),
'jsonl-path': Flags.string({
description: 'Path to JSONL output (requires --jsonl)',
default: 'content/analysis/analyses.jsonl',
}),
json: jsonFlag(),
};
protected async execute(flags: AnalyzeCompleteFlags): Promise<void> {
this.validateFlags(flags);
const files = await this.findAnalyzedFiles();
if (files.length === 0) {
this.log('No analyzed files found in temp/');
return;
}
await withDatabase(async ({ repo }) => {
const writer = new AnalysisWriter({ outputDir: 'content/analysis' });
const results = await this.saveResults(files, repo, writer, {
keepTemp: flags['keep-temp'],
writeJsonl: flags.jsonl,
jsonlPath: flags['jsonl-path'],
});
this.displayResults(results, flags.json);
});
}
private validateFlags(flags: AnalyzeCompleteFlags): void {
if (flags['jsonl-path'] !== 'content/analysis/analyses.jsonl' && !flags.jsonl) {
throw new Error('--jsonl-path requires --jsonl');
}
}
private async findAnalyzedFiles(): Promise<string[]> {
const allFiles = await glob('temp/*.jsonl');
return allFiles.filter((file) => {
if (!existsSync(file)) return false;
const content = readFileSync(file, 'utf-8').trim();
if (!content) return false;
const data = parseJsonSafely<EnrichedResult>(content);
return data?.analysis !== undefined;
});
}
private async saveResults(
files: string[],
repo: AnalysisQueueRepository,
writer: AnalysisWriter,
options: { keepTemp: boolean; writeJsonl: boolean; jsonlPath: string }
): Promise<SaveResults> {
let saved = 0;
let failed = 0;
for (const file of files) {
const success = await this.saveFile(file, repo, writer, options);
if (success) saved++;
else failed++;
}
return { saved, failed, stats: repo.getStats() };
}
private async saveFile(
file: string,
repo: AnalysisQueueRepository,
writer: AnalysisWriter,
options: { keepTemp: boolean; writeJsonl: boolean; jsonlPath: string }
): Promise<boolean> {
const content = readFileSync(file, 'utf-8');
const result = parseJsonSafely<EnrichedResult>(content);
if (!result?.analysis) {
this.warn(`Skipping ${file}: no analysis field`);
return false;
}
const { articleId, articleUrl, articleTitle, savedAt, publishedAt, updatedAt, analysis } = result;
try {
const mdPath = await writer.write(articleId, articleUrl, articleTitle, savedAt, analysis, result.articleSlug);
repo.storeAnalysis(articleId, publishedAt, updatedAt, JSON.stringify(analysis), mdPath);
if (options.writeJsonl) {
await writer.appendToJsonl(options.jsonlPath, buildJsonlRecord(result, analysis, mdPath));
}
if (!options.keepTemp) unlinkSync(file);
this.log(`${this.truncate(articleTitle, 60)}`);
return true;
} catch (err: unknown) {
const message = err instanceof Error ? err.message : String(err);
repo.markFailed(articleId, `Save failed: ${message}`);
this.error(`${this.truncate(articleTitle, 60)} - ${message}`);
return false;
}
}
private displayResults(results: SaveResults, jsonMode: boolean): void {
if (jsonMode) {
this.log(JSON.stringify(results, null, 2));
return;
}
this.log('');
this.log(formatSuccess(`Saved ${results.saved} article(s)`));
if (results.failed > 0) {
this.log(`Failed: ${results.failed}`);
}
this.log('');
this.log(formatHeader('Updated Queue Status'));
displayQueueStats(results.stats);
}
private truncate(text: string, length: number): string {
return text.length > length ? text.substring(0, length) + '...' : text;
}
}

View file

@ -0,0 +1,67 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatSuccess } from '@lib/cli/formatters.js';
/**
* Retry failed article analyses by resetting them to pending status.
* AIDEV-NOTE: CLI command for retrying failed analyses
*/
export default class AnalyzeRetry extends BaseCommand {
static override description = 'Retry failed article analyses';
static override examples = [
'$ omc analyze retry --failed',
'$ omc analyze retry --article-id <id>',
];
static override flags = {
failed: Flags.boolean({
description: 'Retry all failed articles',
default: false,
exclusive: ['article-id'],
}),
'article-id': Flags.string({
description: 'Retry specific article by ID',
exclusive: ['failed'],
}),
json: jsonFlag(),
};
async execute(flags: { failed: boolean; 'article-id'?: string; json: boolean }): Promise<void> {
await withDatabase(async ({ repo }) => {
const count = flags.failed
? this.retryAllFailed(repo)
: this.retrySingleArticle(repo, flags['article-id']);
if (flags.json) {
this.log(JSON.stringify({ retried: count }));
} else {
this.log(formatSuccess(`Reset ${count} failed article${count !== 1 ? 's' : ''} to pending`));
}
});
}
private retryAllFailed(repo: {
getFailed: () => Array<{ articleId: string }>;
resetToPending: (articleId: string) => void;
}): number {
const failed = repo.getFailed();
for (const job of failed) {
repo.resetToPending(job.articleId);
}
return failed.length;
}
private retrySingleArticle(
repo: { resetToPending: (articleId: string) => void },
articleId: string | undefined
): number {
if (!articleId) {
throw new Error('Either use --failed flag or provide --article-id');
}
repo.resetToPending(articleId);
return 1;
}
}

View file

@ -0,0 +1,186 @@
import { Flags } from '@oclif/core';
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { fetchUsername } from '@lib/cli/graphql.js';
import { formatHeader, formatSuccess, formatDivider } from '@lib/cli/formatters.js';
import { displayQueueStats } from '@lib/cli/queue-display.js';
import { getArticle } from '@lib/omnivore/client.js';
import type { AnalysisJob } from '@storage/AnalysisQueueRepository.js';
/**
* OCLIF command: omc analyze run
* Wraps parallel-analyze.ts workflow for queue-based article analysis.
*
* AIDEV-NOTE: tracking-coordination - prepares batch for parallel agent execution
* AIDEV-NOTE: stub-file-creation - fetches full article metadata before analysis
*/
export default class AnalyzeRun extends BaseCommand {
static override description = 'Run parallel content analysis on queued articles';
static override examples = [
'$ omc analyze run',
'$ omc analyze run --batch-size 10',
'$ omc analyze run --article-id abc123',
'$ omc analyze run --all',
'$ omc analyze run --json',
];
static override flags = {
'batch-size': Flags.integer({
char: 'b',
description: 'Number of articles to analyze in parallel',
default: 5,
}),
'article-id': Flags.string({
description: 'Analyze specific article by ID',
exclusive: ['all'],
}),
all: Flags.boolean({
description: 'Analyze all articles regardless of status',
exclusive: ['article-id'],
}),
json: jsonFlag(),
};
protected async execute(flags: Record<string, any>): Promise<void> {
return await withDatabase(async ({ repo }) => {
const jobs = this.selectJobs(repo, flags);
if (jobs.length === 0) {
if (!flags.json) this.log('No jobs to process');
return;
}
if (!flags.json) {
const stats = repo.getStats();
this.log(formatHeader('Analysis Queue Status'));
displayQueueStats(stats);
}
this.markJobsInProgress(repo, jobs);
const username = await fetchUsername();
const agentParams = await this.processJobs(jobs, username, flags.json);
this.outputResults(agentParams, jobs.length, flags.json);
});
}
// AIDEV-NOTE: job-selection - handles --article-id, --all, or default pending
private selectJobs(repo: any, flags: Record<string, any>): AnalysisJob[] {
if (flags['article-id']) {
const job = repo.getByArticleId(flags['article-id']);
if (!job) {
throw new Error(`Article with ID ${flags['article-id']} not found`);
}
return [job];
}
if (flags.all) {
// Get all pending jobs (don't include in_progress to avoid overwriting)
const pending = repo.getByStatus('pending');
const failed = repo.getByStatus('failed');
const combined = [...pending, ...failed];
return flags['batch-size'] ? combined.slice(0, flags['batch-size']) : combined;
}
return repo.getPending(flags['batch-size']);
}
// AIDEV-NOTE: tracking-lock - marks jobs to prevent duplicate analysis
private markJobsInProgress(repo: any, jobs: AnalysisJob[]): void {
for (const job of jobs) {
repo.markInProgress(job.articleId);
}
}
private async processJobs(
jobs: AnalysisJob[],
username: string,
silent: boolean
): Promise<any[]> {
this.ensureTempDir();
const agentParams = [];
for (let i = 0; i < jobs.length; i++) {
const job = jobs[i];
if (!silent) {
this.log(`Fetching article ${i + 1}/${jobs.length}: ${this.truncate(job.articleTitle, 60)}...`);
}
const param = await this.createStubFile(job, username);
if (param) agentParams.push(param);
}
return agentParams;
}
private async fetchArticleData(slug: string, username: string): Promise<any | null> {
try {
const result = await getArticle(slug, username);
if (!result?.article) {
this.warn(`Failed to fetch article: ${slug}`);
return null;
}
return result.article;
} catch (error) {
this.warn(`Failed to fetch article: ${slug}`);
return null;
}
}
private buildStubObject(article: any, job: AnalysisJob, username: string): any {
return {
articleId: job.articleId,
articleSlug: job.articleSlug,
username,
articleUrl: article.url,
articleTitle: article.title,
savedAt: article.savedAt,
publishedAt: article.publishedAt || null,
updatedAt: article.updatedAt || null,
};
}
// AIDEV-NOTE: stub-file-creation - writes complete metadata for agent
private async createStubFile(job: AnalysisJob, username: string): Promise<any | null> {
const article = await this.fetchArticleData(job.articleSlug, username);
if (!article) return null;
const stubPath = join('temp', `${job.articleSlug}.jsonl`);
const stub = this.buildStubObject(article, job, username);
writeFileSync(stubPath, JSON.stringify(stub) + '\n', 'utf-8');
return {
filename: stubPath,
articleId: job.articleId,
articleSlug: job.articleSlug,
username,
articleTitle: this.truncate(article.title, 60),
};
}
private outputResults(agentParams: any[], count: number, jsonMode: boolean): void {
if (jsonMode) {
this.log(JSON.stringify(agentParams, null, 2));
} else {
this.log(`\n${formatDivider()}`);
this.log(formatSuccess(`Prepared ${count} articles for analysis`));
this.log('\nAgent parameters (copy for Task tool invocation):');
this.log(JSON.stringify(agentParams, null, 2));
}
}
private ensureTempDir(): void {
if (!existsSync('temp')) {
mkdirSync('temp', { recursive: true });
}
}
private truncate(text: string, length: number): string {
return text.length > length ? text.substring(0, length) + '...' : text;
}
}

View file

@ -0,0 +1,73 @@
import { readdirSync } from 'node:fs';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader, formatTable } from '@lib/cli/formatters.js';
import { QUEUE_STATUS } from '@lib/cli/constants.js';
import type { AnalysisJob } from '@storage/AnalysisQueueRepository.js';
/**
* OCLIF command: omc analyze status
* Show current analysis batch progress.
* AIDEV-NOTE: tracking-coordination - displays in-progress jobs and stub files
*/
export default class AnalyzeStatus extends BaseCommand {
static override description = 'Show current analysis batch progress';
static override examples = ['$ omc analyze status', '$ omc analyze status --json'];
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: Record<string, any>): Promise<void> {
return await withDatabase(async ({ repo }) => {
const jobs = repo.getByStatus(QUEUE_STATUS.IN_PROGRESS);
const stubs = this.getStubFiles();
const display = this.prepareDisplay(jobs, stubs);
this.output(display, flags.json);
});
}
private getStubFiles(): string[] {
try {
return readdirSync('temp').filter((f) => f.endsWith('.jsonl'));
} catch {
return [];
}
}
private prepareDisplay(jobs: AnalysisJob[], stubs: string[]): any {
return {
inProgress: jobs.map((j) => ({
articleId: j.articleId,
status: j.status,
elapsed: this.elapsed(j.assignedAt),
})),
stubFiles: stubs,
count: jobs.length,
};
}
private output(display: any, jsonMode: boolean): void {
if (jsonMode) {
this.log(JSON.stringify(display, null, 2));
return;
}
this.log(formatHeader('Analysis Status'));
if (display.count === 0) {
this.log('\nNo analysis in progress');
return;
}
this.log(formatTable(display.inProgress, ['articleId', 'status', 'elapsed']));
}
private elapsed(assignedAt?: string): string {
if (!assignedAt) return 'N/A';
const ms = Date.now() - new Date(assignedAt).getTime();
return `${Math.floor(ms / 1000)}s`;
}
}

View file

@ -0,0 +1,66 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader, formatDivider } from '@lib/cli/formatters.js';
import { displayQueueStats } from '@lib/cli/queue-display.js';
/**
* OCLIF command: omc analyze watch
* Real-time monitoring of analysis progress.
* AIDEV-NOTE: polling-monitor - updates stats every interval until completion
*/
export default class AnalyzeWatch extends BaseCommand {
static override description = 'Real-time monitoring of analysis progress';
static override examples = ['$ omc analyze watch', '$ omc analyze watch --interval 5000'];
static override flags = {
interval: Flags.integer({
char: 'i',
description: 'Polling interval in milliseconds',
default: 2000,
}),
};
protected async execute(flags: Record<string, any>): Promise<void> {
let running = true;
this.setupCleanExit(() => (running = false));
while (running) {
const done = await this.pollStats();
if (done) break;
await this.sleep(flags.interval);
}
}
private async pollStats(): Promise<boolean> {
return await withDatabase(async ({ repo }) => {
const stats = repo.getStats();
this.display(stats);
return stats.inProgress === 0 && stats.pending === 0;
});
}
private display(stats: any): void {
this.clear();
this.log(formatHeader('Analysis Progress (Ctrl+C to exit)'));
displayQueueStats(stats);
this.log(formatDivider());
}
private setupCleanExit(callback: () => void): void {
process.on('SIGINT', () => {
this.log('\nExiting...');
callback();
process.exit(0);
});
}
private clear(): void {
process.stdout.write('\x1Bc');
}
private sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
}

View file

@ -0,0 +1,57 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { formatHeader } from '@lib/cli/formatters.js';
import { readdirSync } from 'fs';
/**
* List available environments.
* AIDEV-NOTE: Shows .env files with active indicator
*/
export default class ConfigEnvList extends BaseCommand {
static override description = 'List available environments';
static override examples = [
'$ omc config env list',
'$ omc config env list --json',
];
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: any): Promise<void> {
const environments = this.findEnvironments();
if (flags.json) {
this.log(JSON.stringify(environments, null, 2));
} else {
this.displayEnvironments(environments);
}
}
private findEnvironments(): Array<{ name: string; file: string; active: boolean }> {
const cwd = process.cwd();
const files = readdirSync(cwd);
const envFiles = files.filter(f => f.startsWith('.env'));
return envFiles.map(file => ({
name: file === '.env' ? 'local' : file.replace('.env.', ''),
file,
active: file === '.env',
}));
}
private displayEnvironments(environments: Array<{ name: string; file: string; active: boolean }>): void {
this.log(formatHeader('Available Environments'));
if (environments.length === 0) {
this.log('No environment files found');
return;
}
for (const env of environments) {
const indicator = env.active ? '* ' : ' ';
this.log(`${indicator}${env.name.padEnd(15)} (${env.file})`);
}
}
}

View file

@ -0,0 +1,45 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { Args } from '@oclif/core';
import { formatSuccess } from '@lib/cli/formatters.js';
import { copyFileSync, existsSync } from 'fs';
import { join } from 'path';
/**
* Switch environment.
* AIDEV-NOTE: Copies .env.{environment} to .env
*/
export default class ConfigEnvUse extends BaseCommand {
static override description = 'Switch environment';
static override examples = [
'$ omc config env use dev',
'$ omc config env use prod',
'$ omc config env use local',
];
static override args = {
environment: Args.string({
description: 'Environment to switch to (dev|prod|local)',
required: true,
options: ['dev', 'prod', 'local'],
}),
};
protected async execute(flags: { environment: string }): Promise<void> {
this.switchEnvironment(flags.environment);
this.log(formatSuccess(`Switched to ${flags.environment} environment`));
}
private switchEnvironment(environment: string): void {
const cwd = process.cwd();
const sourceFile = environment === 'local' ? '.env.local' : `.env.${environment}`;
const sourcePath = join(cwd, sourceFile);
const targetPath = join(cwd, '.env');
if (!existsSync(sourcePath)) {
throw new Error(`Environment file not found: ${sourceFile}`);
}
copyFileSync(sourcePath, targetPath);
}
}

View file

@ -0,0 +1,62 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { Args } from '@oclif/core';
import { readFileSync, existsSync } from 'fs';
import { join } from 'path';
/**
* Get specific configuration value.
* AIDEV-NOTE: Masks sensitive values for security
*/
export default class ConfigGet extends BaseCommand {
static override description = 'Get specific configuration value';
static override examples = [
'$ omc config get OMNIVORE_API_KEY',
'$ omc config get CONTENT_OUTPUT_DIR',
];
static override args = {
key: Args.string({
description: 'Configuration key to retrieve',
required: true,
}),
};
protected async execute(flags: { key: string }): Promise<void> {
const value = this.getConfigValue(flags.key);
if (value === undefined) {
throw new Error(`Configuration key '${flags.key}' not found`);
}
this.log(this.maskValue(flags.key, value));
}
private getConfigValue(key: string): string | undefined {
const envPath = join(process.cwd(), '.env');
if (!existsSync(envPath)) {
throw new Error('.env file not found');
}
const content = readFileSync(envPath, 'utf-8');
for (const line of content.split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const [envKey, ...valueParts] = trimmed.split('=');
if (envKey === key && valueParts.length > 0) {
return valueParts.join('=');
}
}
return undefined;
}
private maskValue(key: string, value: string): string {
const sensitiveKeys = ['KEY', 'TOKEN', 'SECRET', 'PASSWORD'];
if (sensitiveKeys.some(s => key.includes(s)) && value.length > 8) {
return `${value.slice(0, 4)}...${value.slice(-4)}`;
}
return value;
}
}

View file

@ -0,0 +1,73 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { Args } from '@oclif/core';
import { formatSuccess } from '@lib/cli/formatters.js';
import { readFileSync, writeFileSync, existsSync } from 'fs';
import { join } from 'path';
/**
* Set configuration value.
* AIDEV-NOTE: Validates key exists in .env.example before updating
*/
export default class ConfigSet extends BaseCommand {
static override description = 'Set configuration value';
static override examples = [
'$ omc config set CONTENT_OUTPUT_DIR ./my-content',
'$ omc config set AUTO_PUBLISH true',
];
static override args = {
key: Args.string({
description: 'Configuration key to set',
required: true,
}),
value: Args.string({
description: 'Value to set',
required: true,
}),
};
protected async execute(flags: { key: string; value: string }): Promise<void> {
this.validateKey(flags.key);
this.updateEnvFile(flags.key, flags.value);
this.log(formatSuccess(`Set ${flags.key} = ${flags.value}`));
}
private validateKey(key: string): void {
const examplePath = join(process.cwd(), '.env.example');
if (!existsSync(examplePath)) return;
const content = readFileSync(examplePath, 'utf-8');
const validKeys = content
.split('\n')
.filter(line => !line.trim().startsWith('#') && line.includes('='))
.map(line => line.split('=')[0]);
if (!validKeys.includes(key)) {
throw new Error(`Unknown config key: ${key}`);
}
}
private updateEnvFile(key: string, value: string): void {
const envPath = join(process.cwd(), '.env');
let content = existsSync(envPath) ? readFileSync(envPath, 'utf-8') : '';
const lines = content.split('\n');
let updated = false;
for (let i = 0; i < lines.length; i++) {
const trimmed = lines[i].trim();
if (trimmed.startsWith(key + '=')) {
lines[i] = `${key}=${value}`;
updated = true;
break;
}
}
if (!updated) {
lines.push(`${key}=${value}`);
}
writeFileSync(envPath, lines.join('\n'));
}
}

View file

@ -0,0 +1,62 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { formatHeader } from '@lib/cli/formatters.js';
import { loadEnvFile } from '@lib/cli/command-utils.js';
import { join } from 'path';
/**
* Show all configuration values.
* AIDEV-NOTE: Masks sensitive values (API keys show first/last 4 chars)
*/
export default class ConfigShow extends BaseCommand {
static override description = 'Show all configuration values';
static override examples = [
'$ omc config show',
'$ omc config show --json',
];
static override flags = {
json: jsonFlag(),
};
async execute(flags: any): Promise<void> {
const envPath = join(process.cwd(), '.env');
const rawConfig = loadEnvFile(envPath);
if (Object.keys(rawConfig).length === 0) {
throw new Error('.env file not found');
}
const config = this.maskSensitiveValues(rawConfig);
if (flags.json) {
this.log(JSON.stringify(config, null, 2));
} else {
this.displayConfig(config);
}
}
private maskSensitiveValues(config: Record<string, string>): Record<string, string> {
const masked: Record<string, string> = {};
for (const [key, value] of Object.entries(config)) {
masked[key] = this.maskValue(key, value);
}
return masked;
}
private maskValue(key: string, value: string): string {
const sensitiveKeys = ['KEY', 'TOKEN', 'SECRET', 'PASSWORD'];
if (sensitiveKeys.some(s => key.includes(s)) && value.length > 8) {
return `${value.slice(0, 4)}...${value.slice(-4)}`;
}
return value;
}
private displayConfig(config: Record<string, string>): void {
this.log(formatHeader('Configuration'));
for (const [key, value] of Object.entries(config)) {
this.log(`${key.padEnd(30)} ${value}`);
}
}
}

View file

@ -0,0 +1,31 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { formatSuccess, formatError } from '@lib/cli/formatters.js';
import { getMe } from '@lib/omnivore/client.js';
/**
* Test API connection.
* AIDEV-NOTE: Uses testConnection pattern from lib/omnivore/client.js
*/
export default class ConfigTest extends BaseCommand {
static override description = 'Test API connection';
static override examples = [
'$ omc config test',
];
protected async execute(flags: any): Promise<void> {
void flags;
try {
const user = await getMe();
this.log(formatSuccess('Connected to Omnivore API'));
this.log(` User: ${user.name} (${user.email})`);
this.log(` Username: ${user.profile?.username ?? 'N/A'}`);
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
this.log(formatError('Failed to connect to Omnivore API'));
this.log(` ${message}`);
throw error;
}
}
}

View file

@ -0,0 +1,96 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { formatHeader, formatSuccess, formatError } from '@lib/cli/formatters.js';
import { loadEnvFile } from '@lib/cli/command-utils.js';
import { join } from 'path';
/**
* Validate all configuration values.
* AIDEV-NOTE: Checks required keys exist and validates formats
*/
export default class ConfigValidate extends BaseCommand {
static override description = 'Validate all configuration values';
static override examples = [
'$ omc config validate',
'$ omc config validate --json',
];
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: any): Promise<void> {
const issues = this.validateConfig();
if (flags.json) {
this.log(JSON.stringify({ valid: issues.length === 0, issues }, null, 2));
} else {
this.displayValidation(issues);
}
if (issues.length > 0) {
throw new Error('Configuration validation failed');
}
}
private validateConfig(): string[] {
const issues: string[] = [];
const envPath = join(process.cwd(), '.env');
const config = loadEnvFile(envPath);
if (Object.keys(config).length === 0) {
issues.push('.env file not found');
return issues;
}
this.checkRequired(config, issues);
this.checkFormats(config, issues);
return issues;
}
private checkRequired(config: Record<string, string>, issues: string[]): void {
const required = ['OMNIVORE_API_KEY'];
for (const key of required) {
if (!config[key] || config[key].trim() === '') {
issues.push(`Missing required key: ${key}`);
}
}
}
private checkFormats(config: Record<string, string>, issues: string[]): void {
if (config.OMNIVORE_API_URL && !this.isValidUrl(config.OMNIVORE_API_URL)) {
issues.push('Invalid URL format: OMNIVORE_API_URL');
}
const boolKeys = ['GENERATE_SEO', 'AUTO_PUBLISH', 'CACHE_ENABLED'];
for (const key of boolKeys) {
if (config[key] && !['true', 'false'].includes(config[key])) {
issues.push(`Invalid boolean value: ${key} (must be 'true' or 'false')`);
}
}
}
private isValidUrl(value: string): boolean {
try {
new URL(value);
return true;
} catch {
return false;
}
}
private displayValidation(issues: string[]): void {
this.log(formatHeader('Configuration Validation'));
if (issues.length === 0) {
this.log(formatSuccess('All configuration valid'));
} else {
this.log(formatError(`Found ${issues.length} issue(s):\n`));
for (const issue of issues) {
this.log(` - ${issue}`);
}
}
}
}

View file

@ -0,0 +1,44 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { Flags } from '@oclif/core';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
import { withDatabase } from '@lib/cli/database.js';
import { join } from 'path';
/**
* Create database backup.
* AIDEV-NOTE: Uses better-sqlite3 backup API for safe copy
*/
export default class DbBackup extends BaseCommand {
static override description = 'Create database backup';
static override examples = [
'$ omc db backup',
'$ omc db backup --destination data/backup.db',
'$ omc db backup --json',
];
static override flags = {
json: jsonFlag(),
destination: Flags.string({
description: 'Backup file path',
}),
};
protected async execute(flags: any): Promise<void> {
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const dest = flags.destination || join(process.cwd(), `data/backup-${timestamp}.db`);
await withDatabase(async ({ db }) => {
db.backup(dest);
const result = { destination: dest, timestamp };
if (flags.json) {
this.log(JSON.stringify(result, null, 2));
} else {
this.log(formatHeader('Database Backup'));
this.log(formatSuccess(`Backup created: ${dest}`));
}
});
}
}

View file

@ -0,0 +1,54 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader, formatSuccess, formatError } from '@lib/cli/formatters.js';
/**
* Verify database integrity.
* AIDEV-NOTE: Runs PRAGMA checks for corruption detection
*/
export default class DbCheck extends BaseCommand {
static override description = 'Verify database integrity';
static override examples = [
'$ omc db check',
'$ omc db check --json',
];
static override flags = {
json: jsonFlag(),
};
async execute(flags: any): Promise<void> {
await withDatabase(async ({ db }) => {
const integrity = this.checkIntegrity(db);
const foreignKeys = this.checkForeignKeys(db);
const isHealthy = integrity.ok && foreignKeys.length === 0;
const result = { ok: isHealthy, integrity, foreignKeys };
if (flags.json) {
this.log(JSON.stringify(result, null, 2));
} else {
this.displayCheckResults(integrity, foreignKeys, isHealthy);
}
});
}
private checkIntegrity(db: any): { ok: boolean; message: string } {
const result = db.pragma('integrity_check');
const message = result[0]?.integrity_check || 'ok';
return { ok: message === 'ok', message };
}
private checkForeignKeys(db: any): any[] {
return db.pragma('foreign_key_check');
}
private displayCheckResults(integrity: any, fkIssues: any[], ok: boolean): void {
this.log(formatHeader('Database Integrity Check'));
this.log(`Integrity: ${integrity.message}`);
this.log(`Foreign Keys: ${fkIssues.length === 0 ? 'OK' : `${fkIssues.length} issues`}`);
this.log(ok ? formatSuccess('Database is healthy') : formatError('Issues detected'));
}
}

View file

@ -0,0 +1,62 @@
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
import { withDatabase } from '@lib/cli/database.js';
/**
* Run database migrations by executing tracking schema.
* AIDEV-NOTE: Idempotent - safe to run multiple times (CREATE IF NOT EXISTS)
*/
export default class DbMigrate extends BaseCommand {
static override description = 'Run database migrations (idempotent schema execution)';
static override examples = [
'$ omc db migrate',
'$ omc db migrate --json',
];
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: any): Promise<void> {
await withDatabase(async ({ db }) => {
const schemaPath = this.getSchemaPath();
const schema = readFileSync(schemaPath, 'utf-8');
db.exec(schema);
const tables = this.getTrackingTables(db);
if (flags.json) {
this.log(JSON.stringify({ tables }, null, 2));
} else {
this.displayMigrationResults(tables);
}
});
}
private getSchemaPath(): string {
// AIDEV-NOTE: Schema must be in source tree, not dist (not bundled by esbuild)
const projectRoot = process.cwd();
return join(projectRoot, 'src/storage/schema/tracking-schema.sql');
}
private getTrackingTables(db: any): string[] {
const result = db.prepare(`
SELECT name FROM sqlite_master
WHERE type='table' AND name NOT LIKE 'sqlite_%'
ORDER BY name
`).all() as Array<{ name: string }>;
return result.map(t => t.name);
}
private displayMigrationResults(tables: string[]): void {
this.log(formatSuccess('Database schema migrated'));
this.log(formatHeader('Tables'));
tables.forEach(t => this.log(`${t}`));
}
}

View file

@ -0,0 +1,70 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { Flags } from '@oclif/core';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
import { listTables, isOmnivoreTable } from '@storage/database.js';
import { readFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
/**
* Reset tracking tables (preserves Omnivore tables).
* AIDEV-NOTE: Boundary protection - never drops Omnivore tables
*/
export default class DbReset extends BaseCommand {
static override description = 'Drop and recreate tracking tables';
static override examples = [
'$ omc db reset --force',
'$ omc db reset --force --json',
];
static override flags = {
json: jsonFlag(),
force: Flags.boolean({ description: 'Required for safety', default: false }),
};
async execute(flags: any): Promise<void> {
if (!flags.force) {
this.error('--force flag required for reset operation');
}
await withDatabase(async (db) => {
const trackingTables = this.dropTrackingTables(db);
this.recreateSchema(db);
this.displayResult(flags, trackingTables);
});
}
private dropTrackingTables(db: any): string[] {
const tables = listTables(db);
const trackingTables = tables.filter(t => !isOmnivoreTable(db, t));
trackingTables.forEach(t => db.exec(`DROP TABLE IF EXISTS ${t}`));
return trackingTables;
}
private recreateSchema(db: any): void {
const schemaPath = this.getSchemaPath();
const schema = readFileSync(schemaPath, 'utf-8');
db.exec(schema);
}
private displayResult(flags: any, droppedTables: string[]): void {
const result = { droppedTables, recreated: true };
if (flags.json) {
this.log(JSON.stringify(result, null, 2));
} else {
this.log(formatHeader('Database Reset'));
this.log(`Dropped ${droppedTables.length} tracking tables`);
this.log(formatSuccess('Schema recreated'));
}
}
private getSchemaPath(): string {
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
return join(__dirname, '../../../storage/schema/tracking-schema.sql');
}
}

View file

@ -0,0 +1,63 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { Args, Flags } from '@oclif/core';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { formatHeader, formatSuccess, formatError } from '@lib/cli/formatters.js';
import { copyFileSync, existsSync } from 'fs';
import { join } from 'path';
/**
* Restore database from backup.
* AIDEV-NOTE: Requires --force flag for safety
*/
export default class DbRestore extends BaseCommand {
static override description = 'Restore database from backup';
static override examples = [
'$ omc db restore data/backup.db --force',
'$ omc db restore data/backup.db --force --json',
];
static override args = {
backupPath: Args.string({ description: 'Path to backup file', required: true }),
};
static override flags = {
json: jsonFlag(),
force: Flags.boolean({ description: 'Skip confirmation', default: false }),
};
async execute(flags: any): Promise<void> {
const { args } = await this.parse(DbRestore);
const backupPath = args.backupPath;
this.validateInputs(backupPath, flags.force);
const dbPath = this.performRestore(backupPath);
this.displayResult(flags, backupPath, dbPath);
}
private validateInputs(backupPath: string, force: boolean): void {
if (!existsSync(backupPath)) {
this.error(formatError(`Backup file not found: ${backupPath}`));
}
if (!force) {
this.error('--force flag required for restore operation');
}
}
private performRestore(backupPath: string): string {
const dbPath = join(process.cwd(), 'data/omnivore-content.db');
copyFileSync(backupPath, dbPath);
return dbPath;
}
private displayResult(flags: any, backupPath: string, dbPath: string): void {
const result = { restored: true, from: backupPath, to: dbPath };
if (flags.json) {
this.log(JSON.stringify(result, null, 2));
} else {
this.log(formatHeader('Database Restore'));
this.log(formatSuccess(`Restored from: ${backupPath}`));
}
}
}

View file

@ -0,0 +1,52 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader } from '@lib/cli/formatters.js';
import { listTables } from '@storage/database.js';
/**
* Display current database schema.
* AIDEV-NOTE: Shows table structure with columns and types
*/
export default class DbSchema extends BaseCommand {
static override description = 'Show current database schema';
static override examples = [
'$ omc db schema',
'$ omc db schema --json',
];
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: any): Promise<void> {
await withDatabase(async ({ db }) => {
const tables = listTables(db);
const schema = this.buildSchemaInfo(db, tables);
if (flags.json) {
this.log(JSON.stringify(schema, null, 2));
} else {
this.displaySchema(schema);
}
});
}
private buildSchemaInfo(db: any, tables: string[]) {
return tables.map(table => ({
name: table,
columns: db.prepare(`PRAGMA table_info(${table})`).all(),
}));
}
private displaySchema(schema: any[]): void {
this.log(formatHeader('Database Schema'));
for (const table of schema) {
this.log(`\n${table.name}:`);
for (const col of table.columns) {
this.log(` ${col.name.padEnd(30)} ${col.type}`);
}
}
}
}

View file

@ -0,0 +1,73 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { formatSuccess } from '@lib/cli/formatters.js';
import { withDatabase } from '@lib/cli/database.js';
/**
* Seed database with sample articles for testing.
* AIDEV-NOTE: Creates 10 diverse sample articles with varied statuses
*/
export default class DbSeed extends BaseCommand {
static override description = 'Seed database with sample data';
static override examples = [
'$ omc db seed',
'$ omc db seed --json',
];
static override flags = {
json: jsonFlag(),
};
async execute(flags: any): Promise<void> {
await withDatabase(async (db) => {
const samples = this.createSampleData();
const inserted = this.seedArticles(db, samples);
if (flags.json) {
this.log(JSON.stringify({ inserted, total: samples.length }, null, 2));
} else {
this.log(formatSuccess(`Seeded ${inserted} sample articles`));
}
});
}
private seedArticles(db: any, samples: any[]): number {
const stmt = db.prepare(`
INSERT OR IGNORE INTO analysis_queue (
article_id, article_slug, article_url, article_title,
saved_at, status, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))
`);
let inserted = 0;
samples.forEach(article => {
const result = stmt.run(
article.articleId,
article.articleSlug,
article.articleUrl,
article.articleTitle,
article.savedAt,
article.status
);
if (result.changes > 0) inserted++;
});
return inserted;
}
private createSampleData() {
return [
{ articleId: 'seed-001', articleSlug: 'intro-to-llms', articleUrl: 'https://omnivore.app/test/intro-to-llms', articleTitle: 'Introduction to Large Language Models', savedAt: '2025-01-01T10:00:00Z', status: 'pending' },
{ articleId: 'seed-002', articleSlug: 'docker-best-practices', articleUrl: 'https://omnivore.app/test/docker-best-practices', articleTitle: 'Docker Best Practices 2025', savedAt: '2025-01-02T10:00:00Z', status: 'pending' },
{ articleId: 'seed-003', articleSlug: 'rust-async-await', articleUrl: 'https://omnivore.app/test/rust-async-await', articleTitle: 'Understanding Async/Await in Rust', savedAt: '2025-01-03T10:00:00Z', status: 'pending' },
{ articleId: 'seed-004', articleSlug: 'kubernetes-scaling', articleUrl: 'https://omnivore.app/test/kubernetes-scaling', articleTitle: 'Kubernetes Autoscaling Guide', savedAt: '2025-01-04T10:00:00Z', status: 'in_progress' },
{ articleId: 'seed-005', articleSlug: 'typescript-generics', articleUrl: 'https://omnivore.app/test/typescript-generics', articleTitle: 'Mastering TypeScript Generics', savedAt: '2025-01-05T10:00:00Z', status: 'in_progress' },
{ articleId: 'seed-006', articleSlug: 'graphql-federation', articleUrl: 'https://omnivore.app/test/graphql-federation', articleTitle: 'GraphQL Federation Explained', savedAt: '2025-01-06T10:00:00Z', status: 'completed' },
{ articleId: 'seed-007', articleSlug: 'postgres-performance', articleUrl: 'https://omnivore.app/test/postgres-performance', articleTitle: 'PostgreSQL Performance Tuning', savedAt: '2025-01-07T10:00:00Z', status: 'completed' },
{ articleId: 'seed-008', articleSlug: 'react-server-components', articleUrl: 'https://omnivore.app/test/react-server-components', articleTitle: 'React Server Components Deep Dive', savedAt: '2025-01-08T10:00:00Z', status: 'completed' },
{ articleId: 'seed-009', articleSlug: 'distributed-tracing', articleUrl: 'https://omnivore.app/test/distributed-tracing', articleTitle: 'Distributed Tracing with OpenTelemetry', savedAt: '2025-01-09T10:00:00Z', status: 'failed' },
{ articleId: 'seed-010', articleSlug: 'webassembly-performance', articleUrl: 'https://omnivore.app/test/webassembly-performance', articleTitle: 'WebAssembly Performance Benchmarks', savedAt: '2025-01-10T10:00:00Z', status: 'failed' },
];
}
}

View file

@ -0,0 +1,59 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader } from '@lib/cli/formatters.js';
import { getTableCounts } from '@storage/database.js';
import { existsSync, statSync } from 'fs';
import { join } from 'path';
/**
* Display database statistics.
* AIDEV-NOTE: Shows table counts and file size
*/
export default class DbStats extends BaseCommand {
static override description = 'Show database statistics';
static override examples = [
'$ omc db stats',
'$ omc db stats --json',
];
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: any): Promise<void> {
await withDatabase(async ({ db }) => {
const counts = getTableCounts(db);
const dbPath = join(process.cwd(), 'data/omnivore-content.db');
const fileSize = this.getFileSize(dbPath);
const result = { tables: counts, fileSize };
if (flags.json) {
this.log(JSON.stringify(result, null, 2));
} else {
this.displayStats(counts, fileSize);
}
});
}
private getFileSize(path: string): number {
return existsSync(path) ? statSync(path).size : 0;
}
private displayStats(counts: Record<string, number>, size: number): void {
this.log(formatHeader('Database Statistics'));
this.log(`File size: ${this.formatBytes(size)}\n`);
this.log('Table Row Counts:');
for (const [table, count] of Object.entries(counts)) {
this.log(` ${table.padEnd(30)} ${count.toLocaleString()}`);
}
}
private formatBytes(bytes: number): string {
return bytes >= 1024 * 1024
? `${(bytes / 1024 / 1024).toFixed(2)} MB`
: `${(bytes / 1024).toFixed(2)} KB`;
}
}

View file

@ -0,0 +1,73 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
import { existsSync, statSync } from 'fs';
import { join } from 'path';
/**
* Optimize database by running VACUUM.
* AIDEV-NOTE: Reclaims unused space and defragments
*/
export default class DbVacuum extends BaseCommand {
static override description = 'Optimize database with VACUUM';
static override examples = [
'$ omc db vacuum',
'$ omc db vacuum --json',
];
static override flags = {
json: jsonFlag(),
};
// AIDEV-NOTE: Execute split into helpers for function length compliance
protected async execute(flags: any): Promise<void> {
const dbPath = join(process.cwd(), 'data/omnivore-content.db');
const sizeBefore = this.getFileSize(dbPath);
await withDatabase(async ({ db }) => {
db.exec('VACUUM');
});
const sizeAfter = this.getFileSize(dbPath);
const result = this.buildResult(sizeBefore, sizeAfter);
this.displayResult(result, flags.json);
}
private buildResult(
sizeBefore: number,
sizeAfter: number
): { sizeBefore: number; sizeAfter: number; savedBytes: number } {
return {
sizeBefore,
sizeAfter,
savedBytes: sizeBefore - sizeAfter,
};
}
private displayResult(
result: { sizeBefore: number; sizeAfter: number; savedBytes: number },
asJson: boolean
): void {
if (asJson) {
this.log(JSON.stringify(result, null, 2));
} else {
this.log(formatHeader('Database Vacuum'));
this.log(`Size before: ${this.formatBytes(result.sizeBefore)}`);
this.log(`Size after: ${this.formatBytes(result.sizeAfter)}`);
this.log(formatSuccess(`Saved ${this.formatBytes(result.savedBytes)}`));
}
}
private getFileSize(path: string): number {
return existsSync(path) ? statSync(path).size : 0;
}
private formatBytes(bytes: number): string {
return bytes >= 1024 * 1024
? `${(bytes / 1024 / 1024).toFixed(2)} MB`
: `${(bytes / 1024).toFixed(2)} KB`;
}
}

View file

@ -0,0 +1,115 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { outputResult } from '@lib/cli/command-utils.js';
import { initDatabase } from '@storage/database.js';
import { testConnection } from '@lib/omnivore/client.js';
import { existsSync } from 'fs';
import { config } from 'dotenv';
/**
* System health check for omnivore-content-system
* AIDEV-NOTE: doctor-command - diagnostic tool for troubleshooting setup issues
*/
export default class Doctor extends BaseCommand {
static override description = 'Check system health and configuration';
static override examples = [
'<%= config.bin %> <%= command.id %>',
'<%= config.bin %> <%= command.id %> --json',
];
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: { json: boolean }): Promise<void> {
const checks = {
database: await this.checkDatabase(),
apiKey: this.checkApiKey(),
apiConnection: await this.checkApiConnection(),
directories: this.checkDirectories(),
dependencies: this.checkDependencies(),
};
const allHealthy = Object.values(checks).every(check => check.healthy);
if (flags.json) {
outputResult(this, { checks, healthy: allHealthy }, '', true);
} else {
this.displayHealthReport(checks, allHealthy);
}
if (!allHealthy) {
process.exit(1);
}
}
private async checkDatabase(): Promise<CheckResult> {
try {
initDatabase('data/omnivore-content.db');
return { healthy: true, message: 'Database connection OK' };
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return { healthy: false, message: `Database error: ${message}` };
}
}
private checkApiKey(): CheckResult {
config();
if (process.env.OMNIVORE_API_KEY) {
return { healthy: true, message: 'OMNIVORE_API_KEY found' };
}
return { healthy: false, message: 'OMNIVORE_API_KEY not set' };
}
private async checkApiConnection(): Promise<CheckResult> {
config();
if (!process.env.OMNIVORE_API_KEY) {
return { healthy: false, message: 'Cannot test - API key missing' };
}
const connected = await testConnection();
if (connected) {
return { healthy: true, message: 'API connection successful' };
}
return { healthy: false, message: 'API connection failed' };
}
private checkDirectories(): CheckResult {
const required = ['data', 'content', 'temp'];
const missing = required.filter(dir => !existsSync(dir));
if (missing.length === 0) {
return { healthy: true, message: 'All required directories exist' };
}
return { healthy: false, message: `Missing directories: ${missing.join(', ')}` };
}
private checkDependencies(): CheckResult {
const nodeVersion = process.version;
const requiredMajor = 18;
const currentMajor = parseInt(nodeVersion.slice(1).split('.')[0]);
if (currentMajor >= requiredMajor) {
return { healthy: true, message: `Node ${nodeVersion} (>= ${requiredMajor})` };
}
return { healthy: false, message: `Node ${nodeVersion} (requires >= ${requiredMajor})` };
}
private displayHealthReport(checks: Record<string, CheckResult>, allHealthy: boolean): void {
this.log('\nSystem Health Report\n');
for (const [name, check] of Object.entries(checks)) {
const icon = check.healthy ? '✓' : '✗';
const label = name.padEnd(20);
this.log(`${icon} ${label} ${check.message}`);
}
this.log(`\nOverall Status: ${allHealthy ? '✓ Healthy' : '✗ Issues detected'}`);
}
}
interface CheckResult {
healthy: boolean;
message: string;
}

View file

@ -0,0 +1,118 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { formatSuccess, formatError } from '@lib/cli/formatters.js';
import { initDatabase } from '@storage/database.js';
import { testConnection } from '@lib/omnivore/client.js';
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
import { config } from 'dotenv';
/**
* Interactive setup wizard for omnivore-content-system
* AIDEV-NOTE: init-command - one-time setup for new installations
*/
export default class Init extends BaseCommand {
static override description = 'Initialize omnivore-content-system setup';
static override examples = [
'<%= config.bin %> <%= command.id %>',
'<%= config.bin %> <%= command.id %> --force',
'<%= config.bin %> <%= command.id %> --api-key YOUR_KEY',
];
static override flags = {
force: Flags.boolean({
description: 'Reinitialize existing setup',
default: false,
}),
'api-key': Flags.string({
description: 'Set OMNIVORE_API_KEY non-interactively',
}),
};
protected async execute(flags: { force: boolean; 'api-key'?: string }): Promise<void> {
this.log('Initializing omnivore-content-system...\n');
await this.ensureDirectories();
await this.initializeDatabase(flags.force);
await this.setupEnvironment(flags['api-key']);
await this.testApiConnection();
this.log('\n' + formatSuccess('Setup complete! Run "omc --help" to get started.'));
}
private async ensureDirectories(): Promise<void> {
const dirs = ['data', 'content', 'temp', 'content/articles', 'content/analysis', 'content/generated'];
for (const dir of dirs) {
if (!existsSync(dir)) {
mkdirSync(dir, { recursive: true });
this.log(formatSuccess(`Created directory: ${dir}`));
}
}
}
private async initializeDatabase(force: boolean): Promise<void> {
try {
initDatabase('data/omnivore-content.db');
this.log(formatSuccess('Database initialized'));
} catch (error) {
if (force) {
this.log(formatSuccess('Database reinitialized'));
} else {
throw error;
}
}
}
private async setupEnvironment(apiKey?: string): Promise<void> {
this.createEnvFile();
if (apiKey) {
this.updateEnvFile('OMNIVORE_API_KEY', apiKey);
this.log(formatSuccess('OMNIVORE_API_KEY set'));
} else {
this.checkApiKey();
}
}
private createEnvFile(): void {
if (!existsSync('.env') && existsSync('.env.example')) {
const example = readFileSync('.env.example', 'utf-8');
writeFileSync('.env', example);
this.log(formatSuccess('Created .env from .env.example'));
} else if (!existsSync('.env')) {
this.log(formatError('.env.example not found'));
}
}
private checkApiKey(): void {
config();
if (!process.env.OMNIVORE_API_KEY) {
this.log(formatError('OMNIVORE_API_KEY not set in .env'));
this.log('Please add your API key to .env file');
}
}
private async testApiConnection(): Promise<void> {
config();
if (process.env.OMNIVORE_API_KEY) {
const connected = await testConnection();
if (!connected) {
this.log(formatError('API connection test failed'));
}
}
}
private updateEnvFile(key: string, value: string): void {
let envContent = existsSync('.env') ? readFileSync('.env', 'utf-8') : '';
const regex = new RegExp(`^${key}=.*$`, 'm');
if (regex.test(envContent)) {
envContent = envContent.replace(regex, `${key}=${value}`);
} else {
envContent += `\n${key}=${value}\n`;
}
writeFileSync('.env', envContent);
}
}

View file

@ -0,0 +1,65 @@
import { Args, Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { fetchUsername, checkGraphQLResult } from '@lib/cli/graphql.js';
import { getArticle } from '@lib/omnivore/client.js';
import type { OmnivoreArticle } from '@omc-types/omnivore.js';
/**
* Fetch article by slug.
* AIDEV-NOTE: Direct Omnivore API access for article retrieval
*/
export default class OmnivoreGet extends BaseCommand {
static override description = 'Fetch article by slug';
static override examples = [
'$ omc omnivore get my-article-slug',
'$ omc omnivore get my-article-slug --json',
'$ omc omnivore get my-article-slug --username different-user',
];
static override args = {
slug: Args.string({ description: 'Article slug', required: true }),
};
static override flags = {
username: Flags.string({ char: 'u', description: 'Username (defaults to authenticated user)' }),
content: Flags.boolean({
description: 'Print raw article content to stdout (agent-friendly)',
default: false,
}),
json: jsonFlag(),
};
protected async execute(flags: { slug: string; username?: string; content: boolean; json: boolean }): Promise<void> {
const username = flags.username || await fetchUsername();
const result = await getArticle(flags.slug, username);
checkGraphQLResult({ data: { article: result }, errors: undefined });
if (!result.article) {
throw new Error('Article not found');
}
if (flags.content && flags.json) {
throw new Error('Use either --content or --json (not both)');
}
if (flags.content) {
this.log(result.article?.content ?? '');
return;
}
this.outputArticle(result.article, flags.json);
}
private outputArticle(article: OmnivoreArticle, json: boolean): void {
if (json) {
this.log(JSON.stringify(article, null, 2));
} else {
this.log(`Title: ${article.title}`);
this.log(`URL: ${article.url}`);
this.log(`Author: ${article.author || 'N/A'}`);
this.log(`Description: ${article.description || 'N/A'}`);
}
}
}

View file

@ -0,0 +1,53 @@
import { Args, Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { checkGraphQLResult } from '@lib/cli/graphql.js';
import { createHighlight } from '@lib/omnivore/client.js';
/**
* Add highlight to article.
* AIDEV-NOTE: Creates HIGHLIGHT-type with quote (optional annotation)
*/
export default class HighlightAdd extends BaseCommand {
static override description = 'Add highlight to article';
static override examples = [
'$ omc omnivore highlight add article-id "quote text"',
'$ omc omnivore highlight add article-id "quote" --annotation "my note"',
'$ omc omnivore highlight add article-id "quote" --color "#FF0000" --json',
];
static override args = {
articleId: Args.string({ description: 'Article ID', required: true }),
quote: Args.string({ description: 'Highlighted text', required: true }),
};
static override flags = {
annotation: Flags.string({ char: 'a', description: 'Optional annotation' }),
color: Flags.string({ char: 'c', description: 'Highlight color', default: '#FFD700' }),
json: jsonFlag(),
};
protected async execute(flags: { articleId: string; quote: string; annotation?: string; color: string; json: boolean }): Promise<void> {
const result = await createHighlight({
id: crypto.randomUUID(),
shortId: crypto.randomUUID().substring(0, 8),
articleId: flags.articleId,
quote: flags.quote,
annotation: flags.annotation,
color: flags.color,
type: 'HIGHLIGHT',
});
checkGraphQLResult({ data: { createHighlight: result }, errors: undefined });
this.outputResult(result, flags.json);
}
private outputResult(result: any, json: boolean): void {
if (json) {
this.log(JSON.stringify(result.highlight, null, 2));
} else {
this.log('Highlight added successfully');
}
}
}

View file

@ -0,0 +1,57 @@
import { Args } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { fetchUsername } from '@lib/cli/graphql.js';
import { getHighlights } from '@lib/omnivore/client.js';
type HighlightRecord = {
id: string;
quote?: string | null;
annotation?: string | null;
createdAt?: string;
type?: string;
};
/**
* List highlights for article.
* AIDEV-NOTE: Filters to show only HIGHLIGHT type (excludes NOTEs)
*/
export default class HighlightList extends BaseCommand {
static override description = 'List highlights for article';
static override examples = [
'$ omc omnivore highlight list article-id',
'$ omc omnivore highlight list article-id --json',
];
static override args = {
articleId: Args.string({ description: 'Article ID', required: true }),
};
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: { articleId: string; json: boolean }): Promise<void> {
const username = await fetchUsername();
const allHighlights = (await getHighlights(flags.articleId, username)) as HighlightRecord[];
const highlights = this.filterHighlights(allHighlights);
this.outputHighlights(highlights, flags.json);
}
private filterHighlights(highlights: HighlightRecord[]): HighlightRecord[] {
return highlights.filter((h) => h.type === 'HIGHLIGHT' || Boolean(h.quote));
}
private outputHighlights(highlights: HighlightRecord[], json: boolean): void {
if (json) {
this.log(JSON.stringify(highlights, null, 2));
} else {
highlights.forEach((h) => {
this.log(`- "${h.quote ?? ''}"`);
if (h.annotation) this.log(` Note: ${h.annotation}`);
});
}
}
}

View file

@ -0,0 +1,38 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { checkGraphQLResult } from '@lib/cli/graphql.js';
import { createLabel } from '@lib/omnivore/client.js';
/**
* Create a label.
* AIDEV-NOTE: Replacement for standalone scripts/test-create-label.js and migrate helpers.
*/
export default class OmnivoreLabelCreate extends BaseCommand {
static override description = 'Create a label';
static override examples = [
'$ omc omnivore label create --name "ai"',
'$ omc omnivore label create --name "ai" --color "#3B82F6" --description "AI/ML" --json',
];
static override flags = {
name: Flags.string({ description: 'Label name', required: true }),
color: Flags.string({ description: 'Hex color (e.g. #3B82F6)' }),
description: Flags.string({ description: 'Description' }),
json: jsonFlag(),
};
protected async execute(flags: { name: string; color?: string; description?: string; json: boolean }): Promise<void> {
const result = await createLabel({ name: flags.name, color: flags.color, description: flags.description });
checkGraphQLResult({ data: { createLabel: result }, errors: undefined });
if (flags.json) {
this.log(JSON.stringify(result.label ?? result, null, 2));
return;
}
this.log(`Label created: ${result.label?.name ?? flags.name}`);
}
}

View file

@ -0,0 +1,36 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { checkGraphQLResult } from '@lib/cli/graphql.js';
import { getLabels } from '@lib/omnivore/client.js';
/**
* List labels.
* AIDEV-NOTE: Integrates legacy Omnivore label scripts into the OMC CLI.
*/
export default class OmnivoreLabelList extends BaseCommand {
static override description = 'List labels';
static override examples = [
'$ omc omnivore label list',
'$ omc omnivore label list --json',
];
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: { json: boolean }): Promise<void> {
const labels = await getLabels();
checkGraphQLResult({ data: { labels }, errors: undefined });
if (flags.json) {
this.log(JSON.stringify(labels, null, 2));
return;
}
for (const label of labels) {
this.log(`- ${label.name} (${label.id})`);
}
}
}

View file

@ -0,0 +1,51 @@
import { Args, Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { checkGraphQLResult } from '@lib/cli/graphql.js';
import { setLabels } from '@lib/omnivore/client.js';
/**
* Set labels on a page (replaces existing labels).
* AIDEV-NOTE: Integrates apply-single-label.js / apply-labels*.js into OMC CLI surface.
*/
export default class OmnivoreLabelSet extends BaseCommand {
static override description = 'Set labels on a page (replaces existing labels)';
static override examples = [
'$ omc omnivore label set <page-id> --label "ai"',
'$ omc omnivore label set <page-id> --label "ai" --label "devops"',
'$ omc omnivore label set <page-id> --label "ai" --source "omc" --json',
];
static override args = {
pageId: Args.string({ description: 'Page/Article ID', required: true }),
};
static override flags = {
label: Flags.string({
description: 'Label name (repeatable)',
multiple: true,
required: true,
}),
source: Flags.string({
description: 'Optional source string (stored by Omnivore)',
default: 'omc',
}),
json: jsonFlag(),
};
protected async execute(flags: { pageId: string; label: string[]; source: string; json: boolean }): Promise<void> {
const labels = flags.label.map((name) => ({ name }));
const result = await setLabels({ pageId: flags.pageId, labels, source: flags.source });
checkGraphQLResult({ data: { setLabels: result }, errors: undefined });
if (flags.json) {
this.log(JSON.stringify(result, null, 2));
return;
}
const applied = (result.labels ?? []).map((l: any) => l.name).join(', ');
this.log(`Labels set: ${applied || '(none)'}`);
}
}

View file

@ -0,0 +1,44 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { checkGraphQLResult } from '@lib/cli/graphql.js';
import { getRecentArticles } from '@lib/omnivore/client.js';
import type { SearchResult } from '@omc-types/omnivore.js';
/**
* List recent articles.
* AIDEV-NOTE: Time-filtered article listing
*/
export default class OmnivoreList extends BaseCommand {
static override description = 'List recent articles';
static override examples = [
'$ omc omnivore list',
'$ omc omnivore list --hours 48',
'$ omc omnivore list --hours 24 --limit 20 --json',
];
static override flags = {
hours: Flags.integer({ char: 'h', description: 'Hours to look back', default: 24 }),
limit: Flags.integer({ char: 'l', description: 'Max results', default: 10 }),
json: jsonFlag(),
};
protected async execute(flags: Record<string, any>): Promise<void> {
const result = await getRecentArticles(flags.hours, flags.limit);
checkGraphQLResult({ data: { search: result }, errors: undefined });
this.outputResults(result.edges, flags.json);
}
private outputResults(edges: SearchResult['edges'], json: boolean): void {
if (json) {
this.log(JSON.stringify(edges.map((e) => e.node), null, 2));
} else {
edges.forEach(({ node }) => {
this.log(`- ${node.title}`);
this.log(` Saved: ${node.savedAt}`);
});
}
}
}

View file

@ -0,0 +1,92 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { searchArticles } from '@lib/omnivore/client.js';
import Database from 'better-sqlite3';
import { mkdirSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
/**
* Download a URL -> pageId mapping into a local SQLite DB.
* AIDEV-NOTE: Integrates scripts/download-items-mapping.js functionality into OMC CLI.
*/
export default class OmnivoreMappingDownload extends BaseCommand {
static override description = 'Download Omnivore URL-to-ID mapping into SQLite';
static override examples = [
'$ omc omnivore mapping download',
'$ omc omnivore mapping download --destination temp/url-id-mapping.sqlite',
'$ omc omnivore mapping download --limit 1000 --json',
];
static override flags = {
destination: Flags.string({
description: 'Output SQLite path',
default: 'temp/url-id-mapping.sqlite',
}),
limit: Flags.integer({
description: 'Optional max items (for testing)',
required: false,
}),
json: jsonFlag(),
};
protected async execute(flags: { destination: string; limit?: number; json: boolean }): Promise<void> {
const destination = resolve(flags.destination);
mkdirSync(dirname(destination), { recursive: true });
const db = new Database(destination);
try {
this.initSchema(db);
const downloaded = await this.downloadInto(db, flags.limit);
const result = { destination, downloaded };
if (flags.json) this.log(JSON.stringify(result, null, 2));
else this.log(`Saved ${downloaded} items to ${destination}`);
} finally {
db.close();
}
}
private initSchema(db: Database.Database): void {
db.exec(`
CREATE TABLE IF NOT EXISTS item_mapping (
id TEXT PRIMARY KEY,
url TEXT NOT NULL,
title TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_item_mapping_url ON item_mapping(url);
DELETE FROM item_mapping;
`);
}
private async downloadInto(db: Database.Database, limit?: number): Promise<number> {
const insert = db.prepare('INSERT OR REPLACE INTO item_mapping (id, url, title) VALUES (?, ?, ?)');
const insertMany = db.transaction((rows: Array<{ id: string; url: string; title?: string }>) => {
for (const r of rows) insert.run(r.id, r.url, r.title ?? null);
});
let after: string | undefined;
let total = 0;
while (true) {
const first = 100;
const result: any = await searchArticles({ query: '', first, after: after ?? '', includeContent: false });
const edges = result.edges ?? [];
let rows = edges.map((e: any) => ({ id: e.node.id, url: e.node.url, title: e.node.title }));
if (limit && total + rows.length > limit) {
rows = rows.slice(0, Math.max(0, limit - total));
}
insertMany(rows);
total += rows.length;
if (limit && total >= limit) return total;
const pageInfo = result.pageInfo;
if (!pageInfo?.hasNextPage) return total;
after = pageInfo.endCursor;
}
}
}

View file

@ -0,0 +1,48 @@
import { Args } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { checkGraphQLResult } from '@lib/cli/graphql.js';
import { createHighlight } from '@lib/omnivore/client.js';
/**
* Add note to article.
* AIDEV-NOTE: Creates NOTE-type highlight (no quote required)
*/
export default class NoteAdd extends BaseCommand {
static override description = 'Add note to article';
static override examples = [
'$ omc omnivore note add article-id "My note content"',
'$ omc omnivore note add article-id "Note" --json',
];
static override args = {
articleId: Args.string({ description: 'Article ID', required: true }),
note: Args.string({ description: 'Note content', required: true }),
};
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: { articleId: string; note: string; json: boolean }): Promise<void> {
const result = await createHighlight({
id: crypto.randomUUID(),
shortId: crypto.randomUUID().substring(0, 8),
articleId: flags.articleId,
annotation: flags.note,
type: 'NOTE',
});
checkGraphQLResult({ data: { createHighlight: result }, errors: undefined });
this.outputResult(result, flags.json);
}
private outputResult(result: any, json: boolean): void {
if (json) {
this.log(JSON.stringify(result.highlight, null, 2));
} else {
this.log('Note added successfully');
}
}
}

View file

@ -0,0 +1,57 @@
import { Args } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { fetchUsername } from '@lib/cli/graphql.js';
import { getHighlights } from '@lib/omnivore/client.js';
type NoteRecord = { annotation?: string | null; createdAt?: string; quote?: string | null; type?: string };
/**
* Get notes for article.
* AIDEV-NOTE: Filters highlights to show only NOTE type
*/
export default class NoteGet extends BaseCommand {
static override description = 'Get notes for article';
static override examples = [
'$ omc omnivore note get article-id',
'$ omc omnivore note get article-id --json',
];
static override args = {
articleId: Args.string({ description: 'Article ID', required: true }),
};
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: { articleId: string; json: boolean }): Promise<void> {
await withDatabase(async ({ repo }) => {
const job = repo.getByArticleId(flags.articleId);
if (!job) {
this.error(`Article not found in database: ${flags.articleId}`);
}
const username = await fetchUsername();
const highlights = (await getHighlights(job.articleSlug, username)) as NoteRecord[];
const notes = this.filterNotes(highlights);
this.outputNotes(notes, flags.json);
});
}
private filterNotes(highlights: NoteRecord[]): NoteRecord[] {
return highlights.filter((h) => h.type === 'NOTE' || (!h.quote && h.annotation));
}
private outputNotes(notes: NoteRecord[], json: boolean): void {
if (json) {
this.log(JSON.stringify(notes, null, 2));
} else {
notes.forEach((note) => {
this.log(`- ${note.annotation ?? ''}`);
if (note.createdAt) this.log(` Created: ${note.createdAt}`);
});
}
}
}

View file

@ -0,0 +1,45 @@
import { Args } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { checkGraphQLResult } from '@lib/cli/graphql.js';
import { updateHighlight } from '@lib/omnivore/client.js';
/**
* Update note content.
* AIDEV-NOTE: Updates annotation field of highlight
*/
export default class NoteUpdate extends BaseCommand {
static override description = 'Update note content';
static override examples = [
'$ omc omnivore note update highlight-id "Updated note"',
'$ omc omnivore note update highlight-id "New content" --json',
];
static override args = {
highlightId: Args.string({ description: 'Highlight ID', required: true }),
note: Args.string({ description: 'New note content', required: true }),
};
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: { highlightId: string; note: string; json: boolean }): Promise<void> {
const result = await updateHighlight({
highlightId: flags.highlightId,
annotation: flags.note,
});
checkGraphQLResult({ data: { updateHighlight: result }, errors: undefined });
this.outputResult(result, flags.json);
}
private outputResult(result: any, json: boolean): void {
if (json) {
this.log(JSON.stringify(result.highlight, null, 2));
} else {
this.log('Note updated successfully');
}
}
}

View file

@ -0,0 +1,47 @@
import { Args, Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { checkGraphQLResult } from '@lib/cli/graphql.js';
import { searchArticles } from '@lib/omnivore/client.js';
import type { SearchResult } from '@omc-types/omnivore.js';
/**
* Search articles.
* AIDEV-NOTE: Omnivore search with query DSL support
*/
export default class OmnivoreSearch extends BaseCommand {
static override description = 'Search articles';
static override examples = [
'$ omc omnivore search "AI machine learning"',
'$ omc omnivore search "label:tech" --limit 20',
'$ omc omnivore search "in:inbox" --json',
];
static override args = {
query: Args.string({ description: 'Search query', required: true }),
};
static override flags = {
limit: Flags.integer({ char: 'l', description: 'Max results', default: 10 }),
json: jsonFlag(),
};
protected async execute(flags: { query: string; limit: number; json: boolean }): Promise<void> {
const result = await searchArticles({ query: flags.query, first: flags.limit });
checkGraphQLResult({ data: { search: result }, errors: undefined });
this.outputResults(result.edges, flags.json);
}
private outputResults(edges: SearchResult['edges'], json: boolean): void {
if (json) {
this.log(JSON.stringify(edges.map((e) => e.node), null, 2));
} else {
edges.forEach(({ node }) => {
this.log(`- ${node.title}`);
this.log(` URL: ${node.url}`);
});
}
}
}

View file

@ -0,0 +1,56 @@
import { Args, Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { checkGraphQLResult } from '@lib/cli/graphql.js';
import { updatePage } from '@lib/omnivore/client.js';
/**
* Update article metadata.
* AIDEV-NOTE: Modify article title/description via Omnivore API
*/
export default class OmnivoreUpdate extends BaseCommand {
static override description = 'Update article metadata';
static override examples = [
'$ omc omnivore update article-id --title "New Title"',
'$ omc omnivore update article-id --description "Summary"',
'$ omc omnivore update article-id --title "Title" --description "Desc" --json',
];
static override args = {
articleId: Args.string({ description: 'Article ID', required: true }),
};
static override flags = {
title: Flags.string({ char: 't', description: 'New title' }),
description: Flags.string({ char: 'd', description: 'New description' }),
json: jsonFlag(),
};
protected async execute(flags: { articleId: string; title?: string; description?: string; json: boolean }): Promise<void> {
this.validateFlags(flags);
const result = await updatePage({
pageId: flags.articleId,
title: flags.title,
description: flags.description,
});
checkGraphQLResult({ data: { updatePage: result }, errors: undefined });
this.outputResult(result, flags.json);
}
private validateFlags(flags: { title?: string; description?: string }): void {
if (!flags.title && !flags.description) {
throw new Error('At least one of --title or --description is required');
}
}
private outputResult(result: any, json: boolean): void {
if (json) {
this.log(JSON.stringify(result.updatedPage, null, 2));
} else {
this.log('Article updated successfully');
}
}
}

View file

@ -0,0 +1,193 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatSuccess } from '@lib/cli/formatters.js';
import { fetchUsername, checkGraphQLResult } from '@lib/cli/graphql.js';
import { searchArticles, getArticle, getArticlesByLabel } from '@lib/omnivore/client.js';
import type { OmnivoreArticle, SearchResult } from '@omc-types/omnivore.js';
interface QueueAddFlags {
hours?: number;
url?: string;
label?: string;
slug?: string;
json: boolean;
}
type QueueSourceArticle = Pick<OmnivoreArticle, 'id' | 'url' | 'title' | 'savedAt'> & { slug: string };
/**
* Add articles to the analysis queue.
* AIDEV-NOTE: Entry point for queue population from Omnivore API
*/
export default class QueueAdd extends BaseCommand {
static override description = 'Add articles to the analysis queue';
static override examples = [
'$ omc queue add --hours 24',
'$ omc queue add --hours 168',
'$ omc queue add --label AI',
'$ omc queue add --slug my-article-slug',
'$ omc queue add --url https://omnivore.app/username/article-slug',
'$ omc queue add --hours 24 --json',
];
static override flags = {
hours: Flags.integer({
char: 'h',
description: 'Add articles from last N hours',
exclusive: ['url', 'label', 'slug'],
}),
url: Flags.string({
char: 'u',
description: 'Add single article by Omnivore URL',
exclusive: ['hours', 'label', 'slug'],
}),
label: Flags.string({
char: 'l',
description: 'Add articles with specific label',
exclusive: ['hours', 'url', 'slug'],
}),
slug: Flags.string({
char: 's',
description: 'Add single article by slug',
exclusive: ['hours', 'url', 'label'],
}),
json: jsonFlag(),
};
protected async execute(flags: QueueAddFlags): Promise<void> {
this.validateFlags(flags);
await withDatabase(async ({ repo }) => {
let articles: QueueSourceArticle[] = [];
if (flags.label) {
articles = await this.fetchByLabel(flags.label);
} else if (flags.url) {
articles = await this.fetchByUrl(flags.url);
} else if (flags.slug) {
articles = await this.fetchBySlug(flags.slug);
} else {
articles = await this.fetchByHours(this.requireHours(flags.hours));
}
const queueArticles = this.formatArticlesForQueue(articles);
const inserted = repo.initializeQueue(queueArticles);
this.outputResults(inserted, articles.length, flags.json);
});
}
private validateFlags(flags: QueueAddFlags): void {
if (!flags.hours && !flags.url && !flags.label && !flags.slug) {
throw new Error('One of --hours, --url, --label, or --slug is required');
}
}
private requireHours(hours: number | undefined): number {
if (hours === undefined) throw new Error('--hours is required');
return hours;
}
private async fetchByHours(hours: number): Promise<QueueSourceArticle[]> {
const hoursAgo = new Date(Date.now() - hours * 60 * 60 * 1000);
const collected: QueueSourceArticle[] = [];
let after: string | undefined;
let shouldContinue = true;
while (shouldContinue) {
const result = await this.fetchSearchPage(after);
const { items, reachedCutoff, nextCursor } = this.collectWithinCutoff(result, hoursAgo);
collected.push(...items);
shouldContinue = !reachedCutoff && Boolean(nextCursor);
after = nextCursor ?? undefined;
}
return collected;
}
private async fetchSearchPage(after?: string): Promise<SearchResult> {
const result = await searchArticles({
query: 'in:all sort:saved-desc',
first: 100,
after: after ?? '',
});
if (!isSearchResult(result)) throw new Error('Unexpected searchArticles response shape');
return result;
}
private collectWithinCutoff(result: SearchResult, cutoff: Date): { items: QueueSourceArticle[]; reachedCutoff: boolean; nextCursor?: string } {
const items: QueueSourceArticle[] = [];
for (const edge of result.edges) {
const node = edge.node;
if (new Date(node.savedAt) < cutoff) return { items, reachedCutoff: true };
if (isQueueSourceArticle(node)) items.push(node);
}
return { items, reachedCutoff: false, nextCursor: result.pageInfo.endCursor };
}
private async fetchByLabel(labelName: string): Promise<QueueSourceArticle[]> {
const result = await getArticlesByLabel(labelName, 50);
if (!isSearchResult(result)) throw new Error('Unexpected getArticlesByLabel response shape');
checkGraphQLResult({ data: { search: result }, errors: undefined });
return result.edges.map((edge) => this.toQueueSourceArticle(edge.node));
}
private async fetchBySlug(slug: string): Promise<QueueSourceArticle[]> {
const username = await fetchUsername();
const result = await getArticle(slug, username);
checkGraphQLResult({ data: { article: result }, errors: undefined });
if (!result.article) {
throw new Error('Article not found');
}
return [this.toQueueSourceArticle({ ...result.article, slug })];
}
private async fetchByUrl(url: string): Promise<QueueSourceArticle[]> {
// AIDEV-NOTE: URL format is https://omnivore.app/{username}/{slug}
const urlPattern = /https:\/\/omnivore\.app\/([^/]+)\/(.+)/;
const match = url.match(urlPattern);
if (!match) {
throw new Error('Invalid Omnivore URL. Expected format: https://omnivore.app/{username}/{slug}');
}
const [, username, slug] = match;
const result = await getArticle(slug, username);
checkGraphQLResult({ data: { article: result }, errors: undefined });
if (!result.article) {
throw new Error('Article not found');
}
return [this.toQueueSourceArticle({ ...result.article, slug })];
}
private toQueueSourceArticle(article: OmnivoreArticle): QueueSourceArticle {
if (!isQueueSourceArticle(article)) throw new Error('Article missing required queue fields');
return { id: article.id, slug: article.slug, url: article.url, title: article.title, savedAt: article.savedAt };
}
private formatArticlesForQueue(articles: QueueSourceArticle[]): Array<{ id: string; slug: string; url: string; title: string; savedAt: string }> {
return articles.map(({ id, slug, url, title, savedAt }) => ({ id, slug, url, title, savedAt }));
}
private outputResults(inserted: number, total: number, json: boolean): void {
if (json) {
this.log(JSON.stringify({ added: inserted, total }));
} else {
this.log(formatSuccess(`Added ${inserted} articles to queue (${total} total found)`));
}
}
}
function isSearchResult(value: unknown): value is SearchResult {
return typeof value === 'object' && value !== null && 'edges' in value && 'pageInfo' in value;
}
function isQueueSourceArticle(value: unknown): value is QueueSourceArticle {
if (typeof value !== 'object' || value === null) return false;
const v = value as { id?: unknown; slug?: unknown; url?: unknown; title?: unknown; savedAt?: unknown };
return typeof v.id === 'string' && typeof v.slug === 'string' && typeof v.url === 'string' && typeof v.title === 'string' && typeof v.savedAt === 'string';
}

View file

@ -0,0 +1,61 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatSuccess } from '@lib/cli/formatters.js';
/**
* Clear articles from queue by status or all.
* AIDEV-NOTE: CLI command for bulk queue clearing
*/
export default class QueueClear extends BaseCommand {
static override description = 'Clear articles from queue';
static override examples = [
'$ omc queue clear --status failed',
'$ omc queue clear --status completed',
'$ omc queue clear --all',
'$ omc queue clear --all --json',
];
static override flags = {
status: Flags.string({
description: 'Clear articles with specific status (pending|in_progress|completed|failed)',
exclusive: ['all'],
}),
all: Flags.boolean({
description: 'Clear all articles from queue',
exclusive: ['status'],
}),
json: jsonFlag(),
};
private validateFlags(flags: { status?: string; all: boolean; json: boolean }): void {
if (!flags.status && !flags.all) {
throw new Error('Either --status or --all flag is required');
}
if (flags.status) {
const validStatuses = ['pending', 'in_progress', 'completed', 'failed'];
if (!validStatuses.includes(flags.status)) {
throw new Error(`Invalid status: ${flags.status}. Must be one of: ${validStatuses.join(', ')}`);
}
}
}
protected async execute(flags: { status?: string; all: boolean; json: boolean }): Promise<void> {
this.validateFlags(flags);
await withDatabase(async ({ repo }) => {
const clearedCount = flags.all
? repo.clearAll()
: repo.clearByStatus(flags.status as string);
if (flags.json) {
this.log(JSON.stringify({ cleared: clearedCount }));
} else {
this.log(formatSuccess(`Cleared ${clearedCount} articles from queue`));
}
});
}
}

View file

@ -0,0 +1,59 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag, statusFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import type { AnalysisJob } from '@storage/AnalysisQueueRepository.js';
/**
* Export queue to JSONL format.
* AIDEV-NOTE: Exports queue jobs for backup/transfer (one JSON object per line)
*/
export default class QueueExport extends BaseCommand {
static override description = 'Export queue to JSONL format';
static override examples = [
'$ omc queue export',
'$ omc queue export --status pending',
'$ omc queue export --json',
];
static override flags = {
status: statusFlag(),
json: jsonFlag(),
};
async execute(flags: any): Promise<void> {
await withDatabase(async ({ repo }) => {
const jobs = await this.fetchJobs(repo, flags.status);
if (flags.json) {
this.outputJsonArray(jobs);
} else {
this.outputJsonl(jobs);
}
});
}
private async fetchJobs(repo: any, status?: string): Promise<AnalysisJob[]> {
if (status) {
return repo.getByStatus(status);
}
const allJobs: AnalysisJob[] = [];
for (const s of ['pending', 'in_progress', 'completed', 'failed']) {
allJobs.push(...repo.getByStatus(s));
}
return allJobs;
}
private outputJsonArray(jobs: AnalysisJob[]): void {
this.log(JSON.stringify(jobs, null, 2));
}
private outputJsonl(jobs: AnalysisJob[]): void {
for (const job of jobs) {
this.log(JSON.stringify(job));
}
}
}

View file

@ -0,0 +1,70 @@
import { Args } from '@oclif/core';
import { readFileSync } from 'fs';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { outputResult, parseJsonSafely } from '@lib/cli/command-utils.js';
/**
* Import queue from JSONL file.
* AIDEV-NOTE: Imports queue jobs from backup/transfer file (one JSON object per line)
*/
export default class QueueImport extends BaseCommand {
static override description = 'Import queue from JSONL file';
static override examples = [
'$ omc queue import queue-backup.jsonl',
'$ omc queue import queue-backup.jsonl --json',
];
static override args = {
file: Args.string({
description: 'Path to JSONL file',
required: true,
}),
};
static override flags = {
json: jsonFlag(),
};
async execute(flags: any): Promise<void> {
const file = await this.parse(QueueImport).then((p) => p.args.file);
const articles = this.readJsonl(file);
await withDatabase(async ({ repo }) => {
const count = repo.initializeQueue(articles);
outputResult(this, { imported: count }, `Imported ${count} articles`, flags.json);
});
}
private readJsonl(filePath: string): Array<{ id: string; slug: string; url: string; title: string; savedAt: string }> {
const content = readFileSync(filePath, 'utf-8');
const lines = content.trim().split('\n');
return lines.map(line => this.parseJobLine(line));
}
private parseJobLine(line: string): { id: string; slug: string; url: string; title: string; savedAt: string } {
const job = parseJsonSafely<any>(line);
if (!job) throw new Error('Failed to parse JSONL line');
this.validateJob(job);
return {
id: job.articleId,
slug: job.articleSlug,
url: job.articleUrl,
title: job.articleTitle,
savedAt: job.savedAt,
};
}
private validateJob(job: any): void {
const required = ['articleId', 'articleSlug', 'articleUrl', 'articleTitle', 'savedAt'];
for (const field of required) {
if (!job[field]) {
throw new Error(`Missing required field: ${field}`);
}
}
}
}

View file

@ -0,0 +1,49 @@
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader } from '@lib/cli/formatters.js';
import { displayJobs } from '@lib/cli/queue-display.js';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag, statusFlag } from '@lib/cli/shared-flags.js';
/**
* List articles in the analysis queue.
* AIDEV-NOTE: Displays queue contents with optional status filtering
*/
export default class QueueList extends BaseCommand {
static override description = 'List articles in the analysis queue';
static override examples = [
'$ omc queue list',
'$ omc queue list --status pending',
'$ omc queue list --status completed',
'$ omc queue list --json',
];
static override flags = {
status: statusFlag(),
json: jsonFlag(),
};
async execute(flags: any): Promise<void> {
await withDatabase(async ({ repo }) => {
let jobs;
if (flags.status) {
jobs = repo.getByStatus(flags.status);
} else {
// Get all jobs from all statuses
const pending = repo.getByStatus('pending');
const inProgress = repo.getByStatus('in_progress');
const completed = repo.getByStatus('completed');
const failed = repo.getByStatus('failed');
jobs = [...pending, ...inProgress, ...completed, ...failed];
}
if (flags.json) {
this.log(JSON.stringify(jobs, null, 2));
} else {
this.log(formatHeader('Queue Listing'));
displayJobs(jobs);
}
});
}
}

View file

@ -0,0 +1,47 @@
import { Args } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatSuccess } from '@lib/cli/formatters.js';
/**
* Remove article from analysis queue.
* AIDEV-NOTE: CLI command for queue article removal
*/
export default class QueueRemove extends BaseCommand {
static override description = 'Remove article from queue';
static override examples = [
'$ omc queue remove <article-id>',
];
static override args = {
articleId: Args.string({
description: 'Article ID to remove',
required: true,
}),
};
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: Record<string, any>): Promise<void> {
const { args } = await this.parse(QueueRemove);
await withDatabase(async ({ repo }) => {
const removed = repo.removeArticle(args.articleId);
if (removed === 0) {
this.warn(`Article ${args.articleId} not found in queue`);
return;
}
if (flags.json) {
this.log(JSON.stringify({ removed, articleId: args.articleId }));
} else {
this.log(formatSuccess(`Removed article ${args.articleId} from queue`));
}
});
}
}

View file

@ -0,0 +1,74 @@
import { Args, Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatSuccess } from '@lib/cli/formatters.js';
/**
* Reset article(s) to pending status for reprocessing.
* AIDEV-NOTE: Useful for retrying stuck in_progress jobs
*/
export default class QueueReset extends BaseCommand {
static override description = 'Reset article to pending status';
static override examples = [
'$ omc queue reset <article-id>',
'$ omc queue reset --all-in-progress',
];
static override args = {
articleId: Args.string({
description: 'Article ID to reset',
required: false,
}),
};
static override flags = {
'all-in-progress': Flags.boolean({
description: 'Reset all in_progress articles to pending',
default: false,
}),
json: jsonFlag(),
};
// AIDEV-NOTE: Extracted helper to keep execute() under 20 lines
private resetAllInProgress(repo: any, flags: any): void {
const jobs = repo.getByStatus('in_progress');
for (const job of jobs) {
repo.resetToPending(job.articleId);
}
if (flags.json) {
this.log(JSON.stringify({ reset: jobs.length }));
} else {
this.log(formatSuccess(`Reset ${jobs.length} in_progress articles to pending`));
}
}
// AIDEV-NOTE: Extracted helper to keep execute() under 20 lines
private resetSingleArticle(repo: any, articleId: string, flags: any): void {
repo.resetToPending(articleId);
if (flags.json) {
this.log(JSON.stringify({ reset: 1, articleId }));
} else {
this.log(formatSuccess(`Reset article ${articleId} to pending`));
}
}
async execute(flags: any): Promise<void> {
await withDatabase(async ({ repo }) => {
if (flags['all-in-progress']) {
this.resetAllInProgress(repo, flags);
return;
}
if (!flags.articleId) {
throw new Error('Either provide article-id or use --all-in-progress flag');
}
this.resetSingleArticle(repo, flags.articleId, flags);
});
}
}

View file

@ -0,0 +1,95 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader, formatDivider } from '@lib/cli/formatters.js';
import { displayQueueStats } from '@lib/cli/queue-display.js';
/**
* Show queue statistics.
* AIDEV-NOTE: Displays aggregate queue metrics for monitoring
*/
export default class QueueStats extends BaseCommand {
static override description = 'Show queue statistics';
static override examples = [
'$ omc queue stats',
'$ omc queue stats --detailed',
'$ omc queue stats --json',
];
static override flags = {
detailed: Flags.boolean({
description: 'Show detailed per-status breakdown',
default: false,
}),
json: jsonFlag(),
};
async execute(flags: any): Promise<void> {
await withDatabase(async ({ repo }) => {
const stats = repo.getStats();
if (flags.json) {
const output = flags.detailed ? this.buildDetailedStats(repo) : stats;
this.log(JSON.stringify(output, null, 2));
} else {
this.log(formatHeader('Queue Statistics'));
displayQueueStats(stats);
if (flags.detailed) {
this.displayDetailedBreakdown(repo);
}
}
});
}
private buildDetailedStats(repo: any): any {
const stats = repo.getStats();
const detailed: any = { ...stats, breakdown: {} };
for (const status of ['pending', 'in_progress', 'completed', 'failed']) {
const jobs = repo.getByStatus(status);
detailed.breakdown[status] = this.analyzeJobs(jobs);
}
return detailed;
}
private displayDetailedBreakdown(repo: any): void {
this.log('\n' + formatDivider());
this.log('Detailed Breakdown\n');
for (const status of ['pending', 'in_progress', 'completed', 'failed']) {
const jobs = repo.getByStatus(status);
this.displayStatusAnalysis(status, jobs);
}
}
private displayStatusAnalysis(status: string, jobs: any[]): void {
const analysis = this.analyzeJobs(jobs);
this.log(`${status.toUpperCase()}:`);
this.log(` Count: ${analysis.count}`);
if (analysis.oldestDate) this.log(` Oldest: ${analysis.oldestDate}`);
if (analysis.newestDate) this.log(` Newest: ${analysis.newestDate}`);
if (status === 'failed') this.log(` Total Retries: ${analysis.totalRetries}`);
this.log('');
}
private analyzeJobs(jobs: any[]): any {
if (jobs.length === 0) {
return { count: 0 };
}
const dates = jobs.map(j => new Date(j.createdAt).getTime()).filter(d => !isNaN(d));
const totalRetries = jobs.reduce((sum, j) => sum + (j.retryCount || 0), 0);
return {
count: jobs.length,
oldestDate: dates.length > 0 ? new Date(Math.min(...dates)).toISOString() : null,
newestDate: dates.length > 0 ? new Date(Math.max(...dates)).toISOString() : null,
totalRetries,
};
}
}

View file

@ -0,0 +1,103 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader } from '@lib/cli/formatters.js';
import { parseJsonSafely } from '@lib/cli/command-utils.js';
import type { ContentAnalysis } from '@omc-types/analysis.js';
/**
* Full corpus analysis report.
* AIDEV-NOTE: Aggregates topic frequencies, sentiment distribution, content types
*/
export default class ReportCorpus extends BaseCommand {
static override description = 'Generate full corpus analysis report';
static override examples = [
'$ omc report corpus',
'$ omc report corpus --json',
];
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: any): Promise<void> {
await withDatabase(async ({ repo }) => {
const jobs = repo.getCompletedWithAnalysis();
const analyses = this.parseAnalyses(jobs);
if (flags.json) {
this.log(JSON.stringify(this.buildCorpusStats(analyses), null, 2));
} else {
this.displayCorpusReport(analyses);
}
});
}
private parseAnalyses(jobs: any[]): ContentAnalysis[] {
return jobs.map(job => parseJsonSafely<ContentAnalysis>(job.analysisJson))
.filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A');
}
private buildCorpusStats(analyses: ContentAnalysis[]): any {
const topicFreq = this.countTopics(analyses);
const sentimentDist = this.countSentiment(analyses);
const contentTypeDist = this.countContentTypes(analyses);
return {
totalAnalyses: analyses.length,
topicDistribution: topicFreq,
sentimentDistribution: sentimentDist,
contentTypeDistribution: contentTypeDist,
};
}
private displayCorpusReport(analyses: ContentAnalysis[]): void {
this.log(formatHeader('Corpus Analysis Report'));
this.log(`\nTotal Analyses: ${analyses.length}\n`);
this.log('Topic Distribution:');
const topics = this.countTopics(analyses);
for (const [topic, count] of Object.entries(topics).sort((a, b) => b[1] - a[1])) {
this.log(` ${topic.padEnd(30)} ${count}`);
}
this.log('\nSentiment Distribution:');
const sentiment = this.countSentiment(analyses);
for (const [sent, count] of Object.entries(sentiment)) {
this.log(` ${sent}: ${count}`);
}
this.log('\nContent Types:');
const types = this.countContentTypes(analyses);
for (const [type, count] of Object.entries(types).sort((a, b) => b[1] - a[1])) {
this.log(` ${type.padEnd(30)} ${count}`);
}
}
private countTopics(analyses: ContentAnalysis[]): Record<string, number> {
const counts: Record<string, number> = {};
for (const a of analyses) {
for (const topic of a.topics) {
counts[topic] = (counts[topic] || 0) + 1;
}
}
return counts;
}
private countSentiment(analyses: ContentAnalysis[]): Record<string, number> {
const counts: Record<string, number> = {};
for (const a of analyses) {
counts[a.sentiment] = (counts[a.sentiment] || 0) + 1;
}
return counts;
}
private countContentTypes(analyses: ContentAnalysis[]): Record<string, number> {
const counts: Record<string, number> = {};
for (const a of analyses) {
counts[a.contentType] = (counts[a.contentType] || 0) + 1;
}
return counts;
}
}

View file

@ -0,0 +1,84 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader } from '@lib/cli/formatters.js';
import { parseJsonSafely } from '@lib/cli/command-utils.js';
import type { ContentAnalysis } from '@omc-types/analysis.js';
/**
* Custom SQL-based report.
* AIDEV-NOTE: Allows filtering analyses with custom SQL WHERE clause
*/
export default class ReportCustom extends BaseCommand {
static override description = 'Run custom SQL query against analyses';
static override examples = [
'$ omc report custom --query "article_title LIKE \'%AI%\'"',
'$ omc report custom --query "completed_at > \'2024-01-01\'" --json',
];
static override flags = {
query: Flags.string({
description: 'SQL WHERE clause for filtering',
required: true,
}),
json: jsonFlag(),
};
protected async execute(flags: any): Promise<void> {
await withDatabase(async ({ db }) => {
const results = this.executeCustomQuery(db, flags.query);
if (flags.json) {
this.log(JSON.stringify(results, null, 2));
} else {
this.displayResults(results);
}
});
}
private executeCustomQuery(db: any, whereClause: string): any[] {
try {
const sql = this.buildQuerySQL(whereClause);
const rows = db.prepare(sql).all();
return this.mapQueryResults(rows);
} catch (error) {
throw new Error(`SQL query failed: ${error instanceof Error ? error.message : String(error)}`);
}
}
private buildQuerySQL(whereClause: string): string {
return `
SELECT article_id as articleId, article_title as articleTitle,
analysis_json as analysisJson, completed_at as completedAt
FROM analysis_queue
WHERE status = 'completed' AND analysis_json IS NOT NULL AND (${whereClause})
ORDER BY completed_at DESC
`;
}
private mapQueryResults(rows: any[]): any[] {
return rows.map((row: any) => {
const analysis = parseJsonSafely<ContentAnalysis>(row.analysisJson);
return {
articleId: row.articleId,
articleTitle: row.articleTitle,
completedAt: row.completedAt,
analysis,
};
}).filter(r => r.analysis);
}
private displayResults(results: any[]): void {
this.log(formatHeader('Custom Query Results'));
this.log(`\nFound ${results.length} matching analyses\n`);
for (const { articleTitle, completedAt, analysis } of results) {
this.log(`${articleTitle}`);
this.log(` Completed: ${completedAt}`);
this.log(` Topics: ${analysis.topics.join(', ')}`);
this.log(` Sentiment: ${analysis.sentiment}\n`);
}
}
}

View file

@ -0,0 +1,127 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { withDatabase } from '@lib/cli/database.js';
import { parseJsonSafely } from '@lib/cli/command-utils.js';
import type { ContentAnalysis } from '@omc-types/analysis.js';
/**
* Export report data.
* AIDEV-NOTE: Supports json|csv|markdown output formats for different use cases
*/
export default class ReportExport extends BaseCommand {
static override description = 'Export report data to file or stdout';
static override examples = [
'$ omc report export --format json',
'$ omc report export --format csv --report-type topics',
'$ omc report export --format markdown',
];
static override flags = {
format: Flags.string({
description: 'Output format',
required: true,
options: ['json', 'csv', 'markdown'],
}),
'report-type': Flags.string({
description: 'Type of report to export',
default: 'corpus',
options: ['corpus', 'topics', 'sentiment', 'monetization'],
}),
};
protected async execute(flags: any): Promise<void> {
await withDatabase(async ({ repo }) => {
const jobs = repo.getCompletedWithAnalysis();
const analyses = this.parseAnalyses(jobs);
const output = this.formatOutput(analyses, flags.format, flags['report-type']);
this.log(output);
});
}
private parseAnalyses(jobs: any[]): ContentAnalysis[] {
return jobs.map(job => parseJsonSafely<ContentAnalysis>(job.analysisJson))
.filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A');
}
private formatOutput(analyses: ContentAnalysis[], format: string, reportType: string): string {
if (format === 'json') return this.exportJSON(analyses, reportType);
if (format === 'csv') return this.exportCSV(analyses, reportType);
return this.exportMarkdown(analyses, reportType);
}
private exportJSON(analyses: ContentAnalysis[], reportType: string): string {
const data = reportType === 'topics' ? this.buildTopicsData(analyses) :
reportType === 'sentiment' ? this.buildSentimentData(analyses) :
reportType === 'monetization' ? this.buildMonetizationData(analyses) :
{ analyses };
return JSON.stringify(data, null, 2);
}
private exportCSV(analyses: ContentAnalysis[], reportType: string): string {
if (reportType === 'topics') {
const topics = this.buildTopicsData(analyses);
return 'Topic,Frequency,AvgScore\n' +
topics.map((t: any) => `${t.topic},${t.frequency},${t.avgScore}`).join('\n');
}
return 'ArticleId,Topics,Sentiment,ContentType\n' +
analyses.map(a => `${a.articleId},"${a.topics.join(';')}",${a.sentiment},${a.contentType}`).join('\n');
}
private exportMarkdown(analyses: ContentAnalysis[], reportType: string): string {
let md = `# ${reportType.charAt(0).toUpperCase() + reportType.slice(1)} Report\n\n`;
md += `Total Analyses: ${analyses.length}\n\n`;
if (reportType === 'topics') {
md += '## Topic Distribution\n\n';
const topics = this.buildTopicsData(analyses);
for (const t of topics) {
md += `- **${t.topic}**: ${t.frequency} articles (avg score: ${t.avgScore})\n`;
}
} else {
md += '## Articles\n\n';
for (const a of analyses.slice(0, 10)) {
md += `### ${a.articleId}\n`;
md += `- Topics: ${a.topics.join(', ')}\n`;
md += `- Sentiment: ${a.sentiment}\n\n`;
}
}
return md;
}
private buildTopicsData(analyses: ContentAnalysis[]): any[] {
const topicData: Record<string, { count: number; scoreSum: number }> = {};
for (const a of analyses) {
for (const topic of a.topics) {
if (!topicData[topic]) topicData[topic] = { count: 0, scoreSum: 0 };
topicData[topic].count++;
topicData[topic].scoreSum += a.topicScores[topic] || 0;
}
}
return Object.entries(topicData)
.map(([topic, data]) => ({
topic,
frequency: data.count,
avgScore: (data.scoreSum / data.count).toFixed(2),
}))
.sort((a, b) => b.frequency - a.frequency);
}
private buildSentimentData(analyses: ContentAnalysis[]): any {
const counts: Record<string, number> = {};
for (const a of analyses) {
counts[a.sentiment] = (counts[a.sentiment] || 0) + 1;
}
return { distribution: counts };
}
private buildMonetizationData(analyses: ContentAnalysis[]): any[] {
return analyses.map(a => ({
articleId: a.articleId,
monetizationAngle: a.monetizationAngle,
topics: a.topics,
sentiment: a.sentiment,
}));
}
}

View file

@ -0,0 +1,95 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader } from '@lib/cli/formatters.js';
import { parseJsonSafely } from '@lib/cli/command-utils.js';
import type { ContentAnalysis } from '@omc-types/analysis.js';
/**
* Monetization opportunities report.
* AIDEV-NOTE: Extracts monetizationAngle from analyses, ranks by potential
*/
export default class ReportMonetization extends BaseCommand {
static override description = 'Show monetization opportunities across analyses';
static override examples = [
'$ omc report monetization',
'$ omc report monetization --json',
];
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: any): Promise<void> {
await withDatabase(async ({ repo }) => {
const jobs = repo.getCompletedWithAnalysis();
const analyses = this.parseAnalyses(jobs);
const opportunities = this.extractOpportunities(analyses);
if (flags.json) {
this.log(JSON.stringify(opportunities, null, 2));
} else {
this.displayOpportunities(opportunities);
}
});
}
private parseAnalyses(jobs: any[]): ContentAnalysis[] {
return jobs.map(job => parseJsonSafely<ContentAnalysis>(job.analysisJson))
.filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A');
}
private extractOpportunities(analyses: ContentAnalysis[]): any[] {
const themeGroups: Record<string, any[]> = {};
for (const a of analyses) {
const theme = this.extractTheme(a.monetizationAngle);
if (!themeGroups[theme]) themeGroups[theme] = [];
themeGroups[theme].push({
articleId: a.articleId,
angle: a.monetizationAngle,
sentiment: a.sentiment,
topics: a.topics,
});
}
return Object.entries(themeGroups)
.map(([theme, items]) => ({
theme,
count: items.length,
items,
potential: this.calculatePotential(items),
}))
.sort((a, b) => b.potential - a.potential);
}
private extractTheme(angle: string): string {
const lower = angle.toLowerCase();
if (lower.includes('comparison') || lower.includes('compare')) return 'comparison';
if (lower.includes('roundup') || lower.includes('weekly')) return 'roundup';
if (lower.includes('tutorial') || lower.includes('guide')) return 'tutorial';
if (lower.includes('review')) return 'review';
return 'other';
}
private calculatePotential(items: any[]): number {
const positiveCount = items.filter(i => i.sentiment === 'positive').length;
return items.length * 10 + positiveCount * 5;
}
private displayOpportunities(opportunities: any[]): void {
this.log(formatHeader('Monetization Opportunities Report'));
this.log('');
for (const { theme, count, items, potential } of opportunities) {
this.log(`${theme.toUpperCase()} (${count} articles, potential: ${potential})`);
for (const item of items.slice(0, 3)) {
this.log(`${item.angle}`);
}
if (items.length > 3) this.log(` ... and ${items.length - 3} more\n`);
else this.log('');
}
}
}

View file

@ -0,0 +1,94 @@
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader } from '@lib/cli/formatters.js';
import { parseJsonSafely } from '@lib/cli/command-utils.js';
import type { ContentAnalysis } from '@omc-types/analysis.js';
/**
* Sentiment analysis report.
* AIDEV-NOTE: Sentiment distribution correlated with topics, identifies outliers
*/
export default class ReportSentiment extends BaseCommand {
static override description = 'Show sentiment analysis across corpus';
static override examples = [
'$ omc report sentiment',
'$ omc report sentiment --json',
];
static override flags = {
json: jsonFlag(),
};
protected async execute(flags: any): Promise<void> {
await withDatabase(async ({ repo }) => {
const jobs = repo.getCompletedWithAnalysis();
const analyses = this.parseAnalyses(jobs);
const sentimentData = this.analyzeSentiment(analyses);
if (flags.json) {
this.log(JSON.stringify(sentimentData, null, 2));
} else {
this.displaySentimentReport(sentimentData);
}
});
}
private parseAnalyses(jobs: any[]): ContentAnalysis[] {
return jobs.map(job => parseJsonSafely<ContentAnalysis>(job.analysisJson))
.filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A');
}
private analyzeSentiment(analyses: ContentAnalysis[]): any {
const distribution = this.countSentiment(analyses);
const topicCorrelation = this.correlateSentimentWithTopics(analyses);
return { distribution, topicCorrelation };
}
private countSentiment(analyses: ContentAnalysis[]): Record<string, number> {
const counts: Record<string, number> = {};
for (const a of analyses) {
counts[a.sentiment] = (counts[a.sentiment] || 0) + 1;
}
return counts;
}
private correlateSentimentWithTopics(analyses: ContentAnalysis[]): any[] {
const topicSentiment: Record<string, Record<string, number>> = {};
for (const a of analyses) {
for (const topic of a.topics) {
if (!topicSentiment[topic]) {
topicSentiment[topic] = { positive: 0, neutral: 0, negative: 0 };
}
topicSentiment[topic][a.sentiment]++;
}
}
return Object.entries(topicSentiment)
.map(([topic, sentiments]) => ({
topic,
positive: sentiments.positive || 0,
neutral: sentiments.neutral || 0,
negative: sentiments.negative || 0,
total: (sentiments.positive || 0) + (sentiments.neutral || 0) + (sentiments.negative || 0),
}))
.sort((a, b) => b.total - a.total);
}
private displaySentimentReport(data: any): void {
this.log(formatHeader('Sentiment Analysis Report'));
this.log('\nOverall Distribution:');
for (const [sentiment, count] of Object.entries(data.distribution)) {
this.log(` ${sentiment}: ${count}`);
}
this.log('\nSentiment by Topic:');
for (const { topic, positive, neutral, negative, total } of data.topicCorrelation) {
this.log(` ${topic.padEnd(30)} +${positive} ~${neutral} -${negative} (total: ${total})`);
}
}
}

View file

@ -0,0 +1,94 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader } from '@lib/cli/formatters.js';
import { parseJsonSafely } from '@lib/cli/command-utils.js';
import type { ContentAnalysis } from '@omc-types/analysis.js';
/**
* Topic distribution report.
* AIDEV-NOTE: Groups analyses by topics with frequency and average scores
*/
export default class ReportTopics extends BaseCommand {
static override description = 'Show topic distribution across all analyses';
static override examples = [
'$ omc report topics',
'$ omc report topics --min-score 0.8',
'$ omc report topics --json',
];
static override flags = {
'min-score': Flags.string({
description: 'Minimum topic score threshold (0-1)',
default: '0.0',
}),
json: jsonFlag(),
};
protected async execute(flags: any): Promise<void> {
await withDatabase(async ({ repo }) => {
const jobs = repo.getCompletedWithAnalysis();
const analyses = this.parseAnalyses(jobs);
const minScore = parseFloat(flags['min-score']);
const topicStats = this.buildTopicStats(analyses, minScore);
if (flags.json) {
this.log(JSON.stringify(topicStats, null, 2));
} else {
this.displayTopicReport(topicStats);
}
});
}
private parseAnalyses(jobs: any[]): ContentAnalysis[] {
return jobs.map(job => parseJsonSafely<ContentAnalysis>(job.analysisJson))
.filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A');
}
private buildTopicStats(analyses: ContentAnalysis[], minScore: number): any[] {
const topicData = this.aggregateTopicData(analyses, minScore);
return this.formatTopicStats(topicData);
}
private aggregateTopicData(analyses: ContentAnalysis[], minScore: number): Record<string, { count: number; scoreSum: number; articles: string[] }> {
const topicData: Record<string, { count: number; scoreSum: number; articles: string[] }> = {};
for (const a of analyses) {
for (const topic of a.topics) {
const score = a.topicScores[topic] || 0;
if (score < minScore) continue;
if (!topicData[topic]) {
topicData[topic] = { count: 0, scoreSum: 0, articles: [] };
}
topicData[topic].count++;
topicData[topic].scoreSum += score;
topicData[topic].articles.push(a.articleId);
}
}
return topicData;
}
private formatTopicStats(topicData: Record<string, { count: number; scoreSum: number; articles: string[] }>): any[] {
return Object.entries(topicData)
.map(([topic, data]) => ({
topic,
frequency: data.count,
avgScore: data.count > 0 ? (data.scoreSum / data.count).toFixed(2) : '0',
articles: data.articles,
}))
.sort((a, b) => b.frequency - a.frequency);
}
private displayTopicReport(stats: any[]): void {
this.log(formatHeader('Topic Distribution Report'));
this.log('');
for (const { topic, frequency, avgScore } of stats) {
this.log(`${topic.padEnd(30)} ${frequency} articles (avg score: ${avgScore})`);
}
}
}

View file

@ -0,0 +1,109 @@
import { Flags } from '@oclif/core';
import { BaseCommand } from '@lib/cli/base-command.js';
import { jsonFlag } from '@lib/cli/shared-flags.js';
import { withDatabase } from '@lib/cli/database.js';
import { formatHeader } from '@lib/cli/formatters.js';
import { parseJsonSafely } from '@lib/cli/command-utils.js';
import type { ContentAnalysis } from '@omc-types/analysis.js';
/**
* Trending topics over time.
* AIDEV-NOTE: Tracks topic frequency changes, identifies rising/falling trends
*/
export default class ReportTrends extends BaseCommand {
static override description = 'Show trending topics over time';
static override examples = [
'$ omc report trends',
'$ omc report trends --period week',
'$ omc report trends --json',
];
static override flags = {
period: Flags.string({
description: 'Time period for grouping (day|week|month)',
default: 'week',
options: ['day', 'week', 'month'],
}),
json: jsonFlag(),
};
protected async execute(flags: any): Promise<void> {
await withDatabase(async ({ repo }) => {
const jobs = repo.getCompletedWithAnalysis();
const trends = this.buildTrendAnalysis(jobs, flags.period);
if (flags.json) {
this.log(JSON.stringify(trends, null, 2));
} else {
this.displayTrends(trends);
}
});
}
private buildTrendAnalysis(jobs: any[], period: string): any {
const periodData: Record<string, Record<string, number>> = {};
for (const job of jobs) {
const analysis = parseJsonSafely<ContentAnalysis>(job.analysisJson);
if (!analysis || !analysis.topics || analysis.topics[0] === 'N/A') continue;
const periodKey = this.getPeriodKey(job.completedAt, period);
if (!periodData[periodKey]) periodData[periodKey] = {};
for (const topic of analysis.topics) {
periodData[periodKey][topic] = (periodData[periodKey][topic] || 0) + 1;
}
}
return this.calculateTrends(periodData);
}
private getPeriodKey(date: string, period: string): string {
const d = new Date(date);
if (period === 'day') return d.toISOString().split('T')[0];
if (period === 'week') {
const weekNum = Math.floor(d.getTime() / (7 * 24 * 60 * 60 * 1000));
return `Week-${weekNum}`;
}
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}`;
}
private calculateTrends(periodData: Record<string, Record<string, number>>): any {
const periods = Object.keys(periodData).sort();
const allTopics = this.extractAllTopics(periodData);
const trends = Array.from(allTopics).map(topic => this.calculateTopicTrend(topic, periods, periodData));
return { periods, trends: trends.sort((a, b) => parseFloat(b.avgSecond) - parseFloat(a.avgSecond)) };
}
private extractAllTopics(periodData: Record<string, Record<string, number>>): Set<string> {
const topics = new Set<string>();
for (const topicCounts of Object.values(periodData)) {
for (const topic of Object.keys(topicCounts)) topics.add(topic);
}
return topics;
}
private calculateTopicTrend(topic: string, periods: string[], periodData: Record<string, Record<string, number>>): any {
const counts = periods.map(p => periodData[p][topic] || 0);
const firstHalf = counts.slice(0, Math.floor(counts.length / 2));
const secondHalf = counts.slice(Math.floor(counts.length / 2));
const avgFirst = firstHalf.reduce((a, b) => a + b, 0) / (firstHalf.length || 1);
const avgSecond = secondHalf.reduce((a, b) => a + b, 0) / (secondHalf.length || 1);
const trend = avgSecond > avgFirst ? 'rising' : avgSecond < avgFirst ? 'falling' : 'stable';
return { topic, trend, avgFirst: avgFirst.toFixed(1), avgSecond: avgSecond.toFixed(1) };
}
private displayTrends(data: any): void {
this.log(formatHeader('Topic Trends Report'));
this.log(`\nPeriods analyzed: ${data.periods.join(', ')}\n`);
for (const { topic, trend, avgFirst, avgSecond } of data.trends) {
const arrow = trend === 'rising' ? '↑' : trend === 'falling' ? '↓' : '→';
this.log(`${arrow} ${topic.padEnd(30)} ${avgFirst}${avgSecond} (${trend})`);
}
}
}

Some files were not shown because too many files have changed in this diff Show more