mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
Add omc-cron sidecar and vendor OMC package
This commit is contained in:
parent
6ff55c4043
commit
fdae98d3fe
132 changed files with 19386 additions and 2 deletions
|
|
@ -5,6 +5,7 @@
|
|||
**/Dockerfile
|
||||
**/.dockerignore
|
||||
**/*.yaml
|
||||
!self-hosting/omc/pnpm-lock.yaml
|
||||
.secrets*.yaml
|
||||
apple
|
||||
android
|
||||
|
|
|
|||
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -35,6 +35,9 @@ package-lock.json
|
|||
.env.local
|
||||
.env.production
|
||||
|
||||
# OMC cron sidecar secrets
|
||||
self-hosting/docker-compose/omc.env
|
||||
|
||||
|
||||
# build & dist dirs
|
||||
build
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
## Docker Compose
|
||||
|
||||
We recommend using Docker-compose for the simplest way to deploy Omnivore. We have provided a configuration in the `self-hosting/docker-compose` folder.
|
||||
|
||||
|
||||
All networking and persistent storage is handled by the docker-compose file.
|
||||
|
||||
### Requirements
|
||||
|
|
@ -62,6 +62,26 @@ When the service is ready you can access the web-app by using localhost:3000
|
|||
|
||||
With the default .env file you will be able to use Omnivore, add RSS Feeds, add stories etc.
|
||||
|
||||
### Optional: OMC Cron Sidecar (Content Analysis + Notes)
|
||||
|
||||
This repo includes an optional `omc-cron` service that runs the Omnivore Content System (OMC) on a schedule:
|
||||
|
||||
- Fetch recent saves into an analysis queue
|
||||
- Run `omc analyze auto` (non-interactive) using `codex exec`
|
||||
- Classify analyzed items (adds topic/sentiment/type labels)
|
||||
- Sync summaries + optional NOTE highlights back into Omnivore
|
||||
- Run non-destructive hygiene (backup + integrity checks + daily corpus report)
|
||||
|
||||
To enable:
|
||||
|
||||
1. In `self-hosting/docker-compose/`, create `omc.env` from the example:
|
||||
- `cp omc.env.example omc.env`
|
||||
2. Fill in:
|
||||
- `OMNIVORE_API_KEY` (from Omnivore Settings → API Keys)
|
||||
- `OPENAI_API_KEY` (for `codex exec`)
|
||||
3. Start the stack:
|
||||
- `docker compose up -d`
|
||||
|
||||
|
||||
### Additional Services used:
|
||||
|
||||
|
|
@ -391,4 +411,3 @@ To learn more about setting up the OpenAI Api key, read here: https://openai.com
|
|||
In future releases we would like to be able to open this up to use different LLMs, such as Anthropic, Mistral, Bedrock, or any of the other myriad LLM Services.
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -153,6 +153,23 @@ services:
|
|||
condition: service_healthy
|
||||
restart: always
|
||||
|
||||
omc-cron:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: self-hosting/docker-compose/omc/Dockerfile
|
||||
container_name: "omnivore-omc-cron"
|
||||
env_file:
|
||||
- ./omc.env
|
||||
volumes:
|
||||
- omc_data:/opt/omc/data
|
||||
- omc_content:/opt/omc/content
|
||||
- omc_temp:/opt/omc/temp
|
||||
- omc_logs:/var/log/omc
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
restart: always
|
||||
|
||||
print-server:
|
||||
profiles: ["print"]
|
||||
build:
|
||||
|
|
@ -188,6 +205,10 @@ volumes:
|
|||
pgdata:
|
||||
redis_data:
|
||||
minio_data:
|
||||
omc_data:
|
||||
omc_content:
|
||||
omc_temp:
|
||||
omc_logs:
|
||||
print_spool:
|
||||
|
||||
networks:
|
||||
|
|
|
|||
21
self-hosting/docker-compose/omc.env.example
Normal file
21
self-hosting/docker-compose/omc.env.example
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
# Omnivore Content System (OMC) cron sidecar
|
||||
#
|
||||
# Copy to `omc.env` and fill in secrets:
|
||||
# cp omc.env.example omc.env
|
||||
|
||||
# Omnivore GraphQL endpoint (inside compose network)
|
||||
OMNIVORE_API_URL=http://api:8080/api/graphql
|
||||
|
||||
# Omnivore API key (generate in Omnivore UI: Settings → API Keys)
|
||||
OMNIVORE_API_KEY=
|
||||
|
||||
# Codex CLI auth (used by `codex exec` during `omc analyze auto`)
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# Optional tuning
|
||||
OMC_QUEUE_HOURS=2
|
||||
OMC_BATCH_SIZE=5
|
||||
OMC_JSONL_PATH=content/analysis/analyses.jsonl
|
||||
OMC_PREP_SINCE_HOURS=24
|
||||
OMC_PREP_LIMIT=50
|
||||
|
||||
44
self-hosting/docker-compose/omc/Dockerfile
Normal file
44
self-hosting/docker-compose/omc/Dockerfile
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
FROM node:18-bookworm-slim
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
ENV PNPM_HOME=/pnpm
|
||||
ENV PATH=$PNPM_HOME:$PATH
|
||||
ENV OMC_DIR=/opt/omc
|
||||
|
||||
WORKDIR /opt/omc
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
bash \
|
||||
ca-certificates \
|
||||
curl \
|
||||
python3 \
|
||||
make \
|
||||
g++ \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# pnpm (Corepack-managed, pinned to the OMC repo's packageManager)
|
||||
RUN corepack enable && corepack prepare pnpm@10.14.0 --activate
|
||||
|
||||
# Codex CLI (OMC's default analyzer shells out to `codex exec`)
|
||||
RUN pnpm add -g @openai/codex@0.93.0
|
||||
|
||||
# supercronic (cron runner that inherits container env)
|
||||
RUN curl -fsSL -o /usr/local/bin/supercronic "https://github.com/aptible/supercronic/releases/download/v0.2.29/supercronic-linux-${TARGETARCH}" \
|
||||
&& chmod +x /usr/local/bin/supercronic
|
||||
|
||||
# Install/build OMC (vendored into this repo under self-hosting/omc/)
|
||||
COPY self-hosting/omc/package.json self-hosting/omc/pnpm-lock.yaml self-hosting/omc/.npmrc ./
|
||||
RUN pnpm install --frozen-lockfile
|
||||
COPY self-hosting/omc/ ./
|
||||
RUN pnpm run build
|
||||
|
||||
RUN mkdir -p /etc/omc /var/log/omc /var/lock/omc /opt/omc/data /opt/omc/content /opt/omc/temp
|
||||
|
||||
COPY self-hosting/docker-compose/omc/omc.crontab /etc/omc/omc.crontab
|
||||
COPY self-hosting/docker-compose/omc/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
||||
|
||||
7
self-hosting/docker-compose/omc/docker-entrypoint.sh
Normal file
7
self-hosting/docker-compose/omc/docker-entrypoint.sh
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p /var/log/omc /var/lock/omc /opt/omc/data /opt/omc/content /opt/omc/temp /opt/omc/content/corpus-reports
|
||||
|
||||
exec /usr/local/bin/supercronic -passthrough-logs /etc/omc/omc.crontab
|
||||
|
||||
14
self-hosting/docker-compose/omc/omc.crontab
Normal file
14
self-hosting/docker-compose/omc/omc.crontab
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
SHELL=/bin/bash
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
# Global lock to avoid overlapping runs.
|
||||
# supercronic runs jobs with container env (so OMNIVORE_API_KEY, OMNIVORE_API_URL, OPENAI_API_KEY work).
|
||||
|
||||
# Every 30 minutes: ingest recent items -> analyze -> retry failures.
|
||||
*/30 * * * * flock -n /var/lock/omc/omc.lock bash -lc 'cd /opt/omc && node dist/bin/omc.js queue add --hours "${OMC_QUEUE_HOURS:-2}" && node dist/bin/omc.js analyze auto --batch-size "${OMC_BATCH_SIZE:-5}" --jsonl --jsonl-path "${OMC_JSONL_PATH:-content/analysis/analyses.jsonl}" && node dist/bin/omc.js analyze retry --failed' >> /var/log/omc/analyze.log 2>&1
|
||||
|
||||
# Hourly: content prep (classification + sync summaries/notes back to Omnivore).
|
||||
10 * * * * flock -n /var/lock/omc/omc.lock bash -lc 'cd /opt/omc && node dist/bin/omc.js content classify --all --since-hours "${OMC_PREP_SINCE_HOURS:-24}" --limit "${OMC_PREP_LIMIT:-50}" && node dist/bin/omc.js content sync --all --since-hours "${OMC_PREP_SINCE_HOURS:-24}" --limit "${OMC_PREP_LIMIT:-50}" --create-notes' >> /var/log/omc/prep.log 2>&1
|
||||
|
||||
# Daily hygiene (non-destructive): backup + integrity check + summary report.
|
||||
0 2 * * * flock -n /var/lock/omc/omc.lock bash -lc 'cd /opt/omc && node dist/bin/omc.js db backup && node dist/bin/omc.js db check && node dist/bin/omc.js doctor && mkdir -p content/corpus-reports && node dist/bin/omc.js report corpus > "content/corpus-reports/$(date -I)-daily.md"' >> /var/log/omc/hygiene.log 2>&1
|
||||
67
self-hosting/omc/.gitignore
vendored
Normal file
67
self-hosting/omc/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
# Environment variables
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
# Dependencies
|
||||
node_modules/
|
||||
.pnpm-store/
|
||||
pnpm-lock.yaml
|
||||
package-lock.json
|
||||
yarn.lock
|
||||
|
||||
# Runtime data
|
||||
data/
|
||||
*.sqlite
|
||||
*.sqlite-journal
|
||||
*.db
|
||||
|
||||
# Test outputs
|
||||
test-scripts/test-output/
|
||||
|
||||
# Cache
|
||||
cache/
|
||||
.cache/
|
||||
|
||||
# Generated content
|
||||
generated/
|
||||
output/
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
*.log
|
||||
npm-debug.log*
|
||||
pnpm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# Test coverage
|
||||
coverage/
|
||||
|
||||
# Build output
|
||||
dist/
|
||||
build/
|
||||
|
||||
# GraphQL codegen output (currently not used by runtime)
|
||||
src/types/generated/
|
||||
|
||||
# Temporary files
|
||||
tmp/
|
||||
temp/
|
||||
*.tmp
|
||||
|
||||
# API keys and secrets (extra safety)
|
||||
**/secrets/
|
||||
**/*secret*
|
||||
**/*key*.json
|
||||
1
self-hosting/omc/.npmrc
Normal file
1
self-hosting/omc/.npmrc
Normal file
|
|
@ -0,0 +1 @@
|
|||
enable-pre-post-scripts=true
|
||||
40
self-hosting/omc/AGENTS.md
Normal file
40
self-hosting/omc/AGENTS.md
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
# Agent Instructions
|
||||
|
||||
This project uses **bd** (beads) for issue tracking. Run `bd onboard` to get started.
|
||||
|
||||
## Quick Reference
|
||||
|
||||
```bash
|
||||
bd ready # Find available work
|
||||
bd show <id> # View issue details
|
||||
bd update <id> --status in_progress # Claim work
|
||||
bd close <id> # Complete work
|
||||
bd sync # Sync with git
|
||||
```
|
||||
|
||||
## Landing the Plane (Session Completion)
|
||||
|
||||
**When ending a work session**, you MUST complete ALL steps below. Work is NOT complete until `git push` succeeds.
|
||||
|
||||
**MANDATORY WORKFLOW:**
|
||||
|
||||
1. **File issues for remaining work** - Create issues for anything that needs follow-up
|
||||
2. **Run quality gates** (if code changed) - Tests, linters, builds
|
||||
3. **Update issue status** - Close finished work, update in-progress items
|
||||
4. **PUSH TO REMOTE** - This is MANDATORY:
|
||||
```bash
|
||||
git pull --rebase
|
||||
bd sync
|
||||
git push
|
||||
git status # MUST show "up to date with origin"
|
||||
```
|
||||
5. **Clean up** - Clear stashes, prune remote branches
|
||||
6. **Verify** - All changes committed AND pushed
|
||||
7. **Hand off** - Provide context for next session
|
||||
|
||||
**CRITICAL RULES:**
|
||||
- Work is NOT complete until `git push` succeeds
|
||||
- NEVER stop before pushing - that leaves work stranded locally
|
||||
- NEVER say "ready to push when you are" - YOU must push
|
||||
- If push fails, resolve and retry until it succeeds
|
||||
|
||||
524
self-hosting/omc/CLAUDE.md
Normal file
524
self-hosting/omc/CLAUDE.md
Normal file
|
|
@ -0,0 +1,524 @@
|
|||
# Omnivore Content System - Agent Context
|
||||
|
||||
## Project Purpose
|
||||
Transform voracious reading habits into monetizable content through AI-powered analysis and generation.
|
||||
|
||||
## Content Strategy
|
||||
|
||||
### Primary Topics
|
||||
- **AI & Machine Learning**: LLMs, agents, training, deployment
|
||||
- **Tech Infrastructure**: Cloud, DevOps, databases, system design
|
||||
- **Software Engineering**: Best practices, tools, frameworks
|
||||
- **Startup/Business**: Product, growth, monetization strategies
|
||||
|
||||
### Content Goals
|
||||
1. **Weekly Blog Roundup**: Top 5-10 AI/Tech stories with analysis
|
||||
2. **Deep Dive Posts**: Multi-article synthesis into original analysis
|
||||
3. **Newsletter**: Curated links with personal commentary
|
||||
4. **SEO Optimization**: Drive organic traffic through strategic keywords
|
||||
5. **Monetization**: Build audience → affiliate links → sponsorships
|
||||
|
||||
## Writing Style
|
||||
|
||||
### Voice & Tone
|
||||
- **Authoritative yet accessible**: Technical depth without jargon overload
|
||||
- **Opinionated**: Strong takes backed by evidence from reading
|
||||
- **Practical**: Focus on actionable insights and implications
|
||||
- **Conversational**: Write like talking to a smart colleague
|
||||
|
||||
### Content Structure
|
||||
- **Hook**: Start with surprising insight or provocative question
|
||||
- **Context**: Brief background from articles
|
||||
- **Analysis**: Your unique take connecting multiple sources
|
||||
- **Implications**: What this means for readers
|
||||
- **CTA**: Subscribe, share, engage
|
||||
|
||||
### Writing Guidelines
|
||||
- Use active voice
|
||||
- Short paragraphs (2-3 sentences max)
|
||||
- Subheadings for scannability
|
||||
- Bullet points for lists
|
||||
- Examples and analogies
|
||||
- Link to sources (Omnivore articles)
|
||||
|
||||
## Content Types
|
||||
|
||||
### 1. Weekly Roundup (Fridays)
|
||||
**Format**: "AI/Tech This Week: Top 5 Stories You Need to Know"
|
||||
- 5-10 articles from the week
|
||||
- 100-150 words per story
|
||||
- Overall theme/trend analysis
|
||||
- 800-1200 words total
|
||||
|
||||
### 2. Deep Dive (Monthly)
|
||||
**Format**: "The Complete Guide to [Topic]"
|
||||
- Synthesize 10-20 articles on single topic
|
||||
- Original analysis and insights
|
||||
- Code examples, diagrams
|
||||
- 2000-3000 words
|
||||
|
||||
### 3. Newsletter (Sundays)
|
||||
**Format**: "Weekend Reading: AI/Tech Digest"
|
||||
- 7-10 curated links
|
||||
- Personal commentary (50-75 words each)
|
||||
- Quick hits section (5-10 line items)
|
||||
- 600-800 words total
|
||||
|
||||
### 4. Social Media
|
||||
**Twitter Threads**: Key insights from deep dives (8-10 tweets)
|
||||
**LinkedIn Posts**: Professional takeaways (300-500 words)
|
||||
|
||||
## SEO Strategy
|
||||
|
||||
### Primary Keywords
|
||||
- "AI trends 2025"
|
||||
- "LLM development"
|
||||
- "Claude agents tutorial"
|
||||
- "Tech infrastructure best practices"
|
||||
- "Startup engineering advice"
|
||||
|
||||
### Content Optimization
|
||||
- Title: 60 chars, keyword-rich, curiosity-driven
|
||||
- Meta description: 155 chars, compelling summary
|
||||
- H2 headers: Question-based, keyword-targeted
|
||||
- Internal links: Cross-reference related posts
|
||||
- External links: High-authority sources (Omnivore articles)
|
||||
|
||||
## Monetization Path
|
||||
|
||||
### Phase 1: Audience Building (Months 1-3)
|
||||
- Publish 2x/week (roundup + deep dive)
|
||||
- Email list growth (newsletter signups)
|
||||
- SEO optimization for organic traffic
|
||||
- Social media distribution
|
||||
|
||||
### Phase 2: Engagement (Months 4-6)
|
||||
- Reader surveys and topic requests
|
||||
- Comments and discussions
|
||||
- Guest posts and collaborations
|
||||
- Community building
|
||||
|
||||
### Phase 3: Monetization (Months 7+)
|
||||
- Affiliate links (tools, books, courses)
|
||||
- Sponsored posts (relevant products)
|
||||
- Premium newsletter tier
|
||||
- Consulting/advisory services
|
||||
|
||||
## Quality Standards
|
||||
|
||||
### Before Publishing
|
||||
- [ ] Fact-check all claims against source articles
|
||||
- [ ] Verify all links work
|
||||
- [ ] Run through Grammarly or similar
|
||||
- [ ] Check SEO optimization (Yoast, etc.)
|
||||
- [ ] Preview formatting on target platform
|
||||
- [ ] Add cover image (if applicable)
|
||||
|
||||
### Performance Tracking
|
||||
- Google Analytics: Traffic, engagement, conversions
|
||||
- Email metrics: Open rate, click rate, growth
|
||||
- Social metrics: Shares, comments, saves
|
||||
- Database: Track which articles → which posts → performance
|
||||
|
||||
## Article Analysis Workflow
|
||||
|
||||
### How to Process Queued Articles
|
||||
|
||||
When user asks to analyze articles, use the unified CLI:
|
||||
|
||||
**Automated (cron/launchd):**
|
||||
```bash
|
||||
# End-to-end, non-interactive (uses codex exec in read-only mode)
|
||||
omc queue add --hours 24
|
||||
omc analyze auto --batch-size 5
|
||||
|
||||
# Retry failures (optional)
|
||||
omc analyze retry --failed
|
||||
```
|
||||
|
||||
**Basic Analysis:**
|
||||
```bash
|
||||
# Prepare batch (creates stub files, outputs agent params)
|
||||
omc analyze run --batch-size 5
|
||||
|
||||
# Prepare specific article
|
||||
omc analyze run --article-id <article-id>
|
||||
|
||||
# After agents complete: save results to database + markdown
|
||||
omc analyze complete
|
||||
|
||||
# Save with keeping temp files for debugging
|
||||
omc analyze complete --keep-temp
|
||||
```
|
||||
|
||||
**Queue Management:**
|
||||
```bash
|
||||
# Add articles to queue
|
||||
omc queue add --hours 24 # Last 24 hours
|
||||
omc queue add --label "ai-ml" # By label
|
||||
omc queue add --url <url> # Single article
|
||||
omc queue add --slug <slug> # By slug
|
||||
|
||||
# Check queue status
|
||||
omc queue list # All queued articles
|
||||
omc queue list --status pending # Filter by status
|
||||
omc queue stats # Statistics
|
||||
|
||||
# Retry failed analyses
|
||||
omc analyze retry --failed # All failed
|
||||
omc analyze retry --article-id <id> # Specific article
|
||||
```
|
||||
|
||||
**Automation notes:**
|
||||
- `omc analyze auto` shells out to `codex exec -s read-only` and expects a single JSON object response.
|
||||
- For schedulers, we set `CODEX_HOME=temp/codex-home` so runs don’t depend on `~/.codex` permissions.
|
||||
- Prefer running installs/builds via Corepack (`corepack pnpm ...`) to avoid native-module ABI mismatches.
|
||||
|
||||
**View Results:**
|
||||
```bash
|
||||
# Show analysis for specific article
|
||||
omc content show <article-id>
|
||||
|
||||
# List all analyzed content
|
||||
omc content list
|
||||
|
||||
# Search analyses
|
||||
omc content search "keyword"
|
||||
|
||||
# Generate reports
|
||||
omc report corpus # Full corpus analysis
|
||||
omc report topics # Topic distribution
|
||||
omc report trends # Trending topics
|
||||
```
|
||||
|
||||
**Sync to Omnivore (Create Notebook Notes):**
|
||||
```bash
|
||||
# Sync single article with note creation
|
||||
omc content sync <article-id> --create-notes
|
||||
|
||||
# Sync all analyzed articles with notes
|
||||
omc content sync --all --create-notes
|
||||
|
||||
# Sync without creating notes (metadata only)
|
||||
omc content sync <article-id>
|
||||
```
|
||||
|
||||
### Complete Workflow (Agent-Assisted Analysis)
|
||||
|
||||
**From Reading to Published Notes:**
|
||||
|
||||
1. **Add to Queue**: `omc queue add --hours 168` (or by label, URL, slug)
|
||||
2. **Prepare Batch**: `omc analyze run --batch-size 5`
|
||||
- Creates stub files in `temp/*.jsonl`
|
||||
- Marks jobs as "in_progress"
|
||||
- Outputs agent parameters (copy these)
|
||||
3. **Invoke Agents**: Copy agent parameters and invoke article-content-analyzer agents via Task tool
|
||||
- Agents fetch content via `omc omnivore get`
|
||||
- Agents analyze and enrich stub files
|
||||
- Run 5 agents in parallel for performance
|
||||
4. **Save Results**: `omc analyze complete`
|
||||
- Reads enriched JSONL from temp/
|
||||
- Writes markdown to `content/analysis/*.md`
|
||||
- Saves to database
|
||||
- Marks jobs "completed"
|
||||
5. **Review**: `omc content show <article-id>` or `omc content list`
|
||||
6. **Sync to Omnivore**: `omc content sync --all --create-notes`
|
||||
- Updates article descriptions
|
||||
- Creates notebook notes with full analysis
|
||||
7. **Repeat**: Go back to step 2 until queue empty
|
||||
|
||||
**Key Commands:**
|
||||
- `omc analyze run --batch-size 5` - Prepare next batch
|
||||
- `omc analyze complete` - Save completed analyses
|
||||
- `omc content sync --all --create-notes` - Push to Omnivore
|
||||
|
||||
### Legacy Scripts (Archived)
|
||||
Old workflow scripts have been archived in `cli/archived-scripts/`. Use the CLI commands above instead.
|
||||
|
||||
## Code Quality Tools
|
||||
|
||||
### OACC (Omniarcs Code Checker)
|
||||
The project has access to `oacc analyze` for automated code quality analysis. **Use this instead of manual analysis.**
|
||||
|
||||
**Available via Bash tool (auto-approved):**
|
||||
```bash
|
||||
oacc analyze <file_path>
|
||||
oacc analyze --functions <file_path>
|
||||
```
|
||||
|
||||
**What oacc provides:**
|
||||
- **Function length analysis**: Automatic counting with thresholds
|
||||
- GREEN: ≤20 lines (target)
|
||||
- YELLOW: 21-25 lines (warning)
|
||||
- RED: >25 lines (must fix)
|
||||
- **Function-level reports**: Shows each function with line count
|
||||
- **Exit codes**: 0 = GREEN, 1 = YELLOW/RED
|
||||
- **FTA scores**: Code quality metrics
|
||||
|
||||
**DO NOT manually:**
|
||||
- Write Python/shell scripts to count lines
|
||||
- Parse files with regex to find functions
|
||||
- Manually count function lengths
|
||||
- Create custom analysis tools
|
||||
|
||||
**DO use oacc:**
|
||||
```bash
|
||||
# Check single file
|
||||
oacc analyze src/commands/queue/export.ts
|
||||
|
||||
# Get function-level details
|
||||
oacc analyze --functions src/commands/queue/export.ts
|
||||
|
||||
# Analyze multiple files in quality-guard agent
|
||||
for file in src/commands/**/*.ts; do
|
||||
oacc analyze "$file"
|
||||
done
|
||||
```
|
||||
|
||||
**Quality-guard agent**: Always use `oacc analyze` for function length validation. It's pre-approved and faster than manual analysis.
|
||||
|
||||
## CLI Development Ground Truths
|
||||
|
||||
### Architecture Patterns (Established 2025-01-05)
|
||||
|
||||
**1. All Commands MUST Extend BaseCommand**
|
||||
- Location: `src/lib/cli/base-command.ts:16`
|
||||
- Pattern: `export default class MyCommand extends BaseCommand`
|
||||
- Why: Provides standard error handling, run() method, and OCLIF integration
|
||||
- Evidence: All 63 commands use this pattern (verified 2025-01-05)
|
||||
|
||||
**2. Database Operations MUST Use withDatabase()**
|
||||
- Location: `src/lib/cli/database.ts:17`
|
||||
- Pattern: `await withDatabase(async (db, repo) => { ... })`
|
||||
- Why: Ensures proper connection cleanup and error handling
|
||||
- Evidence: 45/46 database commands use this (db/backup.ts fixed 2025-01-05)
|
||||
|
||||
**3. JSON Parsing MUST Use parseJsonSafely()**
|
||||
- Location: `src/lib/cli/command-utils.ts:45`
|
||||
- Pattern: `const data = parseJsonSafely<ContentAnalysis>(job.analysisJson)`
|
||||
- Why: Prevents crashes on malformed JSON, provides type safety
|
||||
- Evidence: 15+ commands updated to use this (2025-01-05)
|
||||
|
||||
**4. Shared Flags MUST Use Utility Functions**
|
||||
- `jsonFlag()`: `src/lib/cli/shared-flags.ts:14` - Standard JSON output flag
|
||||
- `statusFlag()`: `src/lib/cli/shared-flags.ts:8` - Queue status filtering
|
||||
- Why: DRY principle, consistent flag behavior
|
||||
- Evidence: 45/63 commands use jsonFlag()
|
||||
|
||||
**5. Output Formatting MUST Use Shared Formatters**
|
||||
- `formatHeader()`: `src/lib/cli/formatters.ts:18` - Section headers
|
||||
- `formatSuccess()`: `src/lib/cli/formatters.ts:37` - Success messages
|
||||
- `formatError()`: `src/lib/cli/formatters.ts:48` - Error messages
|
||||
- Why: Consistent user experience, single source of truth
|
||||
- Evidence: All 63 commands use these formatters
|
||||
|
||||
**6. GraphQL Operations MUST Use checkGraphQLResult()**
|
||||
- Location: `src/lib/cli/graphql.ts:21`
|
||||
- Pattern: `checkGraphQLResult(result)` after every GraphQL call
|
||||
- Why: Checks both result.errors AND domain errorCodes
|
||||
- Evidence: All 9 omnivore commands use this
|
||||
|
||||
**7. Environment Variables MUST Use loadEnvFile()**
|
||||
- Location: `src/lib/cli/command-utils.ts:58`
|
||||
- Pattern: `const env = loadEnvFile('.env')`
|
||||
- Why: Centralized .env parsing, no duplicate implementations
|
||||
- Evidence: 2 duplicate implementations removed (2025-01-05)
|
||||
|
||||
### Shared Utilities Reference
|
||||
|
||||
**Database Utilities:**
|
||||
```typescript
|
||||
// src/lib/cli/database.ts
|
||||
withDatabase<T>(callback: (db, repo) => Promise<T>): Promise<T>
|
||||
```
|
||||
|
||||
**Data Utilities:**
|
||||
```typescript
|
||||
// src/lib/cli/command-utils.ts
|
||||
parseJsonSafely<T>(jsonString: string, fallback?: T): T | undefined
|
||||
loadEnvFile(envPath: string = '.env'): Record<string, string>
|
||||
handleCommandError(command: Command, error: unknown): void
|
||||
outputResult(command: Command, data: any, successMessage: string, jsonMode: boolean): void
|
||||
```
|
||||
|
||||
**Flag Utilities:**
|
||||
```typescript
|
||||
// src/lib/cli/shared-flags.ts
|
||||
jsonFlag(): Flags.Boolean
|
||||
statusFlag(): Flags.String
|
||||
```
|
||||
|
||||
**Formatting Utilities:**
|
||||
```typescript
|
||||
// src/lib/cli/formatters.ts
|
||||
formatHeader(title: string): string
|
||||
formatSuccess(message: string): string
|
||||
formatError(message: string): string
|
||||
formatDivider(): string
|
||||
```
|
||||
|
||||
**GraphQL Utilities:**
|
||||
```typescript
|
||||
// src/lib/cli/graphql.ts
|
||||
checkGraphQLResult(result: any): void
|
||||
fetchUsername(): Promise<string>
|
||||
```
|
||||
|
||||
**Display Utilities:**
|
||||
```typescript
|
||||
// src/lib/cli/queue-display.ts
|
||||
displayQueueStats(stats: QueueStats): void
|
||||
displayJobs(jobs: AnalysisJob[]): void
|
||||
```
|
||||
|
||||
### Command Structure Pattern
|
||||
|
||||
**Every command follows this structure:**
|
||||
```typescript
|
||||
import { Args, Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { formatSuccess } from '@lib/cli/formatters.js';
|
||||
|
||||
export default class MyCommand extends BaseCommand {
|
||||
static override description = 'Clear description';
|
||||
|
||||
static override examples = [
|
||||
'$ omc command arg --flag'
|
||||
];
|
||||
|
||||
static override args = {
|
||||
myArg: Args.string({ description: 'Arg description', required: true })
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
myFlag: Flags.string({ description: 'Flag description' })
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async (db, repo) => {
|
||||
// 1. Validate input
|
||||
// 2. Execute business logic
|
||||
// 3. Format output
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
this.log(formatSuccess('Success message'));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Complete Command Inventory (63 total)
|
||||
|
||||
**Queue Management (9 commands):**
|
||||
- `queue:add` - Add articles to queue
|
||||
- `queue:list` - List queued articles
|
||||
- `queue:stats` - Show queue statistics (with --detailed)
|
||||
- `queue:reset` - Reset article status
|
||||
- `queue:remove` - Remove from queue
|
||||
- `queue:clear` - Bulk clear operations
|
||||
- `queue:export` - Export queue state
|
||||
- `queue:import` - Import queue state
|
||||
- `queue:retry` - Retry failed items
|
||||
|
||||
**Analysis Operations (4 commands):**
|
||||
- `analyze:run` - Process articles
|
||||
- `analyze:retry` - Retry failed analyses
|
||||
- `analyze:status` - Show analysis status
|
||||
- `analyze:watch` - Real-time monitoring
|
||||
|
||||
**Omnivore Integration (9 commands):**
|
||||
- `omnivore:get` - Fetch article by slug
|
||||
- `omnivore:search` - Search articles
|
||||
- `omnivore:list` - List recent articles
|
||||
- `omnivore:update` - Update article metadata
|
||||
- `omnivore:note:add` - Add note
|
||||
- `omnivore:note:get` - Get notes
|
||||
- `omnivore:note:update` - Update note
|
||||
- `omnivore:highlight:add` - Add highlight
|
||||
- `omnivore:highlight:list` - List highlights
|
||||
|
||||
**Content Operations (5 commands):**
|
||||
- `content:show` - Display analysis
|
||||
- `content:list` - List analyzed content
|
||||
- `content:search` - Full-text search
|
||||
- `content:sync` - Sync to Omnivore
|
||||
- `content:export` - Export for blog
|
||||
|
||||
**Database Management (9 commands):**
|
||||
- `db:migrate` - Run migrations
|
||||
- `db:schema` - Show schema
|
||||
- `db:seed` - Load sample data
|
||||
- `db:vacuum` - Optimize database
|
||||
- `db:backup` - Create backup
|
||||
- `db:restore` - Restore from backup
|
||||
- `db:reset` - Drop and recreate
|
||||
- `db:check` - Integrity check
|
||||
- `db:stats` - Database statistics
|
||||
|
||||
**Reporting (7 commands):**
|
||||
- `report:corpus` - Full corpus analysis
|
||||
- `report:topics` - Topic distribution
|
||||
- `report:trends` - Trending topics
|
||||
- `report:monetization` - Opportunities
|
||||
- `report:sentiment` - Sentiment analysis
|
||||
- `report:custom` - Custom queries
|
||||
- `report:export` - Export reports
|
||||
|
||||
**Configuration (7 commands):**
|
||||
- `config:show` - Display config
|
||||
- `config:get` - Get value
|
||||
- `config:set` - Set value
|
||||
- `config:test` - Test API connection
|
||||
- `config:validate` - Validate config
|
||||
- `config:env:list` - List environments
|
||||
- `config:env:use` - Switch environment
|
||||
|
||||
**System (3 commands):**
|
||||
- `init` - Initialize project
|
||||
- `doctor` - System health check
|
||||
- `version` - Version info
|
||||
|
||||
### DRY Principles (Zero Tolerance)
|
||||
|
||||
**Violations to Avoid:**
|
||||
1. ❌ Direct `JSON.parse()` without error handling → Use `parseJsonSafely()`
|
||||
2. ❌ Manual database init/close → Use `withDatabase()`
|
||||
3. ❌ Duplicate flag definitions → Use `jsonFlag()`, `statusFlag()`
|
||||
4. ❌ Custom error handling → Extend `BaseCommand`
|
||||
5. ❌ Duplicate .env parsing → Use `loadEnvFile()`
|
||||
6. ❌ Direct Command extension → Extend `BaseCommand`
|
||||
7. ❌ Custom formatters → Use formatHeader/Success/Error
|
||||
8. ❌ GraphQL without error check → Use `checkGraphQLResult()`
|
||||
|
||||
**Evidence of Compliance:**
|
||||
- All 63 commands extend BaseCommand ✓
|
||||
- All 45 database commands use withDatabase() ✓
|
||||
- All 15+ JSON parsing uses parseJsonSafely() ✓
|
||||
- Zero duplicate helper functions ✓
|
||||
- (Last verified: 2025-01-05)
|
||||
|
||||
### Quality Metrics
|
||||
|
||||
**Function Length Distribution (2025-01-05):**
|
||||
- GREEN (≤20 lines): 52 commands (83%)
|
||||
- YELLOW (21-25 lines): 11 commands (17%)
|
||||
- RED (>25 lines): 0 commands (0%)
|
||||
|
||||
**Build Status:**
|
||||
- TypeScript: Strict mode enabled
|
||||
- Compilation: 0 errors, 0 warnings
|
||||
- Bundle size: Optimized with ESBuild
|
||||
- Commands registered: 63/63 (100%)
|
||||
|
||||
## Notes for Agents
|
||||
- **Always cite sources**: Link back to Omnivore articles
|
||||
- **Maintain authenticity**: Sound like the human, not generic AI
|
||||
- **Be selective**: Quality content over quantity
|
||||
- **Stay current**: Focus on recent articles (last 7-30 days)
|
||||
- **Think monetization**: Every piece should serve the business goal
|
||||
- **Use oacc for code analysis**: Don't reinvent code quality tools
|
||||
140
self-hosting/omc/EXTRACTION_CHECKLIST.md
Normal file
140
self-hosting/omc/EXTRACTION_CHECKLIST.md
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
# Extraction Checklist
|
||||
|
||||
## ✅ Pre-Extraction Verification
|
||||
|
||||
Run these commands to verify everything is ready:
|
||||
|
||||
```bash
|
||||
cd scripts/omnivore-content-system
|
||||
|
||||
# 1. Dependencies resolved
|
||||
pnpm install
|
||||
# Should complete without errors
|
||||
|
||||
# 2. TypeScript compiles
|
||||
pnpm run build
|
||||
# Should create dist/ directory
|
||||
|
||||
# 3. Type checking passes
|
||||
pnpm run typecheck
|
||||
# Should show no errors
|
||||
|
||||
# 4. Omnivore client works
|
||||
node lib/omnivore/client.js --test
|
||||
# Should connect and show user info
|
||||
```
|
||||
|
||||
## 📦 Extraction Commands
|
||||
|
||||
```bash
|
||||
# From omnivore repo root
|
||||
cd /Volumes/devel/personal/keybase/edgerouter/mac-mini/home/omnivore/omnivore
|
||||
|
||||
# Copy to new location
|
||||
cp -r scripts/omnivore-content-system /path/to/new-location/
|
||||
|
||||
# OR move (if extracting permanently)
|
||||
mv scripts/omnivore-content-system /path/to/new-location/
|
||||
```
|
||||
|
||||
## 🔧 Post-Extraction Setup
|
||||
|
||||
```bash
|
||||
cd /path/to/new-location/omnivore-content-system
|
||||
|
||||
# 1. Install dependencies
|
||||
pnpm install
|
||||
|
||||
# 2. Configure environment
|
||||
cp .env.example .env
|
||||
# Edit .env with your API keys
|
||||
|
||||
# 3. Test connection
|
||||
node lib/omnivore/client.js --test
|
||||
|
||||
# 4. Verify build
|
||||
pnpm run build
|
||||
pnpm run typecheck
|
||||
|
||||
# 5. Initialize Git (if new repo)
|
||||
git init
|
||||
git add .
|
||||
git commit -m "Initial commit: Omnivore content monetization system"
|
||||
```
|
||||
|
||||
## 🔍 Verification After Extraction
|
||||
|
||||
### Check 1: No External Dependencies
|
||||
```bash
|
||||
# Should find no references to parent repo paths
|
||||
grep -r "\.\./\.\./self-hosting" .
|
||||
grep -r "omnivore/scripts" .
|
||||
# Both should return no results (or only in IMPLEMENTATION_PLAN.md for documentation)
|
||||
```
|
||||
|
||||
### Check 2: All Files Present
|
||||
```bash
|
||||
# Required files
|
||||
ls -la .env.example # ✅ Should exist
|
||||
ls -la package.json # ✅ Should exist
|
||||
ls -la tsconfig.json # ✅ Should exist
|
||||
ls -la .gitignore # ✅ Should exist
|
||||
ls -la README.md # ✅ Should exist
|
||||
ls -la IMPLEMENTATION_PLAN.md # ✅ Should exist
|
||||
ls -la CLAUDE.md # ✅ Should exist
|
||||
|
||||
# Required directories
|
||||
ls -d lib/ # ✅ Should exist
|
||||
ls -d src/ # ✅ Should exist
|
||||
ls -d content/ # ✅ Should exist
|
||||
```
|
||||
|
||||
### Check 3: Build Works
|
||||
```bash
|
||||
pnpm run build
|
||||
# Should succeed and create dist/types/
|
||||
ls -la dist/
|
||||
```
|
||||
|
||||
### Check 4: Client Works
|
||||
```bash
|
||||
# After configuring .env with real API keys
|
||||
node lib/omnivore/client.js --test
|
||||
# Should output:
|
||||
# ✅ Connected to Omnivore API
|
||||
# User: Your Name (your@email.com)
|
||||
# Username: yourusername
|
||||
```
|
||||
|
||||
## ❌ What Should NOT Be Included
|
||||
|
||||
These remain in the omnivore repo:
|
||||
- ❌ `/scripts/migrate-omnivore.js`
|
||||
- ❌ `/scripts/import-pocket.js`
|
||||
- ❌ `/self-hosting/` directory
|
||||
- ❌ Parent repo's package.json
|
||||
|
||||
## 📝 Final Checklist
|
||||
|
||||
Before considering extraction complete:
|
||||
|
||||
- [ ] All dependencies install without errors
|
||||
- [ ] TypeScript compiles successfully
|
||||
- [ ] Type checking passes with no errors
|
||||
- [ ] Omnivore client connects to API
|
||||
- [ ] .env.example is present with correct structure
|
||||
- [ ] README.md reflects current status
|
||||
- [ ] IMPLEMENTATION_PLAN.md is up to date
|
||||
- [ ] .gitignore properly configured
|
||||
- [ ] No symlinks to parent repo
|
||||
- [ ] No hardcoded paths to parent repo structure
|
||||
|
||||
## 🚀 Ready When
|
||||
|
||||
All checkboxes above are checked ✅
|
||||
|
||||
## 📚 Documentation References
|
||||
|
||||
- [README.md](./README.md) - Overview and quick start
|
||||
- [IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md) - Detailed roadmap
|
||||
- [CLAUDE.md](./CLAUDE.md) - Agent context and strategy
|
||||
1612
self-hosting/omc/IMPLEMENTATION_PLAN.md
Normal file
1612
self-hosting/omc/IMPLEMENTATION_PLAN.md
Normal file
File diff suppressed because it is too large
Load diff
278
self-hosting/omc/README.md
Normal file
278
self-hosting/omc/README.md
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
# Omnivore Content Monetization System
|
||||
|
||||
AI-powered content generation system that transforms your Omnivore reading into monetizable content: blog posts, newsletters, and social media posts.
|
||||
|
||||
## 🔎 Current Reality (Audit: 2026-01-30)
|
||||
|
||||
- The CLI command set is largely implemented (`omc queue/*`, `omc analyze/*`, `omc content/*`, `omc report/*`, `omc omnivore/*`, `omc db/*`, `omc config/*`).
|
||||
- There are still critical fixups needed before this repo is “clean build + reliable run”; see `docs/_meta/current-state.md`.
|
||||
|
||||
## 🚀 Features
|
||||
|
||||
### ✅ Currently Working
|
||||
- **Omnivore API Client**: Fetch/search/update articles, highlights, and notes
|
||||
- **Queue + Tracking DB**: SQLite-backed analysis queue (`data/omnivore-content.db`)
|
||||
- **Analysis Workflow**: Prepare stub files → run external analyzer → persist results to Markdown + DB
|
||||
- **Reporting**: Topic/sentiment/trend aggregation across saved analyses
|
||||
- **Sync Back to Omnivore**: Push summaries (and optional NOTE highlights) back to Omnivore
|
||||
|
||||
### 🚧 Planned Features (NOT YET IMPLEMENTED)
|
||||
- **Blog Post Generation** *(Phase 5)*: Create weekly roundups, deep dives, tutorials from your saved articles
|
||||
- **Newsletter Creation** *(Phase 5)*: Generate curated newsletters with your commentary
|
||||
- **SEO Optimization** *(Phase 5)*: Automatic title, description, and tag generation
|
||||
- **Trend Tracking** *(Phase 7)*: Identify emerging topics before they peak
|
||||
- **Multi-Agent System** *(Phase 6-7)*: Specialized Claude agents for each content type
|
||||
- **Publishing Integration** *(Phase 6)*: Ghost, WordPress, Medium support
|
||||
- **First-Class Content Generation Pipeline**: Turn analyses into publishable posts (beyond reporting/export)
|
||||
|
||||
## 📋 Prerequisites
|
||||
|
||||
- Node.js 18+
|
||||
- Omnivore account with API key
|
||||
- Anthropic API key (Claude)
|
||||
- Self-hosted Omnivore instance OR Omnivore cloud account
|
||||
|
||||
## 📦 Current Status
|
||||
|
||||
**Phase 0: Foundation ✅ COMPLETE**
|
||||
- TypeScript toolchain configured
|
||||
- Omnivore GraphQL client working
|
||||
- Directory structure created
|
||||
- Ready for standalone extraction
|
||||
|
||||
**Phases 1-6: In Progress**
|
||||
- See [IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md) for detailed roadmap
|
||||
|
||||
## 🛠️ Installation
|
||||
|
||||
1. **Extract to standalone directory** (if from omnivore repo)
|
||||
```bash
|
||||
cp -r omnivore/scripts/omnivore-content-system /path/to/new-repo/
|
||||
cd /path/to/new-repo/omnivore-content-system
|
||||
```
|
||||
|
||||
2. **Install dependencies**
|
||||
```bash
|
||||
pnpm install
|
||||
```
|
||||
|
||||
3. **Configure environment**
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# Edit .env with your API keys:
|
||||
# - OMNIVORE_API_KEY
|
||||
# - OMNIVORE_API_URL
|
||||
# - ANTHROPIC_API_KEY
|
||||
```
|
||||
|
||||
4. **Test setup**
|
||||
```bash
|
||||
# Test Omnivore connection
|
||||
node lib/omnivore/client.js --test
|
||||
|
||||
# Test TypeScript build
|
||||
pnpm run build
|
||||
|
||||
# Note: `pnpm run typecheck` is currently failing due to known alias/import issues.
|
||||
# See: docs/_meta/current-state.md
|
||||
```
|
||||
|
||||
## ⚙️ Configuration
|
||||
|
||||
### Required Environment Variables
|
||||
|
||||
```bash
|
||||
OMNIVORE_API_KEY=your_omnivore_api_key
|
||||
OMNIVORE_API_URL=https://your-omnivore-instance.com/api/graphql
|
||||
ANTHROPIC_API_KEY=your_claude_api_key
|
||||
```
|
||||
|
||||
### Optional Configuration (NOT IMPLEMENTED YET)
|
||||
|
||||
- **Publishing**: Configure Ghost, WordPress, or Medium credentials *(Phase 6)*
|
||||
- **Topics**: Edit `config/topics.json` to customize AI/Tech categories *(Phase 1)*
|
||||
- **Templates**: Modify templates in `templates/` for your style *(Phase 5)*
|
||||
- **Workflows**: Adjust workflow schedules in `config/workflows.json` *(Phase 9)*
|
||||
|
||||
## 🎯 Usage (Current)
|
||||
|
||||
### Currently Working
|
||||
```bash
|
||||
# Setup checks
|
||||
omc doctor
|
||||
|
||||
# Add recent articles to queue
|
||||
omc queue add --hours 24
|
||||
|
||||
# Prepare stub files for external analysis
|
||||
omc analyze run --batch-size 5
|
||||
|
||||
# After your external analyzer writes `analysis` into temp/*.jsonl:
|
||||
omc analyze complete
|
||||
|
||||
# View/search
|
||||
omc content list
|
||||
omc content search "kubernetes"
|
||||
|
||||
# Categorized reports
|
||||
omc report topics
|
||||
omc report trends
|
||||
```
|
||||
|
||||
For a deeper breakdown (what works, what’s broken, and what to fix next), see `docs/_meta/current-state.md`.
|
||||
|
||||
## 📁 Project Structure
|
||||
|
||||
```
|
||||
omnivore-content-system/
|
||||
├── lib/ # ✅ Omnivore GraphQL client (WORKING)
|
||||
│ └── omnivore/
|
||||
│ ├── client.js # ✅ Full-featured GraphQL client
|
||||
│ └── queries.js # ✅ Query builders
|
||||
├── src/ # ✅ TypeScript source (READY)
|
||||
│ ├── types/ # (Phase 1 - not started)
|
||||
│ ├── storage/ # (Phase 3 - not started)
|
||||
│ ├── analysis/ # (Phase 4 - not started)
|
||||
│ ├── generation/ # (Phase 5 - not started)
|
||||
│ ├── publishing/ # (Phase 6 - not started)
|
||||
│ ├── workflows/ # (Phase 9 - not started)
|
||||
│ └── utils/ # (Phase 2 - not started)
|
||||
├── content/ # ✅ Storage directories (READY)
|
||||
│ ├── articles/ # (empty - Phase 3)
|
||||
│ ├── analysis/ # (empty - Phase 4)
|
||||
│ └── generated/ # (empty - Phase 5)
|
||||
├── test-scripts/ # ✅ Test scripts (READY)
|
||||
├── cli/ # (Phase 2-6 - not started)
|
||||
├── tests/ # ✅ Test directory (READY)
|
||||
├── legacy-scripts/ # ✅ Original migration scripts (preserved)
|
||||
├── templates/ # (empty - Phase 5 needed)
|
||||
├── .claude/ # (Agent SDK config - Phase 6+)
|
||||
├── tsconfig.json # ✅ TypeScript config (WORKING)
|
||||
├── package.json # ✅ Dependencies (WORKING)
|
||||
└── .env.example # ✅ Config template (WORKING)
|
||||
```
|
||||
|
||||
**Legend:**
|
||||
- ✅ = Working and ready to use
|
||||
- (empty) = Directory created but no files yet
|
||||
- (Phase X) = Implementation planned, not started
|
||||
- (WORKING) = Fully functional
|
||||
|
||||
## 🤖 Agent Architecture (NOT IMPLEMENTED - Phase 6-8)
|
||||
|
||||
### Orchestrator Agent *(NOT IMPLEMENTED)*
|
||||
Main controller that coordinates all subagents and workflows.
|
||||
|
||||
### Specialized Agents *(NOT IMPLEMENTED)*
|
||||
- **Content Analyzer** *(Phase 7)*: Analyzes reading patterns, extracts themes
|
||||
- **Blog Writer** *(Phase 7)*: Generates blog posts from analyzed content
|
||||
- **Newsletter Creator** *(Phase 7)*: Creates newsletters with commentary
|
||||
- **SEO Optimizer** *(Phase 7)*: Optimizes titles, descriptions, tags
|
||||
- **Trend Tracker** *(Phase 7)*: Identifies emerging topics and opportunities
|
||||
|
||||
### MCP Servers *(NOT IMPLEMENTED - Phase 8)*
|
||||
- **Omnivore MCP**: Tools for searching, fetching articles, highlights
|
||||
- **Content DB MCP**: Tools for saving and tracking generated content
|
||||
|
||||
## 📝 Content Templates (NOT IMPLEMENTED - Phase 5)
|
||||
|
||||
### Blog Post Types *(NOT IMPLEMENTED)*
|
||||
- **Weekly Roundup**: Top 5-10 articles from the week
|
||||
- **Deep Dive**: Multi-article synthesis into long-form analysis
|
||||
- **Tutorial**: How-to content extracted from technical articles
|
||||
- **Comparison**: "X vs Y" posts from related articles
|
||||
|
||||
### Newsletter Formats *(NOT IMPLEMENTED)*
|
||||
- **Weekly Digest**: Curated links with your commentary
|
||||
- **Themed Edition**: Deep dive into single topic
|
||||
|
||||
### Social Media *(NOT IMPLEMENTED)*
|
||||
- **Twitter Threads**: Key insights from articles
|
||||
- **LinkedIn Posts**: Professional summaries
|
||||
|
||||
## 🔧 Legacy Scripts
|
||||
|
||||
All original Omnivore scripts are preserved in `legacy-scripts/`:
|
||||
- `import-pocket.js` - Import from Pocket
|
||||
- `apply-labels.js` - Label management
|
||||
- `migrate-omnivore.js` - Data migration
|
||||
- And more...
|
||||
|
||||
These can be called from agents as needed or run independently.
|
||||
|
||||
## 💾 Content Storage (Phase 3 - NOT IMPLEMENTED)
|
||||
|
||||
Will use **Markdown + front-matter + Git** (no database):
|
||||
- **content/articles/** *(empty)*: Omnivore articles saved as Markdown
|
||||
- **content/analysis/** *(empty)*: AI analysis results
|
||||
- **content/generated/** *(empty)*: Generated blog posts and newsletters
|
||||
- **Front-matter**: YAML metadata in each file
|
||||
- **Git**: Version control and history
|
||||
|
||||
Directories are created but no storage implementation yet.
|
||||
|
||||
## 🚀 Publishing (Phase 6 - NOT IMPLEMENTED)
|
||||
|
||||
### Ghost CMS *(NOT IMPLEMENTED)*
|
||||
```bash
|
||||
# Configuration ready in .env.example, but no publisher code yet
|
||||
GHOST_API_URL=https://your-blog.ghost.io
|
||||
GHOST_API_KEY=your_admin_api_key
|
||||
```
|
||||
|
||||
### WordPress *(NOT IMPLEMENTED)*
|
||||
```bash
|
||||
WORDPRESS_URL=https://your-site.com
|
||||
WORDPRESS_API_KEY=your_api_key
|
||||
```
|
||||
|
||||
### Medium *(NOT IMPLEMENTED)*
|
||||
```bash
|
||||
MEDIUM_TOKEN=your_integration_token
|
||||
```
|
||||
|
||||
## 🐛 Troubleshooting
|
||||
|
||||
### API Connection Issues (Currently Applicable)
|
||||
- Verify `OMNIVORE_API_KEY` is correct
|
||||
- Check `OMNIVORE_API_URL` is accessible
|
||||
- Test with: `node lib/omnivore/client.js --test`
|
||||
|
||||
### TypeScript Build Issues (Currently Applicable)
|
||||
- Run `pnpm install` to ensure dependencies are installed
|
||||
- Run `pnpm run typecheck` to verify no type errors
|
||||
- Check that Node.js version is ≥18.0.0
|
||||
|
||||
### Agent Issues *(NOT APPLICABLE YET - Phase 6+)*
|
||||
- Check `ANTHROPIC_API_KEY` is set
|
||||
- Verify Claude Agent SDK is installed
|
||||
- Review logs in `data/logs/`
|
||||
|
||||
### Storage Issues *(NOT APPLICABLE YET - Phase 3+)*
|
||||
- Check permissions on `content/` directory
|
||||
- Verify Git is initialized if using version control
|
||||
|
||||
## 📚 Documentation
|
||||
|
||||
- **[IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md)** - ✅ Detailed implementation roadmap (current)
|
||||
- **[EXTRACTION_CHECKLIST.md](./EXTRACTION_CHECKLIST.md)** - ✅ Steps to move to standalone repo (current)
|
||||
- **[CLAUDE.md](./CLAUDE.md)** - ✅ Agent context and content strategy (current)
|
||||
- ~~Setup Guide~~ *(NOT CREATED - covered in this README)*
|
||||
- ~~Workflows~~ *(NOT CREATED - Phase 9)*
|
||||
- ~~Agent Architecture~~ *(NOT CREATED - Phase 6-8)*
|
||||
- ~~API Documentation~~ *(NOT CREATED - Phase 1-7)*
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
This is a personal content system, but suggestions welcome!
|
||||
|
||||
## 📄 License
|
||||
|
||||
MIT
|
||||
|
||||
## 🙏 Credits
|
||||
|
||||
Built with:
|
||||
- [Claude Agent SDK](https://github.com/anthropics/claude-agent-sdk-typescript) by Anthropic
|
||||
- [Omnivore](https://omnivore.app/) for reading management
|
||||
- Your voracious reading habit!
|
||||
8
self-hosting/omc/bin/omc.ts
Executable file
8
self-hosting/omc/bin/omc.ts
Executable file
|
|
@ -0,0 +1,8 @@
|
|||
#!/usr/bin/env node --import tsx
|
||||
/**
|
||||
* OMC (Omnivore Content System) CLI Executable
|
||||
* AIDEV-NOTE: Main executable entry point for the OMC CLI system
|
||||
*/
|
||||
|
||||
// Import and run the main CLI
|
||||
import '../index.js';
|
||||
23
self-hosting/omc/cli/archived-scripts/README.md
Normal file
23
self-hosting/omc/cli/archived-scripts/README.md
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# Archived Scripts
|
||||
|
||||
These scripts have been replaced by the new OCLIF-based CLI (`omc`). They are preserved here for reference but should not be used.
|
||||
|
||||
## Migration Map
|
||||
|
||||
| Old Script | New CLI Command |
|
||||
|-----------|-----------------|
|
||||
| `parallel-analyze.ts` | `omc analyze run` |
|
||||
| `save-analysis-results.ts` | Internal (called by analyze run) |
|
||||
| `retry-failed.ts` | `omc analyze retry` |
|
||||
| `corpus-report.ts` | `omc report corpus` |
|
||||
| `get-article-content.ts` | `omc omnivore get <slug>` |
|
||||
| `get-article-notes.ts` | `omc omnivore note get <article-id>` |
|
||||
| `test-update-article-notes.ts` | `omc omnivore note update <article-id>` |
|
||||
| `update-note-test.ts` | `omc omnivore note update <article-id>` |
|
||||
| `migrate-database.ts` | `omc db migrate` |
|
||||
|
||||
## Archived Date
|
||||
2025-01-05
|
||||
|
||||
## Reason
|
||||
All functionality has been migrated to the unified CLI system with better error handling, help text, and consistent interfaces.
|
||||
127
self-hosting/omc/cli/archived-scripts/corpus-report.ts
Normal file
127
self-hosting/omc/cli/archived-scripts/corpus-report.ts
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
#!/usr/bin/env tsx
|
||||
|
||||
/**
|
||||
* Generate corpus analysis report
|
||||
* AIDEV-NOTE: tracking-db - reads from SQLite analysis_queue table
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
|
||||
console.log('\n' + '═'.repeat(80));
|
||||
console.log('CORPUS ANALYSIS REPORT');
|
||||
console.log('═'.repeat(80));
|
||||
|
||||
// AIDEV-NOTE: tracking-db - read from SQLite, not deprecated batch file
|
||||
const db = new Database('data/omnivore-content.db', { readonly: true });
|
||||
|
||||
// Queue status from tracking DB
|
||||
const statusCounts = db.prepare(`
|
||||
SELECT status, COUNT(*) as count
|
||||
FROM analysis_queue
|
||||
GROUP BY status
|
||||
`).all() as { status: string; count: number }[];
|
||||
|
||||
const total = statusCounts.reduce((sum, row) => sum + row.count, 0);
|
||||
|
||||
console.log(`\n📊 Queue Status:`);
|
||||
console.log(` Total articles: ${total}`);
|
||||
for (const { status, count } of statusCounts) {
|
||||
console.log(` ${status}: ${count}`);
|
||||
}
|
||||
|
||||
// Load completed analyses from SQLite
|
||||
const rows = db.prepare(`
|
||||
SELECT article_id, article_title, analysis_json
|
||||
FROM analysis_queue
|
||||
WHERE status = 'completed' AND analysis_json IS NOT NULL
|
||||
`).all() as { article_id: string; article_title: string; analysis_json: string }[];
|
||||
|
||||
console.log(`\n📚 Analyzed Articles: ${rows.length}\n`);
|
||||
|
||||
// Parse analysis JSON from each row
|
||||
const analyses = rows.map(row => ({
|
||||
articleId: row.article_id,
|
||||
articleTitle: row.article_title,
|
||||
...JSON.parse(row.analysis_json)
|
||||
})).filter(a => a.topics && a.topics[0] !== 'N/A'); // Filter out failed analyses
|
||||
|
||||
// Topic distribution
|
||||
const topicCounts: Record<string, number> = {};
|
||||
const topicScoreTotals: Record<string, number> = {};
|
||||
|
||||
for (const analysis of analyses) {
|
||||
for (const topic of analysis.topics) {
|
||||
topicCounts[topic] = (topicCounts[topic] || 0) + 1;
|
||||
const score = analysis.topicScores[topic] || 0;
|
||||
topicScoreTotals[topic] = (topicScoreTotals[topic] || 0) + score;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('═'.repeat(80));
|
||||
console.log('TOPIC DISTRIBUTION');
|
||||
console.log('═'.repeat(80));
|
||||
|
||||
const sortedTopics = Object.entries(topicCounts)
|
||||
.sort((a, b) => b[1] - a[1]);
|
||||
|
||||
console.log(`\n📈 Topics by frequency:\n`);
|
||||
for (const [topic, count] of sortedTopics) {
|
||||
const avgScore = (topicScoreTotals[topic] / count).toFixed(2);
|
||||
console.log(` ${topic.padEnd(30)} ${count} articles (avg score: ${avgScore})`);
|
||||
}
|
||||
|
||||
// Sentiment distribution
|
||||
const sentiments: Record<string, number> = {};
|
||||
for (const analysis of analyses) {
|
||||
const sentiment = analysis.sentiment;
|
||||
sentiments[sentiment] = (sentiments[sentiment] || 0) + 1;
|
||||
}
|
||||
|
||||
console.log(`\n😊 Sentiment distribution:\n`);
|
||||
for (const [sentiment, count] of Object.entries(sentiments)) {
|
||||
console.log(` ${sentiment}: ${count}`);
|
||||
}
|
||||
|
||||
// Identify clusters (articles with shared topics)
|
||||
console.log('\n' + '═'.repeat(80));
|
||||
console.log('TOPIC CLUSTERS (Content Opportunities)');
|
||||
console.log('═'.repeat(80));
|
||||
|
||||
for (const [topic, count] of sortedTopics) {
|
||||
if (count < 2) continue; // Only show topics with 2+ articles
|
||||
|
||||
console.log(`\n📂 ${topic} (${count} articles):`);
|
||||
|
||||
const articlesWithTopic = analyses.filter(a =>
|
||||
a.topics.includes(topic)
|
||||
);
|
||||
|
||||
for (const analysis of articlesWithTopic) {
|
||||
const score = analysis.topicScores[topic].toFixed(2);
|
||||
console.log(` [${score}] ${analysis.articleTitle.substring(0, 70)}`);
|
||||
}
|
||||
|
||||
// Content opportunity
|
||||
if (count >= 3) {
|
||||
console.log(` 💡 OPPORTUNITY: Weekly roundup or comparison post possible`);
|
||||
} else if (count === 2) {
|
||||
console.log(` 💡 OPPORTUNITY: Comparison or "X vs Y" post possible`);
|
||||
}
|
||||
}
|
||||
|
||||
// Monetization angles summary
|
||||
console.log('\n' + '═'.repeat(80));
|
||||
console.log('MONETIZATION ANGLES');
|
||||
console.log('═'.repeat(80) + '\n');
|
||||
|
||||
for (const analysis of analyses) {
|
||||
console.log(`📝 ${analysis.articleTitle.substring(0, 70)}`);
|
||||
console.log(` ${analysis.monetizationAngle}\n`);
|
||||
}
|
||||
|
||||
console.log('═'.repeat(80));
|
||||
console.log('END OF REPORT');
|
||||
console.log('═'.repeat(80) + '\n');
|
||||
|
||||
// AIDEV-NOTE: tracking-db - close connection after report
|
||||
db.close();
|
||||
33
self-hosting/omc/cli/archived-scripts/get-article-content.ts
Normal file
33
self-hosting/omc/cli/archived-scripts/get-article-content.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#!/usr/bin/env tsx
|
||||
// AIDEV-NOTE: agent-helper - fetches article content for agent analysis
|
||||
// AIDEV-NOTE: omnivore-boundary - GraphQL API only
|
||||
|
||||
import { getArticle } from '../lib/omnivore/client.js';
|
||||
import 'dotenv/config';
|
||||
|
||||
async function main() {
|
||||
const articleSlug = process.argv[2];
|
||||
const username = process.argv[3];
|
||||
|
||||
if (!articleSlug || !username) {
|
||||
console.error('Usage: tsx cli/get-article-content.ts <articleSlug> <username>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await getArticle(articleSlug, username);
|
||||
|
||||
if (!result?.article?.content) {
|
||||
console.error('No content found');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Output ONLY content to stdout (no JSON wrapper)
|
||||
console.log(result.article.content);
|
||||
} catch (err: any) {
|
||||
console.error(`Error fetching article: ${err.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
84
self-hosting/omc/cli/archived-scripts/get-article-notes.ts
Normal file
84
self-hosting/omc/cli/archived-scripts/get-article-notes.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
#!/usr/bin/env tsx
|
||||
|
||||
/**
|
||||
* Fetch and display notes for an article
|
||||
*/
|
||||
|
||||
import fetch from 'node-fetch';
|
||||
import { config } from 'dotenv';
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = process.env.OMNIVORE_API_URL || 'https://api-prod.omnivore.app/api/graphql';
|
||||
const API_KEY = process.env.OMNIVORE_API_KEY;
|
||||
|
||||
const articleId = process.argv[2] || '5977ff9f-01ea-4977-aa0e-1dbe43cd2a20';
|
||||
|
||||
console.log(`\n📝 Fetching notes for article: ${articleId}\n`);
|
||||
|
||||
const query = `
|
||||
query GetArticle($id: ID!) {
|
||||
article(id: $id) {
|
||||
... on ArticleSuccess {
|
||||
article {
|
||||
id
|
||||
title
|
||||
highlights {
|
||||
id
|
||||
type
|
||||
quote
|
||||
annotation
|
||||
createdAt
|
||||
updatedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
... on ArticleError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const response = await fetch(API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': API_KEY,
|
||||
},
|
||||
body: JSON.stringify({ query, variables: { id: articleId } }),
|
||||
});
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.errors) {
|
||||
console.error('GraphQL errors:', JSON.stringify(result.errors, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const article = result.data?.article?.article;
|
||||
|
||||
if (!article) {
|
||||
console.error('Article not found or error:', result.data?.article?.errorCodes);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Article: ${article.title}\n`);
|
||||
|
||||
const highlights = article.highlights || [];
|
||||
const notes = highlights.filter((h: any) => h.type === 'NOTE');
|
||||
|
||||
console.log(`Found ${notes.length} notes:\n`);
|
||||
|
||||
for (const note of notes) {
|
||||
console.log('═'.repeat(80));
|
||||
console.log(`Note ID: ${note.id}`);
|
||||
console.log(`Type: ${note.type}`);
|
||||
console.log(`Created: ${note.createdAt}`);
|
||||
console.log(`Updated: ${note.updatedAt}`);
|
||||
console.log('─'.repeat(80));
|
||||
console.log('Annotation (raw Markdown):');
|
||||
console.log(note.annotation);
|
||||
console.log('═'.repeat(80));
|
||||
console.log();
|
||||
}
|
||||
55
self-hosting/omc/cli/archived-scripts/migrate-database.ts
Normal file
55
self-hosting/omc/cli/archived-scripts/migrate-database.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
#!/usr/bin/env tsx
|
||||
// AIDEV-NOTE: database-migration - adds new columns to existing analysis_queue table
|
||||
|
||||
import { initDatabase } from '../src/storage/database';
|
||||
|
||||
async function main() {
|
||||
console.log('Starting database migration...\n');
|
||||
|
||||
const db = initDatabase();
|
||||
|
||||
try {
|
||||
// Check current schema
|
||||
const columns = db.prepare(`PRAGMA table_info(analysis_queue)`).all() as Array<{ name: string }>;
|
||||
const columnNames = columns.map(c => c.name);
|
||||
|
||||
console.log('Current columns:', columnNames.join(', '));
|
||||
|
||||
// Add missing columns
|
||||
const migrations = [
|
||||
{ name: 'saved_at', sql: 'ALTER TABLE analysis_queue ADD COLUMN saved_at TEXT NOT NULL DEFAULT ""' },
|
||||
{ name: 'published_at', sql: 'ALTER TABLE analysis_queue ADD COLUMN published_at TEXT' },
|
||||
{ name: 'updated_at_article', sql: 'ALTER TABLE analysis_queue ADD COLUMN updated_at_article TEXT' },
|
||||
{ name: 'analysis_json', sql: 'ALTER TABLE analysis_queue ADD COLUMN analysis_json TEXT' },
|
||||
{ name: 'markdown_path', sql: 'ALTER TABLE analysis_queue ADD COLUMN markdown_path TEXT' },
|
||||
];
|
||||
|
||||
let added = 0;
|
||||
for (const migration of migrations) {
|
||||
if (!columnNames.includes(migration.name)) {
|
||||
console.log(`Adding column: ${migration.name}`);
|
||||
db.exec(migration.sql);
|
||||
added++;
|
||||
} else {
|
||||
console.log(`Column ${migration.name} already exists, skipping`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n✓ Migration complete: ${added} columns added`);
|
||||
|
||||
// Show final schema
|
||||
const finalColumns = db.prepare(`PRAGMA table_info(analysis_queue)`).all() as Array<{ name: string, type: string }>;
|
||||
console.log('\nFinal schema:');
|
||||
finalColumns.forEach(col => {
|
||||
console.log(` - ${col.name} (${col.type})`);
|
||||
});
|
||||
|
||||
} catch (err: any) {
|
||||
console.error('Migration failed:', err.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
98
self-hosting/omc/cli/archived-scripts/parallel-analyze.ts
Normal file
98
self-hosting/omc/cli/archived-scripts/parallel-analyze.ts
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
#!/usr/bin/env tsx
|
||||
// AIDEV-NOTE: tracking-coordination - uses SQLite queue for parallel execution
|
||||
// AIDEV-NOTE: analysis-output-boundary - results written to Markdown/JSONL, NOT database
|
||||
// AIDEV-NOTE: omnivore-boundary - fetches articles via GraphQL API, never local cache
|
||||
|
||||
import { getArticle, getMe } from '../lib/omnivore/client.js';
|
||||
import { initDatabase } from '../src/storage/database';
|
||||
import { AnalysisQueueRepository } from '../src/storage/AnalysisQueueRepository';
|
||||
import { AnalysisWriter } from '../src/storage/AnalysisWriter';
|
||||
import type { ContentAnalysis } from '../src/types/analysis';
|
||||
import { writeFileSync, mkdirSync } from 'fs';
|
||||
import 'dotenv/config';
|
||||
|
||||
const BATCH_SIZE = 5; // Process 5 articles in parallel
|
||||
|
||||
async function main() {
|
||||
// AIDEV-NOTE: tracking-db - coordination only, not analysis storage
|
||||
const db = initDatabase();
|
||||
const queueRepo = new AnalysisQueueRepository(db);
|
||||
|
||||
const stats = queueRepo.getStats();
|
||||
console.log('═'.repeat(80));
|
||||
console.log('Analysis Queue Status');
|
||||
console.log('═'.repeat(80));
|
||||
console.log(` Total: ${stats.total}`);
|
||||
console.log(` Pending: ${stats.pending}`);
|
||||
console.log(` In Progress: ${stats.inProgress}`);
|
||||
console.log(` Completed: ${stats.completed}`);
|
||||
console.log(` Failed: ${stats.failed}`);
|
||||
|
||||
if (stats.pending === 0) {
|
||||
console.log('\n✓ No pending analyses');
|
||||
db.close();
|
||||
return;
|
||||
}
|
||||
|
||||
// Get next batch
|
||||
// AIDEV-NOTE: tracking-batch - fetch jobs for parallel processing
|
||||
const jobs = queueRepo.getPending(BATCH_SIZE);
|
||||
console.log(`\n${'─'.repeat(80)}`);
|
||||
console.log(`Created ${jobs.length} stub files for parallel analysis...`);
|
||||
console.log('─'.repeat(80));
|
||||
|
||||
// Mark as in_progress (coordination lock)
|
||||
// AIDEV-NOTE: tracking-lock - prevents duplicate analysis by concurrent runs
|
||||
for (const job of jobs) {
|
||||
queueRepo.markInProgress(job.articleId);
|
||||
}
|
||||
|
||||
// AIDEV-NOTE: gql-article-query - get username from authenticated user
|
||||
const me = await getMe();
|
||||
const username = me.profile.username;
|
||||
|
||||
// Create temp directory for stub files
|
||||
mkdirSync('temp', { recursive: true });
|
||||
|
||||
// AIDEV-NOTE: stub-file-creation - fetch articles and populate complete metadata
|
||||
const agentParams = [];
|
||||
|
||||
for (let index = 0; index < jobs.length; index++) {
|
||||
const job = jobs[index];
|
||||
const filename = `temp/${job.articleSlug}.jsonl`;
|
||||
|
||||
console.log(`Fetching article ${index + 1}/${jobs.length}: ${job.articleTitle.substring(0, 60)}...`);
|
||||
|
||||
// Fetch article to get publishedAt/updatedAt
|
||||
const result = await getArticle(job.articleSlug, username);
|
||||
|
||||
// Write complete stub JSONL with all metadata
|
||||
const stub = {
|
||||
articleId: job.articleId,
|
||||
articleSlug: job.articleSlug,
|
||||
username: username,
|
||||
articleUrl: result.article.url,
|
||||
articleTitle: result.article.title,
|
||||
savedAt: result.article.savedAt,
|
||||
publishedAt: result.article.publishedAt || null,
|
||||
updatedAt: result.article.updatedAt || null
|
||||
};
|
||||
|
||||
writeFileSync(filename, JSON.stringify(stub) + '\n', 'utf-8');
|
||||
|
||||
// Return metadata for agent invocation
|
||||
agentParams.push({
|
||||
filename: filename,
|
||||
articleId: job.articleId,
|
||||
articleSlug: job.articleSlug,
|
||||
username: username,
|
||||
articleTitle: result.article.title.substring(0, 60) + '...' // truncated for display
|
||||
});
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(agentParams, null, 2));
|
||||
|
||||
db.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
52
self-hosting/omc/cli/archived-scripts/retry-failed.ts
Normal file
52
self-hosting/omc/cli/archived-scripts/retry-failed.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
#!/usr/bin/env tsx
|
||||
// AIDEV-NOTE: tracking-retry - resets failed jobs to pending for another attempt
|
||||
|
||||
import { initDatabase } from '../src/storage/database';
|
||||
import { AnalysisQueueRepository } from '../src/storage/AnalysisQueueRepository';
|
||||
|
||||
const MAX_RETRIES = 3;
|
||||
|
||||
async function main() {
|
||||
const db = initDatabase();
|
||||
const queueRepo = new AnalysisQueueRepository(db);
|
||||
|
||||
const failed = queueRepo.getFailed();
|
||||
|
||||
if (failed.length === 0) {
|
||||
console.log('✓ No failed jobs to retry');
|
||||
db.close();
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('═'.repeat(80));
|
||||
console.log(`Found ${failed.length} failed jobs`);
|
||||
console.log('═'.repeat(80));
|
||||
|
||||
let reset = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const job of failed) {
|
||||
if (job.retryCount < MAX_RETRIES) {
|
||||
queueRepo.resetToPending(job.articleId);
|
||||
console.log(`✓ Reset: ${job.articleTitle.substring(0, 50)}... (attempt ${job.retryCount + 2})`);
|
||||
reset++;
|
||||
} else {
|
||||
console.log(`✗ Skip: ${job.articleTitle.substring(0, 50)}... (max retries exceeded)`);
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n' + '═'.repeat(80));
|
||||
console.log('Summary:');
|
||||
console.log(` Reset for retry: ${reset}`);
|
||||
console.log(` Skipped (max retries): ${skipped}`);
|
||||
console.log('═'.repeat(80));
|
||||
|
||||
if (reset > 0) {
|
||||
console.log('\nRun: pnpm analyze:parallel');
|
||||
}
|
||||
|
||||
db.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
106
self-hosting/omc/cli/archived-scripts/save-analysis-results.ts
Normal file
106
self-hosting/omc/cli/archived-scripts/save-analysis-results.ts
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
#!/usr/bin/env tsx
|
||||
// AIDEV-NOTE: analysis-output-boundary - saves results to git-tracked Markdown/JSONL
|
||||
// AIDEV-NOTE: tracking-update - updates SQLite queue status after save
|
||||
|
||||
import { readFileSync, unlinkSync } from 'fs';
|
||||
import { initDatabase } from '../src/storage/database';
|
||||
import { AnalysisQueueRepository } from '../src/storage/AnalysisQueueRepository';
|
||||
import { AnalysisWriter } from '../src/storage/AnalysisWriter';
|
||||
import type { ContentAnalysis } from '../src/types/analysis';
|
||||
|
||||
interface EnrichedResult {
|
||||
articleId: string;
|
||||
articleSlug: string;
|
||||
username: string;
|
||||
articleUrl: string;
|
||||
articleTitle: string;
|
||||
savedAt: string;
|
||||
publishedAt: string | null;
|
||||
updatedAt: string | null;
|
||||
analysis: ContentAnalysis;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const patterns = process.argv.slice(2);
|
||||
|
||||
if (patterns.length === 0) {
|
||||
patterns.push('temp/*.jsonl'); // Default: all JSONL files in temp/
|
||||
}
|
||||
|
||||
// Read all matching JSONL files from all patterns
|
||||
const { globSync } = await import('glob');
|
||||
const files = patterns.flatMap(pattern => globSync(pattern));
|
||||
|
||||
if (files.length === 0) {
|
||||
console.error(`No files found matching patterns: ${patterns.join(', ')}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const resultsWithFiles: Array<{ result: EnrichedResult; file: string }> = files.map(file => {
|
||||
const content = readFileSync(file, 'utf-8');
|
||||
return { result: JSON.parse(content), file };
|
||||
});
|
||||
|
||||
console.log(`Loaded ${resultsWithFiles.length} enriched analysis results from ${files.length} files`);
|
||||
|
||||
// AIDEV-NOTE: tracking-db - coordination only
|
||||
const db = initDatabase();
|
||||
const queueRepo = new AnalysisQueueRepository(db);
|
||||
|
||||
// AIDEV-NOTE: analysis-output - permanent git-tracked storage
|
||||
const writer = new AnalysisWriter({ outputDir: 'content/analysis' });
|
||||
|
||||
let saved = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const { result, file } of resultsWithFiles) {
|
||||
const { articleId, articleUrl, articleTitle, savedAt, publishedAt, updatedAt, analysis } = result;
|
||||
|
||||
try {
|
||||
// Write to Markdown (git-tracked, human-editable)
|
||||
const mdPath = await writer.write(
|
||||
articleId,
|
||||
articleUrl,
|
||||
articleTitle,
|
||||
savedAt,
|
||||
analysis
|
||||
);
|
||||
|
||||
// Store in database (immutable AI snapshot)
|
||||
queueRepo.storeAnalysis(
|
||||
articleId,
|
||||
publishedAt,
|
||||
updatedAt,
|
||||
JSON.stringify(analysis),
|
||||
mdPath
|
||||
);
|
||||
|
||||
// Remove temp file after successful save
|
||||
unlinkSync(file);
|
||||
|
||||
console.log(`✓ Saved: ${articleTitle.substring(0, 60)}...`);
|
||||
saved++;
|
||||
} catch (err: any) {
|
||||
// Mark as failed in tracking DB
|
||||
queueRepo.markFailed(articleId, `Save failed: ${err.message}`);
|
||||
console.error(`✗ Failed: ${articleTitle.substring(0, 60)}... - ${err.message}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
// Show updated stats
|
||||
const stats = queueRepo.getStats();
|
||||
console.log(`\n${'═'.repeat(80)}`);
|
||||
console.log('Results:');
|
||||
console.log(` Saved: ${saved}`);
|
||||
console.log(` Failed: ${failed}`);
|
||||
console.log('\nUpdated Queue Status:');
|
||||
console.log(` Pending: ${stats.pending}`);
|
||||
console.log(` Completed: ${stats.completed}`);
|
||||
console.log(` Failed: ${stats.failed}`);
|
||||
console.log('═'.repeat(80));
|
||||
|
||||
db.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
#!/usr/bin/env tsx
|
||||
|
||||
/**
|
||||
* Test updating article description AND creating notebook notes
|
||||
*/
|
||||
|
||||
import { updatePage, createHighlight } from '../lib/omnivore/client.js';
|
||||
import { randomUUID } from 'crypto';
|
||||
|
||||
// Use the LLM Observability article
|
||||
const articleId = '5977ff9f-01ea-4977-aa0e-1dbe43cd2a20';
|
||||
const articleTitle = 'LLM Observability in the Wild - Why OpenTelemetry should be the Standard | SigNoz';
|
||||
|
||||
console.log(`\n🧪 Testing article notes update for: ${articleTitle}\n`);
|
||||
|
||||
// Test 1: Update article description (the "info" section in Omnivore UI)
|
||||
console.log('1️⃣ Updating article description (Info section)...');
|
||||
|
||||
const updateResult = await updatePage({
|
||||
pageId: articleId,
|
||||
description: '🔍 WHY I SAVED THIS: Excellent deep dive into OpenTelemetry patterns for LLM observability. Potential for comparison post: OpenTelemetry vs proprietary solutions (Langfuse, Weights & Biases). Key insight: standardization matters more in LLM ops than traditional observability.'
|
||||
});
|
||||
|
||||
console.log(' ✅ Description updated:', updateResult.updatedPage.description);
|
||||
|
||||
// Test 2: Create a NOTE highlight (the "Notebook" section in Omnivore UI)
|
||||
console.log('\n2️⃣ Creating notebook note (NOTE highlight)...');
|
||||
|
||||
const noteId = randomUUID();
|
||||
const shortId = Math.random().toString(36).substring(2, 10); // 8-char random string
|
||||
const noteResult = await createHighlight({
|
||||
id: noteId,
|
||||
shortId: shortId,
|
||||
articleId: articleId,
|
||||
type: 'NOTE',
|
||||
annotation: '💡 CONTENT STRATEGY NOTE:\n\nThis article fits perfectly into our AI Infrastructure theme for the weekly roundup. Key angles:\n\n1. Comparison piece: OpenTelemetry vs Langfuse vs W&B\n2. Tutorial: Setting up OTel for LLM apps\n3. Opinion piece: Why standardization matters in AI ops\n\nTarget audience: ML engineers building production LLM apps\nEstimated value: High - trending topic, practical advice',
|
||||
quote: '', // Empty quote = standalone note
|
||||
patch: '',
|
||||
prefix: '',
|
||||
suffix: ''
|
||||
});
|
||||
|
||||
console.log(' Result:', JSON.stringify(noteResult, null, 2));
|
||||
|
||||
console.log('\n📌 Check Omnivore UI to verify:');
|
||||
console.log(' 1. Description field shows "WHY I SAVED THIS" note');
|
||||
console.log(' 2. Notebook section shows the content strategy note');
|
||||
console.log(` Article ID: ${articleId}\n`);
|
||||
42
self-hosting/omc/cli/archived-scripts/update-note-test.ts
Normal file
42
self-hosting/omc/cli/archived-scripts/update-note-test.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
#!/usr/bin/env tsx
|
||||
|
||||
/**
|
||||
* Update a note with Markdown to test format
|
||||
*/
|
||||
|
||||
import { updateHighlight } from '../lib/omnivore/client.js';
|
||||
|
||||
const noteId = '922361d2-f09c-4b81-8826-396de3690c5d';
|
||||
|
||||
const markdown = `# Analysis Report
|
||||
|
||||
**Article**: LLM Observability
|
||||
**Status**: High Priority
|
||||
|
||||
## Key Insights
|
||||
|
||||
1. **OpenTelemetry vs OpenInference** - competing standards
|
||||
2. **Ruby SDK gap** - No OpenInference support
|
||||
3. **Production debugging** - Visibility challenges
|
||||
|
||||
## Content Strategy
|
||||
|
||||
- [ ] Write comparison post
|
||||
- [ ] Tutorial on OTel setup
|
||||
- [ ] Interview Pranav
|
||||
|
||||
### Target Audience
|
||||
ML engineers building production LLM apps
|
||||
|
||||
*Last updated: 2025-10-02*
|
||||
`;
|
||||
|
||||
console.log('Updating note with Markdown...\n');
|
||||
|
||||
const result = await updateHighlight({
|
||||
highlightId: noteId,
|
||||
annotation: markdown
|
||||
});
|
||||
|
||||
console.log('Result:', JSON.stringify(result, null, 2));
|
||||
console.log('\nCheck Omnivore UI to see how the Markdown renders!');
|
||||
58
self-hosting/omc/codegen.yml
Normal file
58
self-hosting/omc/codegen.yml
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
# GraphQL Code Generator Configuration
|
||||
# Generates TypeScript types from GraphQL schema and operations
|
||||
|
||||
schema: './docs/graphql-schema/schema.graphql'
|
||||
|
||||
# Watch mode for development
|
||||
watch: false
|
||||
|
||||
# The CLI runtime currently uses `lib/omnivore/client.js` (string-based queries).
|
||||
# Typed GraphQL operations are not maintained right now; reintroduce documents as part of a future migration.
|
||||
documents: []
|
||||
|
||||
# Output configuration
|
||||
generates:
|
||||
# Generate TypeScript types for the schema
|
||||
src/types/generated/graphql.ts:
|
||||
plugins:
|
||||
- typescript
|
||||
config:
|
||||
# Type-safe configuration
|
||||
strictScalars: true
|
||||
scalars:
|
||||
Date: string
|
||||
JSON: 'Record<string, unknown>'
|
||||
# Naming conventions
|
||||
enumsAsTypes: true
|
||||
# Add helpful comments
|
||||
addUnderscoreToArgsType: true
|
||||
skipTypename: false
|
||||
|
||||
# Generate TypeScript types for operations (queries/mutations)
|
||||
src/types/generated/operations.ts:
|
||||
plugins:
|
||||
- typescript
|
||||
- typescript-operations
|
||||
- typed-document-node
|
||||
config:
|
||||
# Type-safe configuration
|
||||
strictScalars: true
|
||||
scalars:
|
||||
Date: string
|
||||
JSON: 'Record<string, unknown>'
|
||||
# Avoid duplicating types from schema
|
||||
avoidOptionals: false
|
||||
# Better type names
|
||||
namingConvention:
|
||||
typeNames: pascal-case#pascalCase
|
||||
enumValues: upper-case#upperCase
|
||||
# Fragment configuration
|
||||
fragmentVariablePrefix: ''
|
||||
fragmentVariableSuffix: 'Fragment'
|
||||
# Deduplication
|
||||
dedupeFragments: true
|
||||
|
||||
# Hooks to run after generation
|
||||
# hooks:
|
||||
# afterAllFileWrite:
|
||||
# - prettier --write
|
||||
1042
self-hosting/omc/docs/CLI_DESIGN.md
Normal file
1042
self-hosting/omc/docs/CLI_DESIGN.md
Normal file
File diff suppressed because it is too large
Load diff
395
self-hosting/omc/docs/_meta/architecture.md
Normal file
395
self-hosting/omc/docs/_meta/architecture.md
Normal file
|
|
@ -0,0 +1,395 @@
|
|||
# System Architecture
|
||||
|
||||
**Purpose**: Architectural design decisions, storage boundaries, and structural patterns for the Omnivore Content System.
|
||||
|
||||
**Last Updated**: 2026-01-30
|
||||
|
||||
**Status**: Architectural intent remains valid, but some implementation details have drifted (notably JSONL output and the legacy workflow scripts). See `docs/_meta/current-state.md` for the full audit.
|
||||
|
||||
## Overview
|
||||
|
||||
The analysis engine processes articles from Omnivore through the `@article-content-analyzer` agent, extracting structured insights for content monetization. The system uses a three-layer architecture designed to maintain clear boundaries and prevent data corruption.
|
||||
|
||||
## Three-Layer Architecture
|
||||
|
||||
The system enforces strict separation between source data, coordination, and permanent storage:
|
||||
|
||||
**Layer 1: Omnivore (Source of Truth)**
|
||||
- Always fetch articles via GraphQL API
|
||||
- Never duplicate article content locally
|
||||
- Omnivore labels used as intent hints for analysis
|
||||
|
||||
**Layer 2: SQLite (Immutable Snapshots + Tracking)**
|
||||
- Database: `data/omnivore-content.db` (gitignored)
|
||||
- Purpose: Store immutable AI analysis snapshots + coordinate parallel execution
|
||||
- Tracks: pending/in_progress/completed/failed status
|
||||
- Tables: `analysis_queue`, `analysis_sessions`
|
||||
- Existing Omnivore tables (if present): READ-ONLY, never modify
|
||||
|
||||
**Layer 3: Markdown (Permanent Storage)**
|
||||
- Analysis results: `content/analysis/*.md` (human-readable)
|
||||
- Git-tracked for version history
|
||||
- Only value-added content stored
|
||||
|
||||
### What Gets Stored
|
||||
|
||||
- ✅ **Analysis results** (`content/analysis/*.md`) - Topics, summaries, key points, monetization angles
|
||||
- ✅ **JSONL records (optional)** (`content/analysis/analyses.jsonl`) - Produced when enabled via CLI flags (`omc analyze complete --jsonl` / `omc analyze auto --jsonl`)
|
||||
- ✅ **Tracking queue** (SQLite) - Job status, retry counts, error messages
|
||||
- ✅ **Generated content** (`content/generated/**/*.md`) - Blog posts, newsletters (future)
|
||||
- ❌ **Source articles** - Query Omnivore API; do NOT duplicate locally
|
||||
|
||||
### Storage Formats
|
||||
|
||||
1. **Markdown with YAML front-matter** - Human-readable, git-trackable
|
||||
2. **JSONL (JSON Lines)** - Append-only, machine-readable (planned/optional; not currently produced by default)
|
||||
3. **SQLite** - Immutable AI snapshots + tracking (gitignored but permanent)
|
||||
|
||||
## Database Schema
|
||||
|
||||
**Location**: `src/storage/schema/tracking-schema.sql`
|
||||
|
||||
**Purpose**: Define tracking tables for job queue coordination.
|
||||
|
||||
### Tables
|
||||
|
||||
**1. `analysis_queue`** - Job tracking for parallel execution
|
||||
- `id` (INTEGER PRIMARY KEY) - Auto-increment job ID
|
||||
- `article_id` (TEXT UNIQUE) - Omnivore article ID
|
||||
- `article_url` (TEXT) - Source article URL
|
||||
- `article_title` (TEXT) - Source article title
|
||||
- `status` (TEXT) - Job status: `pending` | `in_progress` | `completed` | `failed`
|
||||
- `assigned_at` (TEXT) - When marked in_progress (ISO 8601)
|
||||
- `completed_at` (TEXT) - When marked completed (ISO 8601)
|
||||
- `error_message` (TEXT) - Error details if failed
|
||||
- `retry_count` (INTEGER) - Number of retry attempts (default: 0)
|
||||
- `created_at` (TEXT) - When added to queue (ISO 8601)
|
||||
- `updated_at` (TEXT) - Last status change (ISO 8601)
|
||||
|
||||
**2. `analysis_sessions`** - Optional batch metadata
|
||||
- `id` (INTEGER PRIMARY KEY) - Auto-increment session ID
|
||||
- `started_at` (TEXT) - Session start time
|
||||
- `completed_at` (TEXT) - Session completion time
|
||||
- `total_articles` (INTEGER) - Total articles in session
|
||||
- `completed_articles` (INTEGER) - Completed count
|
||||
- `failed_articles` (INTEGER) - Failed count
|
||||
- `notes` (TEXT) - Session notes
|
||||
|
||||
### Indexes
|
||||
|
||||
- `idx_analysis_queue_status` - Fast status filtering
|
||||
- `idx_analysis_queue_article_id` - Fast article lookup
|
||||
- `idx_analysis_queue_created_at` - Chronological ordering
|
||||
|
||||
### AIDEV Annotations
|
||||
|
||||
- `tracking + immutable snapshots` - Stores original AI output + coordination
|
||||
- `analysis-output-boundary` - Analysis results stored in git-tracked Markdown/JSONL, NOT here
|
||||
|
||||
## Database Initialization
|
||||
|
||||
**Location**: `src/storage/database.ts`
|
||||
|
||||
**Purpose**: Initialize SQLite database with boundary enforcement for tracking coordination.
|
||||
|
||||
**Import**:
|
||||
```typescript
|
||||
import { initDatabase } from '@storage/database';
|
||||
```
|
||||
|
||||
### Key Functions
|
||||
|
||||
**1. `initDatabase(dbPath?: string): Database`** - Initialize database connection
|
||||
- Default path: `data/omnivore-content.db`
|
||||
- Enables WAL mode for concurrent access
|
||||
- Creates tracking tables via `tracking-schema.sql`
|
||||
- Returns `better-sqlite3` Database instance
|
||||
|
||||
**2. `listTables(db: Database): string[]`** - List all tables
|
||||
- Returns array of table names
|
||||
- Used to identify Omnivore vs tracking tables
|
||||
|
||||
**3. `isOmnivoreTable(db: Database, tableName: string): boolean`** - Identify Omnivore tables
|
||||
- Detects Core Data tables by Z-prefixed column names
|
||||
- Returns `true` if table belongs to Omnivore cache
|
||||
|
||||
**4. `validateOmnivoreTablesReadOnly(db: Database): void`** - Boundary check
|
||||
- Logs Omnivore tables (READ-ONLY) vs tracking tables (READ-WRITE)
|
||||
- Safety check to ensure no modification of Omnivore data
|
||||
|
||||
**5. `getTableCounts(db: Database): Record<string, number>`** - Get row counts
|
||||
- Returns object mapping table names to row counts
|
||||
- Useful for debugging and monitoring
|
||||
|
||||
### AIDEV Annotations
|
||||
|
||||
- `tracking-db-boundary` - SQLite stores immutable AI snapshots + job tracking
|
||||
- `omnivore-boundary` - Existing Omnivore tables are READ-ONLY, never modify
|
||||
|
||||
### Example Usage
|
||||
|
||||
```typescript
|
||||
import { initDatabase, validateOmnivoreTablesReadOnly, listTables } from '@storage/database';
|
||||
|
||||
// Initialize database
|
||||
const db = initDatabase('data/omnivore-content.db');
|
||||
|
||||
// Verify boundaries
|
||||
validateOmnivoreTablesReadOnly(db);
|
||||
|
||||
// List all tables
|
||||
const tables = listTables(db);
|
||||
console.log('Tables:', tables);
|
||||
// Output: ['analysis_queue', 'analysis_sessions', 'ZMEDIA', 'ZUSER', ...]
|
||||
|
||||
// Close when done
|
||||
db.close();
|
||||
```
|
||||
|
||||
## Storage Formats
|
||||
|
||||
### Markdown File Structure
|
||||
|
||||
**Location**: `content/analysis/YYYY-MM-DD-{slug}-analysis.md`
|
||||
|
||||
**Format**:
|
||||
```markdown
|
||||
---
|
||||
articleId: 1039961b-8de3-4ccf-b3e8-df888d6174b8
|
||||
articleSlug: cchistory-tracking-claude-code-system-prompt
|
||||
articleUrl: https://example.com/article
|
||||
articleTitle: "Article Title"
|
||||
savedAt: 2025-09-30T02:27:46.000Z
|
||||
analyzedAt: 2025-10-01T04:25:16.676Z
|
||||
topics: [developer-tools, ai-tooling, reverse-engineering]
|
||||
topicScores:
|
||||
developer-tools: 0.95
|
||||
ai-tooling: 0.9
|
||||
reverse-engineering: 0.85
|
||||
sentiment: positive
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
2-3 sentence summary capturing the main points and why this matters...
|
||||
|
||||
## Key Points
|
||||
|
||||
- First key takeaway or insight
|
||||
- Second key takeaway or insight
|
||||
- Third key takeaway or insight
|
||||
|
||||
## Monetization Angle
|
||||
|
||||
Specific content opportunity description...
|
||||
```
|
||||
|
||||
**Front-Matter Fields**:
|
||||
- `articleId` (string) - Omnivore article ID (references source)
|
||||
- `articleSlug` (string, optional) - Omnivore slug (preferred filename key)
|
||||
- `articleUrl` (string) - Source article URL
|
||||
- `articleTitle` (string) - Source article title (escaped quotes)
|
||||
- `savedAt` (ISO 8601) - When article was saved to Omnivore
|
||||
- `analyzedAt` (ISO 8601) - When analysis was performed
|
||||
- `topics` (array) - 2-5 topic labels
|
||||
- `topicScores` (object) - Topic → confidence score (0-1)
|
||||
- `sentiment` (enum) - `positive` | `neutral` | `negative`
|
||||
|
||||
**Markdown Sections**:
|
||||
- `## Summary` - Strategic summary (2-3 sentences)
|
||||
- `## Key Points` - Bullet list of insights (3-5 items)
|
||||
- `## Monetization Angle` - Content opportunity description
|
||||
|
||||
### JSONL File Structure
|
||||
|
||||
**Location**: `content/analysis/analyses.jsonl`
|
||||
|
||||
**Status**: Optional. `AnalysisWriter.appendToJsonl()` is invoked when JSONL is enabled via CLI flags.
|
||||
|
||||
**Format**: One JSON object per line (JSON Lines / newline-delimited JSON)
|
||||
|
||||
**Purpose**: Append-only machine-readable format for batch processing
|
||||
|
||||
**Record Structure**:
|
||||
```json
|
||||
{"articleId":"abc-123","articleUrl":"https://example.com/article","articleTitle":"Article Title","savedAt":"2025-09-30T14:12:18.000Z","analyzedAt":"2025-10-01T04:30:53.761Z","topics":["developer-tools","api-design"],"topicScores":{"developer-tools":0.95,"api-design":0.9},"sentiment":"positive","summary":"Article summary...","keyPoints":["First point","Second point"],"monetizationAngle":"Content opportunity..."}
|
||||
```
|
||||
|
||||
**Fields**: Same as front-matter plus `summary`, `keyPoints`, `monetizationAngle` (flattened structure)
|
||||
|
||||
**Advantages**:
|
||||
- Append-only (no file rewriting)
|
||||
- Easy parsing line-by-line
|
||||
- Works with streaming processing
|
||||
- Standard format for data pipelines
|
||||
|
||||
**Usage Example**:
|
||||
```javascript
|
||||
import { readFileSync } from 'fs';
|
||||
|
||||
const lines = readFileSync('content/analysis/analyses.jsonl', 'utf-8')
|
||||
.split('\n')
|
||||
.filter(line => line.trim());
|
||||
|
||||
const analyses = lines.map(line => JSON.parse(line));
|
||||
console.log(`Loaded ${analyses.length} analyses`);
|
||||
```
|
||||
|
||||
## Boundary Enforcement
|
||||
|
||||
The system enforces strict boundaries between three layers to prevent data corruption and maintain clarity.
|
||||
|
||||
### AIDEV Annotation Patterns
|
||||
|
||||
All boundary-critical code uses AIDEV annotations for searchability:
|
||||
|
||||
**Tracking Annotations** (SQLite coordination):
|
||||
- `tracking + immutable snapshots` - Stores original AI output + coordination
|
||||
- `tracking-db-boundary` - SQLite stores immutable AI snapshots + job tracking
|
||||
- `tracking-initialization` - Sets up job queue
|
||||
- `tracking-coordination` - Fetches jobs for parallel processing
|
||||
- `tracking-lock` - Prevents duplicate analysis by concurrent runs
|
||||
- `tracking-completion` - Job done, analysis in git-tracked files
|
||||
- `tracking-error` - Increments retry counter
|
||||
- `tracking-retry` - Resets failed job for another attempt
|
||||
- `tracking-stats` - Shows progress, not analysis content
|
||||
- `tracking-deduplication` - Prevents duplicate queue entries
|
||||
- `tracking-cleanup` - Removes completed jobs
|
||||
- `tracking-inspection` - Shows queue status
|
||||
- `tracking-update` - Updates queue status after save
|
||||
|
||||
**Output Annotations** (Git-tracked storage):
|
||||
- `analysis-output-boundary` - Results written to Markdown (and optionally JSONL), NOT stored as editable content in the database
|
||||
- `git-tracked-output` - Permanent storage for analysis results
|
||||
|
||||
**Omnivore Annotations** (Source of truth):
|
||||
- `omnivore-boundary` - Always fetch via GraphQL, never from local cache
|
||||
|
||||
### Search Annotations
|
||||
|
||||
Find all boundary-critical code:
|
||||
```bash
|
||||
# SQLite tracking code
|
||||
rg "AIDEV-NOTE:.*tracking"
|
||||
|
||||
# Analysis output code
|
||||
rg "AIDEV-NOTE:.*analysis-output"
|
||||
|
||||
# Omnivore API usage
|
||||
rg "AIDEV-NOTE:.*omnivore-boundary"
|
||||
|
||||
# Git-tracked output
|
||||
rg "AIDEV-NOTE:.*git-tracked"
|
||||
```
|
||||
|
||||
### Boundary Rules
|
||||
|
||||
**Rule 1: Omnivore is Source of Truth**
|
||||
- Always fetch articles via GraphQL API
|
||||
- Never duplicate article content locally
|
||||
- Use Omnivore labels as intent hints
|
||||
|
||||
**Rule 2: SQLite Stores Immutable Analysis Snapshots**
|
||||
- Stores original AI analysis output (immutable snapshots) + coordination for parallel execution
|
||||
- Database file (`data/omnivore-content.db`) is gitignored but permanent
|
||||
- Contains valuable AI output that should NOT be deleted
|
||||
- Existing Omnivore tables (if present) are READ-ONLY
|
||||
|
||||
**Rule 3: Markdown/JSONL is Permanent Storage**
|
||||
- Analysis results are git-tracked
|
||||
- Only value-added content stored
|
||||
- Never store source articles
|
||||
|
||||
**Rule 4: No Cross-Layer Leakage**
|
||||
- Analysis results never stored in SQLite
|
||||
- Article content never stored in git
|
||||
- Queue status never persisted in Markdown
|
||||
|
||||
### Validation Functions
|
||||
|
||||
Use database boundary checks to verify compliance:
|
||||
|
||||
```typescript
|
||||
import { initDatabase, validateOmnivoreTablesReadOnly, isOmnivoreTable } from '@storage/database';
|
||||
|
||||
const db = initDatabase();
|
||||
|
||||
// Check for Omnivore tables
|
||||
validateOmnivoreTablesReadOnly(db);
|
||||
// Output:
|
||||
// [BOUNDARY CHECK] Omnivore table detected: ZMEDIA (READ-ONLY)
|
||||
// [BOUNDARY CHECK] Tracking table: analysis_queue (READ-WRITE)
|
||||
|
||||
// Verify table type before operations
|
||||
if (isOmnivoreTable(db, 'ZMEDIA')) {
|
||||
throw new Error('Cannot modify Omnivore table');
|
||||
}
|
||||
```
|
||||
|
||||
## Type Definitions
|
||||
|
||||
### ContentAnalysis
|
||||
|
||||
**Location**: `src/types/analysis.ts`
|
||||
|
||||
**Purpose**: Analysis result from `@article-content-analyzer` agent.
|
||||
|
||||
```typescript
|
||||
export interface ContentAnalysis {
|
||||
articleId: string; // Omnivore article ID
|
||||
topics: string[]; // 2-5 main topics
|
||||
topicScores: Record<string, number>; // Topic → confidence (0-1)
|
||||
summary: string; // 2-3 sentence summary
|
||||
keyPoints: string[]; // 3-5 key takeaways
|
||||
sentiment: 'positive' | 'neutral' | 'negative';
|
||||
monetizationAngle: string; // Content opportunity
|
||||
analyzedAt: string; // ISO 8601 timestamp
|
||||
}
|
||||
```
|
||||
|
||||
**Import**:
|
||||
```typescript
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
// OR
|
||||
import type { ContentAnalysis } from '../types/analysis';
|
||||
```
|
||||
|
||||
### AnalysisFrontMatter
|
||||
|
||||
**Location**: `src/types/content.ts`
|
||||
|
||||
**Purpose**: Front-matter schema for analysis Markdown files.
|
||||
|
||||
```typescript
|
||||
export interface AnalysisFrontMatter {
|
||||
articleId: string; // Omnivore article ID
|
||||
articleUrl: string; // Source article URL
|
||||
articleTitle: string; // Source article title
|
||||
savedAt: string; // ISO 8601
|
||||
analyzedAt: string; // ISO 8601
|
||||
topics: string[]; // Topic labels
|
||||
topicScores: Record<string, number>; // Topic → score
|
||||
sentiment: 'positive' | 'neutral' | 'negative';
|
||||
}
|
||||
```
|
||||
|
||||
### StoredAnalysis
|
||||
|
||||
**Location**: `src/types/content.ts`
|
||||
|
||||
**Purpose**: Complete stored analysis (after reading Markdown file).
|
||||
|
||||
```typescript
|
||||
export interface StoredAnalysis {
|
||||
frontMatter: AnalysisFrontMatter; // Parsed YAML
|
||||
summary: string; // From ## Summary section
|
||||
keyPoints: string[]; // From ## Key Points section
|
||||
monetizationAngle: string; // From ## Monetization Angle section
|
||||
}
|
||||
```
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Workflow Internals](workflow-internals.md) - How the parallel analysis workflow operates
|
||||
- [CLI Reference](cli-reference.md) - Command-line interface documentation
|
||||
- [Foundation & Type System](foundation-and-types.md) - TypeScript setup, type definitions, Omnivore client
|
||||
47
self-hosting/omc/docs/_meta/automation-patterns.md
Normal file
47
self-hosting/omc/docs/_meta/automation-patterns.md
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
# Automation Patterns (Codex CLI)
|
||||
|
||||
**Purpose**: Document the non-interactive automation path for daily analysis, and how it integrates with `codex exec`.
|
||||
|
||||
## Goal
|
||||
|
||||
Enable a fully schedulable workflow:
|
||||
|
||||
```bash
|
||||
omc queue add --hours 24
|
||||
omc analyze auto --batch-size 5
|
||||
```
|
||||
|
||||
This replaces the interactive “Task tool / parallel agents” middle step.
|
||||
|
||||
## Key Design Constraints
|
||||
|
||||
- The analysis step must be **non-interactive** and runnable from cron/launchd.
|
||||
- The LLM caller must **not mutate the repository** (safe-by-default).
|
||||
- The CLI must remain usable as a “tool” inside future Codex sessions (content generation phase).
|
||||
|
||||
## Codex CLI Invocation
|
||||
|
||||
The automated analysis pipeline uses the local Codex CLI in non-interactive mode:
|
||||
|
||||
- Command: `codex exec`
|
||||
- Sandbox: `-s read-only`
|
||||
- Prompt input: stdin (`-`)
|
||||
|
||||
Important runtime behavior:
|
||||
|
||||
- Codex writes session files under `CODEX_HOME`. For scheduled runs, `CODEX_HOME` is set to a repo-local directory (`temp/codex-home`) so we don’t depend on `~/.codex/*` permissions.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- LLM wrapper: `src/lib/ai/codex-cli-client.ts`
|
||||
- Builds a strict “JSON only” prompt.
|
||||
- Runs `codex exec` and parses the first JSON object from stdout.
|
||||
- Analyzer: `src/analysis/ContentAnalyzer.ts`
|
||||
- Loads `src/analysis/prompts/analyze-article.md`
|
||||
- Calls the Codex CLI wrapper and normalizes required fields.
|
||||
- Orchestration: `src/commands/analyze/auto.ts`
|
||||
- Selects jobs from SQLite queue
|
||||
- Fetches article content from Omnivore
|
||||
- Calls `ContentAnalyzer`
|
||||
- Persists results to Markdown + SQLite snapshot
|
||||
|
||||
832
self-hosting/omc/docs/_meta/cli-reference.md
Normal file
832
self-hosting/omc/docs/_meta/cli-reference.md
Normal file
|
|
@ -0,0 +1,832 @@
|
|||
# CLI Reference
|
||||
|
||||
**Purpose**: Comprehensive command-line interface documentation for the Omnivore Content System.
|
||||
|
||||
**Last Updated**: 2026-01-30
|
||||
|
||||
**Status**: This file historically mixed design intent with implementation notes. For a full ground-truth audit (including known breakages and fixes), see `docs/_meta/current-state.md`.
|
||||
|
||||
## Overview
|
||||
|
||||
The CLI provides commands for managing the article analysis queue, fetching content from Omnivore, running analysis, and generating reports. All commands use the `omc` binary.
|
||||
|
||||
## Queue Management Commands
|
||||
|
||||
### queue add
|
||||
|
||||
**Command**: `omc queue add`
|
||||
|
||||
**Purpose**: Fetch articles from Omnivore and populate tracking queue.
|
||||
|
||||
**Usage**:
|
||||
```bash
|
||||
omc queue add --hours 24 # Last 24 hours
|
||||
omc queue add --label "ai-ml" # By label
|
||||
omc queue add --url <url> # Single article
|
||||
omc queue add --slug <slug> # By slug
|
||||
```
|
||||
|
||||
**Parameters**:
|
||||
- `--hours=N` - Fetch articles from last N hours
|
||||
- `--label=X` - Fetch articles with specific label
|
||||
- `--url=X` - Add single article by URL
|
||||
- `--slug=X` - Add single article by slug
|
||||
|
||||
**What It Does (Current)**:
|
||||
1. Fetches articles from Omnivore (by hours / label / url / slug)
|
||||
2. Converts results into `{id, slug, url, title, savedAt}` queue rows
|
||||
3. Inserts into `analysis_queue` via `INSERT OR IGNORE` (deduplication by `article_id`)
|
||||
|
||||
**Notes**:
|
||||
- Content-length filtering and label distribution output are not currently implemented in `omc queue add`.
|
||||
|
||||
**Output Example**:
|
||||
```
|
||||
Added 38 articles to queue (45 total found)
|
||||
```
|
||||
|
||||
**AIDEV Annotations**:
|
||||
- `omnivore-boundary` - Fetches via GraphQL API, populates tracking queue
|
||||
- `tracking-initialization` - Adds articles to SQLite queue for parallel analysis
|
||||
|
||||
## Omnivore Integration Commands
|
||||
|
||||
### omnivore get
|
||||
|
||||
**Command**: `omc omnivore get`
|
||||
|
||||
**Purpose**: Fetch an article by slug (metadata by default; raw content with `--content`; full JSON including `content` with `--json`).
|
||||
|
||||
**Usage**:
|
||||
```bash
|
||||
omc omnivore get <articleSlug>
|
||||
omc omnivore get <articleSlug> --content
|
||||
omc omnivore get <articleSlug> --json
|
||||
```
|
||||
|
||||
**Parameters**:
|
||||
- `articleSlug` - Article slug from Omnivore URL or metadata
|
||||
- Username is loaded from .env file automatically
|
||||
|
||||
**What It Does (Current)**:
|
||||
1. Queries Omnivore GraphQL API using `article(slug, username)`
|
||||
2. Default output is a human-readable summary (title/url/author/description)
|
||||
3. With `--json`, prints the full Article payload (including `content`)
|
||||
4. With `--content`, prints the raw article content to stdout (agent-friendly)
|
||||
|
||||
**Agent usage**:
|
||||
- Prefer `omc omnivore get <slug> --content` for stable, parse-free access to the article body.
|
||||
- If you also need metadata, use `--json` and extract `.content`.
|
||||
|
||||
**Why Needed**:
|
||||
- Agents execute via Bash tool and cannot import TypeScript modules
|
||||
- Provides clean separation between content fetching and analysis
|
||||
- Enables zero-context-pollution agent workflow (each agent fetches independently)
|
||||
|
||||
**Usage in Agent Workflow**:
|
||||
```typescript
|
||||
// Agent reads stub file for slug
|
||||
const stub = JSON.parse(fs.readFileSync(`temp/${filename}`, 'utf-8'));
|
||||
|
||||
// Agent calls CLI tool to get content
|
||||
const contentResult = await bash(`omc omnivore get ${stub.articleSlug}`);
|
||||
const articleContent = contentResult.stdout;
|
||||
|
||||
// Agent analyzes content...
|
||||
```
|
||||
|
||||
**AIDEV Annotations**:
|
||||
- `omnivore-boundary` - Fetches via GraphQL API, outputs to stdout
|
||||
- `agent-workflow` - Enables agents to fetch content without TypeScript imports
|
||||
|
||||
## Analysis Commands
|
||||
|
||||
### analyze auto
|
||||
|
||||
**Command**: `omc analyze auto`
|
||||
|
||||
**Purpose**: Run the full analysis pipeline non-interactively (cron/launchd friendly).
|
||||
|
||||
**Usage**:
|
||||
```bash
|
||||
omc analyze auto --batch-size 5
|
||||
omc analyze auto --article-id <id>
|
||||
omc analyze auto --all --batch-size 5
|
||||
omc analyze auto --json
|
||||
omc analyze auto --batch-size 5 --jsonl
|
||||
```
|
||||
|
||||
**What It Does (Current)**:
|
||||
1. Selects jobs from the SQLite queue (`pending` by default; `pending+failed` with `--all`)
|
||||
2. Fetches article content from Omnivore
|
||||
3. Calls `codex exec` in **read-only** mode to produce a `ContentAnalysis` JSON object
|
||||
4. Writes Markdown under `content/analysis/` and stores an immutable analysis snapshot in SQLite
|
||||
5. Deletes `temp/{slug}.jsonl` unless `--keep-temp`
|
||||
|
||||
**Notes**:
|
||||
- Codex session files are written under `CODEX_HOME`. For automation, the CLI sets `CODEX_HOME=temp/codex-home` so runs don’t depend on `~/.codex` permissions.
|
||||
- JSONL output is optional: enable with `--jsonl` (defaults to `content/analysis/analyses.jsonl`).
|
||||
|
||||
### analyze run
|
||||
|
||||
**Command**: `omc analyze run`
|
||||
|
||||
**Purpose**: Run parallel analysis on queued articles with zero-context-pollution design.
|
||||
|
||||
**Usage**:
|
||||
```bash
|
||||
omc analyze run # Process next 5 articles (default)
|
||||
omc analyze run --batch-size 10 # Custom batch size
|
||||
omc analyze run --article-id <id> # Analyze specific article
|
||||
omc analyze run --all # Process entire queue
|
||||
```
|
||||
|
||||
**What It Does**:
|
||||
1. Fetches next 5 pending jobs from tracking queue
|
||||
2. Marks jobs as `in_progress` (coordination lock)
|
||||
3. Fetches article metadata from Omnivore API using `getArticle(slug, username)`
|
||||
4. Creates slug-based stub files: `temp/{articleSlug}.jsonl` (prevents overwrites when running multiple batches)
|
||||
5. Each stub file contains: `articleId`, `articleSlug`, `username`, `articleUrl`, `articleTitle`, `savedAt`, `publishedAt`, `updatedAt`
|
||||
6. Outputs JSON array with agent parameters for parallel invocation
|
||||
7. Closes database connection (agents get fresh connections with no context pollution)
|
||||
|
||||
**File Naming Strategy**:
|
||||
- Stub files use article slug: `temp/{articleSlug}.jsonl` (e.g., `temp/understanding-llm-fine-tuning.jsonl`)
|
||||
- Prevents overwrites when running multiple batches before agents complete
|
||||
- Enables agents to work independently without coordination
|
||||
|
||||
**Output Example**:
|
||||
```
|
||||
Analysis Queue Status
|
||||
Total: 75
|
||||
Pending: 42
|
||||
In Progress: 0
|
||||
Completed: 30
|
||||
Failed: 3
|
||||
|
||||
Processing batch of 5 articles...
|
||||
|
||||
Fetching article metadata from Omnivore API...
|
||||
✓ Fetched 5 articles
|
||||
✓ Created stub files: temp/*.jsonl
|
||||
|
||||
READY FOR PARALLEL ANALYSIS
|
||||
|
||||
[
|
||||
{
|
||||
"filename": "temp/understanding-llm-fine-tuning.jsonl",
|
||||
"articleId": "abc-123",
|
||||
"articleSlug": "understanding-llm-fine-tuning",
|
||||
"username": "myusername",
|
||||
"articleTitle": "Understanding LLM Fine-Tuning Techniques"
|
||||
},
|
||||
...
|
||||
]
|
||||
|
||||
Instructions:
|
||||
1. Invoke 5 @article-content-analyzer agents in parallel (single message)
|
||||
2. Each agent receives: filename, articleId, articleSlug, username, articleTitle
|
||||
3. Wait for all agents to complete
|
||||
4. Run: `omc analyze complete`
|
||||
```
|
||||
|
||||
**AIDEV Annotations**:
|
||||
- `tracking-coordination` - Uses SQLite queue for parallel execution
|
||||
- `omnivore-boundary` - Fetches articles via GraphQL API, never local cache
|
||||
- `tracking-lock` - Marks jobs in_progress to prevent duplicate analysis
|
||||
- `zero-context-pollution` - Creates stub files, closes DB before agent invocation
|
||||
|
||||
### analyze complete
|
||||
|
||||
**Command**: `omc analyze complete`
|
||||
|
||||
**Purpose**: Persist enriched temp files by writing Markdown + storing an immutable `analysis_json` snapshot in SQLite, then marking jobs completed.
|
||||
|
||||
**Input Format** (enriched temp files):
|
||||
```json
|
||||
{
|
||||
"articleId": "abc-123",
|
||||
"articleSlug": "understanding-llm-fine-tuning",
|
||||
"username": "myusername",
|
||||
"articleUrl": "https://example.com/article",
|
||||
"articleTitle": "Article Title",
|
||||
"savedAt": "2025-09-30T10:00:00Z",
|
||||
"publishedAt": "2025-09-29T14:00:00Z",
|
||||
"updatedAt": "2025-09-30T08:00:00Z",
|
||||
"analysis": {
|
||||
"topics": ["ai", "developer-tools"],
|
||||
"topicScores": { "ai": 0.95, "developer-tools": 0.88 },
|
||||
"summary": "Article summary...",
|
||||
"keyPoints": ["point1", "point2"],
|
||||
"sentiment": "positive",
|
||||
"monetizationAngle": "Content opportunity...",
|
||||
"analyzedAt": "2025-10-01T04:30:00Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**What It Does (Current)**:
|
||||
1. Scans `temp/*.jsonl` for files that contain an `analysis` field
|
||||
2. For each enriched temp file:
|
||||
- Writes Markdown file under `content/analysis/` (date from `savedAt`; slug derived from `articleSlug` when available, otherwise title)
|
||||
- Stores `analysis_json` + `markdown_path` into `analysis_queue`
|
||||
- Updates tracking status to `completed`
|
||||
- Deletes the temp file unless `--keep-temp`
|
||||
3. On error: marks job as `failed` in tracking queue and preserves the temp file
|
||||
|
||||
**JSONL Output (Optional)**:
|
||||
- Use `--jsonl` to append a machine-readable record to `content/analysis/analyses.jsonl` for each completed analysis.
|
||||
|
||||
**File Naming Strategy**:
|
||||
- Markdown: `content/analysis/YYYY-MM-DD-{slug}-analysis.md` (date from `savedAt`)
|
||||
- JSONL: written when `--jsonl` is set (`content/analysis/analyses.jsonl` by default)
|
||||
- Temp files: Automatically deleted after successful save using `unlinkSync(file)`
|
||||
|
||||
**Output Example**:
|
||||
```
|
||||
Loaded 5 analysis results from temp files
|
||||
✓ Saved: Understanding LLM Fine-Tuning Techniques...
|
||||
→ content/analysis/2025-09-30-understanding-llm-fine-tuning-analysis.md
|
||||
→ Deleted: temp/understanding-llm-fine-tuning.jsonl
|
||||
✓ Saved: Building Scalable Microservices with Kubernetes...
|
||||
→ content/analysis/2025-09-29-building-scalable-microservices.md
|
||||
→ Deleted: temp/building-scalable-microservices.jsonl
|
||||
✗ Failed: Another Article... - Parse error
|
||||
→ Preserved: temp/another-article.jsonl
|
||||
|
||||
Results:
|
||||
Saved: 4
|
||||
Failed: 1
|
||||
|
||||
Updated Queue Status:
|
||||
Pending: 37
|
||||
Completed: 34
|
||||
Failed: 4
|
||||
```
|
||||
|
||||
**AIDEV Annotations**:
|
||||
- `analysis-output-boundary` - Saves results to git-tracked Markdown (plus immutable DB snapshot)
|
||||
- `tracking-update` - Updates SQLite queue status after save
|
||||
- `git-tracked-output` - Analysis result stored permanently
|
||||
- `temp-file-cleanup` - Deletes temp files after successful DB save
|
||||
|
||||
### analyze status
|
||||
|
||||
**Command**: `omc analyze status`
|
||||
|
||||
**Purpose**: Show queue statistics and failed/in-progress jobs.
|
||||
|
||||
**Usage**:
|
||||
```bash
|
||||
omc analyze status
|
||||
```
|
||||
|
||||
**What It Does**:
|
||||
- Shows queue statistics (total, pending, in_progress, completed, failed)
|
||||
- Lists failed jobs with error messages and retry counts
|
||||
- Lists in-progress jobs with assignment timestamps
|
||||
|
||||
**Output Example**:
|
||||
```
|
||||
Analysis Queue Status
|
||||
Total: 75
|
||||
Pending: 37 (49%)
|
||||
In Progress: 0
|
||||
Completed: 33 (44%)
|
||||
Failed: 5
|
||||
|
||||
Failed Jobs:
|
||||
|
||||
Article: Article That Failed Analysis Due to Timeout...
|
||||
ID: xyz-789
|
||||
Error: Agent timeout after 120s
|
||||
Retries: 2
|
||||
|
||||
Article: Another Failed Article with Parse Error...
|
||||
ID: abc-321
|
||||
Error: JSON parse error
|
||||
Retries: 1
|
||||
```
|
||||
|
||||
**AIDEV Annotations**:
|
||||
- `tracking-inspection` - Shows queue status, not analysis content
|
||||
|
||||
### analyze retry
|
||||
|
||||
**Command**: `omc analyze retry`
|
||||
|
||||
**Purpose**: Reset failed jobs to pending for retry (max 3 attempts).
|
||||
**Purpose**: Reset failed jobs to `pending` for retry.
|
||||
|
||||
**Usage**:
|
||||
```bash
|
||||
omc analyze retry --failed # Retry all failed
|
||||
omc analyze retry --article-id <id> # Retry specific article
|
||||
```
|
||||
|
||||
**What It Does (Current)**:
|
||||
1. Fetches failed jobs (or a single `--article-id`)
|
||||
2. Resets status to `pending` and clears error/assigned fields
|
||||
3. Does not currently enforce a max retry limit (the DB keeps `retry_count` for visibility)
|
||||
|
||||
**Output Example**:
|
||||
```
|
||||
Found 5 failed jobs
|
||||
|
||||
✓ Reset: Article That Failed Analysis... (attempt 3)
|
||||
✓ Reset: Another Failed Article... (attempt 2)
|
||||
✗ Skip: Permanently Failed Article... (max retries exceeded)
|
||||
|
||||
Summary:
|
||||
Reset for retry: 4
|
||||
Skipped (max retries): 1
|
||||
|
||||
Next step: `omc analyze run`
|
||||
```
|
||||
|
||||
**AIDEV Annotations**:
|
||||
- `tracking-retry` - Resets failed jobs to pending for another attempt
|
||||
|
||||
## Reporting Commands
|
||||
|
||||
### report corpus
|
||||
|
||||
**Command**: `omc report corpus`
|
||||
|
||||
**Purpose**: Generate corpus statistics and topic distribution report.
|
||||
|
||||
**Usage**:
|
||||
```bash
|
||||
omc report corpus
|
||||
```
|
||||
|
||||
**What It Does**:
|
||||
- Loads completed analyses from SQLite (`analysis_queue.analysis_json`) via `AnalysisQueueRepository.getCompletedWithAnalysis()`
|
||||
- Generates statistics: total analyses, topic distribution, sentiment distribution, content-type distribution
|
||||
- Prints the report to stdout (use `--json` for machine-readable output)
|
||||
|
||||
**Note**:
|
||||
- `omc report corpus` reads from SQLite (`analysis_queue.analysis_json`). JSONL is an optional *write-only* output for downstream tooling.
|
||||
|
||||
## Storage Layer Commands
|
||||
|
||||
**Note**: The most reliable source of truth for the storage-layer API is `src/storage/AnalysisQueueRepository.ts` and `src/storage/AnalysisWriter.ts`. Some type snippets below may drift as the code evolves.
|
||||
|
||||
### AnalysisQueueRepository
|
||||
|
||||
**Location**: `src/storage/AnalysisQueueRepository.ts`
|
||||
|
||||
**Purpose**: Repository for analysis job queue management.
|
||||
|
||||
**Import**:
|
||||
```typescript
|
||||
import { AnalysisQueueRepository } from '@storage/AnalysisQueueRepository';
|
||||
import type { AnalysisJob, QueueStats } from '@storage/AnalysisQueueRepository';
|
||||
```
|
||||
|
||||
**Types**:
|
||||
|
||||
```typescript
|
||||
interface AnalysisJob {
|
||||
id: number;
|
||||
articleId: string;
|
||||
articleUrl: string;
|
||||
articleTitle: string;
|
||||
status: 'pending' | 'in_progress' | 'completed' | 'failed';
|
||||
assignedAt?: string;
|
||||
completedAt?: string;
|
||||
errorMessage?: string;
|
||||
retryCount: number;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
interface QueueStats {
|
||||
total: number;
|
||||
pending: number;
|
||||
inProgress: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
}
|
||||
```
|
||||
|
||||
**Constructor**:
|
||||
```typescript
|
||||
const queueRepo = new AnalysisQueueRepository(db);
|
||||
```
|
||||
|
||||
**Methods**:
|
||||
|
||||
1. **`initializeQueue(articles): number`** - Initialize queue from article metadata
|
||||
- Parameter: `articles: Array<{ id: string; url: string; title: string }>`
|
||||
- Returns: Number of articles inserted (duplicates ignored by UNIQUE constraint)
|
||||
- Sets status to `pending` for all new articles
|
||||
- AIDEV: `tracking-initialization`
|
||||
|
||||
2. **`getPending(limit?: number): AnalysisJob[]`** - Get next batch of pending jobs
|
||||
- Default limit: 5
|
||||
- Returns jobs in chronological order (oldest first)
|
||||
- AIDEV: `tracking-coordination`
|
||||
|
||||
3. **`markInProgress(articleId: string): void`** - Lock job for processing
|
||||
- Sets status to `in_progress`
|
||||
- Records `assigned_at` timestamp
|
||||
- Prevents duplicate analysis by parallel workers
|
||||
- AIDEV: `tracking-lock`
|
||||
|
||||
4. **`markCompleted(articleId: string): void`** - Mark job as completed
|
||||
- Sets status to `completed`
|
||||
- Records `completed_at` timestamp
|
||||
- Called AFTER analysis saved to Markdown/JSONL
|
||||
- AIDEV: `tracking-completion`
|
||||
|
||||
5. **`markFailed(articleId: string, errorMessage: string): void`** - Mark job as failed
|
||||
- Sets status to `failed`
|
||||
- Stores error message
|
||||
- Increments `retry_count`
|
||||
- AIDEV: `tracking-error`
|
||||
|
||||
6. **`resetToPending(articleId: string): void`** - Reset failed job for retry
|
||||
- Sets status back to `pending`
|
||||
- Clears error message and `assigned_at`
|
||||
- Does NOT reset `retry_count` (used to enforce max retries)
|
||||
- AIDEV: `tracking-retry`
|
||||
|
||||
7. **`getStats(): QueueStats`** - Get queue statistics
|
||||
- Returns counts for all statuses
|
||||
- Used for progress monitoring
|
||||
- AIDEV: `tracking-stats`
|
||||
|
||||
8. **`getFailed(): AnalysisJob[]`** - Get all failed jobs
|
||||
- Sorted by retry count (most retries first), then updated time
|
||||
- Used for error investigation
|
||||
- AIDEV: `tracking-failures`
|
||||
|
||||
9. **`getByStatus(status: string): AnalysisJob[]`** - Get jobs by status
|
||||
- Returns jobs matching specified status
|
||||
- Sorted by creation time (newest first)
|
||||
|
||||
10. **`getByArticleId(articleId: string): AnalysisJob | null`** - Get specific job
|
||||
- Returns job or `null` if not found
|
||||
|
||||
11. **`hasArticle(articleId: string): boolean`** - Check if article in queue
|
||||
- Returns `true` if article exists (any status)
|
||||
- AIDEV: `tracking-deduplication`
|
||||
|
||||
12. **`clearCompleted(): number`** - Remove completed jobs
|
||||
- Deletes all jobs with `completed` status
|
||||
- Returns number of deleted jobs
|
||||
- AIDEV: `tracking-cleanup`
|
||||
|
||||
**Example Usage**:
|
||||
```typescript
|
||||
import { initDatabase } from '@storage/database';
|
||||
import { AnalysisQueueRepository } from '@storage/AnalysisQueueRepository';
|
||||
|
||||
const db = initDatabase();
|
||||
const queueRepo = new AnalysisQueueRepository(db);
|
||||
|
||||
// Add articles to queue
|
||||
const articles = [
|
||||
{ id: 'abc-123', url: 'https://example.com/1', title: 'Article 1' },
|
||||
{ id: 'def-456', url: 'https://example.com/2', title: 'Article 2' }
|
||||
];
|
||||
const inserted = queueRepo.initializeQueue(articles);
|
||||
console.log(`Added ${inserted} articles`);
|
||||
|
||||
// Get pending jobs
|
||||
const jobs = queueRepo.getPending(5);
|
||||
console.log(`Processing ${jobs.length} jobs`);
|
||||
|
||||
// Mark as in_progress
|
||||
for (const job of jobs) {
|
||||
queueRepo.markInProgress(job.articleId);
|
||||
}
|
||||
|
||||
// After successful analysis → mark completed
|
||||
queueRepo.markCompleted('abc-123');
|
||||
|
||||
// If analysis fails → mark failed
|
||||
queueRepo.markFailed('def-456', 'Agent timeout');
|
||||
|
||||
// Check statistics
|
||||
const stats = queueRepo.getStats();
|
||||
console.log(`Pending: ${stats.pending}, Completed: ${stats.completed}`);
|
||||
|
||||
// Get failed jobs for retry
|
||||
const failed = queueRepo.getFailed();
|
||||
for (const job of failed) {
|
||||
if (job.retryCount < 3) {
|
||||
queueRepo.resetToPending(job.articleId);
|
||||
}
|
||||
}
|
||||
|
||||
db.close();
|
||||
```
|
||||
|
||||
### AnalysisWriter
|
||||
|
||||
**Location**: `src/storage/AnalysisWriter.ts`
|
||||
|
||||
**Purpose**: Write `ContentAnalysis` results to Markdown files with YAML front-matter.
|
||||
|
||||
**Import**:
|
||||
```typescript
|
||||
import { AnalysisWriter } from '@storage/AnalysisWriter';
|
||||
```
|
||||
|
||||
**Configuration**:
|
||||
```typescript
|
||||
const writer = new AnalysisWriter({
|
||||
outputDir: 'content/analysis'
|
||||
});
|
||||
```
|
||||
|
||||
**Methods**:
|
||||
|
||||
1. **`write(articleId, articleUrl, articleTitle, savedAt, analysis): Promise<string>`** - Write analysis to Markdown
|
||||
```typescript
|
||||
const filePath = await writer.write(
|
||||
articleId, // Omnivore article ID
|
||||
articleUrl, // Source article URL
|
||||
articleTitle, // Source article title
|
||||
savedAt, // When saved to Omnivore (ISO 8601)
|
||||
analysis, // ContentAnalysis from agent
|
||||
articleSlug // Optional Omnivore slug (preferred for filename stability)
|
||||
);
|
||||
// Returns: 'content/analysis/2025-09-30-omnivore-slug-analysis.md'
|
||||
```
|
||||
- Creates Markdown file with YAML front-matter
|
||||
- File naming: `YYYY-MM-DD-{slug}-analysis.md`
|
||||
- Date from `savedAt` timestamp
|
||||
- Slug from `articleSlug` when provided; otherwise derived from article title (lowercase, alphanumeric, max 50 chars)
|
||||
- AIDEV: `analysis-output-boundary`, `git-tracked-output`
|
||||
|
||||
2. **`appendToJsonl(jsonlPath, data): Promise<void>`** - Append to JSONL file
|
||||
```typescript
|
||||
await writer.appendToJsonl('content/analysis/analyses.jsonl', {
|
||||
articleId,
|
||||
articleUrl,
|
||||
articleTitle,
|
||||
savedAt,
|
||||
analyzedAt: analysis.analyzedAt,
|
||||
topics: analysis.topics,
|
||||
topicScores: analysis.topicScores,
|
||||
summary: analysis.summary,
|
||||
keyPoints: analysis.keyPoints,
|
||||
sentiment: analysis.sentiment,
|
||||
monetizationAngle: analysis.monetizationAngle
|
||||
});
|
||||
```
|
||||
- Called when JSONL output is enabled (`omc analyze complete --jsonl` / `omc analyze auto --jsonl`)
|
||||
- Appends one JSON line to JSONL file
|
||||
- Creates file if it doesn't exist
|
||||
- Machine-readable format for batch processing
|
||||
- AIDEV: `git-tracked-output`
|
||||
|
||||
**Private Methods**:
|
||||
- `generateSlug(title: string): string` - Create URL-friendly slug
|
||||
- `formatMarkdown(...)` - Build YAML front-matter and markdown body
|
||||
|
||||
**Example Usage**:
|
||||
```typescript
|
||||
import { AnalysisWriter } from '@storage/AnalysisWriter';
|
||||
|
||||
const writer = new AnalysisWriter({ outputDir: 'content/analysis' });
|
||||
|
||||
const analysis = {
|
||||
articleId: 'abc-123',
|
||||
topics: ['ai', 'developer-tools'],
|
||||
topicScores: { ai: 0.95, 'developer-tools': 0.88 },
|
||||
summary: 'Article explores...',
|
||||
keyPoints: ['First insight', 'Second insight'],
|
||||
sentiment: 'positive',
|
||||
monetizationAngle: 'Tutorial series on...',
|
||||
analyzedAt: new Date().toISOString()
|
||||
};
|
||||
|
||||
const filePath = await writer.write(
|
||||
'abc-123',
|
||||
'https://example.com/article',
|
||||
'Example Article Title',
|
||||
'2025-09-30T10:00:00Z',
|
||||
analysis
|
||||
);
|
||||
|
||||
console.log(`Saved to: ${filePath}`);
|
||||
// Saved to: content/analysis/2025-09-30-example-article-title-analysis.md
|
||||
```
|
||||
|
||||
### ContentReader
|
||||
|
||||
**Location**: `src/storage/ContentReader.ts`
|
||||
|
||||
**Purpose**: Read and parse Markdown files with YAML front-matter.
|
||||
|
||||
**Import**:
|
||||
```typescript
|
||||
import { ContentReader } from '@storage/ContentReader';
|
||||
```
|
||||
|
||||
**Configuration**:
|
||||
```typescript
|
||||
const reader = new ContentReader({
|
||||
directory: 'content/analysis'
|
||||
});
|
||||
```
|
||||
|
||||
**Methods**:
|
||||
|
||||
1. **`list(pattern?: string): Promise<string[]>`** - List all Markdown files
|
||||
- Returns array of absolute file paths
|
||||
- Sorted by date (newest first, assumes `YYYY-MM-DD` prefix)
|
||||
- Returns empty array if directory doesn't exist
|
||||
|
||||
2. **`read(filePath: string): Promise<StoredAnalysis>`** - Read and parse file
|
||||
- Uses `gray-matter` to parse YAML front-matter
|
||||
- Extracts markdown sections (Summary, Key Points, Monetization Angle)
|
||||
- Returns `StoredAnalysis` with parsed front-matter and content
|
||||
|
||||
3. **`findByArticleId(articleId: string): Promise<StoredAnalysis | null>`** - Find by article ID
|
||||
- Searches all files for matching `articleId` in front-matter
|
||||
- Returns first match or `null`
|
||||
|
||||
4. **`searchByTopic(topic: string): Promise<StoredAnalysis[]>`** - Search by topic
|
||||
- Returns all analyses containing the topic
|
||||
- Topic must match exactly (case-sensitive)
|
||||
|
||||
**Example Usage**:
|
||||
```typescript
|
||||
import { ContentReader } from '@storage/ContentReader';
|
||||
|
||||
const reader = new ContentReader({ directory: 'content/analysis' });
|
||||
|
||||
// List all analyses
|
||||
const files = await reader.list();
|
||||
console.log(`Found ${files.length} analyses`);
|
||||
|
||||
// Read specific analysis
|
||||
const analysis = await reader.read(files[0]);
|
||||
console.log(analysis.frontMatter.articleTitle);
|
||||
console.log(analysis.summary);
|
||||
console.log(analysis.keyPoints);
|
||||
|
||||
// Find by article ID
|
||||
const found = await reader.findByArticleId('abc-123');
|
||||
if (found) {
|
||||
console.log(`Topics: ${found.frontMatter.topics.join(', ')}`);
|
||||
}
|
||||
|
||||
// Search by topic
|
||||
const aiArticles = await reader.searchByTopic('ai');
|
||||
console.log(`${aiArticles.length} articles about AI`);
|
||||
```
|
||||
|
||||
**Private Method**:
|
||||
- `parseMarkdownSections(content: string)` - Extract sections from markdown body using regex
|
||||
|
||||
## Workflow Examples
|
||||
|
||||
### Complete Parallel Analysis Workflow
|
||||
|
||||
```bash
|
||||
# 1. Fetch articles from Omnivore → tracking queue
|
||||
omc queue add --hours 168
|
||||
|
||||
# 2. Prepare batch (writes temp/*.jsonl stubs + marks jobs in_progress)
|
||||
omc analyze run --batch-size 5
|
||||
|
||||
# 3. User invokes 5 agents in parallel (single message)
|
||||
# Each agent:
|
||||
# - Reads stub file
|
||||
# - Calls: omc omnivore get {slug} --json (extract `.content`)
|
||||
# - Analyzes content
|
||||
# - Enriches stub file with analysis field
|
||||
# - Writes back to temp file
|
||||
|
||||
# 4. Persist results (writes content/analysis/*.md, updates DB, cleans temp files)
|
||||
omc analyze complete
|
||||
|
||||
# 5. Check status / stats
|
||||
omc analyze status
|
||||
omc queue stats --detailed
|
||||
|
||||
# 6. Retry failed jobs
|
||||
omc analyze retry --failed
|
||||
|
||||
# 7. Continue analysis
|
||||
omc analyze run --batch-size 5
|
||||
|
||||
# 8. Generate report
|
||||
omc report corpus
|
||||
```
|
||||
|
||||
### Programmatic Queue Management
|
||||
|
||||
```typescript
|
||||
import { initDatabase } from '@storage/database';
|
||||
import { AnalysisQueueRepository } from '@storage/AnalysisQueueRepository';
|
||||
import { AnalysisWriter } from '@storage/AnalysisWriter';
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
|
||||
// Initialize database and repository
|
||||
const db = initDatabase('data/omnivore-content.db');
|
||||
const queueRepo = new AnalysisQueueRepository(db);
|
||||
const writer = new AnalysisWriter({ outputDir: 'content/analysis' });
|
||||
|
||||
// Add articles to queue
|
||||
const articles = [
|
||||
{ id: 'abc-123', url: 'https://example.com/1', title: 'Article 1' },
|
||||
{ id: 'def-456', url: 'https://example.com/2', title: 'Article 2' }
|
||||
];
|
||||
const inserted = queueRepo.initializeQueue(articles);
|
||||
console.log(`Added ${inserted} articles to queue`);
|
||||
|
||||
// Get pending jobs
|
||||
const jobs = queueRepo.getPending(5);
|
||||
|
||||
// Mark as in_progress
|
||||
for (const job of jobs) {
|
||||
queueRepo.markInProgress(job.articleId);
|
||||
}
|
||||
|
||||
// After agent analysis completes...
|
||||
const analysis: ContentAnalysis = {
|
||||
articleId: 'abc-123',
|
||||
topics: ['ai', 'developer-tools'],
|
||||
topicScores: { ai: 0.95, 'developer-tools': 0.88 },
|
||||
summary: 'Article explores...',
|
||||
keyPoints: ['First insight', 'Second insight'],
|
||||
sentiment: 'positive',
|
||||
monetizationAngle: 'Tutorial series on...',
|
||||
analyzedAt: new Date().toISOString()
|
||||
};
|
||||
|
||||
// Save to Markdown (PERMANENT)
|
||||
const mdPath = await writer.write(
|
||||
'abc-123',
|
||||
'https://example.com/1',
|
||||
'Article 1',
|
||||
'2025-09-30T10:00:00Z',
|
||||
analysis
|
||||
);
|
||||
|
||||
// Append to JSONL (PERMANENT)
|
||||
await writer.appendToJsonl('content/analysis/analyses.jsonl', {
|
||||
articleId: 'abc-123',
|
||||
articleUrl: 'https://example.com/1',
|
||||
articleTitle: 'Article 1',
|
||||
savedAt: '2025-09-30T10:00:00Z',
|
||||
analyzedAt: analysis.analyzedAt,
|
||||
topics: analysis.topics,
|
||||
topicScores: analysis.topicScores,
|
||||
summary: analysis.summary,
|
||||
keyPoints: analysis.keyPoints,
|
||||
sentiment: analysis.sentiment,
|
||||
monetizationAngle: analysis.monetizationAngle
|
||||
});
|
||||
|
||||
// Update tracking status (EPHEMERAL)
|
||||
queueRepo.markCompleted('abc-123');
|
||||
|
||||
// Check statistics
|
||||
const stats = queueRepo.getStats();
|
||||
console.log(`Pending: ${stats.pending}, Completed: ${stats.completed}`);
|
||||
|
||||
db.close();
|
||||
```
|
||||
|
||||
### Reading Analysis Results
|
||||
|
||||
```typescript
|
||||
import { ContentReader } from '@storage/ContentReader';
|
||||
|
||||
const reader = new ContentReader({ directory: 'content/analysis' });
|
||||
|
||||
// List all analyses (newest first)
|
||||
const files = await reader.list();
|
||||
console.log(`Total analyses: ${files.length}`);
|
||||
|
||||
// Read latest analysis
|
||||
const latest = await reader.read(files[0]);
|
||||
console.log(`Title: ${latest.frontMatter.articleTitle}`);
|
||||
console.log(`Topics: ${latest.frontMatter.topics.join(', ')}`);
|
||||
console.log(`Summary: ${latest.summary}`);
|
||||
console.log(`Key Points:\n${latest.keyPoints.map(p => `- ${p}`).join('\n')}`);
|
||||
|
||||
// Find specific article's analysis
|
||||
const found = await reader.findByArticleId('abc-123');
|
||||
if (found) {
|
||||
console.log(`Sentiment: ${found.frontMatter.sentiment}`);
|
||||
}
|
||||
|
||||
// Search by topic
|
||||
const aiArticles = await reader.searchByTopic('ai');
|
||||
console.log(`AI articles: ${aiArticles.length}`);
|
||||
```
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Architecture](architecture.md) - System design and storage boundaries
|
||||
- [Workflow Internals](workflow-internals.md) - How the parallel analysis workflow operates
|
||||
- [Foundation & Type System](foundation-and-types.md) - TypeScript setup, type definitions
|
||||
151
self-hosting/omc/docs/_meta/current-state.md
Normal file
151
self-hosting/omc/docs/_meta/current-state.md
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
# Current State Audit
|
||||
|
||||
**Purpose**: Ground-truth documentation of what exists *in code today*, what is broken, and what needs to be fixed to make the system reliable.
|
||||
|
||||
**Last Updated**: 2026-01-31
|
||||
|
||||
## Executive Summary
|
||||
|
||||
This repository has a fairly complete **command surface area** (queue/analyze/content/report/omnivore/db/config/init/doctor/version), plus an intended “3-layer” architecture:
|
||||
|
||||
1. **Omnivore** as the content source-of-truth (GraphQL)
|
||||
2. **SQLite** as coordination + immutable analysis snapshot storage
|
||||
3. **Git-tracked Markdown** as human-editable, permanent output
|
||||
|
||||
The core analysis pipeline now works end-to-end in a clean checkout:
|
||||
|
||||
- `pnpm run typecheck` / `pnpm test` / `pnpm run build` pass.
|
||||
- The built CLI (`dist/bin/omc.js`) can initialize the tracking DB schema.
|
||||
- Commands follow the `BaseCommand.execute({ ...args, ...flags })` contract.
|
||||
- Documentation reflects the current `omc analyze auto` automation path and optional JSONL output.
|
||||
|
||||
## Architecture (As Implemented)
|
||||
|
||||
### Layer 1: Omnivore (Source of Truth)
|
||||
|
||||
- Implementation: `lib/omnivore/client.js`
|
||||
- Used by CLI commands via the TypeScript re-export surface (`src/lib/omnivore/client.ts`) in most places.
|
||||
- Auth is via `.env` (`OMNIVORE_API_URL`, `OMNIVORE_API_KEY`) loaded by `dotenv` at module import time.
|
||||
|
||||
### Layer 2: SQLite (Coordination + Snapshots)
|
||||
|
||||
- Default DB path (source): `src/storage/database.ts` uses `data/omnivore-content.db`.
|
||||
- Schema: `src/storage/schema/tracking-schema.sql`
|
||||
- Repository: `src/storage/AnalysisQueueRepository.ts`
|
||||
|
||||
The DB is used for:
|
||||
- Queue state (`pending`/`in_progress`/`completed`/`failed`)
|
||||
- Retry counters and error messages
|
||||
- Immutable snapshot of analysis output (`analysis_json`) and pointer to Markdown (`markdown_path`)
|
||||
|
||||
### Layer 3: Markdown Output (Git-tracked)
|
||||
|
||||
- Writer: `src/storage/AnalysisWriter.ts`
|
||||
- The current pipeline writes **Markdown** by default.
|
||||
- Optional machine-readable JSONL is available via flags (`omc analyze auto --jsonl` / `omc analyze complete --jsonl`).
|
||||
- Markdown filename uses `savedAt` date and the Omnivore `articleSlug` when available (falls back to title-derived slug).
|
||||
|
||||
## Command Inventory (As Implemented)
|
||||
|
||||
Commands are defined under `src/commands/**` and built/run via oclif (`index.ts`).
|
||||
|
||||
### `omc queue`
|
||||
|
||||
- `omc queue add` (`src/commands/queue/add.ts`): adds articles to tracking queue (hours/label/url/slug).
|
||||
- `omc queue list` (`src/commands/queue/list.ts`): lists queue entries (optional `--status`).
|
||||
- `omc queue stats` (`src/commands/queue/stats.ts`): queue stats (optional `--detailed`).
|
||||
- `omc queue reset` (`src/commands/queue/reset.ts`): resets a specific job to `pending`.
|
||||
- `omc queue remove` (`src/commands/queue/remove.ts`): deletes a job.
|
||||
- `omc queue clear` (`src/commands/queue/clear.ts`): bulk delete by status or all.
|
||||
- `omc queue export` (`src/commands/queue/export.ts`): prints queue rows as JSONL (or JSON array with `--json`).
|
||||
- `omc queue import` (`src/commands/queue/import.ts`): imports JSONL into the queue.
|
||||
|
||||
### `omc analyze`
|
||||
|
||||
- `omc analyze run` (`src/commands/analyze/run.ts`):
|
||||
- Selects jobs (pending by default; pending+failed with `--all`; single with `--article-id`).
|
||||
- Marks selected jobs `in_progress`.
|
||||
- Fetches Omnivore article metadata and writes `temp/{articleSlug}.jsonl` stub files.
|
||||
- Outputs agent parameter JSON describing the stub files.
|
||||
- `omc analyze complete` (`src/commands/analyze/complete.ts`):
|
||||
- Finds `temp/*.jsonl` files that contain an `analysis` field.
|
||||
- Writes Markdown via `AnalysisWriter`.
|
||||
- Stores `analysis_json` + `markdown_path` in SQLite via `AnalysisQueueRepository.storeAnalysis`.
|
||||
- Deletes temp files unless `--keep-temp`.
|
||||
- `omc analyze status` (`src/commands/analyze/status.ts`): shows in-progress jobs + current `temp/*.jsonl`.
|
||||
- `omc analyze watch` (`src/commands/analyze/watch.ts`): polls queue stats until *queue empty* (pending=0 and in_progress=0).
|
||||
- `omc analyze retry` (`src/commands/analyze/retry.ts`): resets failed jobs to pending.
|
||||
|
||||
### `omc content` (Read/Export/Sync)
|
||||
|
||||
- `omc content list` (`src/commands/content/list.ts`): lists completed jobs; optional filters (since/topic).
|
||||
- `omc content show` (`src/commands/content/show.ts`): prints Markdown for one article ID.
|
||||
- `omc content search` (`src/commands/content/search.ts`): searches completed analyses (title/analysis/content/all).
|
||||
- `omc content export` (`src/commands/content/export.ts`): export data for blogging workflows (implementation varies).
|
||||
- `omc content sync` (`src/commands/content/sync.ts`): syncs analysis summaries back to Omnivore (description field) and optionally creates/updates a NOTE highlight from Markdown.
|
||||
|
||||
### `omc report` (Categorized Analysis / Aggregations)
|
||||
|
||||
This is the “categorized analysis” surface area: reports summarize analyses by topic/sentiment/time.
|
||||
|
||||
- `omc report corpus` (`src/commands/report/corpus.ts`): topic + sentiment + contentType distributions.
|
||||
- `omc report topics` (`src/commands/report/topics.ts`): topic distribution.
|
||||
- `omc report trends` (`src/commands/report/trends.ts`): trends over time (based on saved/completed timestamps).
|
||||
- `omc report sentiment` (`src/commands/report/sentiment.ts`): sentiment distribution.
|
||||
- `omc report monetization` (`src/commands/report/monetization.ts`): monetization angle aggregation.
|
||||
- `omc report custom` (`src/commands/report/custom.ts`): custom SQL against the tracking DB.
|
||||
- `omc report export` (`src/commands/report/export.ts`): export report output (format varies).
|
||||
|
||||
### `omc omnivore`
|
||||
|
||||
Direct API operations:
|
||||
- `omc omnivore list`, `search`, `get`, `update`
|
||||
- highlights: `omnivore highlight add/list`
|
||||
- notes: `omnivore note add/get/update`
|
||||
|
||||
### `omc db`
|
||||
|
||||
DB maintenance:
|
||||
- `migrate`, `schema`, `stats`, `check`, `vacuum`, `backup`, `restore`, `reset`, `seed`
|
||||
|
||||
### `omc config`
|
||||
|
||||
Config management:
|
||||
- `show`, `get`, `set`, `test`, `validate`, `env list`, `env use`
|
||||
|
||||
### Misc
|
||||
|
||||
- `omc init` (`src/commands/init.ts`): basic setup wizard (directories, db, env).
|
||||
- `omc doctor` (`src/commands/doctor.ts`): health checks.
|
||||
- `omc version` (`src/commands/version.ts`): version/system info.
|
||||
|
||||
## End-to-End Workflow (Current)
|
||||
|
||||
1. Populate queue: `omc queue add --hours 24` (or `--label`, `--url`, `--slug`).
|
||||
2. Prepare stubs: `omc analyze run --batch-size 5` (creates `temp/{slug}.jsonl`).
|
||||
3. Run external agent(s): read stub, fetch content, add `analysis` field to the stub JSON.
|
||||
4. Persist results: `omc analyze complete` (writes `content/analysis/*.md`, updates DB, cleans temp files).
|
||||
5. View/search: `omc content list`, `omc content search`, `omc content show <article-id>`.
|
||||
6. Report: `omc report topics` / `omc report trends` / etc.
|
||||
7. Sync back to Omnivore: `omc content sync --all --create-notes`.
|
||||
|
||||
## Issues / Fix List (Prioritized)
|
||||
|
||||
### Resolved (as of 2026-01-31)
|
||||
|
||||
- TypeScript aliases normalized (`@omc-types/*` instead of `@types/*`); `pnpm run typecheck` passes.
|
||||
- Tracking schema resolution works in `dist/` builds; built CLI can initialize the DB.
|
||||
- Commands follow `BaseCommand.execute({ ...args, ...flags })`.
|
||||
- Docs and scripts align with the built CLI and the current analysis workflow.
|
||||
- Agent-friendly raw content output via `omc omnivore get --content`.
|
||||
- Optional JSONL output via `--jsonl` flags (`omc analyze auto --jsonl` / `omc analyze complete --jsonl`).
|
||||
- Markdown filename contract prefers Omnivore slug (falls back to title-derived slug).
|
||||
- `omc queue add --hours` paginates until the cutoff is reached.
|
||||
- Markdown front-matter emitted via `gray-matter` (valid YAML for common edge cases).
|
||||
- GraphQL error handling normalized via `checkGraphQLResult(...)`.
|
||||
- Typed GraphQL documents/codegen removed from the runtime path (single client source-of-truth).
|
||||
|
||||
### Remaining / Next
|
||||
|
||||
- Automated content generation and publishing phases (Phase 5/6).
|
||||
- Decide whether to keep or retire `src/lib/ai/anthropic-client.ts` (Codex CLI is the default analysis provider today).
|
||||
712
self-hosting/omc/docs/_meta/foundation-and-types.md
Normal file
712
self-hosting/omc/docs/_meta/foundation-and-types.md
Normal file
|
|
@ -0,0 +1,712 @@
|
|||
# Foundation and Type System
|
||||
|
||||
**Groundtruth Documentation** - What exists and how to use it.
|
||||
|
||||
## TypeScript Foundation
|
||||
|
||||
### Project Setup
|
||||
|
||||
**TypeScript Configuration** (`tsconfig.json`):
|
||||
```typescript
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"declaration": true,
|
||||
"sourceMap": true,
|
||||
"paths": {
|
||||
"@lib/*": ["./lib/*"],
|
||||
"@storage/*": ["./src/storage/*"],
|
||||
"@analysis/*": ["./src/analysis/*"],
|
||||
"@generation/*": ["./src/generation/*"],
|
||||
"@utils/*": ["./src/utils/*"],
|
||||
"@omc-types/*": ["./src/types/*"]
|
||||
}
|
||||
},
|
||||
"include": ["src/**/*", "lib/**/*"],
|
||||
"exclude": ["node_modules", "dist", "test-scripts", "legacy-scripts"]
|
||||
}
|
||||
```
|
||||
|
||||
**Build System**:
|
||||
- Package manager: pnpm
|
||||
- Build command: `pnpm run build` → compiles to `dist/`
|
||||
- Type check: `pnpm run typecheck` → validates without emitting
|
||||
- Dev mode: `pnpm run dev` → watch mode with tsx
|
||||
|
||||
### Dependencies
|
||||
|
||||
**Runtime** (package.json dependencies):
|
||||
- `node-fetch` (^3.3.2) - HTTP requests for GraphQL API
|
||||
- `dotenv` (^16.4.0) - Environment variable loading
|
||||
- `@anthropic-ai/sdk` (^0.20.0) - Claude API for analysis
|
||||
- `@anthropic-ai/claude-agent-sdk` (^0.1.1) - Agent framework
|
||||
- `gray-matter` (^4.0.3) - YAML front-matter parsing
|
||||
- `markdown-it` (^14.0.0) - Markdown processing
|
||||
- `chalk` (^5.3.0) - Terminal colors
|
||||
- `csv-parse` (^6.1.0) - CSV parsing
|
||||
- `p-limit` (^5.0.0) - Concurrency control
|
||||
- `better-sqlite3` (^12.4.1) - SQLite (for legacy scripts)
|
||||
|
||||
**Development** (package.json devDependencies):
|
||||
- `typescript` (^5.4.0) - TypeScript compiler
|
||||
- `@types/node` (^20.0.0) - Node.js type definitions
|
||||
- `tsx` (^4.0.0) - TypeScript execution
|
||||
- `vitest` (^1.0.0) - Testing framework
|
||||
- `nodemon` (^3.0.0) - File watcher
|
||||
|
||||
## GraphQL Organization
|
||||
|
||||
**Current**:
|
||||
- Runtime GraphQL client: `lib/omnivore/client.js` (string-based queries)
|
||||
- TypeScript import surface: `src/lib/omnivore/client.ts`
|
||||
- In TypeScript code, prefer importing client functions from `@lib/omnivore/client.js`
|
||||
|
||||
**Note**:
|
||||
An older typed GraphQL layer (`src/graphql/**` + `src/types/generated/**`) was removed because it was not integrated into the CLI runtime and created drift risk. See `docs/_meta/graphql-organization.md`.
|
||||
|
||||
### Query Builders (`lib/omnivore/queries.js`)
|
||||
|
||||
**Purpose**: Helper functions for composing Omnivore query strings.
|
||||
|
||||
**NOTE**: Query builders remain in JavaScript (`lib/omnivore/queries.js`) for compatibility with existing client code. They generate query string parameters, not GraphQL query documents.
|
||||
|
||||
**Builder Functions**:
|
||||
|
||||
1. **`buildComplexQuery({ keywords, labels, timeRange, status, hasHighlights, sortBy })`**
|
||||
- Compose multiple filters into single query string
|
||||
- Parameters:
|
||||
- `keywords`: Array of search terms (OR logic)
|
||||
- `labels`: Array of label names (OR logic)
|
||||
- `timeRange`: 'last24hrs' | 'last7days' | 'last30days' | date expression
|
||||
- `status`: 'inbox' | 'archived' | 'unread' | 'read'
|
||||
- `hasHighlights`: boolean (true = has:highlights, false = no:highlights)
|
||||
- `sortBy`: Sort order (default: 'saved-desc')
|
||||
- Returns: Omnivore query string
|
||||
- Example:
|
||||
```javascript
|
||||
buildComplexQuery({
|
||||
keywords: ['ai', 'machine learning'],
|
||||
labels: ['technology'],
|
||||
timeRange: 'last7days',
|
||||
hasHighlights: true
|
||||
})
|
||||
// → 'saved:last7days ("ai" OR "machine learning") label:technology has:highlights sort:saved-desc'
|
||||
```
|
||||
|
||||
2. **`buildTopicQuery(topicKey, timeRange)`**
|
||||
- Generate query for predefined topic
|
||||
- Topic keys: 'ai', 'devops', 'programming', 'databases', 'web', 'cloud', 'startup', 'security'
|
||||
- Each topic has predefined keywords (see TOPIC_QUERIES)
|
||||
- Example:
|
||||
```javascript
|
||||
buildTopicQuery('ai', 'last7days')
|
||||
// → 'saved:last7days ("ai" OR "artificial intelligence" OR "machine learning" OR ...) sort:saved-desc'
|
||||
```
|
||||
|
||||
3. **`buildDateRangeQuery(startDate, endDate, sortBy)`**
|
||||
- Create date range filter
|
||||
- Accepts Date objects or YYYY-MM-DD strings
|
||||
- Example:
|
||||
```javascript
|
||||
buildDateRangeQuery('2025-09-01', '2025-09-30')
|
||||
// → 'saved:>2025-09-01 saved:<2025-09-30 sort:saved-desc'
|
||||
```
|
||||
|
||||
4. **`buildLabelQuery(labels, operator)`**
|
||||
- Create label filter with AND/OR logic
|
||||
- Example:
|
||||
```javascript
|
||||
buildLabelQuery(['ai', 'ml'], 'OR')
|
||||
// → 'label:ai OR label:ml'
|
||||
```
|
||||
|
||||
5. **`buildKeywordQuery(keywords, operator)`**
|
||||
- Create keyword search with AND/OR logic
|
||||
- Automatically quotes multi-word terms
|
||||
- Example:
|
||||
```javascript
|
||||
buildKeywordQuery(['ai', 'machine learning'], 'OR')
|
||||
// → '("ai" OR "machine learning")'
|
||||
```
|
||||
|
||||
**Predefined Patterns** (`QUERY_PATTERNS`):
|
||||
```javascript
|
||||
QUERY_PATTERNS.LAST_7_DAYS // 'saved:last7days sort:saved-desc'
|
||||
QUERY_PATTERNS.INBOX // 'in:inbox sort:saved-desc'
|
||||
QUERY_PATTERNS.WITH_HIGHLIGHTS // 'has:highlights sort:saved-desc'
|
||||
QUERY_PATTERNS.AI_ML // '(ai OR "machine learning" OR llm ...) sort:saved-desc'
|
||||
QUERY_PATTERNS.RECENT_AI // 'saved:last7days (ai OR llm OR ...) sort:saved-desc'
|
||||
// ... more patterns (see lib/omnivore/queries.js:189-221)
|
||||
```
|
||||
|
||||
**Topic Definitions** (`TOPIC_QUERIES`):
|
||||
```javascript
|
||||
TOPIC_QUERIES.ai // keywords: ['ai', 'artificial intelligence', ...]
|
||||
TOPIC_QUERIES.devops // keywords: ['devops', 'kubernetes', ...]
|
||||
TOPIC_QUERIES.programming // keywords: ['programming', 'coding', ...]
|
||||
// ... more topics (see lib/omnivore/queries.js:305-338)
|
||||
```
|
||||
|
||||
**Integration Example**:
|
||||
```javascript
|
||||
import { searchArticles } from '@lib/omnivore/client.js';
|
||||
import { buildComplexQuery, TOPIC_QUERIES } from '../lib/omnivore/queries.js';
|
||||
|
||||
// Build query string
|
||||
const queryString = buildComplexQuery({
|
||||
keywords: TOPIC_QUERIES.ai.keywords.slice(0, 5),
|
||||
labels: ['technology', 'research'],
|
||||
timeRange: 'last7days',
|
||||
hasHighlights: true
|
||||
});
|
||||
|
||||
const result = await searchArticles({ query: queryString, first: 20, includeContent: true });
|
||||
```
|
||||
|
||||
## Omnivore Client Library
|
||||
|
||||
### Location
|
||||
|
||||
```
|
||||
lib/omnivore/
|
||||
├── client.js # GraphQL client using node-fetch
|
||||
└── queries.js # Query builders and patterns (documented above)
|
||||
```
|
||||
|
||||
### Client Module (`lib/omnivore/client.js`)
|
||||
|
||||
**Implementation**: Uses `node-fetch` to make GraphQL requests. NOT using Apollo Client.
|
||||
|
||||
**Core Function**: `graphqlRequest(query, variables)`
|
||||
```javascript
|
||||
async function graphqlRequest(query, variables = {}) {
|
||||
const response = await fetch(API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': API_KEY, // From process.env.OMNIVORE_API_KEY
|
||||
},
|
||||
body: JSON.stringify({ query, variables }),
|
||||
});
|
||||
|
||||
const result = await response.json();
|
||||
return result.data;
|
||||
}
|
||||
```
|
||||
|
||||
**Exported Functions**:
|
||||
|
||||
1. **`getMe()`** - Get authenticated user
|
||||
- Returns: `{ id, name, email, profile: { username } }`
|
||||
- Use: Verify API connection
|
||||
|
||||
2. **`searchArticles({ query, first, after, includeContent })`** - Search articles
|
||||
- Parameters:
|
||||
- `query`: Omnivore query syntax (default: 'in:all')
|
||||
- `first`: Number of results (default: 10)
|
||||
- `after`: Pagination cursor (default: '')
|
||||
- `includeContent`: Include full HTML content (default: false)
|
||||
- Returns: `{ search: { pageInfo, edges: [{ node, cursor }] } }`
|
||||
|
||||
3. **`getArticle({ slug, username })`** - Get single article with content
|
||||
- Returns: Full article with content, highlights, labels
|
||||
|
||||
4. **`getArticlesByDate({ startDate, endDate, first })`** - Time-range queries
|
||||
- Date format: YYYY-MM-DD
|
||||
|
||||
5. **`getArticlesByLabel({ label, first })`** - Filter by label name
|
||||
|
||||
6. **`getRecentArticles({ period, first })`** - Recent articles
|
||||
- Periods: 'last24hours', 'last7days', 'last30days'
|
||||
|
||||
7. **`searchByTopic({ topic, period, first })`** - Topic + time filter
|
||||
- Topics: 'ai', 'devops', 'programming', etc.
|
||||
|
||||
8. **`getUnreadArticles({ first })`** - Inbox/unread items
|
||||
|
||||
9. **`getLabels()`** - All available labels
|
||||
- Returns: Array of `{ id, name, color, description }`
|
||||
|
||||
10. **`getHighlights({ articleId })`** - Article highlights
|
||||
- Returns: Array of `{ id, quote, annotation, createdAt }`
|
||||
|
||||
11. **`testConnection()`** - Verify API connectivity
|
||||
- Calls `getMe()` and logs result
|
||||
- Returns: boolean
|
||||
|
||||
**Import Pattern**:
|
||||
```javascript
|
||||
import { searchArticles, getArticle } from './lib/omnivore/client.js';
|
||||
```
|
||||
|
||||
**NOTE**: Query builders and patterns are documented above in "Query Builders (`lib/omnivore/queries.js`)" section.
|
||||
|
||||
## Type System
|
||||
|
||||
### Type Files
|
||||
|
||||
```
|
||||
src/types/
|
||||
├── omnivore.ts # API response types
|
||||
├── content.ts # Storage/front-matter types
|
||||
├── analysis.ts # Analysis result types
|
||||
└── index.ts # Central exports
|
||||
```
|
||||
|
||||
### Omnivore API Types (`src/types/omnivore.ts`)
|
||||
|
||||
**Core Interfaces**:
|
||||
|
||||
```typescript
|
||||
// Article from API
|
||||
export interface OmnivoreArticle {
|
||||
id: string;
|
||||
title: string;
|
||||
url: string;
|
||||
originalArticleUrl?: string;
|
||||
slug?: string;
|
||||
content?: string; // Only present if includeContent: true
|
||||
description?: string;
|
||||
author?: string;
|
||||
image?: string;
|
||||
siteName?: string;
|
||||
pageType?: string;
|
||||
wordCount?: number;
|
||||
createdAt: string; // ISO 8601
|
||||
savedAt: string; // ISO 8601
|
||||
publishedAt?: string; // ISO 8601
|
||||
updatedAt: string; // ISO 8601
|
||||
readingProgressTopPercent?: number;
|
||||
isArchived: boolean;
|
||||
folder?: string;
|
||||
labels: Label[];
|
||||
highlights: Highlight[];
|
||||
}
|
||||
|
||||
// Label metadata
|
||||
export interface Label {
|
||||
id: string;
|
||||
name: string;
|
||||
color: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
// User highlight/annotation
|
||||
export interface Highlight {
|
||||
id: string;
|
||||
quote: string;
|
||||
annotation?: string;
|
||||
createdAt: string; // ISO 8601
|
||||
}
|
||||
|
||||
// Search parameters
|
||||
export interface SearchParams {
|
||||
query?: string; // Omnivore query syntax
|
||||
first?: number; // Results per page
|
||||
after?: string; // Pagination cursor
|
||||
includeContent?: boolean; // Include article HTML
|
||||
}
|
||||
|
||||
// Pagination metadata
|
||||
export interface PageInfo {
|
||||
hasNextPage: boolean;
|
||||
hasPreviousPage: boolean;
|
||||
startCursor?: string;
|
||||
endCursor?: string;
|
||||
totalCount: number;
|
||||
}
|
||||
|
||||
// Search result wrapper
|
||||
export interface SearchResult {
|
||||
edges: Array<{
|
||||
node: OmnivoreArticle;
|
||||
}>;
|
||||
pageInfo: PageInfo;
|
||||
}
|
||||
|
||||
// User profile
|
||||
export interface OmnivoreUser {
|
||||
id: string;
|
||||
name: string;
|
||||
email: string;
|
||||
profile: {
|
||||
username: string;
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
**Import Pattern**:
|
||||
```typescript
|
||||
import type { OmnivoreArticle, SearchResult, Label } from '@omc-types/omnivore.js';
|
||||
// OR
|
||||
import type { OmnivoreArticle, SearchResult, Label } from '../types/omnivore';
|
||||
```
|
||||
|
||||
### Content Storage Types (`src/types/content.ts`)
|
||||
|
||||
**Front-matter Interfaces** for Markdown files with YAML headers:
|
||||
|
||||
```typescript
|
||||
// Article metadata (content/articles/*.md)
|
||||
export interface ArticleFrontMatter {
|
||||
id: string; // Omnivore article ID
|
||||
url: string;
|
||||
title: string;
|
||||
author?: string;
|
||||
savedAt: string; // ISO 8601
|
||||
publishedAt?: string; // ISO 8601
|
||||
labels: string[]; // Label names (not IDs)
|
||||
highlights: number; // Count
|
||||
wordCount: number;
|
||||
siteName?: string;
|
||||
topics?: string[]; // Added by analysis
|
||||
sentiment?: string; // Added by analysis
|
||||
analyzed?: boolean;
|
||||
}
|
||||
|
||||
// Analysis metadata (content/analysis/*.md)
|
||||
export interface AnalysisFrontMatter {
|
||||
articleId: string;
|
||||
analyzedAt: string; // ISO 8601
|
||||
topics: string[];
|
||||
sentiment: 'positive' | 'neutral' | 'negative';
|
||||
topicScores: Record<string, number>;
|
||||
}
|
||||
|
||||
// Generated content metadata (content/generated/**/*.md)
|
||||
export interface GeneratedContentFrontMatter {
|
||||
title: string;
|
||||
metaDescription: string; // Max 155 chars
|
||||
generatedAt: string; // ISO 8601
|
||||
type: 'blog-post' | 'newsletter';
|
||||
sources: string[]; // Article URLs
|
||||
topics: string[];
|
||||
publishedAt?: string; // ISO 8601
|
||||
slug?: string;
|
||||
}
|
||||
|
||||
// Complete stored article (after reading Markdown file)
|
||||
export interface StoredArticle {
|
||||
frontMatter: ArticleFrontMatter;
|
||||
content: string; // Markdown content
|
||||
highlights?: Array<{
|
||||
quote: string;
|
||||
annotation?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
// Complete stored analysis
|
||||
export interface StoredAnalysis {
|
||||
frontMatter: AnalysisFrontMatter;
|
||||
summary: string;
|
||||
keyPoints: string[];
|
||||
monetizationAngle: string;
|
||||
}
|
||||
|
||||
// Complete generated content
|
||||
export interface StoredGeneratedContent {
|
||||
frontMatter: GeneratedContentFrontMatter;
|
||||
content: string;
|
||||
}
|
||||
|
||||
// Search index entry (content/.metadata/index.json)
|
||||
export interface SearchIndexEntry {
|
||||
id: string;
|
||||
slug: string;
|
||||
title: string;
|
||||
topics: string[];
|
||||
savedAt: string;
|
||||
analyzed: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
### Analysis Types (`src/types/analysis.ts`)
|
||||
|
||||
**AI Analysis Interfaces**:
|
||||
|
||||
```typescript
|
||||
// Analysis result from Claude
|
||||
export interface ContentAnalysis {
|
||||
articleId: string;
|
||||
topics: string[]; // 2-5 main topics
|
||||
topicScores: Record<string, number>; // Topic → confidence (0-1)
|
||||
summary: string; // 2-3 sentences
|
||||
keyPoints: string[]; // 3-5 takeaways
|
||||
sentiment: 'positive' | 'neutral' | 'negative';
|
||||
monetizationAngle: string; // Content opportunity
|
||||
analyzedAt: string; // ISO 8601
|
||||
}
|
||||
|
||||
// Input to Claude for analysis
|
||||
export interface AnalysisRequest {
|
||||
title: string;
|
||||
author?: string;
|
||||
url: string;
|
||||
content: string;
|
||||
wordCount: number;
|
||||
highlights: Array<{
|
||||
quote: string;
|
||||
annotation?: string;
|
||||
}>;
|
||||
publishedAt?: string;
|
||||
}
|
||||
|
||||
// Topic with confidence
|
||||
export interface TopicScore {
|
||||
topic: string;
|
||||
score: number; // 0-1
|
||||
keywords: string[];
|
||||
}
|
||||
|
||||
// Analysis configuration
|
||||
export interface AnalysisConfig {
|
||||
focusTopics?: string[];
|
||||
minTopicScore?: number;
|
||||
maxTopics?: number;
|
||||
includeSentiment?: boolean;
|
||||
}
|
||||
|
||||
// Batch analysis result
|
||||
export interface BatchAnalysisResult {
|
||||
articles: Array<{
|
||||
articleId: string;
|
||||
analysis: ContentAnalysis;
|
||||
}>;
|
||||
commonTopics: string[];
|
||||
trends: Array<{
|
||||
topic: string;
|
||||
frequency: number;
|
||||
averageScore: number;
|
||||
}>;
|
||||
analyzedAt: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Central Exports (`src/types/index.ts`)
|
||||
|
||||
**Re-exports all types** plus utility types:
|
||||
|
||||
```typescript
|
||||
// Re-exports
|
||||
export * from './omnivore';
|
||||
export * from './content';
|
||||
export * from './analysis';
|
||||
|
||||
// Utility types
|
||||
export type DateString = string; // ISO 8601
|
||||
export type UUID = string;
|
||||
export type Slug = string;
|
||||
|
||||
// Generation config
|
||||
export interface BlogPostConfig {
|
||||
type: 'single-article' | 'weekly-roundup' | 'deep-dive';
|
||||
title?: string;
|
||||
targetWordCount?: number;
|
||||
includeSources?: boolean;
|
||||
seoOptimize?: boolean;
|
||||
}
|
||||
|
||||
export interface NewsletterConfig {
|
||||
type: 'weekly' | 'monthly';
|
||||
includeSections?: string[];
|
||||
maxArticles?: number;
|
||||
personalCommentary?: boolean;
|
||||
}
|
||||
|
||||
// Publishing
|
||||
export type PublishingPlatform = 'markdown' | 'ghost' | 'wordpress' | 'medium';
|
||||
|
||||
export interface PublishResult {
|
||||
platform: PublishingPlatform;
|
||||
success: boolean;
|
||||
url?: string;
|
||||
error?: string;
|
||||
publishedAt: string;
|
||||
}
|
||||
```
|
||||
|
||||
**Import Pattern**:
|
||||
```typescript
|
||||
// Import from index for convenience
|
||||
import type {
|
||||
OmnivoreArticle,
|
||||
ContentAnalysis,
|
||||
ArticleFrontMatter
|
||||
} from '@omc-types';
|
||||
|
||||
// OR import from specific files
|
||||
import type { OmnivoreArticle } from '@omc-types/omnivore.js';
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
```
|
||||
|
||||
## Environment Configuration
|
||||
|
||||
### Required Variables
|
||||
|
||||
**File**: `.env` (project root, gitignored)
|
||||
|
||||
```bash
|
||||
# Omnivore API (REQUIRED)
|
||||
OMNIVORE_API_KEY=your_api_key_here
|
||||
OMNIVORE_API_URL=https://omnivore-api.caladan.haus/api/graphql
|
||||
|
||||
# Anthropic API (REQUIRED for analysis/generation phases)
|
||||
ANTHROPIC_API_KEY=your_anthropic_api_key_here
|
||||
|
||||
# Content directories (OPTIONAL - defaults provided)
|
||||
CONTENT_OUTPUT_DIR=./content
|
||||
ARTICLES_DIR=./content/articles
|
||||
ANALYSIS_DIR=./content/analysis
|
||||
GENERATED_DIR=./content/generated
|
||||
```
|
||||
|
||||
### For Self-Hosted Omnivore
|
||||
|
||||
If using self-hosted Omnivore instance:
|
||||
- Set `OMNIVORE_API_URL` to your instance's GraphQL endpoint
|
||||
- Get API key from your instance's settings
|
||||
|
||||
### Example: `.env.example`
|
||||
|
||||
Template file included in repo shows all available configuration options.
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Fetching Articles
|
||||
|
||||
```javascript
|
||||
import { searchArticles } from './lib/omnivore/client.js';
|
||||
import { QUERY_PATTERNS } from './lib/omnivore/queries.js';
|
||||
|
||||
// Fetch AI articles from last 7 days
|
||||
const result = await searchArticles({
|
||||
query: QUERY_PATTERNS.AI_ML + ' ' + QUERY_PATTERNS.LAST_7_DAYS,
|
||||
first: 10,
|
||||
includeContent: true
|
||||
});
|
||||
|
||||
// Access articles
|
||||
const articles = result.search.edges.map(edge => edge.node);
|
||||
articles.forEach(article => {
|
||||
console.log(article.title);
|
||||
console.log(article.labels.map(l => l.name));
|
||||
});
|
||||
```
|
||||
|
||||
### Type-Safe Operations
|
||||
|
||||
```typescript
|
||||
import type { OmnivoreArticle, SearchResult } from '@omc-types';
|
||||
import { searchArticles } from './lib/omnivore/client.js';
|
||||
|
||||
async function getAIArticles(): Promise<OmnivoreArticle[]> {
|
||||
const result = await searchArticles({
|
||||
query: 'label:ai',
|
||||
first: 20
|
||||
}) as SearchResult;
|
||||
|
||||
return result.edges.map(edge => edge.node);
|
||||
}
|
||||
```
|
||||
|
||||
### Testing Connection
|
||||
|
||||
```bash
|
||||
# Test API connection
|
||||
node lib/omnivore/client.js --test
|
||||
|
||||
# Expected output:
|
||||
# ✅ Connected to Omnivore API
|
||||
# User: Your Name (your@email.com)
|
||||
# Username: yourusername
|
||||
```
|
||||
|
||||
## File System Conventions
|
||||
|
||||
### Content Storage
|
||||
|
||||
```
|
||||
content/
|
||||
├── articles/ # Original articles as Markdown
|
||||
│ └── YYYY-MM-DD-{slug}.md
|
||||
├── analysis/ # Analysis results
|
||||
│ └── YYYY-MM-DD-{slug}.md
|
||||
├── generated/ # Generated content
|
||||
│ ├── blog-posts/
|
||||
│ │ └── YYYY-MM-DD-{title}.md
|
||||
│ └── newsletters/
|
||||
│ └── YYYY-WW-roundup.md
|
||||
└── .metadata/
|
||||
└── index.json # Search index
|
||||
```
|
||||
|
||||
### Markdown Format
|
||||
|
||||
**Article File** (`content/articles/2025-09-30-example.md`):
|
||||
```markdown
|
||||
---
|
||||
id: omnivore-abc123
|
||||
url: https://example.com/article
|
||||
title: Example Article
|
||||
author: John Doe
|
||||
savedAt: 2025-09-30T10:00:00Z
|
||||
labels: [ai, machine-learning]
|
||||
highlights: 3
|
||||
wordCount: 2500
|
||||
---
|
||||
|
||||
# Example Article
|
||||
|
||||
Article content here...
|
||||
|
||||
## Highlights
|
||||
|
||||
> Important quote
|
||||
— Note: Why this matters
|
||||
```
|
||||
|
||||
## Build System Details
|
||||
|
||||
### TypeScript Compilation
|
||||
|
||||
```bash
|
||||
# Development build (with source maps & declarations)
|
||||
pnpm run build
|
||||
# Output: dist/ with .js, .d.ts, .js.map files
|
||||
|
||||
# Type check only (no output)
|
||||
pnpm run typecheck
|
||||
|
||||
# Watch mode for development
|
||||
pnpm run dev
|
||||
```
|
||||
|
||||
### Path Aliases
|
||||
|
||||
Configured in tsconfig.json, usable in TypeScript files:
|
||||
- `@lib/*` → `src/lib/*`
|
||||
- `@storage/*` → `src/storage/*`
|
||||
- `@analysis/*` → `src/analysis/*`
|
||||
- `@generation/*` → `src/generation/*`
|
||||
- `@utils/*` → `src/utils/*`
|
||||
- `@omc-types/*` → `src/types/*`
|
||||
|
||||
### Module System
|
||||
|
||||
- **Type**: ESM (ES Modules)
|
||||
- **Import syntax**: `import { x } from './file.js'` (include `.js` extension)
|
||||
- **package.json**: `"type": "module"`
|
||||
|
||||
## What Does NOT Exist Yet
|
||||
|
||||
This groundtruth documents what IS built. These are NOT implemented:
|
||||
- Automated content generation modules (blog posts/newsletters) - Phase 5
|
||||
- Publishing modules (Ghost/WordPress/etc.) - Phase 6
|
||||
- End-to-end scheduled orchestration beyond analysis (e.g., daily generation + publishing)
|
||||
|
||||
See IMPLEMENTATION_PLAN.md for what needs to be built.
|
||||
19
self-hosting/omc/docs/_meta/graphql-organization.md
Normal file
19
self-hosting/omc/docs/_meta/graphql-organization.md
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
# GraphQL Organization (Current)
|
||||
|
||||
This repo currently uses a single runtime GraphQL client:
|
||||
|
||||
- Runtime client: `lib/omnivore/client.js` (string-based GraphQL queries + `node-fetch`)
|
||||
- TypeScript import surface: `src/lib/omnivore/client.ts` (re-exports runtime functions for TS/alias imports)
|
||||
|
||||
## Why
|
||||
|
||||
An earlier, typed GraphQL layer (`src/graphql/**` + `src/types/generated/**`) existed but was **not integrated** into the CLI runtime. Maintaining two parallel query stacks created drift risk without any runtime benefit.
|
||||
|
||||
To reduce duplication and keep docs aligned with reality, the typed layer was removed. If/when we migrate the runtime client to typed operations, we can reintroduce codegen + documents as part of that migration.
|
||||
|
||||
## How To Add/Change Fields
|
||||
|
||||
1. Update the query string in `lib/omnivore/client.js`.
|
||||
2. If needed, update the corresponding TS types in `src/types/omnivore.ts`.
|
||||
3. Prefer importing client functions from `@lib/omnivore/client.js` in TypeScript code.
|
||||
|
||||
613
self-hosting/omc/docs/_meta/workflow-internals.md
Normal file
613
self-hosting/omc/docs/_meta/workflow-internals.md
Normal file
|
|
@ -0,0 +1,613 @@
|
|||
# Workflow Internals
|
||||
|
||||
**Purpose**: Deep dive into how the parallel analysis workflow operates, including zero-context-pollution design, agent invocation patterns, and file handling.
|
||||
|
||||
**Last Updated**: 2026-01-30
|
||||
|
||||
**Status**: This document originally described the legacy `cli/parallel-analyze.ts` + `cli/save-analysis-results.ts` flow. The current workflow is implemented via `omc analyze run` and `omc analyze complete`. See `docs/_meta/current-state.md` for the full audit and issue list.
|
||||
|
||||
## Overview
|
||||
|
||||
The parallel analysis workflow uses a **zero-context-pollution** design where each agent operates independently with fresh database connections and no shared state from the orchestration script. This enables true parallel execution and prevents race conditions.
|
||||
|
||||
## Zero-Context-Pollution Design
|
||||
|
||||
### Design Principles
|
||||
|
||||
1. **Stub files as communication**: Main context creates minimal stub files containing only article metadata
|
||||
2. **Fresh connections**: the orchestrator command closes its DB work before agents run; agents should operate independently
|
||||
3. **Independent content fetching**: agents fetch article content via `omc omnivore get <slug> --json` and extract `.content`
|
||||
4. **File-based results**: Agents enrich stub files with analysis field, no inter-agent communication
|
||||
5. **Slug-based naming**: Temp files named by article slug prevents overwrites across batches
|
||||
|
||||
### Why This Matters
|
||||
|
||||
- **Prevents race conditions**: Each agent reads/writes different files
|
||||
- **Enables true parallelism**: No database locks during agent execution
|
||||
- **Supports batch overlap**: Can run multiple batches before saving results
|
||||
- **Isolates failures**: Failed agent doesn't corrupt others' work
|
||||
- **Scales horizontally**: Can distribute agents across machines
|
||||
|
||||
## Parallel Analysis Workflow
|
||||
|
||||
### Automated Mode (`omc analyze auto`)
|
||||
|
||||
The automated workflow is the non-interactive, schedulable path:
|
||||
|
||||
1. Populate queue: `omc queue add --hours 24`
|
||||
2. Analyze end-to-end: `omc analyze auto --batch-size 5`
|
||||
|
||||
Implementation notes:
|
||||
- The analysis step shells out to `codex exec` in `read-only` sandbox mode and expects a single JSON object response.
|
||||
- `CODEX_HOME` is set to `temp/codex-home` during execution to avoid reliance on `~/.codex` permissions in automation environments.
|
||||
|
||||
### Step 1: Orchestration (`omc analyze run`)
|
||||
|
||||
The orchestration script sets up the batch and prepares for agent invocation:
|
||||
|
||||
```typescript
|
||||
// 1. Fetch pending jobs and mark in_progress
|
||||
const jobs = queueRepo.getPending(5);
|
||||
jobs.forEach(job => queueRepo.markInProgress(job.articleId));
|
||||
|
||||
// 2. Fetch article metadata from Omnivore API
|
||||
const result = await getArticle(job.articleSlug, username);
|
||||
const article = result.article;
|
||||
|
||||
// 3. Create stub file with slug-based name
|
||||
const stubFile = `temp/${job.articleSlug}.jsonl`;
|
||||
fs.writeFileSync(stubFile, JSON.stringify({
|
||||
articleId: job.articleId,
|
||||
articleSlug: job.articleSlug,
|
||||
username: username,
|
||||
articleUrl: article.url,
|
||||
articleTitle: article.title,
|
||||
savedAt: article.savedAt,
|
||||
publishedAt: article.publishedAt,
|
||||
updatedAt: article.updatedAt
|
||||
}));
|
||||
|
||||
// 4. Output agent parameters
|
||||
console.log(JSON.stringify([{
|
||||
filename: stubFile,
|
||||
articleId: article.id,
|
||||
articleSlug: article.slug,
|
||||
username: username,
|
||||
articleTitle: article.title
|
||||
}]));
|
||||
|
||||
// 5. Close database (agents get fresh connections)
|
||||
db.close();
|
||||
```
|
||||
|
||||
**Key Points**:
|
||||
- Database is closed before agents execute
|
||||
- Stub files contain metadata only (no content)
|
||||
- Each stub file is named by article slug
|
||||
- Agent parameters output as JSON array
|
||||
|
||||
### Step 2: Agent Execution (@article-content-analyzer)
|
||||
|
||||
Each agent operates independently with its own workflow:
|
||||
|
||||
```typescript
|
||||
// Each agent receives: filename, articleId, articleSlug, username, articleTitle
|
||||
|
||||
// 1. Read stub file to get metadata
|
||||
const stub = JSON.parse(fs.readFileSync(filename, 'utf-8'));
|
||||
|
||||
// 2. Fetch content via CLI tool (no TypeScript imports)
|
||||
const contentResult = await bash(`omc omnivore get ${stub.articleSlug} --json`);
|
||||
const article = JSON.parse(contentResult.stdout);
|
||||
const articleContent = article.content;
|
||||
|
||||
// 3. Analyze content using agent's prompt
|
||||
const analysis = {
|
||||
topics: [...],
|
||||
topicScores: {...},
|
||||
summary: "...",
|
||||
keyPoints: [...],
|
||||
sentiment: "positive",
|
||||
monetizationAngle: "...",
|
||||
analyzedAt: new Date().toISOString()
|
||||
};
|
||||
|
||||
// 4. Re-read stub file (fresh copy, no race conditions)
|
||||
const enriched = JSON.parse(fs.readFileSync(filename, 'utf-8'));
|
||||
|
||||
// 5. Add analysis field ONLY (preserve all original fields)
|
||||
enriched.analysis = analysis;
|
||||
|
||||
// 6. Write enriched data back to same file
|
||||
fs.writeFileSync(filename, JSON.stringify(enriched, null, 2));
|
||||
```
|
||||
|
||||
**Key Points**:
|
||||
- Agent fetches content via CLI (no direct TypeScript imports)
|
||||
- Re-reads stub before writing (ensures fresh data)
|
||||
- Adds `analysis` field only (preserves metadata)
|
||||
- Each agent writes to unique file (slug-based naming)
|
||||
|
||||
### Step 3: Save Results (`omc analyze complete`)
|
||||
|
||||
The save script processes all enriched files and updates tracking:
|
||||
|
||||
```typescript
|
||||
// For each enriched temp file:
|
||||
for (const file of files) {
|
||||
const data = JSON.parse(fs.readFileSync(file, 'utf-8'));
|
||||
|
||||
// 1) Write to Markdown (git-tracked, human-editable)
|
||||
const markdownPath = await writer.write(
|
||||
data.articleId,
|
||||
data.articleUrl,
|
||||
data.articleTitle,
|
||||
data.savedAt,
|
||||
data.analysis
|
||||
);
|
||||
|
||||
// 2) Store immutable snapshot + mark completed in SQLite
|
||||
queueRepo.storeAnalysis(
|
||||
data.articleId,
|
||||
data.publishedAt ?? null,
|
||||
data.updatedAt ?? null,
|
||||
JSON.stringify(data.analysis),
|
||||
markdownPath
|
||||
);
|
||||
|
||||
// 3) Delete temp file (cleanup)
|
||||
fs.unlinkSync(file);
|
||||
}
|
||||
```
|
||||
|
||||
**Key Points**:
|
||||
- Processes all temp files in one pass
|
||||
- Updates database after successful write
|
||||
- Automatically deletes temp files (cleanup)
|
||||
- Preserves temp files on error for debugging
|
||||
|
||||
**Note**:
|
||||
- JSONL output is optional: enable with `omc analyze complete --jsonl` or `omc analyze auto --jsonl`.
|
||||
|
||||
## File Lifecycle
|
||||
|
||||
### Stub File Creation (`omc analyze run`)
|
||||
|
||||
**File**: `temp/{articleSlug}.jsonl`
|
||||
|
||||
**Content**: Article metadata only (no content, no analysis)
|
||||
|
||||
**Purpose**: Minimal communication between orchestrator and agents
|
||||
|
||||
**Example**:
|
||||
```json
|
||||
{
|
||||
"articleId": "abc-123",
|
||||
"articleSlug": "understanding-llm-fine-tuning",
|
||||
"username": "myusername",
|
||||
"articleUrl": "https://example.com/article",
|
||||
"articleTitle": "Understanding LLM Fine-Tuning",
|
||||
"savedAt": "2025-09-30T10:00:00Z",
|
||||
"publishedAt": "2025-09-29T14:00:00Z",
|
||||
"updatedAt": "2025-09-30T08:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
### Enrichment (agent)
|
||||
|
||||
**Process**:
|
||||
1. Reads stub file
|
||||
2. Fetches content via CLI tool
|
||||
3. Adds `analysis` field to JSON
|
||||
4. Writes back to same file
|
||||
|
||||
**Enriched Format**:
|
||||
```json
|
||||
{
|
||||
"articleId": "abc-123",
|
||||
"articleSlug": "understanding-llm-fine-tuning",
|
||||
"username": "myusername",
|
||||
"articleUrl": "https://example.com/article",
|
||||
"articleTitle": "Understanding LLM Fine-Tuning",
|
||||
"savedAt": "2025-09-30T10:00:00Z",
|
||||
"publishedAt": "2025-09-29T14:00:00Z",
|
||||
"updatedAt": "2025-09-30T08:00:00Z",
|
||||
"analysis": {
|
||||
"topics": ["ai", "machine-learning"],
|
||||
"topicScores": { "ai": 0.95, "machine-learning": 0.88 },
|
||||
"summary": "Article summary...",
|
||||
"keyPoints": ["point1", "point2"],
|
||||
"sentiment": "positive",
|
||||
"monetizationAngle": "Content opportunity...",
|
||||
"analyzedAt": "2025-10-01T04:30:00Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Storage & Cleanup (`omc analyze complete`)
|
||||
|
||||
**Process**:
|
||||
1. Reads enriched file
|
||||
2. Writes Markdown to `content/analysis/YYYY-MM-DD-{slug}-analysis.md` (slug derived from `articleSlug` when available, otherwise title)
|
||||
3. Stores immutable snapshot to SQLite (`analysis_json`, `markdown_path`) and marks job `completed`
|
||||
4. **Deletes temp file** with `unlinkSync(file)`
|
||||
|
||||
**Output Files**:
|
||||
- `content/analysis/2025-09-30-understanding-llm-fine-tuning-analysis.md`
|
||||
|
||||
**Temp File**: Deleted after successful save
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Parse Error
|
||||
|
||||
**Scenario**: Enriched file contains invalid JSON
|
||||
|
||||
**Handling**:
|
||||
- Mark job as `failed` in tracking queue
|
||||
- Preserve temp file for debugging
|
||||
- Store error message in database
|
||||
- Increment retry count
|
||||
|
||||
**Example**:
|
||||
```
|
||||
✗ Failed: Understanding LLM Fine-Tuning... - Parse error
|
||||
→ Preserved: temp/understanding-llm-fine-tuning.jsonl
|
||||
```
|
||||
|
||||
### Agent Timeout
|
||||
|
||||
**Scenario**: Agent doesn't complete within time limit
|
||||
|
||||
**Handling**:
|
||||
- Status remains `in_progress` in database
|
||||
- Temp file preserved (partially enriched)
|
||||
- Manual retry needed via `omc analyze retry`
|
||||
|
||||
### Save Failure
|
||||
|
||||
**Scenario**: Error writing to Markdown or JSONL
|
||||
|
||||
**Handling**:
|
||||
- Mark job as `failed`
|
||||
- Preserve temp file
|
||||
- Log error details
|
||||
- Can retry after fixing issue
|
||||
|
||||
## Parallel Invocation Pattern
|
||||
|
||||
### Single Message, Multiple Agents
|
||||
|
||||
The workflow requires invoking all agents in a single message for true parallelism:
|
||||
|
||||
```
|
||||
User message:
|
||||
Analyze these 5 articles in parallel:
|
||||
|
||||
@article-content-analyzer {
|
||||
filename: "temp/article-1.jsonl",
|
||||
articleId: "abc-123",
|
||||
articleSlug: "article-1",
|
||||
username: "myusername",
|
||||
articleTitle: "Article 1 Title"
|
||||
}
|
||||
|
||||
@article-content-analyzer {
|
||||
filename: "temp/article-2.jsonl",
|
||||
articleId: "def-456",
|
||||
articleSlug: "article-2",
|
||||
username: "myusername",
|
||||
articleTitle: "Article 2 Title"
|
||||
}
|
||||
|
||||
... (3 more agents)
|
||||
```
|
||||
|
||||
### Why Single Message
|
||||
|
||||
- **All agents execute in parallel** (no sequential bottleneck)
|
||||
- **User waits for all results** before saving
|
||||
- **Failure of one agent** doesn't block others
|
||||
- **Claude Code orchestrates** parallel execution
|
||||
|
||||
### Agent Independence
|
||||
|
||||
Each agent:
|
||||
- Gets unique parameters (filename, articleId, slug, username, title)
|
||||
- Reads different stub file
|
||||
- Fetches content independently via CLI
|
||||
- Writes to different temp file
|
||||
- No coordination with other agents
|
||||
|
||||
## Slug-Based Naming Benefits
|
||||
|
||||
### Problem with Sequential IDs
|
||||
|
||||
Using sequential names like `temp/result-1.jsonl`, `temp/result-2.jsonl`:
|
||||
- Running batch 2 overwrites batch 1's files
|
||||
- Must save batch 1 before running batch 2
|
||||
- Forces sequential processing
|
||||
|
||||
### Solution with Slug-Based Names
|
||||
|
||||
Using article slug like `temp/{articleSlug}.jsonl`:
|
||||
- Each article has unique filename based on slug
|
||||
- Multiple batches can run simultaneously
|
||||
- Agents can complete out-of-order
|
||||
- Save script processes all `temp/*.jsonl` files at once
|
||||
|
||||
### Example Workflow
|
||||
|
||||
```bash
|
||||
# Batch 1 creates:
|
||||
temp/understanding-llm-fine-tuning.jsonl
|
||||
temp/kubernetes-best-practices.jsonl
|
||||
temp/rust-performance-tips.jsonl
|
||||
|
||||
# Batch 2 creates (while batch 1 agents still running):
|
||||
temp/python-async-patterns.jsonl
|
||||
temp/distributed-systems-design.jsonl
|
||||
|
||||
# Save all at once:
|
||||
omc analyze complete
|
||||
# Processes all 5 files, deletes after successful save
|
||||
```
|
||||
|
||||
## Agent Content Analyzer
|
||||
|
||||
### Agent Name
|
||||
|
||||
`@article-content-analyzer`
|
||||
|
||||
**Location**: `.claude/agents/article-content-analyzer.md`
|
||||
|
||||
**Model**: Sonnet
|
||||
|
||||
### Core Responsibilities
|
||||
|
||||
1. **Topic Extraction & Scoring** - Identify 2-5 topics with confidence scores (0-1)
|
||||
2. **Strategic Summarization** - 2-3 sentence "so what?" summary
|
||||
3. **Key Point Extraction** - 3-5 actionable insights or surprising facts
|
||||
4. **Sentiment Analysis** - Classify as positive/neutral/negative
|
||||
5. **Monetization Strategy** - Identify specific content opportunities
|
||||
|
||||
### Topic Categories
|
||||
|
||||
Aligned with content strategy:
|
||||
- `developer-tools`, `ai-tooling`, `ai`, `machine-learning`, `llm`
|
||||
- `software-engineering`, `architecture`, `code-quality`
|
||||
- `devops`, `kubernetes`, `docker`, `cloud-native`
|
||||
- `databases`, `sql`, `nosql`
|
||||
- `cloud`, `aws`, `azure`, `serverless`
|
||||
- `startups`, `product`, `growth`, `monetization`
|
||||
- `security`, `auth`, `encryption`
|
||||
|
||||
### Topic Scores
|
||||
|
||||
- **0.90-1.0**: Core focus of article
|
||||
- **0.70-0.89**: Significant discussion
|
||||
- **0.50-0.69**: Mentioned but not central
|
||||
- **Below 0.50**: Excluded
|
||||
|
||||
### Output Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"topics": ["topic1", "topic2", "topic3"],
|
||||
"topicScores": {
|
||||
"topic1": 0.95,
|
||||
"topic2": 0.88,
|
||||
"topic3": 0.75
|
||||
},
|
||||
"summary": "2-3 sentence summary capturing main points and why this matters",
|
||||
"keyPoints": [
|
||||
"First key takeaway or insight",
|
||||
"Second key takeaway or insight",
|
||||
"Third key takeaway or insight"
|
||||
],
|
||||
"sentiment": "positive|neutral|negative",
|
||||
"monetizationAngle": "Specific content opportunity description"
|
||||
}
|
||||
```
|
||||
|
||||
### Critical Rules
|
||||
|
||||
- Returns ONLY JSON object (no markdown code blocks, no explanatory text)
|
||||
- All topic labels from approved category list
|
||||
- Topic scores between 0-1, at least one ≥ 0.70
|
||||
- Summary is 2-3 sentences focusing on implications
|
||||
- 3-5 specific, actionable key points
|
||||
- Monetization angle suggests concrete content opportunity
|
||||
- User highlights receive extra weight if provided
|
||||
|
||||
### Usage in Workflow
|
||||
|
||||
The agent is invoked with parameters from the orchestration script:
|
||||
|
||||
```
|
||||
@article-content-analyzer {
|
||||
filename: "temp/understanding-llm-fine-tuning.jsonl",
|
||||
articleId: "abc-123",
|
||||
articleSlug: "understanding-llm-fine-tuning",
|
||||
username: "myusername",
|
||||
articleTitle: "Understanding LLM Fine-Tuning Techniques"
|
||||
}
|
||||
```
|
||||
|
||||
Agent then:
|
||||
1. Reads stub file for metadata
|
||||
2. Calls `omc omnivore get understanding-llm-fine-tuning --json` for content (extract `.content`)
|
||||
3. Analyzes content using its prompt template
|
||||
4. Enriches stub file with analysis
|
||||
5. Writes back to temp file
|
||||
|
||||
## Analysis Prompt Template
|
||||
|
||||
**Location**: `src/analysis/prompts/analyze.md`
|
||||
|
||||
**Purpose**: Structured prompt template for article analysis (used by agent).
|
||||
|
||||
### Template Variables
|
||||
|
||||
- `{{title}}` - Article title
|
||||
- `{{author}}` - Article author
|
||||
- `{{url}}` - Article URL
|
||||
- `{{wordCount}}` - Word count
|
||||
- `{{publishedAt}}` - Publication date
|
||||
- `{{content}}` - Article content (HTML or text)
|
||||
- `{{#if highlights}}...{{/if}}` - Conditional highlights section
|
||||
- `{{#each highlights}}...{{/each}}` - Iterate over highlights
|
||||
|
||||
### Highlights Format
|
||||
|
||||
If user highlights are present:
|
||||
|
||||
```
|
||||
{{#each highlights}}
|
||||
- "{{quote}}"{{#if annotation}} — Note: {{annotation}}{{/if}}
|
||||
{{/each}}
|
||||
```
|
||||
|
||||
### Output Format
|
||||
|
||||
JSON matching `ContentAnalysis` schema (no markdown code blocks).
|
||||
|
||||
### Example Rendered Prompt
|
||||
|
||||
```markdown
|
||||
# Article Content Analysis Prompt
|
||||
|
||||
You are analyzing an article for content monetization opportunities...
|
||||
|
||||
## Article Details
|
||||
|
||||
**Title**: Understanding LLM Fine-Tuning
|
||||
**Author**: Jane Smith
|
||||
**URL**: https://example.com/llm-finetuning
|
||||
**Word Count**: 3200
|
||||
**Published**: 2025-09-25T10:00:00Z
|
||||
|
||||
**Content**:
|
||||
[Full article content here...]
|
||||
|
||||
**User Highlights**:
|
||||
- "LoRA reduces trainable parameters by 90%" — Note: Key metric for cost savings
|
||||
- "Prompt engineering vs fine-tuning trade-offs"
|
||||
|
||||
## Analysis Task
|
||||
|
||||
Extract the following information and return as JSON:
|
||||
[JSON schema and guidelines...]
|
||||
```
|
||||
|
||||
## Data Flow Diagram
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 1. ORCHESTRATION (omc analyze run) │
|
||||
│ │
|
||||
│ Queue DB → Get 5 pending → Mark in_progress │
|
||||
│ → Fetch metadata from Omnivore API │
|
||||
│ → Create stub files (temp/{slug}.jsonl) │
|
||||
│ → Output agent parameters as JSON │
|
||||
│ → Close DB connection │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 2. PARALLEL AGENT EXECUTION (@article-content-analyzer × 5) │
|
||||
│ │
|
||||
│ Each agent independently: │
|
||||
│ - Read stub file for metadata │
|
||||
│ - Call `omc omnivore get {slug} --json` for content │
|
||||
│ - Analyze content using prompt template │
|
||||
│ - Enrich stub file with analysis field │
|
||||
│ - Write back to temp/{slug}.jsonl │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 3. SAVE RESULTS (omc analyze complete) │
|
||||
│ │
|
||||
│ Read all temp/*.jsonl files │
|
||||
│ For each: │
|
||||
│ - Write to content/analysis/YYYY-MM-DD-{slug}-analysis.md │
|
||||
│ - Store snapshot in DB + mark completed │
|
||||
│ - Delete temp file │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Complete Example
|
||||
|
||||
### Real Analysis Output
|
||||
|
||||
**File**: `content/analysis/2025-09-30-cchistory-tracking-claude-code-system-prompt-and-t-analysis.md`
|
||||
|
||||
```markdown
|
||||
---
|
||||
articleId: 1039961b-8de3-4ccf-b3e8-df888d6174b8
|
||||
articleUrl: https://mariozechner.at/posts/2025-08-03-cchistory/
|
||||
articleTitle: "cchistory: Tracking Claude Code System Prompt and Tool Changes"
|
||||
savedAt: 2025-09-30T02:27:46.000Z
|
||||
analyzedAt: 2025-10-01T04:25:16.676Z
|
||||
topics: [developer-tools, ai-tooling, reverse-engineering, software-debugging]
|
||||
topicScores:
|
||||
developer-tools: 0.95
|
||||
ai-tooling: 0.9
|
||||
reverse-engineering: 0.85
|
||||
software-debugging: 0.75
|
||||
sentiment: positive
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
Developer creates tools to reverse-engineer and track changes in Claude Code's system prompts and tool definitions over time. The cchistory project monitors Claude Code updates by intercepting and archiving system messages, revealing how Anthropic evolves the assistant's behavior and capabilities.
|
||||
|
||||
## Key Points
|
||||
|
||||
- Created claude-trace to intercept and log Claude Code's internal API communications
|
||||
- Built cchistory to automatically track and diff system prompt changes across versions
|
||||
- Discovered Claude Code switched from Sonnet to Haiku for certain operations to reduce costs
|
||||
- System prompts reveal detailed behavioral instructions and tool usage patterns
|
||||
- Tracking changes helps understand how AI coding assistants evolve over time
|
||||
|
||||
## Monetization Angle
|
||||
|
||||
Tutorial series on 'Reverse Engineering AI Tools' - developers want to understand how Claude Code works under the hood. Could create content comparing system prompt evolution, analyzing cost optimization strategies, or teaching prompt engineering through real examples from production AI systems.
|
||||
```
|
||||
|
||||
### Example JSONL Record (Optional Output)
|
||||
|
||||
**File**: `content/analysis/analyses.jsonl` (written when `--jsonl` is enabled)
|
||||
|
||||
```json
|
||||
{"articleId":"1039961b-8de3-4ccf-b3e8-df888d6174b8","articleUrl":"https://mariozechner.at/posts/2025-08-03-cchistory/","articleTitle":"cchistory: Tracking Claude Code System Prompt and Tool Changes","savedAt":"2025-09-30T02:27:46.000Z","analyzedAt":"2025-10-01T04:25:16.676Z","topics":["developer-tools","ai-tooling","reverse-engineering","software-debugging"],"topicScores":{"developer-tools":0.95,"ai-tooling":0.9,"reverse-engineering":0.85,"software-debugging":0.75},"sentiment":"positive","summary":"Developer creates tools to reverse-engineer and track changes in Claude Code's system prompts and tool definitions over time. The cchistory project monitors Claude Code updates by intercepting and archiving system messages, revealing how Anthropic evolves the assistant's behavior and capabilities.","keyPoints":["Created claude-trace to intercept and log Claude Code's internal API communications","Built cchistory to automatically track and diff system prompt changes across versions","Discovered Claude Code switched from Sonnet to Haiku for certain operations to reduce costs","System prompts reveal detailed behavioral instructions and tool usage patterns","Tracking changes helps understand how AI coding assistants evolve over time"],"monetizationAngle":"Tutorial series on 'Reverse Engineering AI Tools' - developers want to understand how Claude Code works under the hood. Could create content comparing system prompt evolution, analyzing cost optimization strategies, or teaching prompt engineering through real examples from production AI systems."}
|
||||
```
|
||||
|
||||
## What Does NOT Exist Yet
|
||||
|
||||
**Implemented**:
|
||||
- ✅ SQLite tracking database for parallel execution
|
||||
- ✅ Job queue with status tracking (pending/in_progress/completed/failed)
|
||||
- ✅ Parallel analysis workflow (batch of 5)
|
||||
- ✅ Error handling with retry logic (max 3 attempts)
|
||||
- ✅ CLI tools for queue management
|
||||
- ✅ JSONL output format
|
||||
- ✅ Boundary enforcement with AIDEV annotations
|
||||
- ✅ Zero-context-pollution design
|
||||
- ✅ Slug-based file naming
|
||||
|
||||
**Future Enhancements**:
|
||||
- **Automated agent invocation** - Currently requires manual copy/paste of prompts; could integrate Claude Agent SDK for direct invocation
|
||||
- **Real-time progress tracking** - Add progress bars during fetch/analysis operations
|
||||
- **Session management** - `analysis_sessions` table exists but not yet used for batch tracking
|
||||
- **Queue cleanup automation** - Manual `clearCompleted()` call; could auto-cleanup after export
|
||||
- **Distributed execution** - Agents across multiple machines
|
||||
- **Advanced error recovery** - Automatic retry with backoff
|
||||
- **Batch prioritization** - Priority queue for time-sensitive articles
|
||||
- **Content pre-filtering** - Skip articles unlikely to be valuable
|
||||
- **Incremental updates** - Re-analyze articles when content changes
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Architecture](architecture.md) - System design and storage boundaries
|
||||
- [CLI Reference](cli-reference.md) - Command-line interface documentation
|
||||
- [Foundation & Type System](foundation-and-types.md) - TypeScript setup, type definitions
|
||||
3510
self-hosting/omc/docs/graphql-schema/schema.graphql
Normal file
3510
self-hosting/omc/docs/graphql-schema/schema.graphql
Normal file
File diff suppressed because it is too large
Load diff
77
self-hosting/omc/esbuild.config.mjs
Executable file
77
self-hosting/omc/esbuild.config.mjs
Executable file
|
|
@ -0,0 +1,77 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* ESBuild Configuration for OMC CLI
|
||||
* AIDEV-NOTE: Bundles OCLIF CLI with ESM support and path aliases
|
||||
*/
|
||||
|
||||
import { build } from 'esbuild';
|
||||
import { glob } from 'glob';
|
||||
import { copyFile, mkdir } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// Find all command files
|
||||
const commandFiles = await glob('src/commands/**/*.ts', { cwd: __dirname });
|
||||
const libFiles = await glob('src/lib/**/*.ts', { cwd: __dirname });
|
||||
const storageFiles = await glob('src/storage/**/*.ts', { cwd: __dirname });
|
||||
const rootLibFiles = await glob('lib/**/*.js', { cwd: __dirname });
|
||||
|
||||
// Build configuration
|
||||
const buildConfig = {
|
||||
entryPoints: [
|
||||
'bin/omc.ts',
|
||||
'index.ts',
|
||||
...commandFiles,
|
||||
...libFiles,
|
||||
...storageFiles,
|
||||
...rootLibFiles,
|
||||
],
|
||||
bundle: true,
|
||||
format: 'esm',
|
||||
platform: 'node',
|
||||
target: 'node18',
|
||||
outdir: 'dist',
|
||||
packages: 'external',
|
||||
splitting: true,
|
||||
sourcemap: true,
|
||||
// Path alias resolution
|
||||
alias: {
|
||||
'@lib': path.join(__dirname, 'src/lib'),
|
||||
'@storage': path.join(__dirname, 'src/storage'),
|
||||
'@analysis': path.join(__dirname, 'src/analysis'),
|
||||
'@generation': path.join(__dirname, 'src/generation'),
|
||||
'@publishing': path.join(__dirname, 'src/publishing'),
|
||||
'@workflows': path.join(__dirname, 'src/workflows'),
|
||||
'@utils': path.join(__dirname, 'src/utils'),
|
||||
'@omc-types': path.join(__dirname, 'src/types'),
|
||||
},
|
||||
// Preserve directory structure
|
||||
outbase: '.',
|
||||
};
|
||||
|
||||
try {
|
||||
await build(buildConfig);
|
||||
|
||||
// Copy schema files to dist (needed by chunk files)
|
||||
const schemaDir = path.join(__dirname, 'dist/schema');
|
||||
await mkdir(schemaDir, { recursive: true });
|
||||
await copyFile(
|
||||
path.join(__dirname, 'src/storage/schema/tracking-schema.sql'),
|
||||
path.join(schemaDir, 'tracking-schema.sql')
|
||||
);
|
||||
|
||||
// Copy analysis prompt files (used at runtime by ContentAnalyzer)
|
||||
const promptDir = path.join(__dirname, 'dist/src/analysis/prompts');
|
||||
await mkdir(promptDir, { recursive: true });
|
||||
await copyFile(
|
||||
path.join(__dirname, 'src/analysis/prompts/analyze-article.md'),
|
||||
path.join(promptDir, 'analyze-article.md')
|
||||
);
|
||||
|
||||
console.log('✅ Build complete');
|
||||
} catch (error) {
|
||||
console.error('❌ Build failed:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
9
self-hosting/omc/index.ts
Executable file
9
self-hosting/omc/index.ts
Executable file
|
|
@ -0,0 +1,9 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* OMC Main CLI Entry Point
|
||||
* AIDEV-NOTE: OCLIF CLI runner for OMC commands
|
||||
*/
|
||||
|
||||
import { run } from '@oclif/core';
|
||||
|
||||
await run(process.argv.slice(2), import.meta.url);
|
||||
96
self-hosting/omc/lib/omnivore/client.d.ts
vendored
Normal file
96
self-hosting/omc/lib/omnivore/client.d.ts
vendored
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
export interface OmnivoreHighlight {
|
||||
id: string;
|
||||
quote: string;
|
||||
annotation?: string | null;
|
||||
createdAt: string;
|
||||
updatedAt?: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
export interface OmnivoreLabel {
|
||||
id: string;
|
||||
name: string;
|
||||
color: string;
|
||||
}
|
||||
|
||||
export interface OmnivoreArticle {
|
||||
id: string;
|
||||
title: string;
|
||||
url: string;
|
||||
content?: string | null;
|
||||
description?: string | null;
|
||||
author?: string | null;
|
||||
publishedAt?: string | null;
|
||||
slug?: string;
|
||||
wordCount?: number;
|
||||
createdAt: string;
|
||||
savedAt: string;
|
||||
updatedAt: string;
|
||||
isArchived?: boolean;
|
||||
folder?: string;
|
||||
labels: OmnivoreLabel[];
|
||||
highlights: OmnivoreHighlight[];
|
||||
}
|
||||
|
||||
export interface GetMeResult {
|
||||
id: string;
|
||||
name?: string;
|
||||
email?: string;
|
||||
profile?: { username?: string };
|
||||
}
|
||||
|
||||
export interface GetArticleResult {
|
||||
article?: OmnivoreArticle;
|
||||
errorCodes?: string[];
|
||||
}
|
||||
|
||||
export function getMe(): Promise<GetMeResult>;
|
||||
export function searchArticles(args?: Record<string, unknown>): Promise<any>;
|
||||
export function getArticle(slug: string, username: string): Promise<GetArticleResult>;
|
||||
export function getArticlesByDate(args?: Record<string, unknown>): Promise<any>;
|
||||
export function getArticlesByLabel(labelName: string, first?: number): Promise<any>;
|
||||
export function getRecentArticles(hours?: number, first?: number): Promise<any>;
|
||||
export function searchByTopic(topic: string, first?: number): Promise<any>;
|
||||
export function getUnreadArticles(first?: number): Promise<any>;
|
||||
export function getLabels(): Promise<any>;
|
||||
export function getHighlights(slug: string, username: string): Promise<any[]>;
|
||||
|
||||
export function updatePage(args: {
|
||||
pageId: string;
|
||||
description?: string;
|
||||
title?: string;
|
||||
byline?: string;
|
||||
publishedAt?: string;
|
||||
savedAt?: string;
|
||||
}): Promise<any>;
|
||||
|
||||
export function createLabel(args: {
|
||||
name: string;
|
||||
color?: string;
|
||||
description?: string;
|
||||
}): Promise<any>;
|
||||
|
||||
export function setLabels(args: {
|
||||
pageId: string;
|
||||
labelIds?: string[];
|
||||
labels?: Array<{ name: string; color?: string; description?: string }>;
|
||||
source?: string;
|
||||
}): Promise<any>;
|
||||
|
||||
export function saveUrl(args: {
|
||||
url: string;
|
||||
source: string;
|
||||
clientRequestId: string;
|
||||
folder?: string;
|
||||
savedAt?: string;
|
||||
publishedAt?: string;
|
||||
labels?: Array<{ name: string; color?: string; description?: string }>;
|
||||
timezone?: string;
|
||||
locale?: string;
|
||||
state?: string;
|
||||
}): Promise<any>;
|
||||
|
||||
export function createHighlight(args: Record<string, unknown>): Promise<any>;
|
||||
export function updateHighlight(args: Record<string, unknown>): Promise<any>;
|
||||
export function deleteHighlight(highlightId: string): Promise<any>;
|
||||
export function testConnection(): Promise<boolean>;
|
||||
585
self-hosting/omc/lib/omnivore/client.js
Normal file
585
self-hosting/omc/lib/omnivore/client.js
Normal file
|
|
@ -0,0 +1,585 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Omnivore API Client
|
||||
* GraphQL client for interacting with Omnivore API
|
||||
*/
|
||||
|
||||
import fetch from 'node-fetch';
|
||||
import { config } from 'dotenv';
|
||||
|
||||
// Load environment variables
|
||||
config();
|
||||
|
||||
const API_URL = process.env.OMNIVORE_API_URL || 'https://api-prod.omnivore.app/api/graphql';
|
||||
const API_KEY = process.env.OMNIVORE_API_KEY;
|
||||
|
||||
if (!API_KEY) {
|
||||
console.error('❌ OMNIVORE_API_KEY not set in environment');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute GraphQL query against Omnivore API
|
||||
*/
|
||||
async function graphqlRequest(query, variables = {}) {
|
||||
try {
|
||||
const response = await fetch(API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
// Omnivore self-hosted commonly expects `Omnivore-Authorization`, while some deployments accept `Authorization`.
|
||||
// Sending both keeps the CLI compatible across environments.
|
||||
'Authorization': API_KEY,
|
||||
'Omnivore-Authorization': API_KEY,
|
||||
},
|
||||
body: JSON.stringify({ query, variables }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.errors) {
|
||||
throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`);
|
||||
}
|
||||
|
||||
return result.data;
|
||||
} catch (error) {
|
||||
console.error('GraphQL Request Error:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current authenticated user
|
||||
*/
|
||||
export async function getMe() {
|
||||
const query = `
|
||||
query {
|
||||
me {
|
||||
id
|
||||
name
|
||||
email
|
||||
profile {
|
||||
username
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const data = await graphqlRequest(query);
|
||||
return data.me;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search articles with filters
|
||||
*/
|
||||
export async function searchArticles({
|
||||
query = 'in:all',
|
||||
first = 10,
|
||||
after = '',
|
||||
includeContent = false,
|
||||
} = {}) {
|
||||
const searchQuery = `
|
||||
query Search($query: String!, $first: Int, $after: String, $includeContent: Boolean) {
|
||||
search(query: $query, first: $first, after: $after, includeContent: $includeContent) {
|
||||
... on SearchSuccess {
|
||||
pageInfo {
|
||||
totalCount
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
edges {
|
||||
cursor
|
||||
node {
|
||||
id
|
||||
slug
|
||||
title
|
||||
url
|
||||
originalArticleUrl
|
||||
createdAt
|
||||
updatedAt
|
||||
publishedAt
|
||||
savedAt
|
||||
author
|
||||
description
|
||||
image
|
||||
siteName
|
||||
pageType
|
||||
wordsCount
|
||||
readingProgressTopPercent
|
||||
isArchived
|
||||
folder
|
||||
content
|
||||
labels {
|
||||
id
|
||||
name
|
||||
color
|
||||
}
|
||||
highlights {
|
||||
id
|
||||
quote
|
||||
annotation
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
... on SearchError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const data = await graphqlRequest(searchQuery, { query, first, after, includeContent });
|
||||
|
||||
if (data.search.errorCodes) {
|
||||
throw new Error(`Search Error: ${data.search.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
return data.search;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get article by ID
|
||||
*/
|
||||
export async function getArticle(slug, username) {
|
||||
const query = `
|
||||
query GetArticle($slug: String!, $username: String!) {
|
||||
article(slug: $slug, username: $username) {
|
||||
... on ArticleSuccess {
|
||||
article {
|
||||
id
|
||||
title
|
||||
url
|
||||
content
|
||||
author
|
||||
description
|
||||
publishedAt
|
||||
updatedAt
|
||||
createdAt
|
||||
savedAt
|
||||
highlights(input: {}) {
|
||||
id
|
||||
quote
|
||||
annotation
|
||||
}
|
||||
labels {
|
||||
id
|
||||
name
|
||||
color
|
||||
}
|
||||
}
|
||||
}
|
||||
... on ArticleError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const data = await graphqlRequest(query, { slug, username });
|
||||
return data.article;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get articles saved in a time range
|
||||
*/
|
||||
export async function getArticlesByDate({
|
||||
startDate,
|
||||
endDate,
|
||||
first = 100,
|
||||
} = {}) {
|
||||
const start = startDate || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
|
||||
const end = endDate || new Date().toISOString();
|
||||
|
||||
const query = `saved:>${start.split('T')[0]} saved:<${end.split('T')[0]} sort:saved-desc`;
|
||||
|
||||
return await searchArticles({ query, first, includeContent: false });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get articles with specific labels
|
||||
*/
|
||||
export async function getArticlesByLabel(labelName, first = 50) {
|
||||
const query = `label:${labelName} sort:saved-desc`;
|
||||
return await searchArticles({ query, first });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recently saved articles
|
||||
*/
|
||||
export async function getRecentArticles(hours = 24, first = 50) {
|
||||
let query;
|
||||
|
||||
if (hours <= 24) {
|
||||
query = 'saved:last24hrs';
|
||||
} else if (hours <= 168) {
|
||||
query = 'saved:last7days';
|
||||
} else {
|
||||
const days = Math.ceil(hours / 24);
|
||||
const date = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
|
||||
query = `saved:>${date}`;
|
||||
}
|
||||
|
||||
query += ' sort:saved-desc';
|
||||
return await searchArticles({ query, first, includeContent: false });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get articles by topic/keyword
|
||||
*/
|
||||
export async function searchByTopic(topic, first = 50) {
|
||||
const query = `${topic} sort:saved-desc`;
|
||||
return await searchArticles({ query, first, includeContent: false });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get unread articles
|
||||
*/
|
||||
export async function getUnreadArticles(first = 50) {
|
||||
const query = 'in:inbox is:unread sort:saved-desc';
|
||||
return await searchArticles({ query, first });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all labels
|
||||
*/
|
||||
export async function getLabels() {
|
||||
const query = `
|
||||
query {
|
||||
labels {
|
||||
... on LabelsSuccess {
|
||||
labels {
|
||||
id
|
||||
name
|
||||
color
|
||||
description
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
... on LabelsError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const data = await graphqlRequest(query);
|
||||
|
||||
if (data.labels.errorCodes) {
|
||||
throw new Error(`Labels Error: ${data.labels.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
return data.labels.labels;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get highlights for an article
|
||||
*/
|
||||
export async function getHighlights(slug, username) {
|
||||
const query = `
|
||||
query GetHighlights($slug: String!, $username: String!) {
|
||||
article(slug: $slug, username: $username) {
|
||||
... on ArticleSuccess {
|
||||
article {
|
||||
highlights {
|
||||
id
|
||||
quote
|
||||
annotation
|
||||
createdAt
|
||||
updatedAt
|
||||
type
|
||||
}
|
||||
}
|
||||
}
|
||||
... on ArticleError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const data = await graphqlRequest(query, { slug, username });
|
||||
return data.article?.article?.highlights || [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Update article/page metadata
|
||||
*/
|
||||
export async function updatePage({ pageId, description, title, byline, publishedAt, savedAt }) {
|
||||
const mutation = `
|
||||
mutation UpdatePage($input: UpdatePageInput!) {
|
||||
updatePage(input: $input) {
|
||||
... on UpdatePageSuccess {
|
||||
updatedPage {
|
||||
id
|
||||
title
|
||||
description
|
||||
author
|
||||
}
|
||||
}
|
||||
... on UpdatePageError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const input = { pageId };
|
||||
if (description !== undefined) input.description = description;
|
||||
if (title !== undefined) input.title = title;
|
||||
if (byline !== undefined) input.byline = byline;
|
||||
if (publishedAt !== undefined) input.publishedAt = publishedAt;
|
||||
if (savedAt !== undefined) input.savedAt = savedAt;
|
||||
|
||||
const data = await graphqlRequest(mutation, { input });
|
||||
return data.updatePage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a label.
|
||||
*/
|
||||
export async function createLabel({ name, color, description }) {
|
||||
const mutation = `
|
||||
mutation CreateLabel($input: CreateLabelInput!) {
|
||||
createLabel(input: $input) {
|
||||
... on CreateLabelSuccess {
|
||||
label {
|
||||
id
|
||||
name
|
||||
color
|
||||
description
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
... on CreateLabelError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const input = { name };
|
||||
if (color !== undefined) input.color = color;
|
||||
if (description !== undefined) input.description = description;
|
||||
|
||||
const data = await graphqlRequest(mutation, { input });
|
||||
return data.createLabel;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set labels for a page (replaces existing labels).
|
||||
* Use either `labelIds` or `labels` (CreateLabelInput[]). If `labels` is used, Omnivore may create missing labels.
|
||||
*/
|
||||
export async function setLabels({ pageId, labelIds, labels, source }) {
|
||||
const mutation = `
|
||||
mutation SetLabels($input: SetLabelsInput!) {
|
||||
setLabels(input: $input) {
|
||||
... on SetLabelsSuccess {
|
||||
labels {
|
||||
id
|
||||
name
|
||||
color
|
||||
}
|
||||
}
|
||||
... on SetLabelsError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const input = { pageId };
|
||||
if (labelIds !== undefined) input.labelIds = labelIds;
|
||||
if (labels !== undefined) input.labels = labels;
|
||||
if (source !== undefined) input.source = source;
|
||||
|
||||
const data = await graphqlRequest(mutation, { input });
|
||||
return data.setLabels;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a URL into Omnivore (creates a new library item / saving request).
|
||||
*/
|
||||
export async function saveUrl({ url, source, folder, savedAt, publishedAt, labels, timezone, locale, state, clientRequestId }) {
|
||||
const mutation = `
|
||||
mutation SaveUrl($input: SaveUrlInput!) {
|
||||
saveUrl(input: $input) {
|
||||
... on SaveSuccess {
|
||||
url
|
||||
clientRequestId
|
||||
}
|
||||
... on SaveError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const input = { url, source, clientRequestId };
|
||||
if (folder !== undefined) input.folder = folder;
|
||||
if (savedAt !== undefined) input.savedAt = savedAt;
|
||||
if (publishedAt !== undefined) input.publishedAt = publishedAt;
|
||||
if (labels !== undefined) input.labels = labels;
|
||||
if (timezone !== undefined) input.timezone = timezone;
|
||||
if (locale !== undefined) input.locale = locale;
|
||||
if (state !== undefined) input.state = state;
|
||||
|
||||
const data = await graphqlRequest(mutation, { input });
|
||||
return data.saveUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create highlight with annotation (or standalone NOTE)
|
||||
*/
|
||||
export async function createHighlight({ id, shortId, articleId, quote = '', annotation, patch = '', prefix = '', suffix = '', color = '#FFD700', type = 'HIGHLIGHT' }) {
|
||||
const mutation = `
|
||||
mutation CreateHighlight($input: CreateHighlightInput!) {
|
||||
createHighlight(input: $input) {
|
||||
... on CreateHighlightSuccess {
|
||||
highlight {
|
||||
id
|
||||
quote
|
||||
annotation
|
||||
color
|
||||
type
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
... on CreateHighlightError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const input = {
|
||||
id,
|
||||
shortId,
|
||||
articleId,
|
||||
patch,
|
||||
prefix,
|
||||
suffix,
|
||||
color,
|
||||
type
|
||||
};
|
||||
|
||||
if (quote) input.quote = quote;
|
||||
if (annotation) input.annotation = annotation;
|
||||
|
||||
const data = await graphqlRequest(mutation, { input });
|
||||
return data.createHighlight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update highlight/note annotation
|
||||
*/
|
||||
export async function updateHighlight({ highlightId, annotation, quote, html, color }) {
|
||||
const mutation = `
|
||||
mutation UpdateHighlight($input: UpdateHighlightInput!) {
|
||||
updateHighlight(input: $input) {
|
||||
... on UpdateHighlightSuccess {
|
||||
highlight {
|
||||
id
|
||||
annotation
|
||||
updatedAt
|
||||
}
|
||||
}
|
||||
... on UpdateHighlightError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const input = { highlightId };
|
||||
if (annotation !== undefined) input.annotation = annotation;
|
||||
if (quote !== undefined) input.quote = quote;
|
||||
if (html !== undefined) input.html = html;
|
||||
if (color !== undefined) input.color = color;
|
||||
|
||||
const data = await graphqlRequest(mutation, { input });
|
||||
return data.updateHighlight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete highlight/note
|
||||
*/
|
||||
export async function deleteHighlight(highlightId) {
|
||||
const mutation = `
|
||||
mutation DeleteHighlight($highlightId: ID!) {
|
||||
deleteHighlight(highlightId: $highlightId) {
|
||||
... on DeleteHighlightSuccess {
|
||||
highlight {
|
||||
id
|
||||
}
|
||||
}
|
||||
... on DeleteHighlightError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const data = await graphqlRequest(mutation, { highlightId });
|
||||
return data.deleteHighlight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test API connection
|
||||
*/
|
||||
export async function testConnection() {
|
||||
try {
|
||||
const user = await getMe();
|
||||
console.log('✅ Connected to Omnivore API');
|
||||
console.log(` User: ${user.name} (${user.email})`);
|
||||
console.log(` Username: ${user.profile.username}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to connect to Omnivore API');
|
||||
console.error(` ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// CLI usage
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.includes('--test')) {
|
||||
await testConnection();
|
||||
} else if (args.includes('--recent')) {
|
||||
const hours = parseInt(args[args.indexOf('--recent') + 1]) || 24;
|
||||
const result = await getRecentArticles(hours);
|
||||
console.log(`Found ${result.pageInfo.totalCount} articles from last ${hours} hours`);
|
||||
result.edges.forEach(({ node }) => {
|
||||
console.log(` - ${node.title}`);
|
||||
console.log(` ${node.url}`);
|
||||
console.log(` Saved: ${node.savedAt}`);
|
||||
});
|
||||
} else {
|
||||
console.log('Usage:');
|
||||
console.log(' node client.js --test # Test API connection');
|
||||
console.log(' node client.js --recent [hours] # Get recent articles');
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
graphqlRequest,
|
||||
getMe,
|
||||
searchArticles,
|
||||
getArticle,
|
||||
getArticlesByDate,
|
||||
getArticlesByLabel,
|
||||
getRecentArticles,
|
||||
searchByTopic,
|
||||
getUnreadArticles,
|
||||
getLabels,
|
||||
getHighlights,
|
||||
testConnection,
|
||||
};
|
||||
369
self-hosting/omc/lib/omnivore/queries.js
Normal file
369
self-hosting/omc/lib/omnivore/queries.js
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
/**
|
||||
* Pre-built GraphQL Queries for Omnivore API
|
||||
* Collection of common query patterns for content analysis
|
||||
*/
|
||||
|
||||
/**
|
||||
* Search query with all fields needed for content analysis
|
||||
*/
|
||||
export const SEARCH_ARTICLES_FULL = `
|
||||
query SearchArticlesFull($query: String!, $first: Int, $after: String) {
|
||||
search(query: $query, first: $first, after: $after, includeContent: false) {
|
||||
... on SearchSuccess {
|
||||
pageInfo {
|
||||
totalCount
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
edges {
|
||||
cursor
|
||||
node {
|
||||
id
|
||||
title
|
||||
url
|
||||
originalArticleUrl
|
||||
slug
|
||||
createdAt
|
||||
updatedAt
|
||||
publishedAt
|
||||
savedAt
|
||||
author
|
||||
description
|
||||
image
|
||||
siteName
|
||||
pageType
|
||||
wordCount
|
||||
readingProgressTopPercent
|
||||
readingProgressBottomPercent
|
||||
isArchived
|
||||
folder
|
||||
labels {
|
||||
id
|
||||
name
|
||||
color
|
||||
}
|
||||
highlights {
|
||||
id
|
||||
quote
|
||||
annotation
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
... on SearchError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
/**
|
||||
* Get article with full content
|
||||
*/
|
||||
export const GET_ARTICLE_FULL = `
|
||||
query GetArticleFull($slug: String!, $username: String!) {
|
||||
article(slug: $slug, username: $username) {
|
||||
... on ArticleSuccess {
|
||||
article {
|
||||
id
|
||||
title
|
||||
url
|
||||
slug
|
||||
content
|
||||
author
|
||||
description
|
||||
image
|
||||
publishedAt
|
||||
createdAt
|
||||
savedAt
|
||||
updatedAt
|
||||
wordCount
|
||||
siteName
|
||||
originalArticleUrl
|
||||
highlights {
|
||||
id
|
||||
quote
|
||||
annotation
|
||||
createdAt
|
||||
}
|
||||
labels {
|
||||
id
|
||||
name
|
||||
color
|
||||
description
|
||||
}
|
||||
}
|
||||
}
|
||||
... on ArticleError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
/**
|
||||
* Get user profile
|
||||
*/
|
||||
export const GET_USER_PROFILE = `
|
||||
query GetUserProfile {
|
||||
me {
|
||||
id
|
||||
name
|
||||
email
|
||||
profile {
|
||||
username
|
||||
pictureUrl
|
||||
bio
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
/**
|
||||
* Get all labels
|
||||
*/
|
||||
export const GET_LABELS = `
|
||||
query GetLabels {
|
||||
labels {
|
||||
... on LabelsSuccess {
|
||||
labels {
|
||||
id
|
||||
name
|
||||
color
|
||||
description
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
... on LabelsError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
/**
|
||||
* Search with content included (for analysis)
|
||||
*/
|
||||
export const SEARCH_WITH_CONTENT = `
|
||||
query SearchWithContent($query: String!, $first: Int, $after: String) {
|
||||
search(query: $query, first: $first, after: $after, includeContent: true) {
|
||||
... on SearchSuccess {
|
||||
pageInfo {
|
||||
totalCount
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
edges {
|
||||
cursor
|
||||
node {
|
||||
id
|
||||
title
|
||||
url
|
||||
content
|
||||
author
|
||||
description
|
||||
publishedAt
|
||||
savedAt
|
||||
siteName
|
||||
labels {
|
||||
name
|
||||
}
|
||||
highlights {
|
||||
quote
|
||||
annotation
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
... on SearchError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
/**
|
||||
* Common search query patterns
|
||||
*/
|
||||
export const QUERY_PATTERNS = {
|
||||
// Time-based
|
||||
LAST_24_HOURS: 'saved:last24hrs sort:saved-desc',
|
||||
LAST_7_DAYS: 'saved:last7days sort:saved-desc',
|
||||
LAST_30_DAYS: 'saved:last30days sort:saved-desc',
|
||||
|
||||
// Status-based
|
||||
INBOX: 'in:inbox sort:saved-desc',
|
||||
ARCHIVED: 'in:archive sort:saved-desc',
|
||||
UNREAD: 'is:unread sort:saved-desc',
|
||||
READ: 'is:read sort:saved-desc',
|
||||
|
||||
// Type-based
|
||||
ARTICLES: 'type:article sort:saved-desc',
|
||||
HIGHLIGHTS: 'type:highlights sort:saved-desc',
|
||||
|
||||
// Content-based
|
||||
WITH_HIGHLIGHTS: 'has:highlights sort:saved-desc',
|
||||
WITH_LABELS: 'has:labels sort:saved-desc',
|
||||
NO_LABELS: 'no:label sort:saved-desc',
|
||||
|
||||
// Topic-based (AI/Tech)
|
||||
AI_ML: '(ai OR "machine learning" OR llm OR "language model") sort:saved-desc',
|
||||
DEVOPS: '(devops OR kubernetes OR docker OR "cloud native") sort:saved-desc',
|
||||
DATABASES: '(database OR postgres OR mongodb OR redis) sort:saved-desc',
|
||||
PROGRAMMING: '(programming OR coding OR "software engineering") sort:saved-desc',
|
||||
STARTUPS: '(startup OR founder OR "venture capital" OR product) sort:saved-desc',
|
||||
|
||||
// Combined patterns
|
||||
RECENT_AI: 'saved:last7days (ai OR llm OR "machine learning") sort:saved-desc',
|
||||
RECENT_TECH: 'saved:last7days (programming OR devops OR database) sort:saved-desc',
|
||||
UNREAD_ARTICLES: 'in:inbox is:unread type:article sort:saved-desc',
|
||||
};
|
||||
|
||||
/**
|
||||
* Build date range query
|
||||
*/
|
||||
export function buildDateRangeQuery(startDate, endDate, sortBy = 'saved-desc') {
|
||||
const start = typeof startDate === 'string' ? startDate : startDate.toISOString().split('T')[0];
|
||||
const end = typeof endDate === 'string' ? endDate : endDate.toISOString().split('T')[0];
|
||||
|
||||
return `saved:>${start} saved:<${end} sort:${sortBy}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build label query
|
||||
*/
|
||||
export function buildLabelQuery(labels, operator = 'OR') {
|
||||
if (Array.isArray(labels)) {
|
||||
return labels.map(l => `label:${l}`).join(` ${operator} `);
|
||||
}
|
||||
return `label:${labels}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build keyword query
|
||||
*/
|
||||
export function buildKeywordQuery(keywords, operator = 'OR') {
|
||||
if (Array.isArray(keywords)) {
|
||||
const quoted = keywords.map(k => `"${k}"`);
|
||||
return `(${quoted.join(` ${operator} `)})`;
|
||||
}
|
||||
return `"${keywords}"`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build complex query
|
||||
*/
|
||||
export function buildComplexQuery({
|
||||
keywords = [],
|
||||
labels = [],
|
||||
timeRange = 'last7days',
|
||||
status = null,
|
||||
hasHighlights = null,
|
||||
sortBy = 'saved-desc',
|
||||
} = {}) {
|
||||
const parts = [];
|
||||
|
||||
// Time range
|
||||
if (timeRange === 'last24hrs' || timeRange === 'last7days' || timeRange === 'last30days') {
|
||||
parts.push(`saved:${timeRange}`);
|
||||
} else if (timeRange.includes('>') || timeRange.includes('<')) {
|
||||
parts.push(timeRange);
|
||||
}
|
||||
|
||||
// Keywords
|
||||
if (keywords.length > 0) {
|
||||
parts.push(buildKeywordQuery(keywords));
|
||||
}
|
||||
|
||||
// Labels
|
||||
if (labels.length > 0) {
|
||||
parts.push(buildLabelQuery(labels));
|
||||
}
|
||||
|
||||
// Status
|
||||
if (status) {
|
||||
if (status === 'inbox') parts.push('in:inbox');
|
||||
else if (status === 'archived') parts.push('in:archive');
|
||||
else if (status === 'unread') parts.push('is:unread');
|
||||
else if (status === 'read') parts.push('is:read');
|
||||
}
|
||||
|
||||
// Highlights
|
||||
if (hasHighlights === true) parts.push('has:highlights');
|
||||
else if (hasHighlights === false) parts.push('no:highlights');
|
||||
|
||||
// Sort
|
||||
if (sortBy) parts.push(`sort:${sortBy}`);
|
||||
|
||||
return parts.join(' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Topic-based query builders
|
||||
*/
|
||||
export const TOPIC_QUERIES = {
|
||||
ai: {
|
||||
keywords: ['ai', 'artificial intelligence', 'machine learning', 'llm', 'language model', 'gpt', 'claude', 'neural network'],
|
||||
description: 'AI and Machine Learning',
|
||||
},
|
||||
devops: {
|
||||
keywords: ['devops', 'kubernetes', 'docker', 'ci/cd', 'cloud native', 'infrastructure', 'terraform'],
|
||||
description: 'DevOps and Infrastructure',
|
||||
},
|
||||
programming: {
|
||||
keywords: ['programming', 'coding', 'software engineering', 'algorithm', 'data structure', 'clean code'],
|
||||
description: 'Programming and Software Engineering',
|
||||
},
|
||||
databases: {
|
||||
keywords: ['database', 'sql', 'nosql', 'postgres', 'mongodb', 'redis', 'query optimization'],
|
||||
description: 'Databases and Data Engineering',
|
||||
},
|
||||
web: {
|
||||
keywords: ['web development', 'javascript', 'typescript', 'react', 'vue', 'frontend', 'backend'],
|
||||
description: 'Web Development',
|
||||
},
|
||||
cloud: {
|
||||
keywords: ['aws', 'azure', 'gcp', 'cloud computing', 'serverless', 'lambda'],
|
||||
description: 'Cloud Computing',
|
||||
},
|
||||
startup: {
|
||||
keywords: ['startup', 'founder', 'entrepreneurship', 'venture capital', 'product market fit', 'growth'],
|
||||
description: 'Startups and Business',
|
||||
},
|
||||
security: {
|
||||
keywords: ['security', 'cybersecurity', 'authentication', 'encryption', 'vulnerability'],
|
||||
description: 'Security',
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Build query for specific topic
|
||||
*/
|
||||
export function buildTopicQuery(topicKey, timeRange = 'last7days') {
|
||||
const topic = TOPIC_QUERIES[topicKey];
|
||||
if (!topic) {
|
||||
throw new Error(`Unknown topic: ${topicKey}`);
|
||||
}
|
||||
|
||||
return buildComplexQuery({
|
||||
keywords: topic.keywords,
|
||||
timeRange,
|
||||
sortBy: 'saved-desc',
|
||||
});
|
||||
}
|
||||
|
||||
export default {
|
||||
SEARCH_ARTICLES_FULL,
|
||||
GET_ARTICLE_FULL,
|
||||
GET_USER_PROFILE,
|
||||
GET_LABELS,
|
||||
SEARCH_WITH_CONTENT,
|
||||
QUERY_PATTERNS,
|
||||
buildDateRangeQuery,
|
||||
buildLabelQuery,
|
||||
buildKeywordQuery,
|
||||
buildComplexQuery,
|
||||
buildTopicQuery,
|
||||
TOPIC_QUERIES,
|
||||
};
|
||||
92
self-hosting/omc/package.json
Normal file
92
self-hosting/omc/package.json
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
{
|
||||
"name": "omnivore-content-system",
|
||||
"version": "1.0.0",
|
||||
"description": "AI-powered content monetization system for Omnivore reading - generates blog posts, newsletters, and tracks trends",
|
||||
"packageManager": "pnpm@10.14.0",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"omc": "./dist/bin/omc.js"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "node esbuild.config.mjs",
|
||||
"prebuild": "rm -rf dist",
|
||||
"build:watch": "tsc -p tsconfig.json --watch",
|
||||
"codegen": "graphql-codegen --config codegen.yml",
|
||||
"codegen:watch": "graphql-codegen --config codegen.yml --watch",
|
||||
"dev": "tsx watch index.ts",
|
||||
"clean": "rm -rf dist",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest --watch",
|
||||
"fetch": "corepack pnpm -s run build && node dist/bin/omc.js queue add --hours 24",
|
||||
"analyze:auto": "corepack pnpm -s run build && node dist/bin/omc.js analyze auto --batch-size 5",
|
||||
"analyze:retry": "corepack pnpm -s run build && node dist/bin/omc.js analyze retry --failed",
|
||||
"report:corpus": "corepack pnpm -s run build && node dist/bin/omc.js report corpus"
|
||||
},
|
||||
"keywords": [
|
||||
"omnivore",
|
||||
"content",
|
||||
"blog",
|
||||
"newsletter",
|
||||
"ai",
|
||||
"claude",
|
||||
"agent"
|
||||
],
|
||||
"author": "",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.1.1",
|
||||
"@anthropic-ai/sdk": "^0.20.0",
|
||||
"@graphql-typed-document-node/core": "^3.2.0",
|
||||
"@oclif/core": "^3.26.6",
|
||||
"@oclif/plugin-help": "^6.0.21",
|
||||
"better-sqlite3": "^12.4.1",
|
||||
"chalk": "^5.3.0",
|
||||
"csv-parse": "^6.1.0",
|
||||
"dotenv": "^16.4.0",
|
||||
"glob": "^11.0.3",
|
||||
"graphql": "^16.11.0",
|
||||
"graphql-tag": "^2.12.6",
|
||||
"gray-matter": "^4.0.3",
|
||||
"markdown-it": "^14.0.0",
|
||||
"node-fetch": "^3.3.2",
|
||||
"p-limit": "^5.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@graphql-codegen/cli": "^6.0.0",
|
||||
"@graphql-codegen/typed-document-node": "^6.0.1",
|
||||
"@graphql-codegen/typescript": "^5.0.1",
|
||||
"@graphql-codegen/typescript-operations": "^5.0.1",
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/node": "^20.0.0",
|
||||
"esbuild": "^0.19.0",
|
||||
"nodemon": "^3.0.0",
|
||||
"tsx": "^4.0.0",
|
||||
"typescript": "^5.4.0",
|
||||
"vitest": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"oclif": {
|
||||
"commands": "./dist/src/commands",
|
||||
"bin": "omc",
|
||||
"dirname": "omc",
|
||||
"plugins": [
|
||||
"@oclif/plugin-help"
|
||||
],
|
||||
"topicSeparator": " ",
|
||||
"additionalHelpFlags": [
|
||||
"-h"
|
||||
],
|
||||
"additionalVersionFlags": [
|
||||
"-v"
|
||||
]
|
||||
},
|
||||
"pnpm": {
|
||||
"onlyBuiltDependencies": [
|
||||
"better-sqlite3",
|
||||
"esbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
47
self-hosting/omc/scripts/cleanup-duplicate-notes.mjs
Executable file
47
self-hosting/omc/scripts/cleanup-duplicate-notes.mjs
Executable file
|
|
@ -0,0 +1,47 @@
|
|||
#!/usr/bin/env node
|
||||
import { execSync } from 'child_process';
|
||||
import { getHighlights, deleteHighlight, getMe } from '../lib/omnivore/client.js';
|
||||
import Database from 'better-sqlite3';
|
||||
|
||||
const articleId = process.argv[2];
|
||||
if (!articleId) {
|
||||
console.error('Usage: node scripts/cleanup-duplicate-notes.mjs <article-id>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Use CLI to get article slug
|
||||
const db = Database(process.env.DATABASE_PATH || 'data/omnivore-content.db');
|
||||
const job = db.prepare('SELECT article_slug FROM analysis_queue WHERE article_id = ?').get(articleId);
|
||||
db.close();
|
||||
|
||||
if (!job) {
|
||||
console.error(`Article not found: ${articleId}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Get username from Omnivore API
|
||||
const user = await getMe();
|
||||
const username = user.profile.username;
|
||||
|
||||
const highlights = await getHighlights(job.article_slug, username);
|
||||
const notes = highlights.filter(h => h.type === 'NOTE');
|
||||
|
||||
console.log(`Found ${notes.length} notes total`);
|
||||
|
||||
if (notes.length > 0) {
|
||||
console.log(`\nFirst note content (first 500 chars):`);
|
||||
console.log(notes[0].annotation?.substring(0, 500));
|
||||
}
|
||||
|
||||
const analysisNotes = notes.filter(h => h.annotation?.includes('_Analysis generated by Omnivore Content System_'));
|
||||
const duplicates = notes.filter(h => !h.annotation?.includes('_Analysis generated by Omnivore Content System_'));
|
||||
|
||||
console.log(`\nAnalysis notes with marker: ${analysisNotes.length}`);
|
||||
console.log(`Duplicate notes to delete: ${duplicates.length}`);
|
||||
|
||||
for (const note of duplicates) {
|
||||
console.log(`Deleting note ${note.id}...`);
|
||||
await deleteHighlight(note.id);
|
||||
}
|
||||
|
||||
console.log('✓ Cleanup complete');
|
||||
25
self-hosting/omc/scripts/daily-analysis.sh
Executable file
25
self-hosting/omc/scripts/daily-analysis.sh
Executable file
|
|
@ -0,0 +1,25 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
REPO_DIR="${REPO_DIR:-/Volumes/devel/personal/omnivore-content-system}"
|
||||
BATCH_SIZE="${BATCH_SIZE:-5}"
|
||||
HOURS="${HOURS:-24}"
|
||||
|
||||
cd "$REPO_DIR"
|
||||
|
||||
mkdir -p content/corpus-reports
|
||||
|
||||
# Ensure dependencies/build are available. Prefer Corepack-managed pnpm so native modules
|
||||
# (better-sqlite3) are built for the active Node version.
|
||||
if command -v corepack >/dev/null 2>&1; then
|
||||
corepack install >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
corepack pnpm -s run build
|
||||
|
||||
node dist/bin/omc.js queue add --hours "$HOURS"
|
||||
node dist/bin/omc.js analyze auto --batch-size "$BATCH_SIZE"
|
||||
node dist/bin/omc.js analyze retry --failed
|
||||
|
||||
node dist/bin/omc.js report corpus > "content/corpus-reports/$(date +%Y-%m-%d)-daily.md"
|
||||
|
||||
85
self-hosting/omc/src/analysis/ContentAnalyzer.ts
Normal file
85
self-hosting/omc/src/analysis/ContentAnalyzer.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
import type { AnalysisRequest, ContentAnalysis } from '@omc-types/analysis.js';
|
||||
import { analyzeArticleWithCodexCli } from '@lib/ai/codex-cli-client.js';
|
||||
|
||||
export interface AnalyzeArticleInput extends AnalysisRequest {
|
||||
articleId: string;
|
||||
}
|
||||
|
||||
export interface ContentAnalyzerOptions {
|
||||
promptPath?: string;
|
||||
model?: string;
|
||||
timeoutMs?: number;
|
||||
maxAttempts?: number;
|
||||
}
|
||||
|
||||
export class ContentAnalyzer {
|
||||
private readonly prompt: string;
|
||||
private readonly options: ContentAnalyzerOptions;
|
||||
|
||||
constructor(options: ContentAnalyzerOptions = {}) {
|
||||
this.options = options;
|
||||
this.prompt = loadPromptText(options.promptPath);
|
||||
}
|
||||
|
||||
async analyzeArticle(input: AnalyzeArticleInput): Promise<ContentAnalysis> {
|
||||
const { articleId, ...request } = input;
|
||||
const { analysis } = await analyzeArticleWithCodexCli(request, this.prompt, {
|
||||
model: this.options.model,
|
||||
timeoutMs: this.options.timeoutMs,
|
||||
maxAttempts: this.options.maxAttempts,
|
||||
});
|
||||
return normalizeAnalysis(articleId, analysis);
|
||||
}
|
||||
}
|
||||
|
||||
function loadPromptText(explicitPath?: string): string {
|
||||
const candidates = [
|
||||
explicitPath,
|
||||
process.env.OMC_ANALYZE_PROMPT_PATH,
|
||||
promptPathNearThisFile(),
|
||||
join(process.cwd(), 'src/analysis/prompts/analyze-article.md'),
|
||||
]
|
||||
.filter(Boolean) as string[];
|
||||
for (const p of candidates) {
|
||||
if (existsSync(p)) return readFileSync(p, 'utf-8');
|
||||
}
|
||||
throw new Error('Could not locate analyze-article.md prompt (set OMC_ANALYZE_PROMPT_PATH)');
|
||||
}
|
||||
|
||||
function promptPathNearThisFile(): string {
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
return join(here, 'prompts', 'analyze-article.md');
|
||||
}
|
||||
|
||||
function normalizeAnalysis(articleId: string, analysis: ContentAnalysis): ContentAnalysis {
|
||||
return {
|
||||
...fillDefaults(analysis),
|
||||
articleId,
|
||||
analyzedAt: analysis.analyzedAt || new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function fillDefaults(analysis: ContentAnalysis): ContentAnalysis {
|
||||
return {
|
||||
...analysis,
|
||||
topics: analysis.topics ?? [],
|
||||
topicScores: analysis.topicScores ?? {},
|
||||
keyPoints: analysis.keyPoints ?? [],
|
||||
contentType: analysis.contentType || 'N/A',
|
||||
problemStatement: analysis.problemStatement || 'N/A',
|
||||
audienceLevel: analysis.audienceLevel || 'N/A',
|
||||
technologiesMentioned: analysis.technologiesMentioned ?? [],
|
||||
companiesMentioned: analysis.companiesMentioned ?? [],
|
||||
peopleMentioned: analysis.peopleMentioned ?? [],
|
||||
conceptsExplained: analysis.conceptsExplained ?? [],
|
||||
relatedTechnologies: analysis.relatedTechnologies ?? [],
|
||||
useCases: analysis.useCases ?? [],
|
||||
targetKeywords: analysis.targetKeywords ?? [],
|
||||
searchQuestions: analysis.searchQuestions ?? [],
|
||||
githubRepo: analysis.githubRepo || 'N/A',
|
||||
releaseInfo: analysis.releaseInfo || 'N/A',
|
||||
};
|
||||
}
|
||||
160
self-hosting/omc/src/analysis/analyze-auto-runner.ts
Normal file
160
self-hosting/omc/src/analysis/analyze-auto-runner.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
import { existsSync, mkdirSync, writeFileSync, unlinkSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import type { AnalysisJob, AnalysisQueueRepository } from '@storage/AnalysisQueueRepository.js';
|
||||
import type { ContentAnalyzer, AnalyzeArticleInput } from '@analysis/ContentAnalyzer.js';
|
||||
import type { AnalysisWriter } from '@storage/AnalysisWriter.js';
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
import type { OmnivoreArticle } from '@omc-types/omnivore.js';
|
||||
import { getArticle } from '@lib/omnivore/client.js';
|
||||
import type { AnalysisJsonlRecord } from '@storage/AnalysisWriter.js';
|
||||
|
||||
export interface AnalyzeAutoRunnerDeps {
|
||||
jobs: AnalysisJob[];
|
||||
username: string;
|
||||
analyzer: ContentAnalyzer;
|
||||
writer: AnalysisWriter;
|
||||
repo: AnalysisQueueRepository;
|
||||
keepTemp: boolean;
|
||||
jsonlPath?: string;
|
||||
}
|
||||
|
||||
export interface AnalyzeAutoRunnerResult {
|
||||
saved: number;
|
||||
failed: number;
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
export async function runAnalyzeAuto(deps: AnalyzeAutoRunnerDeps): Promise<AnalyzeAutoRunnerResult> {
|
||||
ensureTempDir();
|
||||
const totals = { saved: 0, failed: 0, skipped: 0 };
|
||||
for (const job of deps.jobs) {
|
||||
const outcome = await runOne(job, deps);
|
||||
totals[outcome]++;
|
||||
}
|
||||
return totals;
|
||||
}
|
||||
|
||||
async function runOne(job: AnalysisJob, deps: AnalyzeAutoRunnerDeps): Promise<'saved' | 'failed' | 'skipped'> {
|
||||
const article = await fetchArticle(job.articleSlug, deps.username);
|
||||
if (!article) return markFailure(deps.repo, job.articleId, 'Failed to fetch article from Omnivore');
|
||||
if (!article.content) return markFailure(deps.repo, job.articleId, 'Omnivore returned no content for article');
|
||||
|
||||
const input = buildAnalyzeInput(job.articleId, article);
|
||||
const analysis = await deps.analyzer.analyzeArticle(input);
|
||||
const tempPath = writeTempRecord(job, article, deps.username, analysis);
|
||||
return await saveAndMarkComplete(job, article, analysis, tempPath, deps);
|
||||
}
|
||||
|
||||
async function fetchArticle(slug: string, username: string): Promise<OmnivoreArticle | null> {
|
||||
try {
|
||||
const result = await getArticle(slug, username);
|
||||
if (result.errorCodes?.length) return null;
|
||||
return result.article ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function buildAnalyzeInput(articleId: string, article: OmnivoreArticle): AnalyzeArticleInput {
|
||||
const content = String(article.content ?? '');
|
||||
return {
|
||||
articleId,
|
||||
title: article.title,
|
||||
author: article.author ?? undefined,
|
||||
url: article.url,
|
||||
content,
|
||||
wordCount: estimateWordCount(content),
|
||||
highlights: (article.highlights ?? []).map((h) => ({
|
||||
quote: h.quote,
|
||||
annotation: h.annotation ?? undefined,
|
||||
})),
|
||||
publishedAt: article.publishedAt ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function writeTempRecord(job: AnalysisJob, article: OmnivoreArticle, username: string, analysis: ContentAnalysis): string {
|
||||
const path = join('temp', `${job.articleSlug}.jsonl`);
|
||||
const record = {
|
||||
articleId: job.articleId,
|
||||
articleSlug: job.articleSlug,
|
||||
username,
|
||||
articleUrl: article.url,
|
||||
articleTitle: article.title,
|
||||
savedAt: article.savedAt,
|
||||
publishedAt: article.publishedAt ?? null,
|
||||
updatedAt: article.updatedAt ?? null,
|
||||
analysis,
|
||||
};
|
||||
writeFileSync(path, JSON.stringify(record) + '\n', 'utf-8');
|
||||
return path;
|
||||
}
|
||||
|
||||
async function saveAndMarkComplete(
|
||||
job: AnalysisJob,
|
||||
article: OmnivoreArticle,
|
||||
analysis: ContentAnalysis,
|
||||
tempPath: string,
|
||||
deps: AnalyzeAutoRunnerDeps
|
||||
): Promise<'saved' | 'failed'> {
|
||||
try {
|
||||
const mdPath = await deps.writer.write(job.articleId, article.url, article.title, article.savedAt, analysis, job.articleSlug);
|
||||
deps.repo.storeAnalysis(job.articleId, article.publishedAt ?? null, article.updatedAt ?? null, JSON.stringify(analysis), mdPath);
|
||||
if (deps.jsonlPath) {
|
||||
await deps.writer.appendToJsonl(deps.jsonlPath, buildJsonlRecord(job, article, analysis, mdPath));
|
||||
}
|
||||
if (!deps.keepTemp) unlinkSync(tempPath);
|
||||
return 'saved';
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
deps.repo.markFailed(job.articleId, `Save failed: ${message}`);
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
function buildJsonlRecord(job: AnalysisJob, article: OmnivoreArticle, analysis: ContentAnalysis, markdownPath: string): AnalysisJsonlRecord {
|
||||
return {
|
||||
articleId: job.articleId,
|
||||
articleSlug: job.articleSlug,
|
||||
articleUrl: article.url,
|
||||
articleTitle: article.title,
|
||||
savedAt: article.savedAt,
|
||||
publishedAt: article.publishedAt ?? null,
|
||||
updatedAt: article.updatedAt ?? null,
|
||||
markdownPath,
|
||||
analyzedAt: analysis.analyzedAt,
|
||||
topics: analysis.topics,
|
||||
topicScores: analysis.topicScores,
|
||||
sentiment: analysis.sentiment,
|
||||
summary: analysis.summary,
|
||||
keyPoints: analysis.keyPoints,
|
||||
monetizationAngle: analysis.monetizationAngle,
|
||||
contentType: analysis.contentType,
|
||||
problemStatement: analysis.problemStatement,
|
||||
audienceLevel: analysis.audienceLevel,
|
||||
technologiesMentioned: analysis.technologiesMentioned,
|
||||
companiesMentioned: analysis.companiesMentioned,
|
||||
peopleMentioned: analysis.peopleMentioned,
|
||||
conceptsExplained: analysis.conceptsExplained,
|
||||
relatedTechnologies: analysis.relatedTechnologies,
|
||||
useCases: analysis.useCases,
|
||||
targetKeywords: analysis.targetKeywords,
|
||||
searchQuestions: analysis.searchQuestions,
|
||||
githubRepo: analysis.githubRepo,
|
||||
releaseInfo: analysis.releaseInfo,
|
||||
};
|
||||
}
|
||||
|
||||
function markFailure(repo: AnalysisQueueRepository, articleId: string, message: string): 'failed' {
|
||||
repo.markFailed(articleId, message);
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
function ensureTempDir(): void {
|
||||
if (!existsSync('temp')) mkdirSync('temp', { recursive: true });
|
||||
}
|
||||
|
||||
function estimateWordCount(text: string): number {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return 0;
|
||||
return trimmed.split(/\s+/).length;
|
||||
}
|
||||
129
self-hosting/omc/src/analysis/prompts/analyze-article.md
Normal file
129
self-hosting/omc/src/analysis/prompts/analyze-article.md
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
# Article Content Analysis Prompt
|
||||
|
||||
You are an elite content strategist and knowledge graph architect specializing in evidence-based article analysis for content monetization. Your dual expertise lies in extracting monetizable insights while building structured metadata for corpus-wide knowledge linking.
|
||||
|
||||
## Primary Directive: Anti-Hallucination Rule
|
||||
|
||||
This rule applies to EVERY field you extract - it is non-negotiable:
|
||||
|
||||
- If information is NOT explicitly present in the article, use "N/A" (strings) or ["N/A"] (arrays)
|
||||
- DO NOT infer, guess, estimate, or use your training data
|
||||
- Only extract what is directly stated in the article text
|
||||
- When uncertain, always use "N/A" - data integrity trumps completeness
|
||||
|
||||
**Why this matters**: Hallucinated data corrupts the knowledge graph and misleads content strategy. "N/A" enables accurate corpus analysis; invented data causes systematic failures.
|
||||
|
||||
## Task
|
||||
|
||||
Analyze the provided article content and extract structured insights in JSON format. You are PURELY ADDITIVE - you will receive article metadata and ONLY add the `analysis` field.
|
||||
|
||||
## Analysis Fields
|
||||
|
||||
### Core Analysis (Always Required)
|
||||
|
||||
**topics** (array, 2-5 items): Main topics from approved categories:
|
||||
- AI & Machine Learning, Developer Tools, Software Engineering, DevOps & Infrastructure
|
||||
- Databases, Cloud, Startups & Business, Security
|
||||
|
||||
**topicScores** (object): Confidence 0-1 for each topic
|
||||
- 0.90-1.0: Core focus
|
||||
- 0.70-0.89: Significant discussion
|
||||
- 0.50-0.69: Mentioned but not central
|
||||
- <0.50: Exclude
|
||||
|
||||
**summary** (string, 2-3 sentences): Focus on "so what?" - why this matters to developers/tech professionals. **Chain-of-thought**: (1) What's the main point? (2) Why does it matter? (3) What's the implication?
|
||||
|
||||
**keyPoints** (array, 3-5 items): Actionable insights, surprising facts, or important takeaways readers can learn or apply. Not generic observations.
|
||||
|
||||
**sentiment** (string): "positive" (optimistic, solutions-focused), "neutral" (informative, balanced), "negative" (critical, problems-focused)
|
||||
|
||||
**monetizationAngle** (string): Specific content opportunity - comparison post, tutorial, weekly roundup, deep dive, contrarian take
|
||||
|
||||
### Content Planning (Evidence-Based)
|
||||
|
||||
**contentType** (string): Open-ended description of article type - "getting started guide", "comparison review", "release announcement", "case study", "technical deep dive", "news article", "tool documentation", etc. Use "N/A" if unclear.
|
||||
|
||||
**problemStatement** (string): What specific problem does article address? Use "N/A" if not explicitly stated.
|
||||
|
||||
**audienceLevel** (string): Based on technical depth - "beginner", "intermediate", "advanced", or "N/A" if unclear
|
||||
|
||||
### Knowledge Graph (Corpus Linking)
|
||||
|
||||
Extract ONLY what's explicitly mentioned - these fields enable future corpus-wide analysis and RAG:
|
||||
|
||||
**technologiesMentioned** (array): Specific tools, frameworks, languages named (e.g., ["Ray", "Python", "Dask"])
|
||||
|
||||
**companiesMentioned** (array): Organizations, companies mentioned (e.g., ["Anyscale", "OpenAI"])
|
||||
|
||||
**peopleMentioned** (array): Notable people if relevant (founders, researchers, etc.), else ["N/A"]
|
||||
|
||||
**conceptsExplained** (array): Technical concepts or techniques explained (e.g., ["distributed computing", "actor model"])
|
||||
|
||||
**relatedTechnologies** (array): Technologies this compares to or builds upon (e.g., ["Spark", "Dask"])
|
||||
|
||||
**useCases** (array): Specific scenarios described (e.g., ["ML training", "data processing"])
|
||||
|
||||
### SEO Signals (Article-Based Only)
|
||||
|
||||
**targetKeywords** (array): Keywords that appear emphasized/repeated in article, or ["N/A"] if none obvious
|
||||
|
||||
**searchQuestions** (array): If article answers specific questions, list them, else ["N/A"]
|
||||
|
||||
### Trend Signals (If Present)
|
||||
|
||||
**githubRepo** (string): GitHub URL if article is about/links to repo, else "N/A"
|
||||
|
||||
**releaseInfo** (string): Version/release info if article announces one, else "N/A"
|
||||
|
||||
## Output Format
|
||||
|
||||
**Input stub (what you receive):**
|
||||
```json
|
||||
{"articleId":"...","articleSlug":"...","username":"...","articleUrl":"...","articleTitle":"...","savedAt":"...","publishedAt":"...","updatedAt":"..."}
|
||||
```
|
||||
|
||||
**Output (what you return):**
|
||||
```json
|
||||
{"articleId":"...","articleSlug":"...","username":"...","articleUrl":"...","articleTitle":"...","savedAt":"...","publishedAt":"...","updatedAt":"...","analysis":{"topics":[],"topicScores":{},"summary":"","keyPoints":[],"sentiment":"","monetizationAngle":"","contentType":"","problemStatement":"","audienceLevel":"","technologiesMentioned":[],"companiesMentioned":[],"peopleMentioned":[],"conceptsExplained":[],"relatedTechnologies":[],"useCases":[],"targetKeywords":[],"searchQuestions":[],"githubRepo":"","releaseInfo":"","analyzedAt":"ISO-timestamp"}}
|
||||
```
|
||||
|
||||
**What you do**: Add the `analysis` field ONLY. All other fields are pass-through from stub.
|
||||
|
||||
## Decision-Making Framework
|
||||
|
||||
**For each field, ask**:
|
||||
1. Is this information explicitly stated in the article?
|
||||
2. If yes: Extract verbatim or in standardized form
|
||||
3. If no: Use "N/A" or ["N/A"]
|
||||
4. If uncertain: Default to "N/A"
|
||||
|
||||
**Quality checks**:
|
||||
- All topic labels from approved categories?
|
||||
- Summary captures "so what?", not just "what?"
|
||||
- Key points actionable and specific?
|
||||
- Knowledge graph fields evidence-based?
|
||||
- No hallucinated dates, companies, or technologies?
|
||||
|
||||
You are precise, evidence-driven, and committed to data integrity over completeness.
|
||||
|
||||
## Instructions for LLM Integration
|
||||
|
||||
When using this prompt:
|
||||
|
||||
1. **Load article content**: Fetch the full article text from your content source
|
||||
2. **Provide article metadata**: Pass in the stub JSON with article metadata
|
||||
3. **Request analysis**: Ask the LLM to analyze the content and return the complete JSON with added `analysis` field
|
||||
4. **Validate output**: Ensure the returned JSON is valid and contains all required fields
|
||||
5. **Save result**: Write the enriched JSON to your output destination
|
||||
|
||||
**Example request format**:
|
||||
```
|
||||
Using the prompt from analyze-article.md, analyze this article:
|
||||
|
||||
[Article content here]
|
||||
|
||||
Article metadata:
|
||||
[Stub JSON here]
|
||||
|
||||
Return the complete JSON with analysis field added.
|
||||
```
|
||||
147
self-hosting/omc/src/analysis/prompts/analyze.md
Normal file
147
self-hosting/omc/src/analysis/prompts/analyze.md
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
# Article Content Analysis Prompt
|
||||
|
||||
You are analyzing an article for content monetization opportunities. Extract structured insights that can be used to create valuable blog posts, newsletters, and social media content.
|
||||
|
||||
⚠️ **CRITICAL ANTI-HALLUCINATION RULE**
|
||||
|
||||
This rule applies to EVERY field in your analysis:
|
||||
- If information is NOT explicitly present in the article content, use "N/A" (for strings) or ["N/A"] (for arrays)
|
||||
- DO NOT infer, guess, estimate, or use knowledge from your training data
|
||||
- Only extract information that is directly stated in the article text
|
||||
- When in doubt, use "N/A" - it's better to have missing data than invented data
|
||||
|
||||
## Article Details
|
||||
|
||||
**Title**: {{title}}
|
||||
**Author**: {{author}}
|
||||
**URL**: {{url}}
|
||||
**Word Count**: {{wordCount}}
|
||||
**Published**: {{publishedAt}}
|
||||
|
||||
**Content**:
|
||||
{{content}}
|
||||
|
||||
{{#if highlights}}
|
||||
**User Highlights**:
|
||||
{{#each highlights}}
|
||||
- "{{quote}}"{{#if annotation}} — Note: {{annotation}}{{/if}}
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
|
||||
## Analysis Task
|
||||
|
||||
Extract the following information and return as JSON:
|
||||
|
||||
```json
|
||||
{
|
||||
"topics": ["topic1", "topic2"],
|
||||
"topicScores": {
|
||||
"topic1": 0.95,
|
||||
"topic2": 0.88
|
||||
},
|
||||
"summary": "2-3 sentence summary capturing main points and why this matters",
|
||||
"keyPoints": [
|
||||
"First key takeaway",
|
||||
"Second key takeaway",
|
||||
"Third key takeaway"
|
||||
],
|
||||
"sentiment": "positive|neutral|negative",
|
||||
"monetizationAngle": "How to turn this into valuable content",
|
||||
|
||||
"contentType": "tutorial|comparison|announcement|case-study|deep-dive|news|tool-review|other",
|
||||
"publishedDate": "2024-03-15 OR N/A",
|
||||
"updatedDate": "2024-03-20 OR N/A",
|
||||
"problemStatement": "What problem does this article address OR N/A",
|
||||
"audienceLevel": "beginner|intermediate|advanced OR N/A",
|
||||
|
||||
"technologiesMentioned": ["Ray", "Python", "Dask"],
|
||||
"companiesMentioned": ["Anyscale", "OpenAI"],
|
||||
"peopleMentioned": ["author name if notable"],
|
||||
"conceptsExplained": ["distributed computing", "actor model"],
|
||||
"relatedTechnologies": ["Spark", "Dask"],
|
||||
"useCases": ["ML training", "data processing"],
|
||||
|
||||
"targetKeywords": ["distributed python", "ray framework"],
|
||||
"searchQuestions": ["how to scale python workloads"],
|
||||
|
||||
"githubRepo": "https://github.com/org/repo OR N/A",
|
||||
"releaseInfo": "v2.0 released OR N/A"
|
||||
}
|
||||
```
|
||||
|
||||
## Guidelines
|
||||
|
||||
### Topics (2-5 main topics)
|
||||
Prioritize these categories that align with the content strategy:
|
||||
- **AI & Machine Learning**: ai, machine-learning, llm, neural-networks, training, inference
|
||||
- **Developer Tools**: developer-tools, ide, debugging, testing, ci-cd
|
||||
- **Software Engineering**: software-engineering, architecture, design-patterns, code-quality
|
||||
- **DevOps & Infrastructure**: devops, kubernetes, docker, cloud-native, infrastructure
|
||||
- **Databases**: databases, sql, nosql, data-modeling, performance
|
||||
- **Cloud**: cloud, aws, azure, gcp, serverless
|
||||
- **Startups & Business**: startups, product, growth, monetization, business-strategy
|
||||
- **Security**: security, auth, encryption, vulnerabilities
|
||||
|
||||
### Topic Scores (0-1 confidence)
|
||||
- 0.90-1.0: Core focus of article
|
||||
- 0.70-0.89: Significant discussion
|
||||
- 0.50-0.69: Mentioned/relevant but not central
|
||||
- Below 0.50: Don't include
|
||||
|
||||
### Summary
|
||||
Focus on the "so what?" - why does this matter to developers/tech professionals?
|
||||
|
||||
### Key Points
|
||||
Extract actionable insights, surprising facts, or important takeaways. Focus on what readers can learn or apply.
|
||||
|
||||
### Sentiment
|
||||
- **positive**: Optimistic, solutions-focused, celebrates innovation
|
||||
- **neutral**: Informative, balanced, educational
|
||||
- **negative**: Critical, identifies problems, warnings
|
||||
|
||||
### Monetization Angle
|
||||
Identify content opportunities:
|
||||
- "Compare with 2-3 similar tools for comparison post"
|
||||
- "Tutorial on implementing this technique"
|
||||
- "Weekly roundup: combine with X other articles on topic Y"
|
||||
- "Deep dive: expand on concept Z with 10+ articles"
|
||||
- "Contrarian take: why this approach has limitations"
|
||||
|
||||
### Content Type
|
||||
Describe what kind of article this is using open-ended language:
|
||||
- "getting started guide", "comparison review", "release announcement", "case study", "technical deep dive", "news article", "tool documentation", etc.
|
||||
- Use N/A if unclear from content
|
||||
|
||||
### Dates
|
||||
- Extract publishedDate and updatedDate if shown in article (use ISO format YYYY-MM-DD)
|
||||
- Use "N/A" if dates not present
|
||||
|
||||
### Problem Statement
|
||||
- What specific problem does the article address?
|
||||
- Use N/A if not explicitly stated
|
||||
|
||||
### Audience Level
|
||||
- Based on technical depth: "beginner", "intermediate", "advanced"
|
||||
- Use N/A if unclear
|
||||
|
||||
### Knowledge Graph Fields
|
||||
Extract ONLY what's explicitly mentioned:
|
||||
- **technologiesMentioned**: Specific tools, frameworks, languages named in article
|
||||
- **companiesMentioned**: Organizations, companies mentioned
|
||||
- **peopleMentioned**: Notable people mentioned (founders, researchers, etc.) - use N/A if none
|
||||
- **conceptsExplained**: Technical concepts or techniques explained
|
||||
- **relatedTechnologies**: Technologies this compares to or builds upon
|
||||
- **useCases**: Specific use cases or scenarios described
|
||||
|
||||
### SEO Signals
|
||||
- **targetKeywords**: Keywords that appear emphasized/repeated in article (N/A if none obvious)
|
||||
- **searchQuestions**: If article answers specific questions, list them (N/A if not question-format)
|
||||
|
||||
### Trend Signals
|
||||
- **githubRepo**: If article is about or links to a GitHub repository
|
||||
- **releaseInfo**: If article announces a new version or release
|
||||
- Use N/A for both if not applicable
|
||||
|
||||
## Output Format
|
||||
|
||||
Return ONLY valid JSON matching the schema above. No markdown code blocks, no additional text.
|
||||
128
self-hosting/omc/src/commands/analyze/auto.ts
Normal file
128
self-hosting/omc/src/commands/analyze/auto.ts
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { fetchUsername } from '@lib/cli/graphql.js';
|
||||
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { displayQueueStats } from '@lib/cli/queue-display.js';
|
||||
import { AnalysisWriter } from '@storage/AnalysisWriter.js';
|
||||
import type { AnalysisJob, QueueStats, AnalysisQueueRepository } from '@storage/AnalysisQueueRepository.js';
|
||||
import { ContentAnalyzer } from '@analysis/ContentAnalyzer.js';
|
||||
import { runAnalyzeAuto } from '@analysis/analyze-auto-runner.js';
|
||||
|
||||
interface AnalyzeAutoFlags {
|
||||
'batch-size': number;
|
||||
'article-id'?: string;
|
||||
all: boolean;
|
||||
model?: string;
|
||||
'timeout-ms': number;
|
||||
'keep-temp': boolean;
|
||||
jsonl: boolean;
|
||||
'jsonl-path': string;
|
||||
json: boolean;
|
||||
}
|
||||
|
||||
export default class AnalyzeAuto extends BaseCommand {
|
||||
static override description = 'Analyze queued articles end-to-end (non-interactive)';
|
||||
|
||||
static override examples = [
|
||||
'$ omc analyze auto --batch-size 5',
|
||||
'$ omc analyze auto --article-id abc123',
|
||||
'$ omc analyze auto --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
'batch-size': Flags.integer({
|
||||
char: 'b',
|
||||
description: 'Number of articles to analyze',
|
||||
default: 5,
|
||||
}),
|
||||
'article-id': Flags.string({
|
||||
description: 'Analyze specific article by ID',
|
||||
exclusive: ['all'],
|
||||
}),
|
||||
all: Flags.boolean({
|
||||
description: 'Analyze pending + failed jobs (up to batch size)',
|
||||
exclusive: ['article-id'],
|
||||
}),
|
||||
model: Flags.string({
|
||||
description: 'Codex model override (passed to codex exec -m)',
|
||||
}),
|
||||
'timeout-ms': Flags.integer({
|
||||
description: 'Timeout per article analysis (milliseconds)',
|
||||
default: 10 * 60 * 1000,
|
||||
}),
|
||||
'keep-temp': Flags.boolean({
|
||||
description: 'Keep temp/*.jsonl files after successful save',
|
||||
default: false,
|
||||
}),
|
||||
jsonl: Flags.boolean({
|
||||
description: 'Append each completed analysis to content/analysis/analyses.jsonl',
|
||||
default: false,
|
||||
}),
|
||||
'jsonl-path': Flags.string({
|
||||
description: 'Path to JSONL output (requires --jsonl)',
|
||||
default: 'content/analysis/analyses.jsonl',
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: AnalyzeAutoFlags): Promise<void> {
|
||||
return await withDatabase(async ({ repo }) => {
|
||||
const jobs = this.selectJobs(repo, flags);
|
||||
if (jobs.length === 0) return this.outputEmpty(flags.json);
|
||||
|
||||
if (!flags.json) {
|
||||
this.log(formatHeader('Analysis Queue Status'));
|
||||
displayQueueStats(repo.getStats());
|
||||
}
|
||||
|
||||
this.markJobsInProgress(repo, jobs);
|
||||
|
||||
const username = await fetchUsername();
|
||||
const analyzer = new ContentAnalyzer({ model: flags.model, timeoutMs: flags['timeout-ms'] });
|
||||
const writer = new AnalysisWriter({ outputDir: 'content/analysis' });
|
||||
|
||||
const jsonlPath = flags.jsonl ? flags['jsonl-path'] : undefined;
|
||||
const results = await runAnalyzeAuto({ jobs, username, analyzer, writer, repo, keepTemp: flags['keep-temp'], jsonlPath });
|
||||
this.outputResults(results, repo.getStats(), flags.json);
|
||||
});
|
||||
}
|
||||
|
||||
private selectJobs(repo: AnalysisQueueRepository, flags: AnalyzeAutoFlags): AnalysisJob[] {
|
||||
if (flags['article-id']) {
|
||||
const job = repo.getByArticleId(flags['article-id']);
|
||||
if (!job) throw new Error(`Article with ID ${flags['article-id']} not found`);
|
||||
return [job];
|
||||
}
|
||||
if (flags.all) {
|
||||
const combined = [...repo.getByStatus('pending'), ...repo.getByStatus('failed')];
|
||||
return combined.slice(0, flags['batch-size'] ?? combined.length);
|
||||
}
|
||||
return repo.getPending(flags['batch-size']);
|
||||
}
|
||||
|
||||
private outputEmpty(jsonMode: boolean): void {
|
||||
if (jsonMode) this.log(JSON.stringify({ saved: 0, failed: 0, skipped: 0 }, null, 2));
|
||||
else this.log('No jobs to process');
|
||||
}
|
||||
|
||||
private markJobsInProgress(repo: AnalysisQueueRepository, jobs: AnalysisJob[]): void {
|
||||
for (const job of jobs) repo.markInProgress(job.articleId);
|
||||
}
|
||||
|
||||
private outputResults(results: { saved: number; failed: number; skipped: number }, stats: QueueStats, jsonMode: boolean): void {
|
||||
if (jsonMode) {
|
||||
this.log(JSON.stringify({ ...results, stats }, null, 2));
|
||||
return;
|
||||
}
|
||||
this.log('');
|
||||
this.log(formatSuccess(`Saved ${results.saved} article(s)`));
|
||||
if (results.failed) this.log(`Failed: ${results.failed}`);
|
||||
if (results.skipped) this.log(`Skipped: ${results.skipped}`);
|
||||
this.log('');
|
||||
this.log(formatHeader('Updated Queue Status'));
|
||||
displayQueueStats(stats);
|
||||
}
|
||||
|
||||
}
|
||||
212
self-hosting/omc/src/commands/analyze/complete.ts
Normal file
212
self-hosting/omc/src/commands/analyze/complete.ts
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { readFileSync, unlinkSync, existsSync } from 'node:fs';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
|
||||
import { parseJsonSafely } from '@lib/cli/command-utils.js';
|
||||
import { displayQueueStats } from '@lib/cli/queue-display.js';
|
||||
import { AnalysisWriter } from '@storage/AnalysisWriter.js';
|
||||
import { glob } from 'glob';
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
import type { AnalysisJsonlRecord } from '@storage/AnalysisWriter.js';
|
||||
import type { AnalysisQueueRepository, QueueStats } from '@storage/AnalysisQueueRepository.js';
|
||||
|
||||
interface EnrichedResult {
|
||||
articleId: string;
|
||||
articleSlug: string;
|
||||
username: string;
|
||||
articleUrl: string;
|
||||
articleTitle: string;
|
||||
savedAt: string;
|
||||
publishedAt: string | null;
|
||||
updatedAt: string | null;
|
||||
analysis?: ContentAnalysis;
|
||||
}
|
||||
|
||||
interface AnalyzeCompleteFlags {
|
||||
'keep-temp': boolean;
|
||||
jsonl: boolean;
|
||||
'jsonl-path': string;
|
||||
json: boolean;
|
||||
}
|
||||
|
||||
interface SaveResults {
|
||||
saved: number;
|
||||
failed: number;
|
||||
stats: QueueStats;
|
||||
}
|
||||
|
||||
function buildJsonlRecord(result: EnrichedResult, analysis: ContentAnalysis, markdownPath: string): AnalysisJsonlRecord {
|
||||
return {
|
||||
articleId: result.articleId,
|
||||
articleSlug: result.articleSlug,
|
||||
articleUrl: result.articleUrl,
|
||||
articleTitle: result.articleTitle,
|
||||
savedAt: result.savedAt,
|
||||
publishedAt: result.publishedAt,
|
||||
updatedAt: result.updatedAt,
|
||||
markdownPath,
|
||||
analyzedAt: analysis.analyzedAt,
|
||||
topics: analysis.topics,
|
||||
topicScores: analysis.topicScores,
|
||||
sentiment: analysis.sentiment,
|
||||
summary: analysis.summary,
|
||||
keyPoints: analysis.keyPoints,
|
||||
monetizationAngle: analysis.monetizationAngle,
|
||||
contentType: analysis.contentType,
|
||||
problemStatement: analysis.problemStatement,
|
||||
audienceLevel: analysis.audienceLevel,
|
||||
technologiesMentioned: analysis.technologiesMentioned,
|
||||
companiesMentioned: analysis.companiesMentioned,
|
||||
peopleMentioned: analysis.peopleMentioned,
|
||||
conceptsExplained: analysis.conceptsExplained,
|
||||
relatedTechnologies: analysis.relatedTechnologies,
|
||||
useCases: analysis.useCases,
|
||||
targetKeywords: analysis.targetKeywords,
|
||||
searchQuestions: analysis.searchQuestions,
|
||||
githubRepo: analysis.githubRepo,
|
||||
releaseInfo: analysis.releaseInfo,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* OCLIF command: omc analyze complete
|
||||
* Saves agent-analyzed results from temp/ to database and marks jobs complete.
|
||||
* AIDEV-NOTE: analysis-complete - final step in analyze workflow after agents finish
|
||||
*/
|
||||
export default class AnalyzeComplete extends BaseCommand {
|
||||
static override description = 'Save analyzed results and mark jobs complete';
|
||||
|
||||
static override examples = [
|
||||
'$ omc analyze complete',
|
||||
'$ omc analyze complete --keep-temp',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
'keep-temp': Flags.boolean({
|
||||
description: 'Keep temp files after saving',
|
||||
default: false,
|
||||
}),
|
||||
jsonl: Flags.boolean({
|
||||
description: 'Append each completed analysis to content/analysis/analyses.jsonl',
|
||||
default: false,
|
||||
}),
|
||||
'jsonl-path': Flags.string({
|
||||
description: 'Path to JSONL output (requires --jsonl)',
|
||||
default: 'content/analysis/analyses.jsonl',
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: AnalyzeCompleteFlags): Promise<void> {
|
||||
this.validateFlags(flags);
|
||||
const files = await this.findAnalyzedFiles();
|
||||
|
||||
if (files.length === 0) {
|
||||
this.log('No analyzed files found in temp/');
|
||||
return;
|
||||
}
|
||||
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const writer = new AnalysisWriter({ outputDir: 'content/analysis' });
|
||||
const results = await this.saveResults(files, repo, writer, {
|
||||
keepTemp: flags['keep-temp'],
|
||||
writeJsonl: flags.jsonl,
|
||||
jsonlPath: flags['jsonl-path'],
|
||||
});
|
||||
this.displayResults(results, flags.json);
|
||||
});
|
||||
}
|
||||
|
||||
private validateFlags(flags: AnalyzeCompleteFlags): void {
|
||||
if (flags['jsonl-path'] !== 'content/analysis/analyses.jsonl' && !flags.jsonl) {
|
||||
throw new Error('--jsonl-path requires --jsonl');
|
||||
}
|
||||
}
|
||||
|
||||
private async findAnalyzedFiles(): Promise<string[]> {
|
||||
const allFiles = await glob('temp/*.jsonl');
|
||||
return allFiles.filter((file) => {
|
||||
if (!existsSync(file)) return false;
|
||||
const content = readFileSync(file, 'utf-8').trim();
|
||||
if (!content) return false;
|
||||
const data = parseJsonSafely<EnrichedResult>(content);
|
||||
return data?.analysis !== undefined;
|
||||
});
|
||||
}
|
||||
|
||||
private async saveResults(
|
||||
files: string[],
|
||||
repo: AnalysisQueueRepository,
|
||||
writer: AnalysisWriter,
|
||||
options: { keepTemp: boolean; writeJsonl: boolean; jsonlPath: string }
|
||||
): Promise<SaveResults> {
|
||||
let saved = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const file of files) {
|
||||
const success = await this.saveFile(file, repo, writer, options);
|
||||
if (success) saved++;
|
||||
else failed++;
|
||||
}
|
||||
|
||||
return { saved, failed, stats: repo.getStats() };
|
||||
}
|
||||
|
||||
private async saveFile(
|
||||
file: string,
|
||||
repo: AnalysisQueueRepository,
|
||||
writer: AnalysisWriter,
|
||||
options: { keepTemp: boolean; writeJsonl: boolean; jsonlPath: string }
|
||||
): Promise<boolean> {
|
||||
const content = readFileSync(file, 'utf-8');
|
||||
const result = parseJsonSafely<EnrichedResult>(content);
|
||||
|
||||
if (!result?.analysis) {
|
||||
this.warn(`Skipping ${file}: no analysis field`);
|
||||
return false;
|
||||
}
|
||||
|
||||
const { articleId, articleUrl, articleTitle, savedAt, publishedAt, updatedAt, analysis } = result;
|
||||
|
||||
try {
|
||||
const mdPath = await writer.write(articleId, articleUrl, articleTitle, savedAt, analysis, result.articleSlug);
|
||||
repo.storeAnalysis(articleId, publishedAt, updatedAt, JSON.stringify(analysis), mdPath);
|
||||
|
||||
if (options.writeJsonl) {
|
||||
await writer.appendToJsonl(options.jsonlPath, buildJsonlRecord(result, analysis, mdPath));
|
||||
}
|
||||
|
||||
if (!options.keepTemp) unlinkSync(file);
|
||||
|
||||
this.log(`✓ ${this.truncate(articleTitle, 60)}`);
|
||||
return true;
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
repo.markFailed(articleId, `Save failed: ${message}`);
|
||||
this.error(`✗ ${this.truncate(articleTitle, 60)} - ${message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private displayResults(results: SaveResults, jsonMode: boolean): void {
|
||||
if (jsonMode) {
|
||||
this.log(JSON.stringify(results, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
this.log('');
|
||||
this.log(formatSuccess(`Saved ${results.saved} article(s)`));
|
||||
if (results.failed > 0) {
|
||||
this.log(`Failed: ${results.failed}`);
|
||||
}
|
||||
this.log('');
|
||||
this.log(formatHeader('Updated Queue Status'));
|
||||
displayQueueStats(results.stats);
|
||||
}
|
||||
|
||||
private truncate(text: string, length: number): string {
|
||||
return text.length > length ? text.substring(0, length) + '...' : text;
|
||||
}
|
||||
}
|
||||
67
self-hosting/omc/src/commands/analyze/retry.ts
Normal file
67
self-hosting/omc/src/commands/analyze/retry.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatSuccess } from '@lib/cli/formatters.js';
|
||||
|
||||
/**
|
||||
* Retry failed article analyses by resetting them to pending status.
|
||||
* AIDEV-NOTE: CLI command for retrying failed analyses
|
||||
*/
|
||||
export default class AnalyzeRetry extends BaseCommand {
|
||||
static override description = 'Retry failed article analyses';
|
||||
|
||||
static override examples = [
|
||||
'$ omc analyze retry --failed',
|
||||
'$ omc analyze retry --article-id <id>',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
failed: Flags.boolean({
|
||||
description: 'Retry all failed articles',
|
||||
default: false,
|
||||
exclusive: ['article-id'],
|
||||
}),
|
||||
'article-id': Flags.string({
|
||||
description: 'Retry specific article by ID',
|
||||
exclusive: ['failed'],
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
async execute(flags: { failed: boolean; 'article-id'?: string; json: boolean }): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const count = flags.failed
|
||||
? this.retryAllFailed(repo)
|
||||
: this.retrySingleArticle(repo, flags['article-id']);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify({ retried: count }));
|
||||
} else {
|
||||
this.log(formatSuccess(`Reset ${count} failed article${count !== 1 ? 's' : ''} to pending`));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private retryAllFailed(repo: {
|
||||
getFailed: () => Array<{ articleId: string }>;
|
||||
resetToPending: (articleId: string) => void;
|
||||
}): number {
|
||||
const failed = repo.getFailed();
|
||||
for (const job of failed) {
|
||||
repo.resetToPending(job.articleId);
|
||||
}
|
||||
return failed.length;
|
||||
}
|
||||
|
||||
private retrySingleArticle(
|
||||
repo: { resetToPending: (articleId: string) => void },
|
||||
articleId: string | undefined
|
||||
): number {
|
||||
if (!articleId) {
|
||||
throw new Error('Either use --failed flag or provide --article-id');
|
||||
}
|
||||
repo.resetToPending(articleId);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
186
self-hosting/omc/src/commands/analyze/run.ts
Normal file
186
self-hosting/omc/src/commands/analyze/run.ts
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { fetchUsername } from '@lib/cli/graphql.js';
|
||||
import { formatHeader, formatSuccess, formatDivider } from '@lib/cli/formatters.js';
|
||||
import { displayQueueStats } from '@lib/cli/queue-display.js';
|
||||
import { getArticle } from '@lib/omnivore/client.js';
|
||||
import type { AnalysisJob } from '@storage/AnalysisQueueRepository.js';
|
||||
|
||||
/**
|
||||
* OCLIF command: omc analyze run
|
||||
* Wraps parallel-analyze.ts workflow for queue-based article analysis.
|
||||
*
|
||||
* AIDEV-NOTE: tracking-coordination - prepares batch for parallel agent execution
|
||||
* AIDEV-NOTE: stub-file-creation - fetches full article metadata before analysis
|
||||
*/
|
||||
export default class AnalyzeRun extends BaseCommand {
|
||||
static override description = 'Run parallel content analysis on queued articles';
|
||||
|
||||
static override examples = [
|
||||
'$ omc analyze run',
|
||||
'$ omc analyze run --batch-size 10',
|
||||
'$ omc analyze run --article-id abc123',
|
||||
'$ omc analyze run --all',
|
||||
'$ omc analyze run --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
'batch-size': Flags.integer({
|
||||
char: 'b',
|
||||
description: 'Number of articles to analyze in parallel',
|
||||
default: 5,
|
||||
}),
|
||||
'article-id': Flags.string({
|
||||
description: 'Analyze specific article by ID',
|
||||
exclusive: ['all'],
|
||||
}),
|
||||
all: Flags.boolean({
|
||||
description: 'Analyze all articles regardless of status',
|
||||
exclusive: ['article-id'],
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: Record<string, any>): Promise<void> {
|
||||
return await withDatabase(async ({ repo }) => {
|
||||
const jobs = this.selectJobs(repo, flags);
|
||||
|
||||
if (jobs.length === 0) {
|
||||
if (!flags.json) this.log('No jobs to process');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!flags.json) {
|
||||
const stats = repo.getStats();
|
||||
this.log(formatHeader('Analysis Queue Status'));
|
||||
displayQueueStats(stats);
|
||||
}
|
||||
|
||||
this.markJobsInProgress(repo, jobs);
|
||||
|
||||
const username = await fetchUsername();
|
||||
const agentParams = await this.processJobs(jobs, username, flags.json);
|
||||
|
||||
this.outputResults(agentParams, jobs.length, flags.json);
|
||||
});
|
||||
}
|
||||
|
||||
// AIDEV-NOTE: job-selection - handles --article-id, --all, or default pending
|
||||
private selectJobs(repo: any, flags: Record<string, any>): AnalysisJob[] {
|
||||
if (flags['article-id']) {
|
||||
const job = repo.getByArticleId(flags['article-id']);
|
||||
if (!job) {
|
||||
throw new Error(`Article with ID ${flags['article-id']} not found`);
|
||||
}
|
||||
return [job];
|
||||
}
|
||||
|
||||
if (flags.all) {
|
||||
// Get all pending jobs (don't include in_progress to avoid overwriting)
|
||||
const pending = repo.getByStatus('pending');
|
||||
const failed = repo.getByStatus('failed');
|
||||
const combined = [...pending, ...failed];
|
||||
return flags['batch-size'] ? combined.slice(0, flags['batch-size']) : combined;
|
||||
}
|
||||
|
||||
return repo.getPending(flags['batch-size']);
|
||||
}
|
||||
|
||||
// AIDEV-NOTE: tracking-lock - marks jobs to prevent duplicate analysis
|
||||
private markJobsInProgress(repo: any, jobs: AnalysisJob[]): void {
|
||||
for (const job of jobs) {
|
||||
repo.markInProgress(job.articleId);
|
||||
}
|
||||
}
|
||||
|
||||
private async processJobs(
|
||||
jobs: AnalysisJob[],
|
||||
username: string,
|
||||
silent: boolean
|
||||
): Promise<any[]> {
|
||||
this.ensureTempDir();
|
||||
const agentParams = [];
|
||||
|
||||
for (let i = 0; i < jobs.length; i++) {
|
||||
const job = jobs[i];
|
||||
if (!silent) {
|
||||
this.log(`Fetching article ${i + 1}/${jobs.length}: ${this.truncate(job.articleTitle, 60)}...`);
|
||||
}
|
||||
|
||||
const param = await this.createStubFile(job, username);
|
||||
if (param) agentParams.push(param);
|
||||
}
|
||||
|
||||
return agentParams;
|
||||
}
|
||||
|
||||
private async fetchArticleData(slug: string, username: string): Promise<any | null> {
|
||||
try {
|
||||
const result = await getArticle(slug, username);
|
||||
if (!result?.article) {
|
||||
this.warn(`Failed to fetch article: ${slug}`);
|
||||
return null;
|
||||
}
|
||||
return result.article;
|
||||
} catch (error) {
|
||||
this.warn(`Failed to fetch article: ${slug}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private buildStubObject(article: any, job: AnalysisJob, username: string): any {
|
||||
return {
|
||||
articleId: job.articleId,
|
||||
articleSlug: job.articleSlug,
|
||||
username,
|
||||
articleUrl: article.url,
|
||||
articleTitle: article.title,
|
||||
savedAt: article.savedAt,
|
||||
publishedAt: article.publishedAt || null,
|
||||
updatedAt: article.updatedAt || null,
|
||||
};
|
||||
}
|
||||
|
||||
// AIDEV-NOTE: stub-file-creation - writes complete metadata for agent
|
||||
private async createStubFile(job: AnalysisJob, username: string): Promise<any | null> {
|
||||
const article = await this.fetchArticleData(job.articleSlug, username);
|
||||
if (!article) return null;
|
||||
|
||||
const stubPath = join('temp', `${job.articleSlug}.jsonl`);
|
||||
const stub = this.buildStubObject(article, job, username);
|
||||
writeFileSync(stubPath, JSON.stringify(stub) + '\n', 'utf-8');
|
||||
|
||||
return {
|
||||
filename: stubPath,
|
||||
articleId: job.articleId,
|
||||
articleSlug: job.articleSlug,
|
||||
username,
|
||||
articleTitle: this.truncate(article.title, 60),
|
||||
};
|
||||
}
|
||||
|
||||
private outputResults(agentParams: any[], count: number, jsonMode: boolean): void {
|
||||
if (jsonMode) {
|
||||
this.log(JSON.stringify(agentParams, null, 2));
|
||||
} else {
|
||||
this.log(`\n${formatDivider()}`);
|
||||
this.log(formatSuccess(`Prepared ${count} articles for analysis`));
|
||||
this.log('\nAgent parameters (copy for Task tool invocation):');
|
||||
this.log(JSON.stringify(agentParams, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
private ensureTempDir(): void {
|
||||
if (!existsSync('temp')) {
|
||||
mkdirSync('temp', { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
private truncate(text: string, length: number): string {
|
||||
return text.length > length ? text.substring(0, length) + '...' : text;
|
||||
}
|
||||
}
|
||||
73
self-hosting/omc/src/commands/analyze/status.ts
Normal file
73
self-hosting/omc/src/commands/analyze/status.ts
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
import { readdirSync } from 'node:fs';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader, formatTable } from '@lib/cli/formatters.js';
|
||||
import { QUEUE_STATUS } from '@lib/cli/constants.js';
|
||||
import type { AnalysisJob } from '@storage/AnalysisQueueRepository.js';
|
||||
|
||||
/**
|
||||
* OCLIF command: omc analyze status
|
||||
* Show current analysis batch progress.
|
||||
* AIDEV-NOTE: tracking-coordination - displays in-progress jobs and stub files
|
||||
*/
|
||||
export default class AnalyzeStatus extends BaseCommand {
|
||||
static override description = 'Show current analysis batch progress';
|
||||
|
||||
static override examples = ['$ omc analyze status', '$ omc analyze status --json'];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: Record<string, any>): Promise<void> {
|
||||
return await withDatabase(async ({ repo }) => {
|
||||
const jobs = repo.getByStatus(QUEUE_STATUS.IN_PROGRESS);
|
||||
const stubs = this.getStubFiles();
|
||||
const display = this.prepareDisplay(jobs, stubs);
|
||||
|
||||
this.output(display, flags.json);
|
||||
});
|
||||
}
|
||||
|
||||
private getStubFiles(): string[] {
|
||||
try {
|
||||
return readdirSync('temp').filter((f) => f.endsWith('.jsonl'));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private prepareDisplay(jobs: AnalysisJob[], stubs: string[]): any {
|
||||
return {
|
||||
inProgress: jobs.map((j) => ({
|
||||
articleId: j.articleId,
|
||||
status: j.status,
|
||||
elapsed: this.elapsed(j.assignedAt),
|
||||
})),
|
||||
stubFiles: stubs,
|
||||
count: jobs.length,
|
||||
};
|
||||
}
|
||||
|
||||
private output(display: any, jsonMode: boolean): void {
|
||||
if (jsonMode) {
|
||||
this.log(JSON.stringify(display, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
this.log(formatHeader('Analysis Status'));
|
||||
if (display.count === 0) {
|
||||
this.log('\nNo analysis in progress');
|
||||
return;
|
||||
}
|
||||
|
||||
this.log(formatTable(display.inProgress, ['articleId', 'status', 'elapsed']));
|
||||
}
|
||||
|
||||
private elapsed(assignedAt?: string): string {
|
||||
if (!assignedAt) return 'N/A';
|
||||
const ms = Date.now() - new Date(assignedAt).getTime();
|
||||
return `${Math.floor(ms / 1000)}s`;
|
||||
}
|
||||
}
|
||||
66
self-hosting/omc/src/commands/analyze/watch.ts
Normal file
66
self-hosting/omc/src/commands/analyze/watch.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader, formatDivider } from '@lib/cli/formatters.js';
|
||||
import { displayQueueStats } from '@lib/cli/queue-display.js';
|
||||
|
||||
/**
|
||||
* OCLIF command: omc analyze watch
|
||||
* Real-time monitoring of analysis progress.
|
||||
* AIDEV-NOTE: polling-monitor - updates stats every interval until completion
|
||||
*/
|
||||
export default class AnalyzeWatch extends BaseCommand {
|
||||
static override description = 'Real-time monitoring of analysis progress';
|
||||
|
||||
static override examples = ['$ omc analyze watch', '$ omc analyze watch --interval 5000'];
|
||||
|
||||
static override flags = {
|
||||
interval: Flags.integer({
|
||||
char: 'i',
|
||||
description: 'Polling interval in milliseconds',
|
||||
default: 2000,
|
||||
}),
|
||||
};
|
||||
|
||||
protected async execute(flags: Record<string, any>): Promise<void> {
|
||||
let running = true;
|
||||
this.setupCleanExit(() => (running = false));
|
||||
|
||||
while (running) {
|
||||
const done = await this.pollStats();
|
||||
if (done) break;
|
||||
await this.sleep(flags.interval);
|
||||
}
|
||||
}
|
||||
|
||||
private async pollStats(): Promise<boolean> {
|
||||
return await withDatabase(async ({ repo }) => {
|
||||
const stats = repo.getStats();
|
||||
this.display(stats);
|
||||
return stats.inProgress === 0 && stats.pending === 0;
|
||||
});
|
||||
}
|
||||
|
||||
private display(stats: any): void {
|
||||
this.clear();
|
||||
this.log(formatHeader('Analysis Progress (Ctrl+C to exit)'));
|
||||
displayQueueStats(stats);
|
||||
this.log(formatDivider());
|
||||
}
|
||||
|
||||
private setupCleanExit(callback: () => void): void {
|
||||
process.on('SIGINT', () => {
|
||||
this.log('\nExiting...');
|
||||
callback();
|
||||
process.exit(0);
|
||||
});
|
||||
}
|
||||
|
||||
private clear(): void {
|
||||
process.stdout.write('\x1Bc');
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
}
|
||||
57
self-hosting/omc/src/commands/config/env/list.ts
vendored
Normal file
57
self-hosting/omc/src/commands/config/env/list.ts
vendored
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { formatHeader } from '@lib/cli/formatters.js';
|
||||
import { readdirSync } from 'fs';
|
||||
|
||||
/**
|
||||
* List available environments.
|
||||
* AIDEV-NOTE: Shows .env files with active indicator
|
||||
*/
|
||||
export default class ConfigEnvList extends BaseCommand {
|
||||
static override description = 'List available environments';
|
||||
|
||||
static override examples = [
|
||||
'$ omc config env list',
|
||||
'$ omc config env list --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
const environments = this.findEnvironments();
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(environments, null, 2));
|
||||
} else {
|
||||
this.displayEnvironments(environments);
|
||||
}
|
||||
}
|
||||
|
||||
private findEnvironments(): Array<{ name: string; file: string; active: boolean }> {
|
||||
const cwd = process.cwd();
|
||||
const files = readdirSync(cwd);
|
||||
const envFiles = files.filter(f => f.startsWith('.env'));
|
||||
|
||||
return envFiles.map(file => ({
|
||||
name: file === '.env' ? 'local' : file.replace('.env.', ''),
|
||||
file,
|
||||
active: file === '.env',
|
||||
}));
|
||||
}
|
||||
|
||||
private displayEnvironments(environments: Array<{ name: string; file: string; active: boolean }>): void {
|
||||
this.log(formatHeader('Available Environments'));
|
||||
|
||||
if (environments.length === 0) {
|
||||
this.log('No environment files found');
|
||||
return;
|
||||
}
|
||||
|
||||
for (const env of environments) {
|
||||
const indicator = env.active ? '* ' : ' ';
|
||||
this.log(`${indicator}${env.name.padEnd(15)} (${env.file})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
45
self-hosting/omc/src/commands/config/env/use.ts
vendored
Normal file
45
self-hosting/omc/src/commands/config/env/use.ts
vendored
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { Args } from '@oclif/core';
|
||||
import { formatSuccess } from '@lib/cli/formatters.js';
|
||||
import { copyFileSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Switch environment.
|
||||
* AIDEV-NOTE: Copies .env.{environment} to .env
|
||||
*/
|
||||
export default class ConfigEnvUse extends BaseCommand {
|
||||
static override description = 'Switch environment';
|
||||
|
||||
static override examples = [
|
||||
'$ omc config env use dev',
|
||||
'$ omc config env use prod',
|
||||
'$ omc config env use local',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
environment: Args.string({
|
||||
description: 'Environment to switch to (dev|prod|local)',
|
||||
required: true,
|
||||
options: ['dev', 'prod', 'local'],
|
||||
}),
|
||||
};
|
||||
|
||||
protected async execute(flags: { environment: string }): Promise<void> {
|
||||
this.switchEnvironment(flags.environment);
|
||||
this.log(formatSuccess(`Switched to ${flags.environment} environment`));
|
||||
}
|
||||
|
||||
private switchEnvironment(environment: string): void {
|
||||
const cwd = process.cwd();
|
||||
const sourceFile = environment === 'local' ? '.env.local' : `.env.${environment}`;
|
||||
const sourcePath = join(cwd, sourceFile);
|
||||
const targetPath = join(cwd, '.env');
|
||||
|
||||
if (!existsSync(sourcePath)) {
|
||||
throw new Error(`Environment file not found: ${sourceFile}`);
|
||||
}
|
||||
|
||||
copyFileSync(sourcePath, targetPath);
|
||||
}
|
||||
}
|
||||
62
self-hosting/omc/src/commands/config/get.ts
Normal file
62
self-hosting/omc/src/commands/config/get.ts
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { Args } from '@oclif/core';
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Get specific configuration value.
|
||||
* AIDEV-NOTE: Masks sensitive values for security
|
||||
*/
|
||||
export default class ConfigGet extends BaseCommand {
|
||||
static override description = 'Get specific configuration value';
|
||||
|
||||
static override examples = [
|
||||
'$ omc config get OMNIVORE_API_KEY',
|
||||
'$ omc config get CONTENT_OUTPUT_DIR',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
key: Args.string({
|
||||
description: 'Configuration key to retrieve',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
protected async execute(flags: { key: string }): Promise<void> {
|
||||
const value = this.getConfigValue(flags.key);
|
||||
if (value === undefined) {
|
||||
throw new Error(`Configuration key '${flags.key}' not found`);
|
||||
}
|
||||
|
||||
this.log(this.maskValue(flags.key, value));
|
||||
}
|
||||
|
||||
private getConfigValue(key: string): string | undefined {
|
||||
const envPath = join(process.cwd(), '.env');
|
||||
if (!existsSync(envPath)) {
|
||||
throw new Error('.env file not found');
|
||||
}
|
||||
|
||||
const content = readFileSync(envPath, 'utf-8');
|
||||
|
||||
for (const line of content.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
|
||||
const [envKey, ...valueParts] = trimmed.split('=');
|
||||
if (envKey === key && valueParts.length > 0) {
|
||||
return valueParts.join('=');
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private maskValue(key: string, value: string): string {
|
||||
const sensitiveKeys = ['KEY', 'TOKEN', 'SECRET', 'PASSWORD'];
|
||||
if (sensitiveKeys.some(s => key.includes(s)) && value.length > 8) {
|
||||
return `${value.slice(0, 4)}...${value.slice(-4)}`;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
73
self-hosting/omc/src/commands/config/set.ts
Normal file
73
self-hosting/omc/src/commands/config/set.ts
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { Args } from '@oclif/core';
|
||||
import { formatSuccess } from '@lib/cli/formatters.js';
|
||||
import { readFileSync, writeFileSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Set configuration value.
|
||||
* AIDEV-NOTE: Validates key exists in .env.example before updating
|
||||
*/
|
||||
export default class ConfigSet extends BaseCommand {
|
||||
static override description = 'Set configuration value';
|
||||
|
||||
static override examples = [
|
||||
'$ omc config set CONTENT_OUTPUT_DIR ./my-content',
|
||||
'$ omc config set AUTO_PUBLISH true',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
key: Args.string({
|
||||
description: 'Configuration key to set',
|
||||
required: true,
|
||||
}),
|
||||
value: Args.string({
|
||||
description: 'Value to set',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
protected async execute(flags: { key: string; value: string }): Promise<void> {
|
||||
this.validateKey(flags.key);
|
||||
this.updateEnvFile(flags.key, flags.value);
|
||||
this.log(formatSuccess(`Set ${flags.key} = ${flags.value}`));
|
||||
}
|
||||
|
||||
private validateKey(key: string): void {
|
||||
const examplePath = join(process.cwd(), '.env.example');
|
||||
if (!existsSync(examplePath)) return;
|
||||
|
||||
const content = readFileSync(examplePath, 'utf-8');
|
||||
const validKeys = content
|
||||
.split('\n')
|
||||
.filter(line => !line.trim().startsWith('#') && line.includes('='))
|
||||
.map(line => line.split('=')[0]);
|
||||
|
||||
if (!validKeys.includes(key)) {
|
||||
throw new Error(`Unknown config key: ${key}`);
|
||||
}
|
||||
}
|
||||
|
||||
private updateEnvFile(key: string, value: string): void {
|
||||
const envPath = join(process.cwd(), '.env');
|
||||
let content = existsSync(envPath) ? readFileSync(envPath, 'utf-8') : '';
|
||||
|
||||
const lines = content.split('\n');
|
||||
let updated = false;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const trimmed = lines[i].trim();
|
||||
if (trimmed.startsWith(key + '=')) {
|
||||
lines[i] = `${key}=${value}`;
|
||||
updated = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!updated) {
|
||||
lines.push(`${key}=${value}`);
|
||||
}
|
||||
|
||||
writeFileSync(envPath, lines.join('\n'));
|
||||
}
|
||||
}
|
||||
62
self-hosting/omc/src/commands/config/show.ts
Normal file
62
self-hosting/omc/src/commands/config/show.ts
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { formatHeader } from '@lib/cli/formatters.js';
|
||||
import { loadEnvFile } from '@lib/cli/command-utils.js';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Show all configuration values.
|
||||
* AIDEV-NOTE: Masks sensitive values (API keys show first/last 4 chars)
|
||||
*/
|
||||
export default class ConfigShow extends BaseCommand {
|
||||
static override description = 'Show all configuration values';
|
||||
|
||||
static override examples = [
|
||||
'$ omc config show',
|
||||
'$ omc config show --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
async execute(flags: any): Promise<void> {
|
||||
const envPath = join(process.cwd(), '.env');
|
||||
const rawConfig = loadEnvFile(envPath);
|
||||
|
||||
if (Object.keys(rawConfig).length === 0) {
|
||||
throw new Error('.env file not found');
|
||||
}
|
||||
|
||||
const config = this.maskSensitiveValues(rawConfig);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(config, null, 2));
|
||||
} else {
|
||||
this.displayConfig(config);
|
||||
}
|
||||
}
|
||||
|
||||
private maskSensitiveValues(config: Record<string, string>): Record<string, string> {
|
||||
const masked: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(config)) {
|
||||
masked[key] = this.maskValue(key, value);
|
||||
}
|
||||
return masked;
|
||||
}
|
||||
|
||||
private maskValue(key: string, value: string): string {
|
||||
const sensitiveKeys = ['KEY', 'TOKEN', 'SECRET', 'PASSWORD'];
|
||||
if (sensitiveKeys.some(s => key.includes(s)) && value.length > 8) {
|
||||
return `${value.slice(0, 4)}...${value.slice(-4)}`;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private displayConfig(config: Record<string, string>): void {
|
||||
this.log(formatHeader('Configuration'));
|
||||
for (const [key, value] of Object.entries(config)) {
|
||||
this.log(`${key.padEnd(30)} ${value}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
31
self-hosting/omc/src/commands/config/test.ts
Normal file
31
self-hosting/omc/src/commands/config/test.ts
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { formatSuccess, formatError } from '@lib/cli/formatters.js';
|
||||
import { getMe } from '@lib/omnivore/client.js';
|
||||
|
||||
/**
|
||||
* Test API connection.
|
||||
* AIDEV-NOTE: Uses testConnection pattern from lib/omnivore/client.js
|
||||
*/
|
||||
export default class ConfigTest extends BaseCommand {
|
||||
static override description = 'Test API connection';
|
||||
|
||||
static override examples = [
|
||||
'$ omc config test',
|
||||
];
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
void flags;
|
||||
try {
|
||||
const user = await getMe();
|
||||
|
||||
this.log(formatSuccess('Connected to Omnivore API'));
|
||||
this.log(` User: ${user.name} (${user.email})`);
|
||||
this.log(` Username: ${user.profile?.username ?? 'N/A'}`);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Unknown error';
|
||||
this.log(formatError('Failed to connect to Omnivore API'));
|
||||
this.log(` ${message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
96
self-hosting/omc/src/commands/config/validate.ts
Normal file
96
self-hosting/omc/src/commands/config/validate.ts
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { formatHeader, formatSuccess, formatError } from '@lib/cli/formatters.js';
|
||||
import { loadEnvFile } from '@lib/cli/command-utils.js';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Validate all configuration values.
|
||||
* AIDEV-NOTE: Checks required keys exist and validates formats
|
||||
*/
|
||||
export default class ConfigValidate extends BaseCommand {
|
||||
static override description = 'Validate all configuration values';
|
||||
|
||||
static override examples = [
|
||||
'$ omc config validate',
|
||||
'$ omc config validate --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
const issues = this.validateConfig();
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify({ valid: issues.length === 0, issues }, null, 2));
|
||||
} else {
|
||||
this.displayValidation(issues);
|
||||
}
|
||||
|
||||
if (issues.length > 0) {
|
||||
throw new Error('Configuration validation failed');
|
||||
}
|
||||
}
|
||||
|
||||
private validateConfig(): string[] {
|
||||
const issues: string[] = [];
|
||||
const envPath = join(process.cwd(), '.env');
|
||||
const config = loadEnvFile(envPath);
|
||||
|
||||
if (Object.keys(config).length === 0) {
|
||||
issues.push('.env file not found');
|
||||
return issues;
|
||||
}
|
||||
|
||||
this.checkRequired(config, issues);
|
||||
this.checkFormats(config, issues);
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
private checkRequired(config: Record<string, string>, issues: string[]): void {
|
||||
const required = ['OMNIVORE_API_KEY'];
|
||||
for (const key of required) {
|
||||
if (!config[key] || config[key].trim() === '') {
|
||||
issues.push(`Missing required key: ${key}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private checkFormats(config: Record<string, string>, issues: string[]): void {
|
||||
if (config.OMNIVORE_API_URL && !this.isValidUrl(config.OMNIVORE_API_URL)) {
|
||||
issues.push('Invalid URL format: OMNIVORE_API_URL');
|
||||
}
|
||||
|
||||
const boolKeys = ['GENERATE_SEO', 'AUTO_PUBLISH', 'CACHE_ENABLED'];
|
||||
for (const key of boolKeys) {
|
||||
if (config[key] && !['true', 'false'].includes(config[key])) {
|
||||
issues.push(`Invalid boolean value: ${key} (must be 'true' or 'false')`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private isValidUrl(value: string): boolean {
|
||||
try {
|
||||
new URL(value);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private displayValidation(issues: string[]): void {
|
||||
this.log(formatHeader('Configuration Validation'));
|
||||
|
||||
if (issues.length === 0) {
|
||||
this.log(formatSuccess('All configuration valid'));
|
||||
} else {
|
||||
this.log(formatError(`Found ${issues.length} issue(s):\n`));
|
||||
for (const issue of issues) {
|
||||
this.log(` - ${issue}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
44
self-hosting/omc/src/commands/db/backup.ts
Normal file
44
self-hosting/omc/src/commands/db/backup.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { Flags } from '@oclif/core';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Create database backup.
|
||||
* AIDEV-NOTE: Uses better-sqlite3 backup API for safe copy
|
||||
*/
|
||||
export default class DbBackup extends BaseCommand {
|
||||
static override description = 'Create database backup';
|
||||
|
||||
static override examples = [
|
||||
'$ omc db backup',
|
||||
'$ omc db backup --destination data/backup.db',
|
||||
'$ omc db backup --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
destination: Flags.string({
|
||||
description: 'Backup file path',
|
||||
}),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const dest = flags.destination || join(process.cwd(), `data/backup-${timestamp}.db`);
|
||||
|
||||
await withDatabase(async ({ db }) => {
|
||||
db.backup(dest);
|
||||
const result = { destination: dest, timestamp };
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
this.log(formatHeader('Database Backup'));
|
||||
this.log(formatSuccess(`Backup created: ${dest}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
54
self-hosting/omc/src/commands/db/check.ts
Normal file
54
self-hosting/omc/src/commands/db/check.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader, formatSuccess, formatError } from '@lib/cli/formatters.js';
|
||||
|
||||
/**
|
||||
* Verify database integrity.
|
||||
* AIDEV-NOTE: Runs PRAGMA checks for corruption detection
|
||||
*/
|
||||
export default class DbCheck extends BaseCommand {
|
||||
static override description = 'Verify database integrity';
|
||||
|
||||
static override examples = [
|
||||
'$ omc db check',
|
||||
'$ omc db check --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ db }) => {
|
||||
const integrity = this.checkIntegrity(db);
|
||||
const foreignKeys = this.checkForeignKeys(db);
|
||||
const isHealthy = integrity.ok && foreignKeys.length === 0;
|
||||
|
||||
const result = { ok: isHealthy, integrity, foreignKeys };
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
this.displayCheckResults(integrity, foreignKeys, isHealthy);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private checkIntegrity(db: any): { ok: boolean; message: string } {
|
||||
const result = db.pragma('integrity_check');
|
||||
const message = result[0]?.integrity_check || 'ok';
|
||||
return { ok: message === 'ok', message };
|
||||
}
|
||||
|
||||
private checkForeignKeys(db: any): any[] {
|
||||
return db.pragma('foreign_key_check');
|
||||
}
|
||||
|
||||
private displayCheckResults(integrity: any, fkIssues: any[], ok: boolean): void {
|
||||
this.log(formatHeader('Database Integrity Check'));
|
||||
this.log(`Integrity: ${integrity.message}`);
|
||||
this.log(`Foreign Keys: ${fkIssues.length === 0 ? 'OK' : `${fkIssues.length} issues`}`);
|
||||
this.log(ok ? formatSuccess('Database is healthy') : formatError('Issues detected'));
|
||||
}
|
||||
}
|
||||
62
self-hosting/omc/src/commands/db/migrate.ts
Normal file
62
self-hosting/omc/src/commands/db/migrate.ts
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
|
||||
/**
|
||||
* Run database migrations by executing tracking schema.
|
||||
* AIDEV-NOTE: Idempotent - safe to run multiple times (CREATE IF NOT EXISTS)
|
||||
*/
|
||||
export default class DbMigrate extends BaseCommand {
|
||||
static override description = 'Run database migrations (idempotent schema execution)';
|
||||
|
||||
static override examples = [
|
||||
'$ omc db migrate',
|
||||
'$ omc db migrate --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ db }) => {
|
||||
const schemaPath = this.getSchemaPath();
|
||||
const schema = readFileSync(schemaPath, 'utf-8');
|
||||
|
||||
db.exec(schema);
|
||||
|
||||
const tables = this.getTrackingTables(db);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify({ tables }, null, 2));
|
||||
} else {
|
||||
this.displayMigrationResults(tables);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private getSchemaPath(): string {
|
||||
// AIDEV-NOTE: Schema must be in source tree, not dist (not bundled by esbuild)
|
||||
const projectRoot = process.cwd();
|
||||
return join(projectRoot, 'src/storage/schema/tracking-schema.sql');
|
||||
}
|
||||
|
||||
private getTrackingTables(db: any): string[] {
|
||||
const result = db.prepare(`
|
||||
SELECT name FROM sqlite_master
|
||||
WHERE type='table' AND name NOT LIKE 'sqlite_%'
|
||||
ORDER BY name
|
||||
`).all() as Array<{ name: string }>;
|
||||
|
||||
return result.map(t => t.name);
|
||||
}
|
||||
|
||||
private displayMigrationResults(tables: string[]): void {
|
||||
this.log(formatSuccess('Database schema migrated'));
|
||||
this.log(formatHeader('Tables'));
|
||||
tables.forEach(t => this.log(` ✓ ${t}`));
|
||||
}
|
||||
}
|
||||
70
self-hosting/omc/src/commands/db/reset.ts
Normal file
70
self-hosting/omc/src/commands/db/reset.ts
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { Flags } from '@oclif/core';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
|
||||
import { listTables, isOmnivoreTable } from '@storage/database.js';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
/**
|
||||
* Reset tracking tables (preserves Omnivore tables).
|
||||
* AIDEV-NOTE: Boundary protection - never drops Omnivore tables
|
||||
*/
|
||||
export default class DbReset extends BaseCommand {
|
||||
static override description = 'Drop and recreate tracking tables';
|
||||
|
||||
static override examples = [
|
||||
'$ omc db reset --force',
|
||||
'$ omc db reset --force --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
force: Flags.boolean({ description: 'Required for safety', default: false }),
|
||||
};
|
||||
|
||||
async execute(flags: any): Promise<void> {
|
||||
if (!flags.force) {
|
||||
this.error('--force flag required for reset operation');
|
||||
}
|
||||
|
||||
await withDatabase(async (db) => {
|
||||
const trackingTables = this.dropTrackingTables(db);
|
||||
this.recreateSchema(db);
|
||||
this.displayResult(flags, trackingTables);
|
||||
});
|
||||
}
|
||||
|
||||
private dropTrackingTables(db: any): string[] {
|
||||
const tables = listTables(db);
|
||||
const trackingTables = tables.filter(t => !isOmnivoreTable(db, t));
|
||||
trackingTables.forEach(t => db.exec(`DROP TABLE IF EXISTS ${t}`));
|
||||
return trackingTables;
|
||||
}
|
||||
|
||||
private recreateSchema(db: any): void {
|
||||
const schemaPath = this.getSchemaPath();
|
||||
const schema = readFileSync(schemaPath, 'utf-8');
|
||||
db.exec(schema);
|
||||
}
|
||||
|
||||
private displayResult(flags: any, droppedTables: string[]): void {
|
||||
const result = { droppedTables, recreated: true };
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
this.log(formatHeader('Database Reset'));
|
||||
this.log(`Dropped ${droppedTables.length} tracking tables`);
|
||||
this.log(formatSuccess('Schema recreated'));
|
||||
}
|
||||
}
|
||||
|
||||
private getSchemaPath(): string {
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
return join(__dirname, '../../../storage/schema/tracking-schema.sql');
|
||||
}
|
||||
}
|
||||
63
self-hosting/omc/src/commands/db/restore.ts
Normal file
63
self-hosting/omc/src/commands/db/restore.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { Args, Flags } from '@oclif/core';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { formatHeader, formatSuccess, formatError } from '@lib/cli/formatters.js';
|
||||
import { copyFileSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Restore database from backup.
|
||||
* AIDEV-NOTE: Requires --force flag for safety
|
||||
*/
|
||||
export default class DbRestore extends BaseCommand {
|
||||
static override description = 'Restore database from backup';
|
||||
|
||||
static override examples = [
|
||||
'$ omc db restore data/backup.db --force',
|
||||
'$ omc db restore data/backup.db --force --json',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
backupPath: Args.string({ description: 'Path to backup file', required: true }),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
force: Flags.boolean({ description: 'Skip confirmation', default: false }),
|
||||
};
|
||||
|
||||
async execute(flags: any): Promise<void> {
|
||||
const { args } = await this.parse(DbRestore);
|
||||
const backupPath = args.backupPath;
|
||||
|
||||
this.validateInputs(backupPath, flags.force);
|
||||
const dbPath = this.performRestore(backupPath);
|
||||
this.displayResult(flags, backupPath, dbPath);
|
||||
}
|
||||
|
||||
private validateInputs(backupPath: string, force: boolean): void {
|
||||
if (!existsSync(backupPath)) {
|
||||
this.error(formatError(`Backup file not found: ${backupPath}`));
|
||||
}
|
||||
if (!force) {
|
||||
this.error('--force flag required for restore operation');
|
||||
}
|
||||
}
|
||||
|
||||
private performRestore(backupPath: string): string {
|
||||
const dbPath = join(process.cwd(), 'data/omnivore-content.db');
|
||||
copyFileSync(backupPath, dbPath);
|
||||
return dbPath;
|
||||
}
|
||||
|
||||
private displayResult(flags: any, backupPath: string, dbPath: string): void {
|
||||
const result = { restored: true, from: backupPath, to: dbPath };
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
this.log(formatHeader('Database Restore'));
|
||||
this.log(formatSuccess(`Restored from: ${backupPath}`));
|
||||
}
|
||||
}
|
||||
}
|
||||
52
self-hosting/omc/src/commands/db/schema.ts
Normal file
52
self-hosting/omc/src/commands/db/schema.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader } from '@lib/cli/formatters.js';
|
||||
import { listTables } from '@storage/database.js';
|
||||
|
||||
/**
|
||||
* Display current database schema.
|
||||
* AIDEV-NOTE: Shows table structure with columns and types
|
||||
*/
|
||||
export default class DbSchema extends BaseCommand {
|
||||
static override description = 'Show current database schema';
|
||||
|
||||
static override examples = [
|
||||
'$ omc db schema',
|
||||
'$ omc db schema --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ db }) => {
|
||||
const tables = listTables(db);
|
||||
const schema = this.buildSchemaInfo(db, tables);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(schema, null, 2));
|
||||
} else {
|
||||
this.displaySchema(schema);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private buildSchemaInfo(db: any, tables: string[]) {
|
||||
return tables.map(table => ({
|
||||
name: table,
|
||||
columns: db.prepare(`PRAGMA table_info(${table})`).all(),
|
||||
}));
|
||||
}
|
||||
|
||||
private displaySchema(schema: any[]): void {
|
||||
this.log(formatHeader('Database Schema'));
|
||||
for (const table of schema) {
|
||||
this.log(`\n${table.name}:`);
|
||||
for (const col of table.columns) {
|
||||
this.log(` ${col.name.padEnd(30)} ${col.type}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
73
self-hosting/omc/src/commands/db/seed.ts
Normal file
73
self-hosting/omc/src/commands/db/seed.ts
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { formatSuccess } from '@lib/cli/formatters.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
|
||||
/**
|
||||
* Seed database with sample articles for testing.
|
||||
* AIDEV-NOTE: Creates 10 diverse sample articles with varied statuses
|
||||
*/
|
||||
export default class DbSeed extends BaseCommand {
|
||||
static override description = 'Seed database with sample data';
|
||||
|
||||
static override examples = [
|
||||
'$ omc db seed',
|
||||
'$ omc db seed --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async (db) => {
|
||||
const samples = this.createSampleData();
|
||||
const inserted = this.seedArticles(db, samples);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify({ inserted, total: samples.length }, null, 2));
|
||||
} else {
|
||||
this.log(formatSuccess(`Seeded ${inserted} sample articles`));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private seedArticles(db: any, samples: any[]): number {
|
||||
const stmt = db.prepare(`
|
||||
INSERT OR IGNORE INTO analysis_queue (
|
||||
article_id, article_slug, article_url, article_title,
|
||||
saved_at, status, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))
|
||||
`);
|
||||
|
||||
let inserted = 0;
|
||||
samples.forEach(article => {
|
||||
const result = stmt.run(
|
||||
article.articleId,
|
||||
article.articleSlug,
|
||||
article.articleUrl,
|
||||
article.articleTitle,
|
||||
article.savedAt,
|
||||
article.status
|
||||
);
|
||||
if (result.changes > 0) inserted++;
|
||||
});
|
||||
|
||||
return inserted;
|
||||
}
|
||||
|
||||
private createSampleData() {
|
||||
return [
|
||||
{ articleId: 'seed-001', articleSlug: 'intro-to-llms', articleUrl: 'https://omnivore.app/test/intro-to-llms', articleTitle: 'Introduction to Large Language Models', savedAt: '2025-01-01T10:00:00Z', status: 'pending' },
|
||||
{ articleId: 'seed-002', articleSlug: 'docker-best-practices', articleUrl: 'https://omnivore.app/test/docker-best-practices', articleTitle: 'Docker Best Practices 2025', savedAt: '2025-01-02T10:00:00Z', status: 'pending' },
|
||||
{ articleId: 'seed-003', articleSlug: 'rust-async-await', articleUrl: 'https://omnivore.app/test/rust-async-await', articleTitle: 'Understanding Async/Await in Rust', savedAt: '2025-01-03T10:00:00Z', status: 'pending' },
|
||||
{ articleId: 'seed-004', articleSlug: 'kubernetes-scaling', articleUrl: 'https://omnivore.app/test/kubernetes-scaling', articleTitle: 'Kubernetes Autoscaling Guide', savedAt: '2025-01-04T10:00:00Z', status: 'in_progress' },
|
||||
{ articleId: 'seed-005', articleSlug: 'typescript-generics', articleUrl: 'https://omnivore.app/test/typescript-generics', articleTitle: 'Mastering TypeScript Generics', savedAt: '2025-01-05T10:00:00Z', status: 'in_progress' },
|
||||
{ articleId: 'seed-006', articleSlug: 'graphql-federation', articleUrl: 'https://omnivore.app/test/graphql-federation', articleTitle: 'GraphQL Federation Explained', savedAt: '2025-01-06T10:00:00Z', status: 'completed' },
|
||||
{ articleId: 'seed-007', articleSlug: 'postgres-performance', articleUrl: 'https://omnivore.app/test/postgres-performance', articleTitle: 'PostgreSQL Performance Tuning', savedAt: '2025-01-07T10:00:00Z', status: 'completed' },
|
||||
{ articleId: 'seed-008', articleSlug: 'react-server-components', articleUrl: 'https://omnivore.app/test/react-server-components', articleTitle: 'React Server Components Deep Dive', savedAt: '2025-01-08T10:00:00Z', status: 'completed' },
|
||||
{ articleId: 'seed-009', articleSlug: 'distributed-tracing', articleUrl: 'https://omnivore.app/test/distributed-tracing', articleTitle: 'Distributed Tracing with OpenTelemetry', savedAt: '2025-01-09T10:00:00Z', status: 'failed' },
|
||||
{ articleId: 'seed-010', articleSlug: 'webassembly-performance', articleUrl: 'https://omnivore.app/test/webassembly-performance', articleTitle: 'WebAssembly Performance Benchmarks', savedAt: '2025-01-10T10:00:00Z', status: 'failed' },
|
||||
];
|
||||
}
|
||||
}
|
||||
59
self-hosting/omc/src/commands/db/stats.ts
Normal file
59
self-hosting/omc/src/commands/db/stats.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader } from '@lib/cli/formatters.js';
|
||||
import { getTableCounts } from '@storage/database.js';
|
||||
import { existsSync, statSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Display database statistics.
|
||||
* AIDEV-NOTE: Shows table counts and file size
|
||||
*/
|
||||
export default class DbStats extends BaseCommand {
|
||||
static override description = 'Show database statistics';
|
||||
|
||||
static override examples = [
|
||||
'$ omc db stats',
|
||||
'$ omc db stats --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ db }) => {
|
||||
const counts = getTableCounts(db);
|
||||
const dbPath = join(process.cwd(), 'data/omnivore-content.db');
|
||||
const fileSize = this.getFileSize(dbPath);
|
||||
|
||||
const result = { tables: counts, fileSize };
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
this.displayStats(counts, fileSize);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private getFileSize(path: string): number {
|
||||
return existsSync(path) ? statSync(path).size : 0;
|
||||
}
|
||||
|
||||
private displayStats(counts: Record<string, number>, size: number): void {
|
||||
this.log(formatHeader('Database Statistics'));
|
||||
this.log(`File size: ${this.formatBytes(size)}\n`);
|
||||
this.log('Table Row Counts:');
|
||||
for (const [table, count] of Object.entries(counts)) {
|
||||
this.log(` ${table.padEnd(30)} ${count.toLocaleString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
private formatBytes(bytes: number): string {
|
||||
return bytes >= 1024 * 1024
|
||||
? `${(bytes / 1024 / 1024).toFixed(2)} MB`
|
||||
: `${(bytes / 1024).toFixed(2)} KB`;
|
||||
}
|
||||
}
|
||||
73
self-hosting/omc/src/commands/db/vacuum.ts
Normal file
73
self-hosting/omc/src/commands/db/vacuum.ts
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader, formatSuccess } from '@lib/cli/formatters.js';
|
||||
import { existsSync, statSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Optimize database by running VACUUM.
|
||||
* AIDEV-NOTE: Reclaims unused space and defragments
|
||||
*/
|
||||
export default class DbVacuum extends BaseCommand {
|
||||
static override description = 'Optimize database with VACUUM';
|
||||
|
||||
static override examples = [
|
||||
'$ omc db vacuum',
|
||||
'$ omc db vacuum --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
// AIDEV-NOTE: Execute split into helpers for function length compliance
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
const dbPath = join(process.cwd(), 'data/omnivore-content.db');
|
||||
const sizeBefore = this.getFileSize(dbPath);
|
||||
|
||||
await withDatabase(async ({ db }) => {
|
||||
db.exec('VACUUM');
|
||||
});
|
||||
|
||||
const sizeAfter = this.getFileSize(dbPath);
|
||||
const result = this.buildResult(sizeBefore, sizeAfter);
|
||||
|
||||
this.displayResult(result, flags.json);
|
||||
}
|
||||
|
||||
private buildResult(
|
||||
sizeBefore: number,
|
||||
sizeAfter: number
|
||||
): { sizeBefore: number; sizeAfter: number; savedBytes: number } {
|
||||
return {
|
||||
sizeBefore,
|
||||
sizeAfter,
|
||||
savedBytes: sizeBefore - sizeAfter,
|
||||
};
|
||||
}
|
||||
|
||||
private displayResult(
|
||||
result: { sizeBefore: number; sizeAfter: number; savedBytes: number },
|
||||
asJson: boolean
|
||||
): void {
|
||||
if (asJson) {
|
||||
this.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
this.log(formatHeader('Database Vacuum'));
|
||||
this.log(`Size before: ${this.formatBytes(result.sizeBefore)}`);
|
||||
this.log(`Size after: ${this.formatBytes(result.sizeAfter)}`);
|
||||
this.log(formatSuccess(`Saved ${this.formatBytes(result.savedBytes)}`));
|
||||
}
|
||||
}
|
||||
|
||||
private getFileSize(path: string): number {
|
||||
return existsSync(path) ? statSync(path).size : 0;
|
||||
}
|
||||
|
||||
private formatBytes(bytes: number): string {
|
||||
return bytes >= 1024 * 1024
|
||||
? `${(bytes / 1024 / 1024).toFixed(2)} MB`
|
||||
: `${(bytes / 1024).toFixed(2)} KB`;
|
||||
}
|
||||
}
|
||||
115
self-hosting/omc/src/commands/doctor.ts
Normal file
115
self-hosting/omc/src/commands/doctor.ts
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { outputResult } from '@lib/cli/command-utils.js';
|
||||
import { initDatabase } from '@storage/database.js';
|
||||
import { testConnection } from '@lib/omnivore/client.js';
|
||||
import { existsSync } from 'fs';
|
||||
import { config } from 'dotenv';
|
||||
|
||||
/**
|
||||
* System health check for omnivore-content-system
|
||||
* AIDEV-NOTE: doctor-command - diagnostic tool for troubleshooting setup issues
|
||||
*/
|
||||
export default class Doctor extends BaseCommand {
|
||||
static override description = 'Check system health and configuration';
|
||||
|
||||
static override examples = [
|
||||
'<%= config.bin %> <%= command.id %>',
|
||||
'<%= config.bin %> <%= command.id %> --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { json: boolean }): Promise<void> {
|
||||
const checks = {
|
||||
database: await this.checkDatabase(),
|
||||
apiKey: this.checkApiKey(),
|
||||
apiConnection: await this.checkApiConnection(),
|
||||
directories: this.checkDirectories(),
|
||||
dependencies: this.checkDependencies(),
|
||||
};
|
||||
|
||||
const allHealthy = Object.values(checks).every(check => check.healthy);
|
||||
|
||||
if (flags.json) {
|
||||
outputResult(this, { checks, healthy: allHealthy }, '', true);
|
||||
} else {
|
||||
this.displayHealthReport(checks, allHealthy);
|
||||
}
|
||||
|
||||
if (!allHealthy) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private async checkDatabase(): Promise<CheckResult> {
|
||||
try {
|
||||
initDatabase('data/omnivore-content.db');
|
||||
return { healthy: true, message: 'Database connection OK' };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return { healthy: false, message: `Database error: ${message}` };
|
||||
}
|
||||
}
|
||||
|
||||
private checkApiKey(): CheckResult {
|
||||
config();
|
||||
if (process.env.OMNIVORE_API_KEY) {
|
||||
return { healthy: true, message: 'OMNIVORE_API_KEY found' };
|
||||
}
|
||||
return { healthy: false, message: 'OMNIVORE_API_KEY not set' };
|
||||
}
|
||||
|
||||
private async checkApiConnection(): Promise<CheckResult> {
|
||||
config();
|
||||
if (!process.env.OMNIVORE_API_KEY) {
|
||||
return { healthy: false, message: 'Cannot test - API key missing' };
|
||||
}
|
||||
|
||||
const connected = await testConnection();
|
||||
if (connected) {
|
||||
return { healthy: true, message: 'API connection successful' };
|
||||
}
|
||||
return { healthy: false, message: 'API connection failed' };
|
||||
}
|
||||
|
||||
private checkDirectories(): CheckResult {
|
||||
const required = ['data', 'content', 'temp'];
|
||||
const missing = required.filter(dir => !existsSync(dir));
|
||||
|
||||
if (missing.length === 0) {
|
||||
return { healthy: true, message: 'All required directories exist' };
|
||||
}
|
||||
return { healthy: false, message: `Missing directories: ${missing.join(', ')}` };
|
||||
}
|
||||
|
||||
private checkDependencies(): CheckResult {
|
||||
const nodeVersion = process.version;
|
||||
const requiredMajor = 18;
|
||||
const currentMajor = parseInt(nodeVersion.slice(1).split('.')[0]);
|
||||
|
||||
if (currentMajor >= requiredMajor) {
|
||||
return { healthy: true, message: `Node ${nodeVersion} (>= ${requiredMajor})` };
|
||||
}
|
||||
return { healthy: false, message: `Node ${nodeVersion} (requires >= ${requiredMajor})` };
|
||||
}
|
||||
|
||||
private displayHealthReport(checks: Record<string, CheckResult>, allHealthy: boolean): void {
|
||||
this.log('\nSystem Health Report\n');
|
||||
|
||||
for (const [name, check] of Object.entries(checks)) {
|
||||
const icon = check.healthy ? '✓' : '✗';
|
||||
const label = name.padEnd(20);
|
||||
this.log(`${icon} ${label} ${check.message}`);
|
||||
}
|
||||
|
||||
this.log(`\nOverall Status: ${allHealthy ? '✓ Healthy' : '✗ Issues detected'}`);
|
||||
}
|
||||
}
|
||||
|
||||
interface CheckResult {
|
||||
healthy: boolean;
|
||||
message: string;
|
||||
}
|
||||
118
self-hosting/omc/src/commands/init.ts
Normal file
118
self-hosting/omc/src/commands/init.ts
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { formatSuccess, formatError } from '@lib/cli/formatters.js';
|
||||
import { initDatabase } from '@storage/database.js';
|
||||
import { testConnection } from '@lib/omnivore/client.js';
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
|
||||
import { config } from 'dotenv';
|
||||
|
||||
/**
|
||||
* Interactive setup wizard for omnivore-content-system
|
||||
* AIDEV-NOTE: init-command - one-time setup for new installations
|
||||
*/
|
||||
export default class Init extends BaseCommand {
|
||||
static override description = 'Initialize omnivore-content-system setup';
|
||||
|
||||
static override examples = [
|
||||
'<%= config.bin %> <%= command.id %>',
|
||||
'<%= config.bin %> <%= command.id %> --force',
|
||||
'<%= config.bin %> <%= command.id %> --api-key YOUR_KEY',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
force: Flags.boolean({
|
||||
description: 'Reinitialize existing setup',
|
||||
default: false,
|
||||
}),
|
||||
'api-key': Flags.string({
|
||||
description: 'Set OMNIVORE_API_KEY non-interactively',
|
||||
}),
|
||||
};
|
||||
|
||||
protected async execute(flags: { force: boolean; 'api-key'?: string }): Promise<void> {
|
||||
this.log('Initializing omnivore-content-system...\n');
|
||||
|
||||
await this.ensureDirectories();
|
||||
await this.initializeDatabase(flags.force);
|
||||
await this.setupEnvironment(flags['api-key']);
|
||||
await this.testApiConnection();
|
||||
|
||||
this.log('\n' + formatSuccess('Setup complete! Run "omc --help" to get started.'));
|
||||
}
|
||||
|
||||
private async ensureDirectories(): Promise<void> {
|
||||
const dirs = ['data', 'content', 'temp', 'content/articles', 'content/analysis', 'content/generated'];
|
||||
|
||||
for (const dir of dirs) {
|
||||
if (!existsSync(dir)) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
this.log(formatSuccess(`Created directory: ${dir}`));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async initializeDatabase(force: boolean): Promise<void> {
|
||||
try {
|
||||
initDatabase('data/omnivore-content.db');
|
||||
this.log(formatSuccess('Database initialized'));
|
||||
} catch (error) {
|
||||
if (force) {
|
||||
this.log(formatSuccess('Database reinitialized'));
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async setupEnvironment(apiKey?: string): Promise<void> {
|
||||
this.createEnvFile();
|
||||
|
||||
if (apiKey) {
|
||||
this.updateEnvFile('OMNIVORE_API_KEY', apiKey);
|
||||
this.log(formatSuccess('OMNIVORE_API_KEY set'));
|
||||
} else {
|
||||
this.checkApiKey();
|
||||
}
|
||||
}
|
||||
|
||||
private createEnvFile(): void {
|
||||
if (!existsSync('.env') && existsSync('.env.example')) {
|
||||
const example = readFileSync('.env.example', 'utf-8');
|
||||
writeFileSync('.env', example);
|
||||
this.log(formatSuccess('Created .env from .env.example'));
|
||||
} else if (!existsSync('.env')) {
|
||||
this.log(formatError('.env.example not found'));
|
||||
}
|
||||
}
|
||||
|
||||
private checkApiKey(): void {
|
||||
config();
|
||||
if (!process.env.OMNIVORE_API_KEY) {
|
||||
this.log(formatError('OMNIVORE_API_KEY not set in .env'));
|
||||
this.log('Please add your API key to .env file');
|
||||
}
|
||||
}
|
||||
|
||||
private async testApiConnection(): Promise<void> {
|
||||
config();
|
||||
if (process.env.OMNIVORE_API_KEY) {
|
||||
const connected = await testConnection();
|
||||
if (!connected) {
|
||||
this.log(formatError('API connection test failed'));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private updateEnvFile(key: string, value: string): void {
|
||||
let envContent = existsSync('.env') ? readFileSync('.env', 'utf-8') : '';
|
||||
const regex = new RegExp(`^${key}=.*$`, 'm');
|
||||
|
||||
if (regex.test(envContent)) {
|
||||
envContent = envContent.replace(regex, `${key}=${value}`);
|
||||
} else {
|
||||
envContent += `\n${key}=${value}\n`;
|
||||
}
|
||||
|
||||
writeFileSync('.env', envContent);
|
||||
}
|
||||
}
|
||||
65
self-hosting/omc/src/commands/omnivore/get.ts
Normal file
65
self-hosting/omc/src/commands/omnivore/get.ts
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
import { Args, Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { fetchUsername, checkGraphQLResult } from '@lib/cli/graphql.js';
|
||||
import { getArticle } from '@lib/omnivore/client.js';
|
||||
import type { OmnivoreArticle } from '@omc-types/omnivore.js';
|
||||
|
||||
/**
|
||||
* Fetch article by slug.
|
||||
* AIDEV-NOTE: Direct Omnivore API access for article retrieval
|
||||
*/
|
||||
export default class OmnivoreGet extends BaseCommand {
|
||||
static override description = 'Fetch article by slug';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore get my-article-slug',
|
||||
'$ omc omnivore get my-article-slug --json',
|
||||
'$ omc omnivore get my-article-slug --username different-user',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
slug: Args.string({ description: 'Article slug', required: true }),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
username: Flags.string({ char: 'u', description: 'Username (defaults to authenticated user)' }),
|
||||
content: Flags.boolean({
|
||||
description: 'Print raw article content to stdout (agent-friendly)',
|
||||
default: false,
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { slug: string; username?: string; content: boolean; json: boolean }): Promise<void> {
|
||||
const username = flags.username || await fetchUsername();
|
||||
const result = await getArticle(flags.slug, username);
|
||||
checkGraphQLResult({ data: { article: result }, errors: undefined });
|
||||
|
||||
if (!result.article) {
|
||||
throw new Error('Article not found');
|
||||
}
|
||||
|
||||
if (flags.content && flags.json) {
|
||||
throw new Error('Use either --content or --json (not both)');
|
||||
}
|
||||
|
||||
if (flags.content) {
|
||||
this.log(result.article?.content ?? '');
|
||||
return;
|
||||
}
|
||||
|
||||
this.outputArticle(result.article, flags.json);
|
||||
}
|
||||
|
||||
private outputArticle(article: OmnivoreArticle, json: boolean): void {
|
||||
if (json) {
|
||||
this.log(JSON.stringify(article, null, 2));
|
||||
} else {
|
||||
this.log(`Title: ${article.title}`);
|
||||
this.log(`URL: ${article.url}`);
|
||||
this.log(`Author: ${article.author || 'N/A'}`);
|
||||
this.log(`Description: ${article.description || 'N/A'}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
53
self-hosting/omc/src/commands/omnivore/highlight/add.ts
Normal file
53
self-hosting/omc/src/commands/omnivore/highlight/add.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import { Args, Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { checkGraphQLResult } from '@lib/cli/graphql.js';
|
||||
import { createHighlight } from '@lib/omnivore/client.js';
|
||||
|
||||
/**
|
||||
* Add highlight to article.
|
||||
* AIDEV-NOTE: Creates HIGHLIGHT-type with quote (optional annotation)
|
||||
*/
|
||||
export default class HighlightAdd extends BaseCommand {
|
||||
static override description = 'Add highlight to article';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore highlight add article-id "quote text"',
|
||||
'$ omc omnivore highlight add article-id "quote" --annotation "my note"',
|
||||
'$ omc omnivore highlight add article-id "quote" --color "#FF0000" --json',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
articleId: Args.string({ description: 'Article ID', required: true }),
|
||||
quote: Args.string({ description: 'Highlighted text', required: true }),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
annotation: Flags.string({ char: 'a', description: 'Optional annotation' }),
|
||||
color: Flags.string({ char: 'c', description: 'Highlight color', default: '#FFD700' }),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { articleId: string; quote: string; annotation?: string; color: string; json: boolean }): Promise<void> {
|
||||
const result = await createHighlight({
|
||||
id: crypto.randomUUID(),
|
||||
shortId: crypto.randomUUID().substring(0, 8),
|
||||
articleId: flags.articleId,
|
||||
quote: flags.quote,
|
||||
annotation: flags.annotation,
|
||||
color: flags.color,
|
||||
type: 'HIGHLIGHT',
|
||||
});
|
||||
checkGraphQLResult({ data: { createHighlight: result }, errors: undefined });
|
||||
|
||||
this.outputResult(result, flags.json);
|
||||
}
|
||||
|
||||
private outputResult(result: any, json: boolean): void {
|
||||
if (json) {
|
||||
this.log(JSON.stringify(result.highlight, null, 2));
|
||||
} else {
|
||||
this.log('Highlight added successfully');
|
||||
}
|
||||
}
|
||||
}
|
||||
57
self-hosting/omc/src/commands/omnivore/highlight/list.ts
Normal file
57
self-hosting/omc/src/commands/omnivore/highlight/list.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { Args } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { fetchUsername } from '@lib/cli/graphql.js';
|
||||
import { getHighlights } from '@lib/omnivore/client.js';
|
||||
|
||||
type HighlightRecord = {
|
||||
id: string;
|
||||
quote?: string | null;
|
||||
annotation?: string | null;
|
||||
createdAt?: string;
|
||||
type?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* List highlights for article.
|
||||
* AIDEV-NOTE: Filters to show only HIGHLIGHT type (excludes NOTEs)
|
||||
*/
|
||||
export default class HighlightList extends BaseCommand {
|
||||
static override description = 'List highlights for article';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore highlight list article-id',
|
||||
'$ omc omnivore highlight list article-id --json',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
articleId: Args.string({ description: 'Article ID', required: true }),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { articleId: string; json: boolean }): Promise<void> {
|
||||
const username = await fetchUsername();
|
||||
const allHighlights = (await getHighlights(flags.articleId, username)) as HighlightRecord[];
|
||||
const highlights = this.filterHighlights(allHighlights);
|
||||
|
||||
this.outputHighlights(highlights, flags.json);
|
||||
}
|
||||
|
||||
private filterHighlights(highlights: HighlightRecord[]): HighlightRecord[] {
|
||||
return highlights.filter((h) => h.type === 'HIGHLIGHT' || Boolean(h.quote));
|
||||
}
|
||||
|
||||
private outputHighlights(highlights: HighlightRecord[], json: boolean): void {
|
||||
if (json) {
|
||||
this.log(JSON.stringify(highlights, null, 2));
|
||||
} else {
|
||||
highlights.forEach((h) => {
|
||||
this.log(`- "${h.quote ?? ''}"`);
|
||||
if (h.annotation) this.log(` Note: ${h.annotation}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
38
self-hosting/omc/src/commands/omnivore/label/create.ts
Normal file
38
self-hosting/omc/src/commands/omnivore/label/create.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { checkGraphQLResult } from '@lib/cli/graphql.js';
|
||||
import { createLabel } from '@lib/omnivore/client.js';
|
||||
|
||||
/**
|
||||
* Create a label.
|
||||
* AIDEV-NOTE: Replacement for standalone scripts/test-create-label.js and migrate helpers.
|
||||
*/
|
||||
export default class OmnivoreLabelCreate extends BaseCommand {
|
||||
static override description = 'Create a label';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore label create --name "ai"',
|
||||
'$ omc omnivore label create --name "ai" --color "#3B82F6" --description "AI/ML" --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
name: Flags.string({ description: 'Label name', required: true }),
|
||||
color: Flags.string({ description: 'Hex color (e.g. #3B82F6)' }),
|
||||
description: Flags.string({ description: 'Description' }),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { name: string; color?: string; description?: string; json: boolean }): Promise<void> {
|
||||
const result = await createLabel({ name: flags.name, color: flags.color, description: flags.description });
|
||||
checkGraphQLResult({ data: { createLabel: result }, errors: undefined });
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(result.label ?? result, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
this.log(`Label created: ${result.label?.name ?? flags.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
36
self-hosting/omc/src/commands/omnivore/label/list.ts
Normal file
36
self-hosting/omc/src/commands/omnivore/label/list.ts
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { checkGraphQLResult } from '@lib/cli/graphql.js';
|
||||
import { getLabels } from '@lib/omnivore/client.js';
|
||||
|
||||
/**
|
||||
* List labels.
|
||||
* AIDEV-NOTE: Integrates legacy Omnivore label scripts into the OMC CLI.
|
||||
*/
|
||||
export default class OmnivoreLabelList extends BaseCommand {
|
||||
static override description = 'List labels';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore label list',
|
||||
'$ omc omnivore label list --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { json: boolean }): Promise<void> {
|
||||
const labels = await getLabels();
|
||||
checkGraphQLResult({ data: { labels }, errors: undefined });
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(labels, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
for (const label of labels) {
|
||||
this.log(`- ${label.name} (${label.id})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
51
self-hosting/omc/src/commands/omnivore/label/set.ts
Normal file
51
self-hosting/omc/src/commands/omnivore/label/set.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import { Args, Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { checkGraphQLResult } from '@lib/cli/graphql.js';
|
||||
import { setLabels } from '@lib/omnivore/client.js';
|
||||
|
||||
/**
|
||||
* Set labels on a page (replaces existing labels).
|
||||
* AIDEV-NOTE: Integrates apply-single-label.js / apply-labels*.js into OMC CLI surface.
|
||||
*/
|
||||
export default class OmnivoreLabelSet extends BaseCommand {
|
||||
static override description = 'Set labels on a page (replaces existing labels)';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore label set <page-id> --label "ai"',
|
||||
'$ omc omnivore label set <page-id> --label "ai" --label "devops"',
|
||||
'$ omc omnivore label set <page-id> --label "ai" --source "omc" --json',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
pageId: Args.string({ description: 'Page/Article ID', required: true }),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
label: Flags.string({
|
||||
description: 'Label name (repeatable)',
|
||||
multiple: true,
|
||||
required: true,
|
||||
}),
|
||||
source: Flags.string({
|
||||
description: 'Optional source string (stored by Omnivore)',
|
||||
default: 'omc',
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { pageId: string; label: string[]; source: string; json: boolean }): Promise<void> {
|
||||
const labels = flags.label.map((name) => ({ name }));
|
||||
const result = await setLabels({ pageId: flags.pageId, labels, source: flags.source });
|
||||
checkGraphQLResult({ data: { setLabels: result }, errors: undefined });
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(result, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
const applied = (result.labels ?? []).map((l: any) => l.name).join(', ');
|
||||
this.log(`Labels set: ${applied || '(none)'}`);
|
||||
}
|
||||
}
|
||||
|
||||
44
self-hosting/omc/src/commands/omnivore/list.ts
Normal file
44
self-hosting/omc/src/commands/omnivore/list.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { checkGraphQLResult } from '@lib/cli/graphql.js';
|
||||
import { getRecentArticles } from '@lib/omnivore/client.js';
|
||||
import type { SearchResult } from '@omc-types/omnivore.js';
|
||||
|
||||
/**
|
||||
* List recent articles.
|
||||
* AIDEV-NOTE: Time-filtered article listing
|
||||
*/
|
||||
export default class OmnivoreList extends BaseCommand {
|
||||
static override description = 'List recent articles';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore list',
|
||||
'$ omc omnivore list --hours 48',
|
||||
'$ omc omnivore list --hours 24 --limit 20 --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
hours: Flags.integer({ char: 'h', description: 'Hours to look back', default: 24 }),
|
||||
limit: Flags.integer({ char: 'l', description: 'Max results', default: 10 }),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: Record<string, any>): Promise<void> {
|
||||
const result = await getRecentArticles(flags.hours, flags.limit);
|
||||
checkGraphQLResult({ data: { search: result }, errors: undefined });
|
||||
|
||||
this.outputResults(result.edges, flags.json);
|
||||
}
|
||||
|
||||
private outputResults(edges: SearchResult['edges'], json: boolean): void {
|
||||
if (json) {
|
||||
this.log(JSON.stringify(edges.map((e) => e.node), null, 2));
|
||||
} else {
|
||||
edges.forEach(({ node }) => {
|
||||
this.log(`- ${node.title}`);
|
||||
this.log(` Saved: ${node.savedAt}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
92
self-hosting/omc/src/commands/omnivore/mapping/download.ts
Normal file
92
self-hosting/omc/src/commands/omnivore/mapping/download.ts
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { searchArticles } from '@lib/omnivore/client.js';
|
||||
import Database from 'better-sqlite3';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
|
||||
/**
|
||||
* Download a URL -> pageId mapping into a local SQLite DB.
|
||||
* AIDEV-NOTE: Integrates scripts/download-items-mapping.js functionality into OMC CLI.
|
||||
*/
|
||||
export default class OmnivoreMappingDownload extends BaseCommand {
|
||||
static override description = 'Download Omnivore URL-to-ID mapping into SQLite';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore mapping download',
|
||||
'$ omc omnivore mapping download --destination temp/url-id-mapping.sqlite',
|
||||
'$ omc omnivore mapping download --limit 1000 --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
destination: Flags.string({
|
||||
description: 'Output SQLite path',
|
||||
default: 'temp/url-id-mapping.sqlite',
|
||||
}),
|
||||
limit: Flags.integer({
|
||||
description: 'Optional max items (for testing)',
|
||||
required: false,
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { destination: string; limit?: number; json: boolean }): Promise<void> {
|
||||
const destination = resolve(flags.destination);
|
||||
mkdirSync(dirname(destination), { recursive: true });
|
||||
|
||||
const db = new Database(destination);
|
||||
try {
|
||||
this.initSchema(db);
|
||||
const downloaded = await this.downloadInto(db, flags.limit);
|
||||
const result = { destination, downloaded };
|
||||
|
||||
if (flags.json) this.log(JSON.stringify(result, null, 2));
|
||||
else this.log(`Saved ${downloaded} items to ${destination}`);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
private initSchema(db: Database.Database): void {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS item_mapping (
|
||||
id TEXT PRIMARY KEY,
|
||||
url TEXT NOT NULL,
|
||||
title TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_item_mapping_url ON item_mapping(url);
|
||||
DELETE FROM item_mapping;
|
||||
`);
|
||||
}
|
||||
|
||||
private async downloadInto(db: Database.Database, limit?: number): Promise<number> {
|
||||
const insert = db.prepare('INSERT OR REPLACE INTO item_mapping (id, url, title) VALUES (?, ?, ?)');
|
||||
const insertMany = db.transaction((rows: Array<{ id: string; url: string; title?: string }>) => {
|
||||
for (const r of rows) insert.run(r.id, r.url, r.title ?? null);
|
||||
});
|
||||
|
||||
let after: string | undefined;
|
||||
let total = 0;
|
||||
|
||||
while (true) {
|
||||
const first = 100;
|
||||
const result: any = await searchArticles({ query: '', first, after: after ?? '', includeContent: false });
|
||||
|
||||
const edges = result.edges ?? [];
|
||||
let rows = edges.map((e: any) => ({ id: e.node.id, url: e.node.url, title: e.node.title }));
|
||||
if (limit && total + rows.length > limit) {
|
||||
rows = rows.slice(0, Math.max(0, limit - total));
|
||||
}
|
||||
insertMany(rows);
|
||||
|
||||
total += rows.length;
|
||||
if (limit && total >= limit) return total;
|
||||
|
||||
const pageInfo = result.pageInfo;
|
||||
if (!pageInfo?.hasNextPage) return total;
|
||||
after = pageInfo.endCursor;
|
||||
}
|
||||
}
|
||||
}
|
||||
48
self-hosting/omc/src/commands/omnivore/note/add.ts
Normal file
48
self-hosting/omc/src/commands/omnivore/note/add.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
import { Args } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { checkGraphQLResult } from '@lib/cli/graphql.js';
|
||||
import { createHighlight } from '@lib/omnivore/client.js';
|
||||
|
||||
/**
|
||||
* Add note to article.
|
||||
* AIDEV-NOTE: Creates NOTE-type highlight (no quote required)
|
||||
*/
|
||||
export default class NoteAdd extends BaseCommand {
|
||||
static override description = 'Add note to article';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore note add article-id "My note content"',
|
||||
'$ omc omnivore note add article-id "Note" --json',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
articleId: Args.string({ description: 'Article ID', required: true }),
|
||||
note: Args.string({ description: 'Note content', required: true }),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { articleId: string; note: string; json: boolean }): Promise<void> {
|
||||
const result = await createHighlight({
|
||||
id: crypto.randomUUID(),
|
||||
shortId: crypto.randomUUID().substring(0, 8),
|
||||
articleId: flags.articleId,
|
||||
annotation: flags.note,
|
||||
type: 'NOTE',
|
||||
});
|
||||
checkGraphQLResult({ data: { createHighlight: result }, errors: undefined });
|
||||
|
||||
this.outputResult(result, flags.json);
|
||||
}
|
||||
|
||||
private outputResult(result: any, json: boolean): void {
|
||||
if (json) {
|
||||
this.log(JSON.stringify(result.highlight, null, 2));
|
||||
} else {
|
||||
this.log('Note added successfully');
|
||||
}
|
||||
}
|
||||
}
|
||||
57
self-hosting/omc/src/commands/omnivore/note/get.ts
Normal file
57
self-hosting/omc/src/commands/omnivore/note/get.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { Args } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { fetchUsername } from '@lib/cli/graphql.js';
|
||||
import { getHighlights } from '@lib/omnivore/client.js';
|
||||
|
||||
type NoteRecord = { annotation?: string | null; createdAt?: string; quote?: string | null; type?: string };
|
||||
|
||||
/**
|
||||
* Get notes for article.
|
||||
* AIDEV-NOTE: Filters highlights to show only NOTE type
|
||||
*/
|
||||
export default class NoteGet extends BaseCommand {
|
||||
static override description = 'Get notes for article';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore note get article-id',
|
||||
'$ omc omnivore note get article-id --json',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
articleId: Args.string({ description: 'Article ID', required: true }),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { articleId: string; json: boolean }): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const job = repo.getByArticleId(flags.articleId);
|
||||
if (!job) {
|
||||
this.error(`Article not found in database: ${flags.articleId}`);
|
||||
}
|
||||
const username = await fetchUsername();
|
||||
const highlights = (await getHighlights(job.articleSlug, username)) as NoteRecord[];
|
||||
const notes = this.filterNotes(highlights);
|
||||
this.outputNotes(notes, flags.json);
|
||||
});
|
||||
}
|
||||
|
||||
private filterNotes(highlights: NoteRecord[]): NoteRecord[] {
|
||||
return highlights.filter((h) => h.type === 'NOTE' || (!h.quote && h.annotation));
|
||||
}
|
||||
|
||||
private outputNotes(notes: NoteRecord[], json: boolean): void {
|
||||
if (json) {
|
||||
this.log(JSON.stringify(notes, null, 2));
|
||||
} else {
|
||||
notes.forEach((note) => {
|
||||
this.log(`- ${note.annotation ?? ''}`);
|
||||
if (note.createdAt) this.log(` Created: ${note.createdAt}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
45
self-hosting/omc/src/commands/omnivore/note/update.ts
Normal file
45
self-hosting/omc/src/commands/omnivore/note/update.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import { Args } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { checkGraphQLResult } from '@lib/cli/graphql.js';
|
||||
import { updateHighlight } from '@lib/omnivore/client.js';
|
||||
|
||||
/**
|
||||
* Update note content.
|
||||
* AIDEV-NOTE: Updates annotation field of highlight
|
||||
*/
|
||||
export default class NoteUpdate extends BaseCommand {
|
||||
static override description = 'Update note content';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore note update highlight-id "Updated note"',
|
||||
'$ omc omnivore note update highlight-id "New content" --json',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
highlightId: Args.string({ description: 'Highlight ID', required: true }),
|
||||
note: Args.string({ description: 'New note content', required: true }),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { highlightId: string; note: string; json: boolean }): Promise<void> {
|
||||
const result = await updateHighlight({
|
||||
highlightId: flags.highlightId,
|
||||
annotation: flags.note,
|
||||
});
|
||||
checkGraphQLResult({ data: { updateHighlight: result }, errors: undefined });
|
||||
|
||||
this.outputResult(result, flags.json);
|
||||
}
|
||||
|
||||
private outputResult(result: any, json: boolean): void {
|
||||
if (json) {
|
||||
this.log(JSON.stringify(result.highlight, null, 2));
|
||||
} else {
|
||||
this.log('Note updated successfully');
|
||||
}
|
||||
}
|
||||
}
|
||||
47
self-hosting/omc/src/commands/omnivore/search.ts
Normal file
47
self-hosting/omc/src/commands/omnivore/search.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import { Args, Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { checkGraphQLResult } from '@lib/cli/graphql.js';
|
||||
import { searchArticles } from '@lib/omnivore/client.js';
|
||||
import type { SearchResult } from '@omc-types/omnivore.js';
|
||||
|
||||
/**
|
||||
* Search articles.
|
||||
* AIDEV-NOTE: Omnivore search with query DSL support
|
||||
*/
|
||||
export default class OmnivoreSearch extends BaseCommand {
|
||||
static override description = 'Search articles';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore search "AI machine learning"',
|
||||
'$ omc omnivore search "label:tech" --limit 20',
|
||||
'$ omc omnivore search "in:inbox" --json',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
query: Args.string({ description: 'Search query', required: true }),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
limit: Flags.integer({ char: 'l', description: 'Max results', default: 10 }),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { query: string; limit: number; json: boolean }): Promise<void> {
|
||||
const result = await searchArticles({ query: flags.query, first: flags.limit });
|
||||
checkGraphQLResult({ data: { search: result }, errors: undefined });
|
||||
|
||||
this.outputResults(result.edges, flags.json);
|
||||
}
|
||||
|
||||
private outputResults(edges: SearchResult['edges'], json: boolean): void {
|
||||
if (json) {
|
||||
this.log(JSON.stringify(edges.map((e) => e.node), null, 2));
|
||||
} else {
|
||||
edges.forEach(({ node }) => {
|
||||
this.log(`- ${node.title}`);
|
||||
this.log(` URL: ${node.url}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
56
self-hosting/omc/src/commands/omnivore/update.ts
Normal file
56
self-hosting/omc/src/commands/omnivore/update.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import { Args, Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { checkGraphQLResult } from '@lib/cli/graphql.js';
|
||||
import { updatePage } from '@lib/omnivore/client.js';
|
||||
|
||||
/**
|
||||
* Update article metadata.
|
||||
* AIDEV-NOTE: Modify article title/description via Omnivore API
|
||||
*/
|
||||
export default class OmnivoreUpdate extends BaseCommand {
|
||||
static override description = 'Update article metadata';
|
||||
|
||||
static override examples = [
|
||||
'$ omc omnivore update article-id --title "New Title"',
|
||||
'$ omc omnivore update article-id --description "Summary"',
|
||||
'$ omc omnivore update article-id --title "Title" --description "Desc" --json',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
articleId: Args.string({ description: 'Article ID', required: true }),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
title: Flags.string({ char: 't', description: 'New title' }),
|
||||
description: Flags.string({ char: 'd', description: 'New description' }),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: { articleId: string; title?: string; description?: string; json: boolean }): Promise<void> {
|
||||
this.validateFlags(flags);
|
||||
|
||||
const result = await updatePage({
|
||||
pageId: flags.articleId,
|
||||
title: flags.title,
|
||||
description: flags.description,
|
||||
});
|
||||
checkGraphQLResult({ data: { updatePage: result }, errors: undefined });
|
||||
|
||||
this.outputResult(result, flags.json);
|
||||
}
|
||||
|
||||
private validateFlags(flags: { title?: string; description?: string }): void {
|
||||
if (!flags.title && !flags.description) {
|
||||
throw new Error('At least one of --title or --description is required');
|
||||
}
|
||||
}
|
||||
|
||||
private outputResult(result: any, json: boolean): void {
|
||||
if (json) {
|
||||
this.log(JSON.stringify(result.updatedPage, null, 2));
|
||||
} else {
|
||||
this.log('Article updated successfully');
|
||||
}
|
||||
}
|
||||
}
|
||||
193
self-hosting/omc/src/commands/queue/add.ts
Normal file
193
self-hosting/omc/src/commands/queue/add.ts
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatSuccess } from '@lib/cli/formatters.js';
|
||||
import { fetchUsername, checkGraphQLResult } from '@lib/cli/graphql.js';
|
||||
import { searchArticles, getArticle, getArticlesByLabel } from '@lib/omnivore/client.js';
|
||||
import type { OmnivoreArticle, SearchResult } from '@omc-types/omnivore.js';
|
||||
|
||||
interface QueueAddFlags {
|
||||
hours?: number;
|
||||
url?: string;
|
||||
label?: string;
|
||||
slug?: string;
|
||||
json: boolean;
|
||||
}
|
||||
|
||||
type QueueSourceArticle = Pick<OmnivoreArticle, 'id' | 'url' | 'title' | 'savedAt'> & { slug: string };
|
||||
|
||||
/**
|
||||
* Add articles to the analysis queue.
|
||||
* AIDEV-NOTE: Entry point for queue population from Omnivore API
|
||||
*/
|
||||
export default class QueueAdd extends BaseCommand {
|
||||
static override description = 'Add articles to the analysis queue';
|
||||
|
||||
static override examples = [
|
||||
'$ omc queue add --hours 24',
|
||||
'$ omc queue add --hours 168',
|
||||
'$ omc queue add --label AI',
|
||||
'$ omc queue add --slug my-article-slug',
|
||||
'$ omc queue add --url https://omnivore.app/username/article-slug',
|
||||
'$ omc queue add --hours 24 --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
hours: Flags.integer({
|
||||
char: 'h',
|
||||
description: 'Add articles from last N hours',
|
||||
exclusive: ['url', 'label', 'slug'],
|
||||
}),
|
||||
url: Flags.string({
|
||||
char: 'u',
|
||||
description: 'Add single article by Omnivore URL',
|
||||
exclusive: ['hours', 'label', 'slug'],
|
||||
}),
|
||||
label: Flags.string({
|
||||
char: 'l',
|
||||
description: 'Add articles with specific label',
|
||||
exclusive: ['hours', 'url', 'slug'],
|
||||
}),
|
||||
slug: Flags.string({
|
||||
char: 's',
|
||||
description: 'Add single article by slug',
|
||||
exclusive: ['hours', 'url', 'label'],
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: QueueAddFlags): Promise<void> {
|
||||
this.validateFlags(flags);
|
||||
|
||||
await withDatabase(async ({ repo }) => {
|
||||
let articles: QueueSourceArticle[] = [];
|
||||
|
||||
if (flags.label) {
|
||||
articles = await this.fetchByLabel(flags.label);
|
||||
} else if (flags.url) {
|
||||
articles = await this.fetchByUrl(flags.url);
|
||||
} else if (flags.slug) {
|
||||
articles = await this.fetchBySlug(flags.slug);
|
||||
} else {
|
||||
articles = await this.fetchByHours(this.requireHours(flags.hours));
|
||||
}
|
||||
|
||||
const queueArticles = this.formatArticlesForQueue(articles);
|
||||
const inserted = repo.initializeQueue(queueArticles);
|
||||
this.outputResults(inserted, articles.length, flags.json);
|
||||
});
|
||||
}
|
||||
|
||||
private validateFlags(flags: QueueAddFlags): void {
|
||||
if (!flags.hours && !flags.url && !flags.label && !flags.slug) {
|
||||
throw new Error('One of --hours, --url, --label, or --slug is required');
|
||||
}
|
||||
}
|
||||
|
||||
private requireHours(hours: number | undefined): number {
|
||||
if (hours === undefined) throw new Error('--hours is required');
|
||||
return hours;
|
||||
}
|
||||
|
||||
private async fetchByHours(hours: number): Promise<QueueSourceArticle[]> {
|
||||
const hoursAgo = new Date(Date.now() - hours * 60 * 60 * 1000);
|
||||
const collected: QueueSourceArticle[] = [];
|
||||
let after: string | undefined;
|
||||
let shouldContinue = true;
|
||||
|
||||
while (shouldContinue) {
|
||||
const result = await this.fetchSearchPage(after);
|
||||
const { items, reachedCutoff, nextCursor } = this.collectWithinCutoff(result, hoursAgo);
|
||||
collected.push(...items);
|
||||
shouldContinue = !reachedCutoff && Boolean(nextCursor);
|
||||
after = nextCursor ?? undefined;
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
|
||||
private async fetchSearchPage(after?: string): Promise<SearchResult> {
|
||||
const result = await searchArticles({
|
||||
query: 'in:all sort:saved-desc',
|
||||
first: 100,
|
||||
after: after ?? '',
|
||||
});
|
||||
|
||||
if (!isSearchResult(result)) throw new Error('Unexpected searchArticles response shape');
|
||||
return result;
|
||||
}
|
||||
|
||||
private collectWithinCutoff(result: SearchResult, cutoff: Date): { items: QueueSourceArticle[]; reachedCutoff: boolean; nextCursor?: string } {
|
||||
const items: QueueSourceArticle[] = [];
|
||||
for (const edge of result.edges) {
|
||||
const node = edge.node;
|
||||
if (new Date(node.savedAt) < cutoff) return { items, reachedCutoff: true };
|
||||
if (isQueueSourceArticle(node)) items.push(node);
|
||||
}
|
||||
return { items, reachedCutoff: false, nextCursor: result.pageInfo.endCursor };
|
||||
}
|
||||
|
||||
private async fetchByLabel(labelName: string): Promise<QueueSourceArticle[]> {
|
||||
const result = await getArticlesByLabel(labelName, 50);
|
||||
if (!isSearchResult(result)) throw new Error('Unexpected getArticlesByLabel response shape');
|
||||
checkGraphQLResult({ data: { search: result }, errors: undefined });
|
||||
return result.edges.map((edge) => this.toQueueSourceArticle(edge.node));
|
||||
}
|
||||
|
||||
private async fetchBySlug(slug: string): Promise<QueueSourceArticle[]> {
|
||||
const username = await fetchUsername();
|
||||
const result = await getArticle(slug, username);
|
||||
checkGraphQLResult({ data: { article: result }, errors: undefined });
|
||||
if (!result.article) {
|
||||
throw new Error('Article not found');
|
||||
}
|
||||
return [this.toQueueSourceArticle({ ...result.article, slug })];
|
||||
}
|
||||
|
||||
private async fetchByUrl(url: string): Promise<QueueSourceArticle[]> {
|
||||
// AIDEV-NOTE: URL format is https://omnivore.app/{username}/{slug}
|
||||
const urlPattern = /https:\/\/omnivore\.app\/([^/]+)\/(.+)/;
|
||||
const match = url.match(urlPattern);
|
||||
|
||||
if (!match) {
|
||||
throw new Error('Invalid Omnivore URL. Expected format: https://omnivore.app/{username}/{slug}');
|
||||
}
|
||||
|
||||
const [, username, slug] = match;
|
||||
const result = await getArticle(slug, username);
|
||||
checkGraphQLResult({ data: { article: result }, errors: undefined });
|
||||
if (!result.article) {
|
||||
throw new Error('Article not found');
|
||||
}
|
||||
return [this.toQueueSourceArticle({ ...result.article, slug })];
|
||||
}
|
||||
|
||||
private toQueueSourceArticle(article: OmnivoreArticle): QueueSourceArticle {
|
||||
if (!isQueueSourceArticle(article)) throw new Error('Article missing required queue fields');
|
||||
return { id: article.id, slug: article.slug, url: article.url, title: article.title, savedAt: article.savedAt };
|
||||
}
|
||||
|
||||
private formatArticlesForQueue(articles: QueueSourceArticle[]): Array<{ id: string; slug: string; url: string; title: string; savedAt: string }> {
|
||||
return articles.map(({ id, slug, url, title, savedAt }) => ({ id, slug, url, title, savedAt }));
|
||||
}
|
||||
|
||||
private outputResults(inserted: number, total: number, json: boolean): void {
|
||||
if (json) {
|
||||
this.log(JSON.stringify({ added: inserted, total }));
|
||||
} else {
|
||||
this.log(formatSuccess(`Added ${inserted} articles to queue (${total} total found)`));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function isSearchResult(value: unknown): value is SearchResult {
|
||||
return typeof value === 'object' && value !== null && 'edges' in value && 'pageInfo' in value;
|
||||
}
|
||||
|
||||
function isQueueSourceArticle(value: unknown): value is QueueSourceArticle {
|
||||
if (typeof value !== 'object' || value === null) return false;
|
||||
const v = value as { id?: unknown; slug?: unknown; url?: unknown; title?: unknown; savedAt?: unknown };
|
||||
return typeof v.id === 'string' && typeof v.slug === 'string' && typeof v.url === 'string' && typeof v.title === 'string' && typeof v.savedAt === 'string';
|
||||
}
|
||||
61
self-hosting/omc/src/commands/queue/clear.ts
Normal file
61
self-hosting/omc/src/commands/queue/clear.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatSuccess } from '@lib/cli/formatters.js';
|
||||
|
||||
/**
|
||||
* Clear articles from queue by status or all.
|
||||
* AIDEV-NOTE: CLI command for bulk queue clearing
|
||||
*/
|
||||
export default class QueueClear extends BaseCommand {
|
||||
static override description = 'Clear articles from queue';
|
||||
|
||||
static override examples = [
|
||||
'$ omc queue clear --status failed',
|
||||
'$ omc queue clear --status completed',
|
||||
'$ omc queue clear --all',
|
||||
'$ omc queue clear --all --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
status: Flags.string({
|
||||
description: 'Clear articles with specific status (pending|in_progress|completed|failed)',
|
||||
exclusive: ['all'],
|
||||
}),
|
||||
all: Flags.boolean({
|
||||
description: 'Clear all articles from queue',
|
||||
exclusive: ['status'],
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
private validateFlags(flags: { status?: string; all: boolean; json: boolean }): void {
|
||||
if (!flags.status && !flags.all) {
|
||||
throw new Error('Either --status or --all flag is required');
|
||||
}
|
||||
|
||||
if (flags.status) {
|
||||
const validStatuses = ['pending', 'in_progress', 'completed', 'failed'];
|
||||
if (!validStatuses.includes(flags.status)) {
|
||||
throw new Error(`Invalid status: ${flags.status}. Must be one of: ${validStatuses.join(', ')}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected async execute(flags: { status?: string; all: boolean; json: boolean }): Promise<void> {
|
||||
this.validateFlags(flags);
|
||||
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const clearedCount = flags.all
|
||||
? repo.clearAll()
|
||||
: repo.clearByStatus(flags.status as string);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify({ cleared: clearedCount }));
|
||||
} else {
|
||||
this.log(formatSuccess(`Cleared ${clearedCount} articles from queue`));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
59
self-hosting/omc/src/commands/queue/export.ts
Normal file
59
self-hosting/omc/src/commands/queue/export.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag, statusFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import type { AnalysisJob } from '@storage/AnalysisQueueRepository.js';
|
||||
|
||||
/**
|
||||
* Export queue to JSONL format.
|
||||
* AIDEV-NOTE: Exports queue jobs for backup/transfer (one JSON object per line)
|
||||
*/
|
||||
export default class QueueExport extends BaseCommand {
|
||||
static override description = 'Export queue to JSONL format';
|
||||
|
||||
static override examples = [
|
||||
'$ omc queue export',
|
||||
'$ omc queue export --status pending',
|
||||
'$ omc queue export --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
status: statusFlag(),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const jobs = await this.fetchJobs(repo, flags.status);
|
||||
|
||||
if (flags.json) {
|
||||
this.outputJsonArray(jobs);
|
||||
} else {
|
||||
this.outputJsonl(jobs);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async fetchJobs(repo: any, status?: string): Promise<AnalysisJob[]> {
|
||||
if (status) {
|
||||
return repo.getByStatus(status);
|
||||
}
|
||||
|
||||
const allJobs: AnalysisJob[] = [];
|
||||
|
||||
for (const s of ['pending', 'in_progress', 'completed', 'failed']) {
|
||||
allJobs.push(...repo.getByStatus(s));
|
||||
}
|
||||
|
||||
return allJobs;
|
||||
}
|
||||
|
||||
private outputJsonArray(jobs: AnalysisJob[]): void {
|
||||
this.log(JSON.stringify(jobs, null, 2));
|
||||
}
|
||||
|
||||
private outputJsonl(jobs: AnalysisJob[]): void {
|
||||
for (const job of jobs) {
|
||||
this.log(JSON.stringify(job));
|
||||
}
|
||||
}
|
||||
}
|
||||
70
self-hosting/omc/src/commands/queue/import.ts
Normal file
70
self-hosting/omc/src/commands/queue/import.ts
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
import { Args } from '@oclif/core';
|
||||
import { readFileSync } from 'fs';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { outputResult, parseJsonSafely } from '@lib/cli/command-utils.js';
|
||||
|
||||
/**
|
||||
* Import queue from JSONL file.
|
||||
* AIDEV-NOTE: Imports queue jobs from backup/transfer file (one JSON object per line)
|
||||
*/
|
||||
export default class QueueImport extends BaseCommand {
|
||||
static override description = 'Import queue from JSONL file';
|
||||
|
||||
static override examples = [
|
||||
'$ omc queue import queue-backup.jsonl',
|
||||
'$ omc queue import queue-backup.jsonl --json',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
file: Args.string({
|
||||
description: 'Path to JSONL file',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
async execute(flags: any): Promise<void> {
|
||||
const file = await this.parse(QueueImport).then((p) => p.args.file);
|
||||
const articles = this.readJsonl(file);
|
||||
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const count = repo.initializeQueue(articles);
|
||||
outputResult(this, { imported: count }, `Imported ${count} articles`, flags.json);
|
||||
});
|
||||
}
|
||||
|
||||
private readJsonl(filePath: string): Array<{ id: string; slug: string; url: string; title: string; savedAt: string }> {
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
const lines = content.trim().split('\n');
|
||||
return lines.map(line => this.parseJobLine(line));
|
||||
}
|
||||
|
||||
private parseJobLine(line: string): { id: string; slug: string; url: string; title: string; savedAt: string } {
|
||||
const job = parseJsonSafely<any>(line);
|
||||
if (!job) throw new Error('Failed to parse JSONL line');
|
||||
this.validateJob(job);
|
||||
|
||||
return {
|
||||
id: job.articleId,
|
||||
slug: job.articleSlug,
|
||||
url: job.articleUrl,
|
||||
title: job.articleTitle,
|
||||
savedAt: job.savedAt,
|
||||
};
|
||||
}
|
||||
|
||||
private validateJob(job: any): void {
|
||||
const required = ['articleId', 'articleSlug', 'articleUrl', 'articleTitle', 'savedAt'];
|
||||
|
||||
for (const field of required) {
|
||||
if (!job[field]) {
|
||||
throw new Error(`Missing required field: ${field}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
49
self-hosting/omc/src/commands/queue/list.ts
Normal file
49
self-hosting/omc/src/commands/queue/list.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader } from '@lib/cli/formatters.js';
|
||||
import { displayJobs } from '@lib/cli/queue-display.js';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag, statusFlag } from '@lib/cli/shared-flags.js';
|
||||
|
||||
/**
|
||||
* List articles in the analysis queue.
|
||||
* AIDEV-NOTE: Displays queue contents with optional status filtering
|
||||
*/
|
||||
export default class QueueList extends BaseCommand {
|
||||
static override description = 'List articles in the analysis queue';
|
||||
|
||||
static override examples = [
|
||||
'$ omc queue list',
|
||||
'$ omc queue list --status pending',
|
||||
'$ omc queue list --status completed',
|
||||
'$ omc queue list --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
status: statusFlag(),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
let jobs;
|
||||
|
||||
if (flags.status) {
|
||||
jobs = repo.getByStatus(flags.status);
|
||||
} else {
|
||||
// Get all jobs from all statuses
|
||||
const pending = repo.getByStatus('pending');
|
||||
const inProgress = repo.getByStatus('in_progress');
|
||||
const completed = repo.getByStatus('completed');
|
||||
const failed = repo.getByStatus('failed');
|
||||
jobs = [...pending, ...inProgress, ...completed, ...failed];
|
||||
}
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(jobs, null, 2));
|
||||
} else {
|
||||
this.log(formatHeader('Queue Listing'));
|
||||
displayJobs(jobs);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
47
self-hosting/omc/src/commands/queue/remove.ts
Normal file
47
self-hosting/omc/src/commands/queue/remove.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import { Args } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatSuccess } from '@lib/cli/formatters.js';
|
||||
|
||||
/**
|
||||
* Remove article from analysis queue.
|
||||
* AIDEV-NOTE: CLI command for queue article removal
|
||||
*/
|
||||
export default class QueueRemove extends BaseCommand {
|
||||
static override description = 'Remove article from queue';
|
||||
|
||||
static override examples = [
|
||||
'$ omc queue remove <article-id>',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
articleId: Args.string({
|
||||
description: 'Article ID to remove',
|
||||
required: true,
|
||||
}),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: Record<string, any>): Promise<void> {
|
||||
const { args } = await this.parse(QueueRemove);
|
||||
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const removed = repo.removeArticle(args.articleId);
|
||||
|
||||
if (removed === 0) {
|
||||
this.warn(`Article ${args.articleId} not found in queue`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify({ removed, articleId: args.articleId }));
|
||||
} else {
|
||||
this.log(formatSuccess(`Removed article ${args.articleId} from queue`));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
74
self-hosting/omc/src/commands/queue/reset.ts
Normal file
74
self-hosting/omc/src/commands/queue/reset.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
import { Args, Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatSuccess } from '@lib/cli/formatters.js';
|
||||
|
||||
/**
|
||||
* Reset article(s) to pending status for reprocessing.
|
||||
* AIDEV-NOTE: Useful for retrying stuck in_progress jobs
|
||||
*/
|
||||
export default class QueueReset extends BaseCommand {
|
||||
static override description = 'Reset article to pending status';
|
||||
|
||||
static override examples = [
|
||||
'$ omc queue reset <article-id>',
|
||||
'$ omc queue reset --all-in-progress',
|
||||
];
|
||||
|
||||
static override args = {
|
||||
articleId: Args.string({
|
||||
description: 'Article ID to reset',
|
||||
required: false,
|
||||
}),
|
||||
};
|
||||
|
||||
static override flags = {
|
||||
'all-in-progress': Flags.boolean({
|
||||
description: 'Reset all in_progress articles to pending',
|
||||
default: false,
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
|
||||
// AIDEV-NOTE: Extracted helper to keep execute() under 20 lines
|
||||
private resetAllInProgress(repo: any, flags: any): void {
|
||||
const jobs = repo.getByStatus('in_progress');
|
||||
for (const job of jobs) {
|
||||
repo.resetToPending(job.articleId);
|
||||
}
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify({ reset: jobs.length }));
|
||||
} else {
|
||||
this.log(formatSuccess(`Reset ${jobs.length} in_progress articles to pending`));
|
||||
}
|
||||
}
|
||||
|
||||
// AIDEV-NOTE: Extracted helper to keep execute() under 20 lines
|
||||
private resetSingleArticle(repo: any, articleId: string, flags: any): void {
|
||||
repo.resetToPending(articleId);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify({ reset: 1, articleId }));
|
||||
} else {
|
||||
this.log(formatSuccess(`Reset article ${articleId} to pending`));
|
||||
}
|
||||
}
|
||||
|
||||
async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
if (flags['all-in-progress']) {
|
||||
this.resetAllInProgress(repo, flags);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!flags.articleId) {
|
||||
throw new Error('Either provide article-id or use --all-in-progress flag');
|
||||
}
|
||||
|
||||
this.resetSingleArticle(repo, flags.articleId, flags);
|
||||
});
|
||||
}
|
||||
}
|
||||
95
self-hosting/omc/src/commands/queue/stats.ts
Normal file
95
self-hosting/omc/src/commands/queue/stats.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader, formatDivider } from '@lib/cli/formatters.js';
|
||||
import { displayQueueStats } from '@lib/cli/queue-display.js';
|
||||
|
||||
/**
|
||||
* Show queue statistics.
|
||||
* AIDEV-NOTE: Displays aggregate queue metrics for monitoring
|
||||
*/
|
||||
export default class QueueStats extends BaseCommand {
|
||||
static override description = 'Show queue statistics';
|
||||
|
||||
static override examples = [
|
||||
'$ omc queue stats',
|
||||
'$ omc queue stats --detailed',
|
||||
'$ omc queue stats --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
detailed: Flags.boolean({
|
||||
description: 'Show detailed per-status breakdown',
|
||||
default: false,
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const stats = repo.getStats();
|
||||
|
||||
if (flags.json) {
|
||||
const output = flags.detailed ? this.buildDetailedStats(repo) : stats;
|
||||
this.log(JSON.stringify(output, null, 2));
|
||||
} else {
|
||||
this.log(formatHeader('Queue Statistics'));
|
||||
displayQueueStats(stats);
|
||||
|
||||
if (flags.detailed) {
|
||||
this.displayDetailedBreakdown(repo);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private buildDetailedStats(repo: any): any {
|
||||
const stats = repo.getStats();
|
||||
const detailed: any = { ...stats, breakdown: {} };
|
||||
|
||||
for (const status of ['pending', 'in_progress', 'completed', 'failed']) {
|
||||
const jobs = repo.getByStatus(status);
|
||||
detailed.breakdown[status] = this.analyzeJobs(jobs);
|
||||
}
|
||||
|
||||
return detailed;
|
||||
}
|
||||
|
||||
private displayDetailedBreakdown(repo: any): void {
|
||||
this.log('\n' + formatDivider());
|
||||
this.log('Detailed Breakdown\n');
|
||||
|
||||
for (const status of ['pending', 'in_progress', 'completed', 'failed']) {
|
||||
const jobs = repo.getByStatus(status);
|
||||
this.displayStatusAnalysis(status, jobs);
|
||||
}
|
||||
}
|
||||
|
||||
private displayStatusAnalysis(status: string, jobs: any[]): void {
|
||||
const analysis = this.analyzeJobs(jobs);
|
||||
|
||||
this.log(`${status.toUpperCase()}:`);
|
||||
this.log(` Count: ${analysis.count}`);
|
||||
if (analysis.oldestDate) this.log(` Oldest: ${analysis.oldestDate}`);
|
||||
if (analysis.newestDate) this.log(` Newest: ${analysis.newestDate}`);
|
||||
if (status === 'failed') this.log(` Total Retries: ${analysis.totalRetries}`);
|
||||
this.log('');
|
||||
}
|
||||
|
||||
private analyzeJobs(jobs: any[]): any {
|
||||
if (jobs.length === 0) {
|
||||
return { count: 0 };
|
||||
}
|
||||
|
||||
const dates = jobs.map(j => new Date(j.createdAt).getTime()).filter(d => !isNaN(d));
|
||||
const totalRetries = jobs.reduce((sum, j) => sum + (j.retryCount || 0), 0);
|
||||
|
||||
return {
|
||||
count: jobs.length,
|
||||
oldestDate: dates.length > 0 ? new Date(Math.min(...dates)).toISOString() : null,
|
||||
newestDate: dates.length > 0 ? new Date(Math.max(...dates)).toISOString() : null,
|
||||
totalRetries,
|
||||
};
|
||||
}
|
||||
}
|
||||
103
self-hosting/omc/src/commands/report/corpus.ts
Normal file
103
self-hosting/omc/src/commands/report/corpus.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader } from '@lib/cli/formatters.js';
|
||||
import { parseJsonSafely } from '@lib/cli/command-utils.js';
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
|
||||
/**
|
||||
* Full corpus analysis report.
|
||||
* AIDEV-NOTE: Aggregates topic frequencies, sentiment distribution, content types
|
||||
*/
|
||||
export default class ReportCorpus extends BaseCommand {
|
||||
static override description = 'Generate full corpus analysis report';
|
||||
|
||||
static override examples = [
|
||||
'$ omc report corpus',
|
||||
'$ omc report corpus --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const jobs = repo.getCompletedWithAnalysis();
|
||||
const analyses = this.parseAnalyses(jobs);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(this.buildCorpusStats(analyses), null, 2));
|
||||
} else {
|
||||
this.displayCorpusReport(analyses);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private parseAnalyses(jobs: any[]): ContentAnalysis[] {
|
||||
return jobs.map(job => parseJsonSafely<ContentAnalysis>(job.analysisJson))
|
||||
.filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A');
|
||||
}
|
||||
|
||||
private buildCorpusStats(analyses: ContentAnalysis[]): any {
|
||||
const topicFreq = this.countTopics(analyses);
|
||||
const sentimentDist = this.countSentiment(analyses);
|
||||
const contentTypeDist = this.countContentTypes(analyses);
|
||||
|
||||
return {
|
||||
totalAnalyses: analyses.length,
|
||||
topicDistribution: topicFreq,
|
||||
sentimentDistribution: sentimentDist,
|
||||
contentTypeDistribution: contentTypeDist,
|
||||
};
|
||||
}
|
||||
|
||||
private displayCorpusReport(analyses: ContentAnalysis[]): void {
|
||||
this.log(formatHeader('Corpus Analysis Report'));
|
||||
this.log(`\nTotal Analyses: ${analyses.length}\n`);
|
||||
|
||||
this.log('Topic Distribution:');
|
||||
const topics = this.countTopics(analyses);
|
||||
for (const [topic, count] of Object.entries(topics).sort((a, b) => b[1] - a[1])) {
|
||||
this.log(` ${topic.padEnd(30)} ${count}`);
|
||||
}
|
||||
|
||||
this.log('\nSentiment Distribution:');
|
||||
const sentiment = this.countSentiment(analyses);
|
||||
for (const [sent, count] of Object.entries(sentiment)) {
|
||||
this.log(` ${sent}: ${count}`);
|
||||
}
|
||||
|
||||
this.log('\nContent Types:');
|
||||
const types = this.countContentTypes(analyses);
|
||||
for (const [type, count] of Object.entries(types).sort((a, b) => b[1] - a[1])) {
|
||||
this.log(` ${type.padEnd(30)} ${count}`);
|
||||
}
|
||||
}
|
||||
|
||||
private countTopics(analyses: ContentAnalysis[]): Record<string, number> {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const a of analyses) {
|
||||
for (const topic of a.topics) {
|
||||
counts[topic] = (counts[topic] || 0) + 1;
|
||||
}
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
private countSentiment(analyses: ContentAnalysis[]): Record<string, number> {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const a of analyses) {
|
||||
counts[a.sentiment] = (counts[a.sentiment] || 0) + 1;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
private countContentTypes(analyses: ContentAnalysis[]): Record<string, number> {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const a of analyses) {
|
||||
counts[a.contentType] = (counts[a.contentType] || 0) + 1;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
}
|
||||
84
self-hosting/omc/src/commands/report/custom.ts
Normal file
84
self-hosting/omc/src/commands/report/custom.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader } from '@lib/cli/formatters.js';
|
||||
import { parseJsonSafely } from '@lib/cli/command-utils.js';
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
|
||||
/**
|
||||
* Custom SQL-based report.
|
||||
* AIDEV-NOTE: Allows filtering analyses with custom SQL WHERE clause
|
||||
*/
|
||||
export default class ReportCustom extends BaseCommand {
|
||||
static override description = 'Run custom SQL query against analyses';
|
||||
|
||||
static override examples = [
|
||||
'$ omc report custom --query "article_title LIKE \'%AI%\'"',
|
||||
'$ omc report custom --query "completed_at > \'2024-01-01\'" --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
query: Flags.string({
|
||||
description: 'SQL WHERE clause for filtering',
|
||||
required: true,
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ db }) => {
|
||||
const results = this.executeCustomQuery(db, flags.query);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(results, null, 2));
|
||||
} else {
|
||||
this.displayResults(results);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private executeCustomQuery(db: any, whereClause: string): any[] {
|
||||
try {
|
||||
const sql = this.buildQuerySQL(whereClause);
|
||||
const rows = db.prepare(sql).all();
|
||||
return this.mapQueryResults(rows);
|
||||
} catch (error) {
|
||||
throw new Error(`SQL query failed: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private buildQuerySQL(whereClause: string): string {
|
||||
return `
|
||||
SELECT article_id as articleId, article_title as articleTitle,
|
||||
analysis_json as analysisJson, completed_at as completedAt
|
||||
FROM analysis_queue
|
||||
WHERE status = 'completed' AND analysis_json IS NOT NULL AND (${whereClause})
|
||||
ORDER BY completed_at DESC
|
||||
`;
|
||||
}
|
||||
|
||||
private mapQueryResults(rows: any[]): any[] {
|
||||
return rows.map((row: any) => {
|
||||
const analysis = parseJsonSafely<ContentAnalysis>(row.analysisJson);
|
||||
return {
|
||||
articleId: row.articleId,
|
||||
articleTitle: row.articleTitle,
|
||||
completedAt: row.completedAt,
|
||||
analysis,
|
||||
};
|
||||
}).filter(r => r.analysis);
|
||||
}
|
||||
|
||||
private displayResults(results: any[]): void {
|
||||
this.log(formatHeader('Custom Query Results'));
|
||||
this.log(`\nFound ${results.length} matching analyses\n`);
|
||||
|
||||
for (const { articleTitle, completedAt, analysis } of results) {
|
||||
this.log(`${articleTitle}`);
|
||||
this.log(` Completed: ${completedAt}`);
|
||||
this.log(` Topics: ${analysis.topics.join(', ')}`);
|
||||
this.log(` Sentiment: ${analysis.sentiment}\n`);
|
||||
}
|
||||
}
|
||||
}
|
||||
127
self-hosting/omc/src/commands/report/export.ts
Normal file
127
self-hosting/omc/src/commands/report/export.ts
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { parseJsonSafely } from '@lib/cli/command-utils.js';
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
|
||||
/**
|
||||
* Export report data.
|
||||
* AIDEV-NOTE: Supports json|csv|markdown output formats for different use cases
|
||||
*/
|
||||
export default class ReportExport extends BaseCommand {
|
||||
static override description = 'Export report data to file or stdout';
|
||||
|
||||
static override examples = [
|
||||
'$ omc report export --format json',
|
||||
'$ omc report export --format csv --report-type topics',
|
||||
'$ omc report export --format markdown',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
format: Flags.string({
|
||||
description: 'Output format',
|
||||
required: true,
|
||||
options: ['json', 'csv', 'markdown'],
|
||||
}),
|
||||
'report-type': Flags.string({
|
||||
description: 'Type of report to export',
|
||||
default: 'corpus',
|
||||
options: ['corpus', 'topics', 'sentiment', 'monetization'],
|
||||
}),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const jobs = repo.getCompletedWithAnalysis();
|
||||
const analyses = this.parseAnalyses(jobs);
|
||||
|
||||
const output = this.formatOutput(analyses, flags.format, flags['report-type']);
|
||||
this.log(output);
|
||||
});
|
||||
}
|
||||
|
||||
private parseAnalyses(jobs: any[]): ContentAnalysis[] {
|
||||
return jobs.map(job => parseJsonSafely<ContentAnalysis>(job.analysisJson))
|
||||
.filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A');
|
||||
}
|
||||
|
||||
private formatOutput(analyses: ContentAnalysis[], format: string, reportType: string): string {
|
||||
if (format === 'json') return this.exportJSON(analyses, reportType);
|
||||
if (format === 'csv') return this.exportCSV(analyses, reportType);
|
||||
return this.exportMarkdown(analyses, reportType);
|
||||
}
|
||||
|
||||
private exportJSON(analyses: ContentAnalysis[], reportType: string): string {
|
||||
const data = reportType === 'topics' ? this.buildTopicsData(analyses) :
|
||||
reportType === 'sentiment' ? this.buildSentimentData(analyses) :
|
||||
reportType === 'monetization' ? this.buildMonetizationData(analyses) :
|
||||
{ analyses };
|
||||
return JSON.stringify(data, null, 2);
|
||||
}
|
||||
|
||||
private exportCSV(analyses: ContentAnalysis[], reportType: string): string {
|
||||
if (reportType === 'topics') {
|
||||
const topics = this.buildTopicsData(analyses);
|
||||
return 'Topic,Frequency,AvgScore\n' +
|
||||
topics.map((t: any) => `${t.topic},${t.frequency},${t.avgScore}`).join('\n');
|
||||
}
|
||||
return 'ArticleId,Topics,Sentiment,ContentType\n' +
|
||||
analyses.map(a => `${a.articleId},"${a.topics.join(';')}",${a.sentiment},${a.contentType}`).join('\n');
|
||||
}
|
||||
|
||||
private exportMarkdown(analyses: ContentAnalysis[], reportType: string): string {
|
||||
let md = `# ${reportType.charAt(0).toUpperCase() + reportType.slice(1)} Report\n\n`;
|
||||
md += `Total Analyses: ${analyses.length}\n\n`;
|
||||
|
||||
if (reportType === 'topics') {
|
||||
md += '## Topic Distribution\n\n';
|
||||
const topics = this.buildTopicsData(analyses);
|
||||
for (const t of topics) {
|
||||
md += `- **${t.topic}**: ${t.frequency} articles (avg score: ${t.avgScore})\n`;
|
||||
}
|
||||
} else {
|
||||
md += '## Articles\n\n';
|
||||
for (const a of analyses.slice(0, 10)) {
|
||||
md += `### ${a.articleId}\n`;
|
||||
md += `- Topics: ${a.topics.join(', ')}\n`;
|
||||
md += `- Sentiment: ${a.sentiment}\n\n`;
|
||||
}
|
||||
}
|
||||
return md;
|
||||
}
|
||||
|
||||
private buildTopicsData(analyses: ContentAnalysis[]): any[] {
|
||||
const topicData: Record<string, { count: number; scoreSum: number }> = {};
|
||||
for (const a of analyses) {
|
||||
for (const topic of a.topics) {
|
||||
if (!topicData[topic]) topicData[topic] = { count: 0, scoreSum: 0 };
|
||||
topicData[topic].count++;
|
||||
topicData[topic].scoreSum += a.topicScores[topic] || 0;
|
||||
}
|
||||
}
|
||||
return Object.entries(topicData)
|
||||
.map(([topic, data]) => ({
|
||||
topic,
|
||||
frequency: data.count,
|
||||
avgScore: (data.scoreSum / data.count).toFixed(2),
|
||||
}))
|
||||
.sort((a, b) => b.frequency - a.frequency);
|
||||
}
|
||||
|
||||
private buildSentimentData(analyses: ContentAnalysis[]): any {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const a of analyses) {
|
||||
counts[a.sentiment] = (counts[a.sentiment] || 0) + 1;
|
||||
}
|
||||
return { distribution: counts };
|
||||
}
|
||||
|
||||
private buildMonetizationData(analyses: ContentAnalysis[]): any[] {
|
||||
return analyses.map(a => ({
|
||||
articleId: a.articleId,
|
||||
monetizationAngle: a.monetizationAngle,
|
||||
topics: a.topics,
|
||||
sentiment: a.sentiment,
|
||||
}));
|
||||
}
|
||||
}
|
||||
95
self-hosting/omc/src/commands/report/monetization.ts
Normal file
95
self-hosting/omc/src/commands/report/monetization.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader } from '@lib/cli/formatters.js';
|
||||
import { parseJsonSafely } from '@lib/cli/command-utils.js';
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
|
||||
/**
|
||||
* Monetization opportunities report.
|
||||
* AIDEV-NOTE: Extracts monetizationAngle from analyses, ranks by potential
|
||||
*/
|
||||
export default class ReportMonetization extends BaseCommand {
|
||||
static override description = 'Show monetization opportunities across analyses';
|
||||
|
||||
static override examples = [
|
||||
'$ omc report monetization',
|
||||
'$ omc report monetization --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const jobs = repo.getCompletedWithAnalysis();
|
||||
const analyses = this.parseAnalyses(jobs);
|
||||
|
||||
const opportunities = this.extractOpportunities(analyses);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(opportunities, null, 2));
|
||||
} else {
|
||||
this.displayOpportunities(opportunities);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private parseAnalyses(jobs: any[]): ContentAnalysis[] {
|
||||
return jobs.map(job => parseJsonSafely<ContentAnalysis>(job.analysisJson))
|
||||
.filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A');
|
||||
}
|
||||
|
||||
private extractOpportunities(analyses: ContentAnalysis[]): any[] {
|
||||
const themeGroups: Record<string, any[]> = {};
|
||||
|
||||
for (const a of analyses) {
|
||||
const theme = this.extractTheme(a.monetizationAngle);
|
||||
if (!themeGroups[theme]) themeGroups[theme] = [];
|
||||
themeGroups[theme].push({
|
||||
articleId: a.articleId,
|
||||
angle: a.monetizationAngle,
|
||||
sentiment: a.sentiment,
|
||||
topics: a.topics,
|
||||
});
|
||||
}
|
||||
|
||||
return Object.entries(themeGroups)
|
||||
.map(([theme, items]) => ({
|
||||
theme,
|
||||
count: items.length,
|
||||
items,
|
||||
potential: this.calculatePotential(items),
|
||||
}))
|
||||
.sort((a, b) => b.potential - a.potential);
|
||||
}
|
||||
|
||||
private extractTheme(angle: string): string {
|
||||
const lower = angle.toLowerCase();
|
||||
if (lower.includes('comparison') || lower.includes('compare')) return 'comparison';
|
||||
if (lower.includes('roundup') || lower.includes('weekly')) return 'roundup';
|
||||
if (lower.includes('tutorial') || lower.includes('guide')) return 'tutorial';
|
||||
if (lower.includes('review')) return 'review';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
private calculatePotential(items: any[]): number {
|
||||
const positiveCount = items.filter(i => i.sentiment === 'positive').length;
|
||||
return items.length * 10 + positiveCount * 5;
|
||||
}
|
||||
|
||||
private displayOpportunities(opportunities: any[]): void {
|
||||
this.log(formatHeader('Monetization Opportunities Report'));
|
||||
this.log('');
|
||||
|
||||
for (const { theme, count, items, potential } of opportunities) {
|
||||
this.log(`${theme.toUpperCase()} (${count} articles, potential: ${potential})`);
|
||||
for (const item of items.slice(0, 3)) {
|
||||
this.log(` • ${item.angle}`);
|
||||
}
|
||||
if (items.length > 3) this.log(` ... and ${items.length - 3} more\n`);
|
||||
else this.log('');
|
||||
}
|
||||
}
|
||||
}
|
||||
94
self-hosting/omc/src/commands/report/sentiment.ts
Normal file
94
self-hosting/omc/src/commands/report/sentiment.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader } from '@lib/cli/formatters.js';
|
||||
import { parseJsonSafely } from '@lib/cli/command-utils.js';
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
|
||||
/**
|
||||
* Sentiment analysis report.
|
||||
* AIDEV-NOTE: Sentiment distribution correlated with topics, identifies outliers
|
||||
*/
|
||||
export default class ReportSentiment extends BaseCommand {
|
||||
static override description = 'Show sentiment analysis across corpus';
|
||||
|
||||
static override examples = [
|
||||
'$ omc report sentiment',
|
||||
'$ omc report sentiment --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const jobs = repo.getCompletedWithAnalysis();
|
||||
const analyses = this.parseAnalyses(jobs);
|
||||
|
||||
const sentimentData = this.analyzeSentiment(analyses);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(sentimentData, null, 2));
|
||||
} else {
|
||||
this.displaySentimentReport(sentimentData);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private parseAnalyses(jobs: any[]): ContentAnalysis[] {
|
||||
return jobs.map(job => parseJsonSafely<ContentAnalysis>(job.analysisJson))
|
||||
.filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A');
|
||||
}
|
||||
|
||||
private analyzeSentiment(analyses: ContentAnalysis[]): any {
|
||||
const distribution = this.countSentiment(analyses);
|
||||
const topicCorrelation = this.correlateSentimentWithTopics(analyses);
|
||||
|
||||
return { distribution, topicCorrelation };
|
||||
}
|
||||
|
||||
private countSentiment(analyses: ContentAnalysis[]): Record<string, number> {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const a of analyses) {
|
||||
counts[a.sentiment] = (counts[a.sentiment] || 0) + 1;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
private correlateSentimentWithTopics(analyses: ContentAnalysis[]): any[] {
|
||||
const topicSentiment: Record<string, Record<string, number>> = {};
|
||||
|
||||
for (const a of analyses) {
|
||||
for (const topic of a.topics) {
|
||||
if (!topicSentiment[topic]) {
|
||||
topicSentiment[topic] = { positive: 0, neutral: 0, negative: 0 };
|
||||
}
|
||||
topicSentiment[topic][a.sentiment]++;
|
||||
}
|
||||
}
|
||||
|
||||
return Object.entries(topicSentiment)
|
||||
.map(([topic, sentiments]) => ({
|
||||
topic,
|
||||
positive: sentiments.positive || 0,
|
||||
neutral: sentiments.neutral || 0,
|
||||
negative: sentiments.negative || 0,
|
||||
total: (sentiments.positive || 0) + (sentiments.neutral || 0) + (sentiments.negative || 0),
|
||||
}))
|
||||
.sort((a, b) => b.total - a.total);
|
||||
}
|
||||
|
||||
private displaySentimentReport(data: any): void {
|
||||
this.log(formatHeader('Sentiment Analysis Report'));
|
||||
this.log('\nOverall Distribution:');
|
||||
for (const [sentiment, count] of Object.entries(data.distribution)) {
|
||||
this.log(` ${sentiment}: ${count}`);
|
||||
}
|
||||
|
||||
this.log('\nSentiment by Topic:');
|
||||
for (const { topic, positive, neutral, negative, total } of data.topicCorrelation) {
|
||||
this.log(` ${topic.padEnd(30)} +${positive} ~${neutral} -${negative} (total: ${total})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
94
self-hosting/omc/src/commands/report/topics.ts
Normal file
94
self-hosting/omc/src/commands/report/topics.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader } from '@lib/cli/formatters.js';
|
||||
import { parseJsonSafely } from '@lib/cli/command-utils.js';
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
|
||||
/**
|
||||
* Topic distribution report.
|
||||
* AIDEV-NOTE: Groups analyses by topics with frequency and average scores
|
||||
*/
|
||||
export default class ReportTopics extends BaseCommand {
|
||||
static override description = 'Show topic distribution across all analyses';
|
||||
|
||||
static override examples = [
|
||||
'$ omc report topics',
|
||||
'$ omc report topics --min-score 0.8',
|
||||
'$ omc report topics --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
'min-score': Flags.string({
|
||||
description: 'Minimum topic score threshold (0-1)',
|
||||
default: '0.0',
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const jobs = repo.getCompletedWithAnalysis();
|
||||
const analyses = this.parseAnalyses(jobs);
|
||||
const minScore = parseFloat(flags['min-score']);
|
||||
|
||||
const topicStats = this.buildTopicStats(analyses, minScore);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(topicStats, null, 2));
|
||||
} else {
|
||||
this.displayTopicReport(topicStats);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private parseAnalyses(jobs: any[]): ContentAnalysis[] {
|
||||
return jobs.map(job => parseJsonSafely<ContentAnalysis>(job.analysisJson))
|
||||
.filter((a): a is ContentAnalysis => !!a && !!a.topics && a.topics[0] !== 'N/A');
|
||||
}
|
||||
|
||||
private buildTopicStats(analyses: ContentAnalysis[], minScore: number): any[] {
|
||||
const topicData = this.aggregateTopicData(analyses, minScore);
|
||||
return this.formatTopicStats(topicData);
|
||||
}
|
||||
|
||||
private aggregateTopicData(analyses: ContentAnalysis[], minScore: number): Record<string, { count: number; scoreSum: number; articles: string[] }> {
|
||||
const topicData: Record<string, { count: number; scoreSum: number; articles: string[] }> = {};
|
||||
|
||||
for (const a of analyses) {
|
||||
for (const topic of a.topics) {
|
||||
const score = a.topicScores[topic] || 0;
|
||||
if (score < minScore) continue;
|
||||
|
||||
if (!topicData[topic]) {
|
||||
topicData[topic] = { count: 0, scoreSum: 0, articles: [] };
|
||||
}
|
||||
topicData[topic].count++;
|
||||
topicData[topic].scoreSum += score;
|
||||
topicData[topic].articles.push(a.articleId);
|
||||
}
|
||||
}
|
||||
return topicData;
|
||||
}
|
||||
|
||||
private formatTopicStats(topicData: Record<string, { count: number; scoreSum: number; articles: string[] }>): any[] {
|
||||
return Object.entries(topicData)
|
||||
.map(([topic, data]) => ({
|
||||
topic,
|
||||
frequency: data.count,
|
||||
avgScore: data.count > 0 ? (data.scoreSum / data.count).toFixed(2) : '0',
|
||||
articles: data.articles,
|
||||
}))
|
||||
.sort((a, b) => b.frequency - a.frequency);
|
||||
}
|
||||
|
||||
private displayTopicReport(stats: any[]): void {
|
||||
this.log(formatHeader('Topic Distribution Report'));
|
||||
this.log('');
|
||||
|
||||
for (const { topic, frequency, avgScore } of stats) {
|
||||
this.log(`${topic.padEnd(30)} ${frequency} articles (avg score: ${avgScore})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
109
self-hosting/omc/src/commands/report/trends.ts
Normal file
109
self-hosting/omc/src/commands/report/trends.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import { Flags } from '@oclif/core';
|
||||
import { BaseCommand } from '@lib/cli/base-command.js';
|
||||
import { jsonFlag } from '@lib/cli/shared-flags.js';
|
||||
import { withDatabase } from '@lib/cli/database.js';
|
||||
import { formatHeader } from '@lib/cli/formatters.js';
|
||||
import { parseJsonSafely } from '@lib/cli/command-utils.js';
|
||||
import type { ContentAnalysis } from '@omc-types/analysis.js';
|
||||
|
||||
/**
|
||||
* Trending topics over time.
|
||||
* AIDEV-NOTE: Tracks topic frequency changes, identifies rising/falling trends
|
||||
*/
|
||||
export default class ReportTrends extends BaseCommand {
|
||||
static override description = 'Show trending topics over time';
|
||||
|
||||
static override examples = [
|
||||
'$ omc report trends',
|
||||
'$ omc report trends --period week',
|
||||
'$ omc report trends --json',
|
||||
];
|
||||
|
||||
static override flags = {
|
||||
period: Flags.string({
|
||||
description: 'Time period for grouping (day|week|month)',
|
||||
default: 'week',
|
||||
options: ['day', 'week', 'month'],
|
||||
}),
|
||||
json: jsonFlag(),
|
||||
};
|
||||
|
||||
protected async execute(flags: any): Promise<void> {
|
||||
await withDatabase(async ({ repo }) => {
|
||||
const jobs = repo.getCompletedWithAnalysis();
|
||||
const trends = this.buildTrendAnalysis(jobs, flags.period);
|
||||
|
||||
if (flags.json) {
|
||||
this.log(JSON.stringify(trends, null, 2));
|
||||
} else {
|
||||
this.displayTrends(trends);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private buildTrendAnalysis(jobs: any[], period: string): any {
|
||||
const periodData: Record<string, Record<string, number>> = {};
|
||||
|
||||
for (const job of jobs) {
|
||||
const analysis = parseJsonSafely<ContentAnalysis>(job.analysisJson);
|
||||
if (!analysis || !analysis.topics || analysis.topics[0] === 'N/A') continue;
|
||||
|
||||
const periodKey = this.getPeriodKey(job.completedAt, period);
|
||||
if (!periodData[periodKey]) periodData[periodKey] = {};
|
||||
|
||||
for (const topic of analysis.topics) {
|
||||
periodData[periodKey][topic] = (periodData[periodKey][topic] || 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
return this.calculateTrends(periodData);
|
||||
}
|
||||
|
||||
private getPeriodKey(date: string, period: string): string {
|
||||
const d = new Date(date);
|
||||
if (period === 'day') return d.toISOString().split('T')[0];
|
||||
if (period === 'week') {
|
||||
const weekNum = Math.floor(d.getTime() / (7 * 24 * 60 * 60 * 1000));
|
||||
return `Week-${weekNum}`;
|
||||
}
|
||||
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
private calculateTrends(periodData: Record<string, Record<string, number>>): any {
|
||||
const periods = Object.keys(periodData).sort();
|
||||
const allTopics = this.extractAllTopics(periodData);
|
||||
const trends = Array.from(allTopics).map(topic => this.calculateTopicTrend(topic, periods, periodData));
|
||||
|
||||
return { periods, trends: trends.sort((a, b) => parseFloat(b.avgSecond) - parseFloat(a.avgSecond)) };
|
||||
}
|
||||
|
||||
private extractAllTopics(periodData: Record<string, Record<string, number>>): Set<string> {
|
||||
const topics = new Set<string>();
|
||||
for (const topicCounts of Object.values(periodData)) {
|
||||
for (const topic of Object.keys(topicCounts)) topics.add(topic);
|
||||
}
|
||||
return topics;
|
||||
}
|
||||
|
||||
private calculateTopicTrend(topic: string, periods: string[], periodData: Record<string, Record<string, number>>): any {
|
||||
const counts = periods.map(p => periodData[p][topic] || 0);
|
||||
const firstHalf = counts.slice(0, Math.floor(counts.length / 2));
|
||||
const secondHalf = counts.slice(Math.floor(counts.length / 2));
|
||||
|
||||
const avgFirst = firstHalf.reduce((a, b) => a + b, 0) / (firstHalf.length || 1);
|
||||
const avgSecond = secondHalf.reduce((a, b) => a + b, 0) / (secondHalf.length || 1);
|
||||
const trend = avgSecond > avgFirst ? 'rising' : avgSecond < avgFirst ? 'falling' : 'stable';
|
||||
|
||||
return { topic, trend, avgFirst: avgFirst.toFixed(1), avgSecond: avgSecond.toFixed(1) };
|
||||
}
|
||||
|
||||
private displayTrends(data: any): void {
|
||||
this.log(formatHeader('Topic Trends Report'));
|
||||
this.log(`\nPeriods analyzed: ${data.periods.join(', ')}\n`);
|
||||
|
||||
for (const { topic, trend, avgFirst, avgSecond } of data.trends) {
|
||||
const arrow = trend === 'rising' ? '↑' : trend === 'falling' ? '↓' : '→';
|
||||
this.log(`${arrow} ${topic.padEnd(30)} ${avgFirst} → ${avgSecond} (${trend})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue