mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
422aba0837
Complete implementation of background content extraction with BullMQ: Content Extraction Features: - Mozilla Readability integration for article content extraction - Open Graph metadata extraction (title, author, description, thumbnail, etc.) - JSON-LD structured data extraction with schema.org support - HTML sanitization using DOMPurify to prevent XSS attacks - SHA-256 content hashing for duplicate detection - Accurate word count calculation from sanitized HTML - Multi-source metadata merging (JSON-LD > Readability > Open Graph) Queue Infrastructure: - BullMQ worker with @Processor decorator for automatic job processing - Retry logic with exponential backoff (3 attempts, 2s delay) - Job state management (CONTENT_NOT_FETCHED → PROCESSING → SUCCEEDED/FAILED) - Event-driven architecture with EventBusService for monitoring - Configurable job cleanup (24h for completed, 7d for failed) Security & Performance: - HTML sanitization with comprehensive tag/attribute allowlists - Security logging when >20% content removed - URL validation (http/https only) - linkedom for fast server-side DOM parsing - Worker concurrency configuration Testing: - 113 unit tests passing (including 8 new content processor tests) - 5 E2E tests for queue integration workflow - Comprehensive test coverage for error scenarios Files Changed: - src/queue/processors/content-processor.service.ts: Core extraction logic - src/queue/services/html-sanitizer.service.ts: XSS prevention layer - src/queue/queue.module.ts: BullMQ configuration - test/content-extraction.e2e-spec.ts: E2E workflow tests - package.json: Dependencies (dompurify, turndown) Highlight Fixes (from earlier session): - web-vite/src/lib/anchoredHighlights.ts: Click handlers for highlights - web-vite/src/pages/ReaderPage.tsx: Scroll position preservation Related: #21 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
134 lines
4.1 KiB
JSON
134 lines
4.1 KiB
JSON
{
|
|
"name": "@omnivore/api-nest",
|
|
"version": "1.0.0",
|
|
"description": "NestJS API for Omnivore - Migration from Express",
|
|
"main": "dist/main.js",
|
|
"license": "AGPL-3.0-or-later",
|
|
"engines": {
|
|
"node": ">=22.0.0",
|
|
"npm": ">=10.0.0"
|
|
},
|
|
"scripts": {
|
|
"build": "nest build",
|
|
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
|
|
"start": "nest start",
|
|
"start:dev": "nest start --watch",
|
|
"start:debug": "nest start --debug --watch",
|
|
"start:prod": "node dist/main",
|
|
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
|
|
"lint:fix": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
|
|
"test": "jest",
|
|
"test:watch": "jest --watch",
|
|
"test:cov": "jest --coverage",
|
|
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
|
|
"test:e2e": "jest --config ./test/jest-e2e.json",
|
|
"typeorm": "typeorm-ts-node-commonjs -d src/config/typeorm.ts",
|
|
"migration:generate": "npm run typeorm -- migration:generate",
|
|
"migration:run": "npm run typeorm -- migration:run",
|
|
"migration:revert": "npm run typeorm -- migration:revert",
|
|
"schema:drop": "npm run typeorm -- schema:drop"
|
|
},
|
|
"dependencies": {
|
|
"@apollo/server": "^4.11.1",
|
|
"@mozilla/readability": "^0.6.0",
|
|
"@nestjs/apollo": "^12.0.11",
|
|
"@nestjs/bullmq": "^10.0.0",
|
|
"@nestjs/common": "^10.0.0",
|
|
"@nestjs/config": "^3.0.0",
|
|
"@nestjs/core": "^10.0.0",
|
|
"@nestjs/graphql": "^12.0.11",
|
|
"@nestjs/jwt": "^10.0.0",
|
|
"@nestjs/passport": "^10.0.0",
|
|
"@nestjs/platform-express": "^10.0.0",
|
|
"@nestjs/swagger": "^7.1.16",
|
|
"@nestjs/terminus": "^10.0.0",
|
|
"@nestjs/typeorm": "^10.0.0",
|
|
"bcrypt": "^5.1.1",
|
|
"bullmq": "^5.0.0",
|
|
"class-transformer": "^0.5.1",
|
|
"class-validator": "^0.14.0",
|
|
"cross-fetch": "^4.1.0",
|
|
"dataloader": "^2.2.3",
|
|
"dompurify": "^3.2.2",
|
|
"google-auth-library": "^9.0.0",
|
|
"graphql": "^16.11.0",
|
|
"graphql-scalars": "^1.25.0",
|
|
"ioredis": "^5.3.2",
|
|
"joi": "^17.11.0",
|
|
"jwk-to-pem": "^2.0.5",
|
|
"linkedom": "^0.18.5",
|
|
"passport": "^0.6.0",
|
|
"passport-google-oauth20": "^2.0.0",
|
|
"passport-jwt": "^4.0.1",
|
|
"passport-local": "^1.0.0",
|
|
"pg": "^8.11.3",
|
|
"reflect-metadata": "^0.1.13",
|
|
"rxjs": "^7.8.1",
|
|
"swagger-ui-express": "^5.0.0",
|
|
"turndown": "^7.2.2",
|
|
"typeorm": "^0.3.17"
|
|
},
|
|
"devDependencies": {
|
|
"@faker-js/faker": "^10.1.0",
|
|
"@nestjs/cli": "^10.0.0",
|
|
"@nestjs/schematics": "^10.0.0",
|
|
"@nestjs/testing": "^10.0.0",
|
|
"@testcontainers/postgresql": "^11.7.1",
|
|
"@types/bcrypt": "^5.0.2",
|
|
"@types/dompurify": "^3.2.0",
|
|
"@types/express": "^4.17.17",
|
|
"@types/jest": "^29.5.14",
|
|
"@types/joi": "^17.2.3",
|
|
"@types/jwk-to-pem": "^2.0.1",
|
|
"@types/node": "^20.3.1",
|
|
"@types/passport-google-oauth20": "^2.0.11",
|
|
"@types/passport-jwt": "^3.0.9",
|
|
"@types/passport-local": "^1.0.35",
|
|
"@types/pg": "^8.10.9",
|
|
"@types/supertest": "^2.0.12",
|
|
"@types/turndown": "^5.0.5",
|
|
"@typescript-eslint/eslint-plugin": "^6.0.0",
|
|
"@typescript-eslint/parser": "^6.0.0",
|
|
"eslint": "^8.42.0",
|
|
"eslint-config-prettier": "^9.0.0",
|
|
"eslint-plugin-prettier": "^5.0.0",
|
|
"jest": "^29.5.0",
|
|
"prettier": "^3.0.0",
|
|
"source-map-support": "^0.5.21",
|
|
"supertest": "^6.3.3",
|
|
"ts-jest": "^29.1.0",
|
|
"ts-loader": "^9.4.3",
|
|
"ts-node": "^10.9.1",
|
|
"tsconfig-paths": "^4.2.0",
|
|
"typescript": "^5.1.3"
|
|
},
|
|
"jest": {
|
|
"moduleFileExtensions": [
|
|
"js",
|
|
"json",
|
|
"ts"
|
|
],
|
|
"moduleDirectories": [
|
|
"node_modules",
|
|
"../node_modules",
|
|
"../../node_modules"
|
|
],
|
|
"moduleNameMapper": {
|
|
"^typeorm$": "<rootDir>/../../../node_modules/typeorm",
|
|
"^uuid$": "<rootDir>/../../../node_modules/uuid/dist/index.js"
|
|
},
|
|
"rootDir": "src",
|
|
"testRegex": ".*\\.spec\\.ts$",
|
|
"transform": {
|
|
"^.+\\.(t|j)s$": "ts-jest"
|
|
},
|
|
"transformIgnorePatterns": [
|
|
"node_modules/(?!(bullmq|msgpackr)/)"
|
|
],
|
|
"collectCoverageFrom": [
|
|
"**/*.(t|j)s"
|
|
],
|
|
"coverageDirectory": "../coverage",
|
|
"testEnvironment": "node"
|
|
}
|
|
}
|