diff --git a/packages/api-nest/package.json b/packages/api-nest/package.json
index 888346448..ddc7b3032 100644
--- a/packages/api-nest/package.json
+++ b/packages/api-nest/package.json
@@ -29,6 +29,7 @@
},
"dependencies": {
"@apollo/server": "^4.11.1",
+ "@mozilla/readability": "^0.6.0",
"@nestjs/apollo": "^12.0.11",
"@nestjs/bullmq": "^10.0.0",
"@nestjs/common": "^10.0.0",
@@ -43,13 +44,15 @@
"@nestjs/typeorm": "^10.0.0",
"bcrypt": "^5.1.1",
"bullmq": "^5.0.0",
+ "class-transformer": "^0.5.1",
+ "class-validator": "^0.14.0",
+ "cross-fetch": "^4.1.0",
"google-auth-library": "^9.0.0",
"graphql": "^16.11.0",
"ioredis": "^5.3.2",
- "jwk-to-pem": "^2.0.5",
- "class-transformer": "^0.5.1",
- "class-validator": "^0.14.0",
"joi": "^17.11.0",
+ "jwk-to-pem": "^2.0.5",
+ "linkedom": "^0.18.5",
"passport": "^0.6.0",
"passport-google-oauth20": "^2.0.0",
"passport-jwt": "^4.0.1",
@@ -68,12 +71,12 @@
"@types/express": "^4.17.17",
"@types/jest": "^29.5.14",
"@types/joi": "^17.2.3",
+ "@types/jwk-to-pem": "^2.0.1",
"@types/node": "^20.3.1",
"@types/passport-google-oauth20": "^2.0.11",
"@types/passport-jwt": "^3.0.9",
"@types/passport-local": "^1.0.35",
"@types/pg": "^8.10.9",
- "@types/jwk-to-pem": "^2.0.1",
"@types/supertest": "^2.0.12",
"@typescript-eslint/eslint-plugin": "^6.0.0",
"@typescript-eslint/parser": "^6.0.0",
diff --git a/packages/api-nest/src/queue/processors/content-processor.service.ts b/packages/api-nest/src/queue/processors/content-processor.service.ts
index a8d48da52..43bd45d49 100644
--- a/packages/api-nest/src/queue/processors/content-processor.service.ts
+++ b/packages/api-nest/src/queue/processors/content-processor.service.ts
@@ -5,11 +5,14 @@
* web content for saved library items.
*/
-import { Injectable, Logger, OnModuleInit } from '@nestjs/common'
+import { Injectable, Logger, OnModuleInit, OnModuleDestroy } from '@nestjs/common'
import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq'
import { Job } from 'bullmq'
import { InjectRepository } from '@nestjs/typeorm'
import { Repository } from 'typeorm'
+import { Readability } from '@mozilla/readability'
+import { parseHTML } from 'linkedom'
+import fetch from 'cross-fetch'
import { LibraryItemEntity, LibraryItemState } from '../../library/entities/library-item.entity'
import { EventBusService } from '../event-bus.service'
import { EVENT_NAMES } from '../events.constants'
@@ -38,6 +41,9 @@ export interface ContentFetchResult {
publishedDate?: Date
siteIcon?: string
thumbnail?: string
+ description?: string
+ siteName?: string
+ wordCount?: number
error?: string
}
@@ -45,7 +51,7 @@ export interface ContentFetchResult {
@Processor(QUEUE_NAMES.CONTENT_PROCESSING, {
concurrency: JOB_CONFIG.WORKER_CONCURRENCY,
})
-export class ContentProcessorService extends WorkerHost implements OnModuleInit {
+export class ContentProcessorService extends WorkerHost implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(ContentProcessorService.name)
constructor(
@@ -62,6 +68,17 @@ export class ContentProcessorService extends WorkerHost implements OnModuleInit
)
}
+ async onModuleDestroy() {
+ this.logger.log('ContentProcessorService shutting down...')
+ try {
+ // Close the worker gracefully
+ await this.worker?.close()
+ this.logger.log('ContentProcessorService shut down successfully')
+ } catch (error) {
+ this.logger.error(`Error during ContentProcessorService shutdown: ${error}`)
+ }
+ }
+
/**
* Main job processing method
* Called by BullMQ for each job
@@ -184,8 +201,9 @@ export class ContentProcessorService extends WorkerHost implements OnModuleInit
}
/**
- * Fetch content from URL
- * TODO: Implement full content fetching in Phase 3 with Puppeteer/handlers
+ * Fetch content from URL using two-phase extraction:
+ * 1. Open Graph metadata (fast preview)
+ * 2. Mozilla Readability (full content)
*/
private async fetchContent(
url: string,
@@ -194,25 +212,76 @@ export class ContentProcessorService extends WorkerHost implements OnModuleInit
this.logger.log(`Fetching content from ${url}`)
try {
- // STUB: For Phase 2, we'll just create a placeholder result
- // In Phase 3, this will be replaced with actual Puppeteer/handler logic
+ // Phase 1: Fetch HTML content
+ this.logger.debug(`Fetching HTML from ${url}`)
+ const response = await fetch(url, {
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (compatible; Omnivore/1.0; +https://omnivore.app)',
+ },
+ signal: AbortSignal.timeout(30000), // 30 second timeout
+ })
- // Simulate network delay
- await this.delay(1000)
+ if (!response.ok) {
+ throw new Error(`HTTP ${response.status}: ${response.statusText}`)
+ }
+
+ const html = await response.text()
await job.updateProgress(40)
- // Simulate content processing
- await this.delay(1000)
+ // Phase 2: Parse HTML with linkedom
+ this.logger.debug(`Parsing HTML for ${url}`)
+ const { document } = parseHTML(html)
+ await job.updateProgress(45)
+
+ // Phase 3: Extract Open Graph metadata (fast)
+ this.logger.debug(`Extracting Open Graph metadata from ${url}`)
+ const ogData = this.extractOpenGraph(document, url)
+ await job.updateProgress(50)
+
+ // Phase 4: Extract content with Readability
+ this.logger.debug(`Extracting readable content from ${url}`)
+ const reader = new Readability(document, {
+ keepClasses: false,
+ charThreshold: 500,
+ })
+ const article = reader.parse()
await job.updateProgress(60)
- // Return stub data
+ if (!article) {
+ // Readability failed, but we can still use Open Graph data
+ this.logger.warn(`Readability failed for ${url}, using Open Graph data only`)
+ return {
+ success: true,
+ title: ogData.title || new URL(url).hostname,
+ content: ogData.description ? `${ogData.description}
` : '',
+ contentType: 'text/html',
+ author: ogData.author,
+ description: ogData.description,
+ thumbnail: ogData.image,
+ siteName: ogData.siteName,
+ siteIcon: ogData.favicon,
+ publishedDate: ogData.publishedTime ? new Date(ogData.publishedTime) : undefined,
+ wordCount: 0,
+ }
+ }
+
+ // Phase 5: Combine Open Graph + Readability results
+ this.logger.log(
+ `Successfully extracted content from ${url}: ${article.length} words`
+ )
+
return {
success: true,
- title: `Content from ${new URL(url).hostname}`,
- content: 'This is stub content. Real content fetching will be implemented in Phase 3.
',
+ title: article.title || ogData.title || 'Untitled',
+ content: article.content || '',
contentType: 'text/html',
- author: 'Unknown',
- publishedDate: new Date(),
+ author: article.byline || ogData.author,
+ description: article.excerpt || ogData.description,
+ thumbnail: ogData.image,
+ siteName: article.siteName || ogData.siteName,
+ siteIcon: ogData.favicon,
+ publishedDate: ogData.publishedTime ? new Date(ogData.publishedTime) : undefined,
+ wordCount: article.length || 0,
}
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error)
@@ -225,6 +294,52 @@ export class ContentProcessorService extends WorkerHost implements OnModuleInit
}
}
+ /**
+ * Extract Open Graph metadata from HTML document
+ */
+ private extractOpenGraph(
+ document: Document,
+ url: string
+ ): {
+ title?: string
+ description?: string
+ image?: string
+ siteName?: string
+ author?: string
+ publishedTime?: string
+ favicon?: string
+ } {
+ const getMeta = (property: string): string | undefined => {
+ const element = document.querySelector(
+ `meta[property="${property}"], meta[name="${property}"]`
+ )
+ return element?.getAttribute('content') || undefined
+ }
+
+ const getLink = (rel: string): string | undefined => {
+ const element = document.querySelector(`link[rel="${rel}"]`)
+ const href = element?.getAttribute('href')
+ if (!href) return undefined
+
+ // Convert relative URLs to absolute
+ try {
+ return new URL(href, url).toString()
+ } catch {
+ return href
+ }
+ }
+
+ return {
+ title: getMeta('og:title') || getMeta('twitter:title'),
+ description: getMeta('og:description') || getMeta('twitter:description') || getMeta('description'),
+ image: getMeta('og:image') || getMeta('twitter:image'),
+ siteName: getMeta('og:site_name'),
+ author: getMeta('article:author') || getMeta('author'),
+ publishedTime: getMeta('article:published_time'),
+ favicon: getLink('icon') || getLink('shortcut icon'),
+ }
+ }
+
/**
* Save processed content to database
*/
@@ -239,9 +354,12 @@ export class ContentProcessorService extends WorkerHost implements OnModuleInit
title: result.title,
readableContent: result.content,
author: result.author,
+ description: result.description,
publishedAt: result.publishedDate,
+ siteName: result.siteName,
siteIcon: result.siteIcon,
thumbnail: result.thumbnail,
+ wordCount: result.wordCount,
})
this.logger.log(`Content saved for library item ${libraryItemId}`)
diff --git a/yarn.lock b/yarn.lock
index 9a77c3044..d6035d59b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -14954,6 +14954,13 @@ cross-fetch@^3.0.6, cross-fetch@^3.1.5:
dependencies:
node-fetch "^2.7.0"
+cross-fetch@^4.1.0:
+ version "4.1.0"
+ resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-4.1.0.tgz#8f69355007ee182e47fa692ecbaa37a52e43c3d2"
+ integrity sha512-uKm5PU+MHTootlWEY+mZ4vvXoCn4fLQxT9dSc1sXVMSFkINTJVN8cAQROpwcKm8bJ/c7rgZVIBWzH5T78sNZZw==
+ dependencies:
+ node-fetch "^2.7.0"
+
cross-inspect@1.0.1:
version "1.0.1"
resolved "https://registry.npmjs.org/cross-inspect/-/cross-inspect-1.0.1.tgz"
@@ -22806,6 +22813,17 @@ linkedom@^0.16.4, linkedom@^0.16.5:
htmlparser2 "^9.1.0"
uhyphen "^0.2.0"
+linkedom@^0.18.5:
+ version "0.18.12"
+ resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.18.12.tgz#a8b1a1942b567dcb1888093df311055da1349a14"
+ integrity sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==
+ dependencies:
+ css-select "^5.1.0"
+ cssom "^0.5.0"
+ html-escaper "^3.0.3"
+ htmlparser2 "^10.0.0"
+ uhyphen "^0.2.0"
+
linkedom@^0.18.9:
version "0.18.11"
resolved "https://registry.npmjs.org/linkedom/-/linkedom-0.18.11.tgz"