From c2387e72c9224d943589d90ee9694002aa06f019 Mon Sep 17 00:00:00 2001 From: Timothy Atapagra Date: Tue, 28 Oct 2025 20:17:03 -0400 Subject: [PATCH] feat(api-nest): add comprehensive tests for calculateWordCount method - Implemented a suite of tests for the calculateWordCount method in ContentProcessorService to ensure accurate word counting from various HTML and plain text inputs. - Added tests for handling edge cases, including empty content, malformed HTML, and mixed language content. - Enhanced logging for debugging purposes during word count calculations. - Updated the method to improve error handling and whitespace normalization. This commit improves the reliability of content processing by validating the word count functionality against a wide range of scenarios. --- .../content-processor.service.spec.ts | 204 ++++++++++++++++++ .../processors/content-processor.service.ts | 46 +++- 2 files changed, 242 insertions(+), 8 deletions(-) diff --git a/packages/api-nest/src/queue/processors/content-processor.service.spec.ts b/packages/api-nest/src/queue/processors/content-processor.service.spec.ts index 4707d1d77..290eb2ed0 100644 --- a/packages/api-nest/src/queue/processors/content-processor.service.spec.ts +++ b/packages/api-nest/src/queue/processors/content-processor.service.spec.ts @@ -364,6 +364,210 @@ describe('ContentProcessorService', () => { ).rejects.toThrow('Update failed') }) }) + + describe('calculateWordCount', () => { + describe('basic functionality', () => { + it('should count words in simple HTML content', () => { + const html = '

Hello world, this is a test.

' + const count = service.calculateWordCount(html) + expect(count).toBe(6) + }) + + it('should handle empty content', () => { + expect(service.calculateWordCount('')).toBe(0) + expect(service.calculateWordCount(' ')).toBe(0) + }) + + it('should handle HTML with no text content', () => { + const html = '
' + const count = service.calculateWordCount(html) + expect(count).toBe(0) + }) + + it('should count words in plain text', () => { + const text = 'This is plain text without HTML tags' + const count = service.calculateWordCount(text) + expect(count).toBe(7) + }) + }) + + describe('HTML parsing', () => { + it('should strip HTML tags from content', () => { + const html = '

Title

Paragraph with bold text

' + const count = service.calculateWordCount(html) + expect(count).toBe(4) // Title Paragraph bold text (note: 'with' counted separately) + }) + + it('should handle nested HTML elements', () => { + const html = ` +
+

Article Title

+
+

First paragraph with emphasis.

+

Second paragraph with link.

+
+
+ ` + const count = service.calculateWordCount(html) + expect(count).toBe(10) // Article Title First paragraph with emphasis Second paragraph with link + }) + + it('should handle Readability-style HTML fragments', () => { + const html = ` +
+
+

This is content from Readability parser.

+

It comes wrapped in a DIV element.

+
+
+ ` + const count = service.calculateWordCount(html) + expect(count).toBe(13) // Actual count includes all words + }) + + it('should handle HTML with inline styles and attributes', () => { + const html = '

Content here

' + const count = service.calculateWordCount(html) + expect(count).toBe(2) // Content here + }) + }) + + describe('HTML entity decoding', () => { + it('should decode common HTML entities', () => { + const html = '

Tom & Jerry

' + const count = service.calculateWordCount(html) + expect(count).toBe(3) // Tom & Jerry + }) + + it('should decode numeric entities', () => { + const html = '

Hello world

' + const count = service.calculateWordCount(html) + expect(count).toBe(2) // Hello world + }) + + it('should handle special characters', () => { + const html = '

Price: $100 — sold!

' + const count = service.calculateWordCount(html) + expect(count).toBe(4) // Price: $100 — sold! + }) + + it('should handle quotes and apostrophes', () => { + const html = "

"It's" a test

" + const count = service.calculateWordCount(html) + expect(count).toBe(3) // "It's" a test + }) + }) + + describe('whitespace normalization', () => { + it('should normalize multiple spaces', () => { + const html = '

Hello world test

' + const count = service.calculateWordCount(html) + expect(count).toBe(3) + }) + + it('should handle line breaks', () => { + const html = `

First line + Second line + Third line

` + const count = service.calculateWordCount(html) + expect(count).toBe(6) + }) + + it('should trim leading and trailing whitespace', () => { + const html = '

Content

' + const count = service.calculateWordCount(html) + expect(count).toBe(1) + }) + + it('should handle mixed whitespace characters', () => { + const html = '

Word1\t\tWord2\n\nWord3

' + const count = service.calculateWordCount(html) + expect(count).toBe(3) + }) + }) + + describe('edge cases', () => { + it('should handle very long content', () => { + const words = Array(10000).fill('word').join(' ') + const html = `

${words}

` + const count = service.calculateWordCount(html) + expect(count).toBe(10000) + }) + + it('should handle content with only punctuation', () => { + const html = '

... !!! ???

' + const count = service.calculateWordCount(html) + expect(count).toBe(3) // Each punctuation group is a "word" + }) + + it('should handle mixed language content', () => { + const html = '

Hello world 你好世界 Hola mundo

' + const count = service.calculateWordCount(html) + // Note: This counts space-separated tokens, which may not be ideal for all languages + expect(count).toBeGreaterThan(0) + }) + + it('should handle content with URLs', () => { + const html = '

Visit https://example.com for more info

' + const count = service.calculateWordCount(html) + expect(count).toBe(5) // Visit https://example.com for more info + }) + + it('should handle malformed HTML gracefully', () => { + const html = '

Unclosed paragraph

Nested content' + const count = service.calculateWordCount(html) + expect(count).toBe(3) // Unclosed paragraph Nested content + }) + }) + + describe('real-world examples', () => { + it('should accurately count words in article-like content', () => { + const html = ` +
+

The Future of Web Development

+

Web development has evolved significantly over the past decade.

+

Modern frameworks like React and Vue have revolutionized how we build applications.

+

The future looks bright with emerging technologies like WebAssembly and serverless computing.

+
+ ` + const count = service.calculateWordCount(html) + expect(count).toBe(38) // Actual word count + }) + + it('should match word count from known article', () => { + // This is a simplified version of actual Readability output + const html = ` +
+
+

To provide genuinely helpful signals for product decisions, a backlog needs to be well-organized.

+

But organizing a backlog has historically been manual work that doesn't scale.

+
+
+ ` + const count = service.calculateWordCount(html) + expect(count).toBe(26) // Actual word count from the text + }) + }) + + describe('error handling', () => { + it('should return 0 for null input', () => { + const count = service.calculateWordCount(null as any) + expect(count).toBe(0) + }) + + it('should return 0 for undefined input', () => { + const count = service.calculateWordCount(undefined as any) + expect(count).toBe(0) + }) + + it('should handle invalid HTML gracefully', () => { + const html = '<<>><>invalid html<<>>' + const count = service.calculateWordCount(html) + // linkedom should handle this gracefully + expect(count).toBeGreaterThanOrEqual(0) + }) + }) + }) }) /** diff --git a/packages/api-nest/src/queue/processors/content-processor.service.ts b/packages/api-nest/src/queue/processors/content-processor.service.ts index a1217f5b6..f25b85f1e 100644 --- a/packages/api-nest/src/queue/processors/content-processor.service.ts +++ b/packages/api-nest/src/queue/processors/content-processor.service.ts @@ -273,6 +273,7 @@ export class ContentProcessorService // Phase 3: Extract Open Graph metadata (fast) this.logger.debug(`Extracting Open Graph metadata from ${url}`) const ogData = this.extractOpenGraph(document, url) + this.logger.log(`[DEBUG] Open Graph data extracted: ${JSON.stringify({ title: ogData.title, image: ogData.image, siteName: ogData.siteName })}`) await job.updateProgress(50) // Phase 4: Extract content with Readability @@ -312,12 +313,20 @@ export class ContentProcessorService // Phase 5: Calculate accurate word count from content (strips HTML) const actualWordCount = this.calculateWordCount(article.content || '') + // Cross-check: Readability also provides textContent (plain text) + // This helps verify our HTML-to-text word counting is accurate + const readabilityTextLength = article.textContent?.length || 0 + const readabilityWordEstimate = article.textContent + ? article.textContent.trim().split(/\s+/).filter(w => w.length > 0).length + : 0 + // Phase 6: Combine Open Graph + Readability results this.logger.log( - `Successfully extracted content from ${url}: ${actualWordCount} words`, + `Successfully extracted content from ${url}: ${actualWordCount} words ` + + `(Readability textContent estimate: ${readabilityWordEstimate} words, text length: ${readabilityTextLength})` ) - return { + const result = { success: true, title: article.title || ogData.title || 'Untitled', content: article.content || '', @@ -332,6 +341,9 @@ export class ContentProcessorService : undefined, wordCount: actualWordCount, } + + this.logger.log(`[DEBUG] Content fetch result: ${JSON.stringify({ title: result.title, thumbnail: result.thumbnail, wordCount: result.wordCount })}`) + return result } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error) @@ -347,24 +359,39 @@ export class ContentProcessorService /** * Calculate word count from HTML content * Parses HTML to decode entities (e.g.,  , &) before counting words + * * @param htmlContent - HTML content to count words from * @returns Actual word count + * @internal - Public for testing purposes only, not part of public API */ - private calculateWordCount(htmlContent: string): number { - if (!htmlContent) return 0 + public calculateWordCount(htmlContent: string): number { + if (!htmlContent) { + this.logger.debug('[calculateWordCount] No HTML content provided') + return 0 + } try { + // Readability returns HTML fragment (DIV), not a complete document + // Wrap it in a proper HTML structure so linkedom can parse it correctly + const wrappedHtml = `${htmlContent}` + // Parse HTML to decode entities and extract text content - const { document } = parseHTML(htmlContent) + const { document } = parseHTML(wrappedHtml) const textOnly = document.body?.textContent || '' + this.logger.debug(`[calculateWordCount] HTML length: ${htmlContent.length}, Text length: ${textOnly.length}`) + // Remove extra whitespace and normalize const normalized = textOnly.replace(/\s+/g, ' ').trim() - if (!normalized) return 0 + if (!normalized) { + this.logger.debug('[calculateWordCount] Normalized text is empty') + return 0 + } // Split by whitespace and count non-empty words const words = normalized.split(' ').filter((word) => word.length > 0) + this.logger.debug(`[calculateWordCount] Word count: ${words.length}`) return words.length } catch (error) { this.logger.warn(`Failed to calculate word count: ${error}`) @@ -431,7 +458,7 @@ export class ContentProcessorService this.logger.log(`Saving content for library item ${libraryItemId}`) try { - await this.libraryItemRepository.update(libraryItemId, { + const updateData = { title: result.title, readableContent: result.content, author: result.author, @@ -441,7 +468,10 @@ export class ContentProcessorService siteIcon: result.siteIcon, thumbnail: result.thumbnail, wordCount: result.wordCount, - }) + } + + this.logger.log(`[DEBUG] Saving to DB: ${JSON.stringify({ title: updateData.title, thumbnail: updateData.thumbnail, wordCount: updateData.wordCount })}`) + await this.libraryItemRepository.update(libraryItemId, updateData) this.logger.log(`Content saved for library item ${libraryItemId}`) } catch (error) {