From ba8b4d5f4d2460eac2e812776f11a6942291ef37 Mon Sep 17 00:00:00 2001 From: Timothy Atapagra Date: Sat, 11 Oct 2025 19:50:49 -0400 Subject: [PATCH] feat(queue): refactor Redis config in the queue module to utilize a single URL for both local and Docker envs --- packages/api-nest/env.template | 8 +- .../processors/content-processor.service.ts | 12 +- .../api-nest/src/queue/queue.constants.ts | 23 +-- packages/api-nest/src/queue/queue.module.ts | 16 +- packages/web-vite/src/pages/LibraryPage.tsx | 182 ++++++++++++------ 5 files changed, 157 insertions(+), 84 deletions(-) diff --git a/packages/api-nest/env.template b/packages/api-nest/env.template index 3503be820..fed83abea 100644 --- a/packages/api-nest/env.template +++ b/packages/api-nest/env.template @@ -46,13 +46,19 @@ DATABASE_NAME=omnivore # DATABASE_CA_FILE=/path/to/ca.pem # ================================ -# REDIS CONFIGURATION (REQUIRED FOR EMAIL VERIFICATION) +# REDIS CONFIGURATION (REQUIRED FOR QUEUES AND EMAIL VERIFICATION) # ================================ +# Redis connection URL - works for both local and Docker environments REDIS_URL=redis://localhost:6379 + # Optional TLS certificate contents (PEM encoded) # REDIS_TLS_CERT="-----BEGIN CERTIFICATE-----..." +# Optional: Redis Sentinel configuration for production +# REDIS_SENTINEL_NAME=mymaster +# REDIS_SENTINELS=redis1:26379,redis2:26379,redis3:26379 + # ================================ # AUTHENTICATION WORKFLOW # ================================ diff --git a/packages/api-nest/src/queue/processors/content-processor.service.ts b/packages/api-nest/src/queue/processors/content-processor.service.ts index 43bd45d49..42d200cf7 100644 --- a/packages/api-nest/src/queue/processors/content-processor.service.ts +++ b/packages/api-nest/src/queue/processors/content-processor.service.ts @@ -214,9 +214,19 @@ export class ContentProcessorService extends WorkerHost implements OnModuleInit, try { // Phase 1: Fetch HTML content this.logger.debug(`Fetching HTML from ${url}`) + const response = await fetch(url, { headers: { - 'User-Agent': 'Mozilla/5.0 (compatible; Omnivore/1.0; +https://omnivore.app)', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + 'Cache-Control': 'no-cache', + 'Pragma': 'no-cache', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Upgrade-Insecure-Requests': '1', }, signal: AbortSignal.timeout(30000), // 30 second timeout }) diff --git a/packages/api-nest/src/queue/queue.constants.ts b/packages/api-nest/src/queue/queue.constants.ts index 044c0d791..19a859311 100644 --- a/packages/api-nest/src/queue/queue.constants.ts +++ b/packages/api-nest/src/queue/queue.constants.ts @@ -69,30 +69,21 @@ export const JOB_CONFIG = { * Redis Configuration */ export const REDIS_CONFIG = { - // Connection - HOST: process.env.REDIS_HOST || 'localhost', - PORT: parseInt(process.env.REDIS_PORT || '6379', 10), - PASSWORD: process.env.REDIS_PASSWORD, + URL: process.env.REDIS_URL || 'redis://localhost:6379', - // Sentinel configuration (for production) SENTINEL_NAME: process.env.REDIS_SENTINEL_NAME || 'mymaster', SENTINELS: process.env.REDIS_SENTINELS - ? process.env.REDIS_SENTINELS.split(',').map(s => { + ? process.env.REDIS_SENTINELS.split(',').map((s) => { const [host, port] = s.split(':') return { host, port: parseInt(port, 10) } }) : undefined, - // Connection pool - // BullMQ requires null for blocking operations (BRPOPLPUSH, etc.) MAX_RETRIES_PER_REQUEST: null, ENABLE_READY_CHECK: true, ENABLE_OFFLINE_QUEUE: true, - // Key prefixes for different environments - KEY_PREFIX: process.env.NODE_ENV === 'test' - ? 'omnivore:test:' - : 'omnivore:', + KEY_PREFIX: process.env.NODE_ENV === 'test' ? 'omnivore:test:' : 'omnivore:', } as const /** @@ -110,7 +101,7 @@ export const QUEUE_STATE = { /** * Type exports for type-safe usage */ -export type QueueName = typeof QUEUE_NAMES[keyof typeof QUEUE_NAMES] -export type JobType = typeof JOB_TYPES[keyof typeof JOB_TYPES] -export type JobPriority = typeof JOB_PRIORITY[keyof typeof JOB_PRIORITY] -export type QueueState = typeof QUEUE_STATE[keyof typeof QUEUE_STATE] +export type QueueName = (typeof QUEUE_NAMES)[keyof typeof QUEUE_NAMES] +export type JobType = (typeof JOB_TYPES)[keyof typeof JOB_TYPES] +export type JobPriority = (typeof JOB_PRIORITY)[keyof typeof JOB_PRIORITY] +export type QueueState = (typeof QUEUE_STATE)[keyof typeof QUEUE_STATE] diff --git a/packages/api-nest/src/queue/queue.module.ts b/packages/api-nest/src/queue/queue.module.ts index 48e511e85..7f38d2c4f 100644 --- a/packages/api-nest/src/queue/queue.module.ts +++ b/packages/api-nest/src/queue/queue.module.ts @@ -14,17 +14,19 @@ import { EventBusService } from './event-bus.service' import { QueueHealthIndicator } from './queue-health.indicator' import { ContentProcessorService } from './processors/content-processor.service' import { LibraryItemEntity } from '../library/entities/library-item.entity' +import { EnvVariables } from '../config/env-variables' @Module({ imports: [ ConfigModule, TypeOrmModule.forFeature([LibraryItemEntity]), - // Register BullMQ with Redis Sentinel configuration BullModule.forRootAsync({ imports: [ConfigModule], inject: [ConfigService], useFactory: (configService: ConfigService) => { - // Check if we're using Sentinel (production) or standalone Redis (development) + const redisUrl = + configService.get(EnvVariables.REDIS_URL) || REDIS_CONFIG.URL + const useSentinel = configService.get('NODE_ENV') === 'production' && REDIS_CONFIG.SENTINELS @@ -32,19 +34,14 @@ import { LibraryItemEntity } from '../library/entities/library-item.entity' return { connection: useSentinel ? { - // Sentinel configuration (production) sentinels: REDIS_CONFIG.SENTINELS, name: REDIS_CONFIG.SENTINEL_NAME, - password: REDIS_CONFIG.PASSWORD, maxRetriesPerRequest: REDIS_CONFIG.MAX_RETRIES_PER_REQUEST, enableReadyCheck: REDIS_CONFIG.ENABLE_READY_CHECK, enableOfflineQueue: REDIS_CONFIG.ENABLE_OFFLINE_QUEUE, } : { - // Standalone Redis configuration (development) - host: REDIS_CONFIG.HOST, - port: REDIS_CONFIG.PORT, - password: REDIS_CONFIG.PASSWORD, + url: redisUrl, maxRetriesPerRequest: REDIS_CONFIG.MAX_RETRIES_PER_REQUEST, enableReadyCheck: REDIS_CONFIG.ENABLE_READY_CHECK, enableOfflineQueue: REDIS_CONFIG.ENABLE_OFFLINE_QUEUE, @@ -64,7 +61,6 @@ import { LibraryItemEntity } from '../library/entities/library-item.entity' }, }), - // Register individual queues BullModule.registerQueue( { name: QUEUE_NAMES.CONTENT_PROCESSING, @@ -95,7 +91,7 @@ import { LibraryItemEntity } from '../library/entities/library-item.entity' delay: 3000, }, }, - } + }, ), ], providers: [EventBusService, QueueHealthIndicator, ContentProcessorService], diff --git a/packages/web-vite/src/pages/LibraryPage.tsx b/packages/web-vite/src/pages/LibraryPage.tsx index a4b7a34a0..91afb1cbd 100644 --- a/packages/web-vite/src/pages/LibraryPage.tsx +++ b/packages/web-vite/src/pages/LibraryPage.tsx @@ -67,7 +67,8 @@ const LibraryPage: React.FC = () => { const [activeFolder, setActiveFolder] = useState('all') const [sortBy, setSortBy] = useState('SAVED_AT') const [sortOrder, setSortOrder] = useState<'ASC' | 'DESC'>('DESC') - const [toast, setToast] = useState<{ message: string; type: 'success' | 'error' } | null>(null) + const [toast, setToast] = + useState<{ message: string; type: 'success' | 'error' } | null>(null) const [processingItemId, setProcessingItemId] = useState(null) const [selectedItems, setSelectedItems] = useState>(new Set()) const [isMultiSelectMode, setIsMultiSelectMode] = useState(false) @@ -114,13 +115,13 @@ const LibraryPage: React.FC = () => { searchParams.sortBy = sortBy searchParams.sortOrder = sortOrder - const data = await graphqlRequest<{ libraryItems: LibraryItemsConnection }>( - LIBRARY_ITEMS_QUERY, - { - first: INITIAL_PAGE_SIZE, - search: Object.keys(searchParams).length > 0 ? searchParams : undefined - } - ) + const data = await graphqlRequest<{ + libraryItems: LibraryItemsConnection + }>(LIBRARY_ITEMS_QUERY, { + first: INITIAL_PAGE_SIZE, + search: + Object.keys(searchParams).length > 0 ? searchParams : undefined, + }) setItems(data.libraryItems.items) setError(null) @@ -137,7 +138,15 @@ const LibraryPage: React.FC = () => { // Debounce search query - shorter for better UX const debounceTimer = setTimeout(fetchItems, searchQuery ? 300 : 0) return () => clearTimeout(debounceTimer) - }, [user, searchQuery, activeFolder, sortBy, sortOrder, selectedLabelFilters]) + }, [ + user, + searchQuery, + activeFolder, + sortBy, + sortOrder, + selectedLabelFilters, + items.length, + ]) // No client-side filtering needed - using server-side search const filteredItems = items @@ -185,7 +194,10 @@ const LibraryPage: React.FC = () => { } } - const showToast = (message: string, type: 'success' | 'error' = 'success') => { + const showToast = ( + message: string, + type: 'success' | 'error' = 'success' + ) => { setToast({ message, type }) setTimeout(() => setToast(null), 3000) } @@ -194,7 +206,10 @@ const LibraryPage: React.FC = () => { navigate(`/reader/${itemId}`) } - const handleArchive = async (itemId: string, currentState: LibraryItemState) => { + const handleArchive = async ( + itemId: string, + currentState: LibraryItemState + ) => { const isArchived = currentState === 'ARCHIVED' try { @@ -204,7 +219,11 @@ const LibraryPage: React.FC = () => { setItems((prevItems) => prevItems.map((item) => item.id === itemId - ? { ...item, state: isArchived ? 'SUCCEEDED' : 'ARCHIVED', folder: isArchived ? 'inbox' : 'archive' } + ? { + ...item, + state: isArchived ? 'SUCCEEDED' : 'ARCHIVED', + folder: isArchived ? 'inbox' : 'archive', + } : item ) ) @@ -215,9 +234,7 @@ const LibraryPage: React.FC = () => { // Revert optimistic update on error setItems((prevItems) => prevItems.map((item) => - item.id === itemId - ? { ...item, state: currentState } - : item + item.id === itemId ? { ...item, state: currentState } : item ) ) showToast(err instanceof Error ? err.message : 'Action failed', 'error') @@ -280,13 +297,23 @@ const LibraryPage: React.FC = () => { setItems((prevItems) => prevItems.map((item) => selectedItems.has(item.id) - ? { ...item, state: archived ? 'ARCHIVED' : 'SUCCEEDED', folder: archived ? 'archive' : 'inbox' } + ? { + ...item, + state: archived ? 'ARCHIVED' : 'SUCCEEDED', + folder: archived ? 'archive' : 'inbox', + } : item ) ) const result = await bulkArchive(itemIds, archived) - showToast(result.message || `${result.successCount} items ${archived ? 'archived' : 'unarchived'}`, 'success') + showToast( + result.message || + `${result.successCount} items ${ + archived ? 'archived' : 'unarchived' + }`, + 'success' + ) if (result.failureCount > 0 && result.errors) { console.error('Bulk archive errors:', result.errors) @@ -294,7 +321,10 @@ const LibraryPage: React.FC = () => { deselectAll() } catch (err) { - showToast(err instanceof Error ? err.message : 'Bulk archive failed', 'error') + showToast( + err instanceof Error ? err.message : 'Bulk archive failed', + 'error' + ) // Refetch to restore correct state window.location.reload() } @@ -303,7 +333,9 @@ const LibraryPage: React.FC = () => { const handleBulkDelete = async () => { if (selectedItems.size === 0) return - if (!confirm(`Are you sure you want to delete ${selectedItems.size} item(s)?`)) { + if ( + !confirm(`Are you sure you want to delete ${selectedItems.size} item(s)?`) + ) { return } @@ -311,10 +343,15 @@ const LibraryPage: React.FC = () => { const itemIds = Array.from(selectedItems) // Optimistic update - remove from list - setItems((prevItems) => prevItems.filter((item) => !selectedItems.has(item.id))) + setItems((prevItems) => + prevItems.filter((item) => !selectedItems.has(item.id)) + ) const result = await bulkDelete(itemIds) - showToast(result.message || `${result.successCount} items deleted`, 'success') + showToast( + result.message || `${result.successCount} items deleted`, + 'success' + ) if (result.failureCount > 0 && result.errors) { console.error('Bulk delete errors:', result.errors) @@ -322,7 +359,10 @@ const LibraryPage: React.FC = () => { deselectAll() } catch (err) { - showToast(err instanceof Error ? err.message : 'Bulk delete failed', 'error') + showToast( + err instanceof Error ? err.message : 'Bulk delete failed', + 'error' + ) window.location.reload() } } @@ -348,7 +388,10 @@ const LibraryPage: React.FC = () => { ) const result = await bulkMoveToFolder(itemIds, folder) - showToast(result.message || `${result.successCount} items moved to ${folder}`, 'success') + showToast( + result.message || `${result.successCount} items moved to ${folder}`, + 'success' + ) if (result.failureCount > 0 && result.errors) { console.error('Bulk move errors:', result.errors) @@ -356,7 +399,10 @@ const LibraryPage: React.FC = () => { deselectAll() } catch (err) { - showToast(err instanceof Error ? err.message : 'Bulk move failed', 'error') + showToast( + err instanceof Error ? err.message : 'Bulk move failed', + 'error' + ) window.location.reload() } } @@ -377,7 +423,10 @@ const LibraryPage: React.FC = () => { ) const result = await bulkMarkAsRead(itemIds) - showToast(result.message || `${result.successCount} items marked as read`, 'success') + showToast( + result.message || `${result.successCount} items marked as read`, + 'success' + ) if (result.failureCount > 0 && result.errors) { console.error('Bulk mark as read errors:', result.errors) @@ -385,19 +434,25 @@ const LibraryPage: React.FC = () => { deselectAll() } catch (err) { - showToast(err instanceof Error ? err.message : 'Bulk mark as read failed', 'error') + showToast( + err instanceof Error ? err.message : 'Bulk mark as read failed', + 'error' + ) window.location.reload() } } - const handleLabelsUpdate = async (itemId: string, newLabelNames: string[]) => { + const handleLabelsUpdate = async ( + itemId: string, + newLabelNames: string[] + ) => { // Optimistic update - convert label names to Label objects setItems((prevItems) => prevItems.map((item) => { if (item.id === itemId && allLabels) { // Map label names to full Label objects from allLabels const updatedLabels = newLabelNames - .map(name => allLabels.find(l => l.name === name)) + .map((name) => allLabels.find((l) => l.name === name)) .filter((l): l is NonNullable => l !== undefined) return { ...item, labels: updatedLabels } } @@ -422,13 +477,12 @@ const LibraryPage: React.FC = () => { searchParams.sortBy = sortBy searchParams.sortOrder = sortOrder - const data = await graphqlRequest<{ libraryItems: LibraryItemsConnection }>( - LIBRARY_ITEMS_QUERY, - { - first: INITIAL_PAGE_SIZE, - search: Object.keys(searchParams).length > 0 ? searchParams : undefined - } - ) + const data = await graphqlRequest<{ + libraryItems: LibraryItemsConnection + }>(LIBRARY_ITEMS_QUERY, { + first: INITIAL_PAGE_SIZE, + search: Object.keys(searchParams).length > 0 ? searchParams : undefined, + }) setItems(data.libraryItems.items) } catch (err) { @@ -468,13 +522,12 @@ const LibraryPage: React.FC = () => { searchParams.sortBy = sortBy searchParams.sortOrder = sortOrder - const data = await graphqlRequest<{ libraryItems: LibraryItemsConnection }>( - LIBRARY_ITEMS_QUERY, - { - first: INITIAL_PAGE_SIZE, - search: Object.keys(searchParams).length > 0 ? searchParams : undefined - } - ) + const data = await graphqlRequest<{ + libraryItems: LibraryItemsConnection + }>(LIBRARY_ITEMS_QUERY, { + first: INITIAL_PAGE_SIZE, + search: Object.keys(searchParams).length > 0 ? searchParams : undefined, + }) setItems(data.libraryItems.items) } catch (err) { @@ -504,16 +557,19 @@ const LibraryPage: React.FC = () => { return ( {toast && ( -
- {toast.message} -
+
{toast.message}
)}

- Your Library {searching && Searching...} + Your Library{' '} + {searching && ( + Searching... + )} {selectedItems.size > 0 && ( - ({selectedItems.size} selected) + + ({selectedItems.size} selected) + )}

@@ -532,7 +588,9 @@ const LibraryPage: React.FC = () => { className="label-filter-toggle-btn" onClick={() => setShowLabelFilter(!showLabelFilter)} > - 🏷️ Filter by Labels {selectedLabelFilters.length > 0 && `(${selectedLabelFilters.length})`} + 🏷️ Filter by Labels{' '} + {selectedLabelFilters.length > 0 && + `(${selectedLabelFilters.length})`} {showLabelFilter && (
@@ -663,19 +721,25 @@ const LibraryPage: React.FC = () => { All l.name) || []} - onUpdate={(labelNames) => handleLabelsUpdate(item.id, labelNames)} + currentLabels={item.labels?.map((l) => l.name) || []} + onUpdate={(labelNames) => + handleLabelsUpdate(item.id, labelNames) + } />