prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(queue): refactor Redis config in the queue module to utilize a single URL for both local and Docker envs

Browse source
This commit is contained in:
Timothy Atapagra 2025-10-11 19:50:49 -04:00
parent 161a6eab97
commit ba8b4d5f4d
5 changed files with 157 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
packages/api-nest/env.template
View file

@ -46,13 +46,19 @@ DATABASE_NAME=omnivore
# DATABASE_CA_FILE=/path/to/ca.pem
# ================================
# REDIS CONFIGURATION (REQUIRED FOR EMAIL VERIFICATION)
# REDIS CONFIGURATION (REQUIRED FOR QUEUES AND EMAIL VERIFICATION)
# ================================
# Redis connection URL - works for both local and Docker environments
REDIS_URL=redis://localhost:6379
# Optional TLS certificate contents (PEM encoded)
# REDIS_TLS_CERT="-----BEGIN CERTIFICATE-----..."
# Optional: Redis Sentinel configuration for production
# REDIS_SENTINEL_NAME=mymaster
# REDIS_SENTINELS=redis1:26379,redis2:26379,redis3:26379
# ================================
# AUTHENTICATION WORKFLOW
# ================================

12
packages/api-nest/src/queue/processors/content-processor.service.ts
View file

@ -214,9 +214,19 @@ export class ContentProcessorService extends WorkerHost implements OnModuleInit,
try {
// Phase 1: Fetch HTML content
this.logger.debug(`Fetching HTML from ${url}`)
const response = await fetch(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; Omnivore/1.0; +https://omnivore.app)',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'no-cache',
'Pragma': 'no-cache',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Upgrade-Insecure-Requests': '1',
},
signal: AbortSignal.timeout(30000), // 30 second timeout
})

23
packages/api-nest/src/queue/queue.constants.ts
View file

@ -69,30 +69,21 @@ export const JOB_CONFIG = {
* Redis Configuration
*/
export const REDIS_CONFIG = {
// Connection
HOST: process.env.REDIS_HOST || 'localhost',
PORT: parseInt(process.env.REDIS_PORT || '6379', 10),
PASSWORD: process.env.REDIS_PASSWORD,
URL: process.env.REDIS_URL || 'redis://localhost:6379',
// Sentinel configuration (for production)
SENTINEL_NAME: process.env.REDIS_SENTINEL_NAME || 'mymaster',
SENTINELS: process.env.REDIS_SENTINELS
? process.env.REDIS_SENTINELS.split(',').map(s => {
? process.env.REDIS_SENTINELS.split(',').map((s) => {
const [host, port] = s.split(':')
return { host, port: parseInt(port, 10) }
})
: undefined,
// Connection pool
// BullMQ requires null for blocking operations (BRPOPLPUSH, etc.)
MAX_RETRIES_PER_REQUEST: null,
ENABLE_READY_CHECK: true,
ENABLE_OFFLINE_QUEUE: true,
// Key prefixes for different environments
KEY_PREFIX: process.env.NODE_ENV === 'test'
? 'omnivore:test:'
: 'omnivore:',
KEY_PREFIX: process.env.NODE_ENV === 'test' ? 'omnivore:test:' : 'omnivore:',
} as const
/**
@ -110,7 +101,7 @@ export const QUEUE_STATE = {
/**
* Type exports for type-safe usage
*/
export type QueueName = typeof QUEUE_NAMES[keyof typeof QUEUE_NAMES]
export type JobType = typeof JOB_TYPES[keyof typeof JOB_TYPES]
export type JobPriority = typeof JOB_PRIORITY[keyof typeof JOB_PRIORITY]
export type QueueState = typeof QUEUE_STATE[keyof typeof QUEUE_STATE]
export type QueueName = (typeof QUEUE_NAMES)[keyof typeof QUEUE_NAMES]
export type JobType = (typeof JOB_TYPES)[keyof typeof JOB_TYPES]
export type JobPriority = (typeof JOB_PRIORITY)[keyof typeof JOB_PRIORITY]
export type QueueState = (typeof QUEUE_STATE)[keyof typeof QUEUE_STATE]

16
packages/api-nest/src/queue/queue.module.ts
View file

@ -14,17 +14,19 @@ import { EventBusService } from './event-bus.service'
import { QueueHealthIndicator } from './queue-health.indicator'
import { ContentProcessorService } from './processors/content-processor.service'
import { LibraryItemEntity } from '../library/entities/library-item.entity'
import { EnvVariables } from '../config/env-variables'
@Module({
imports: [
ConfigModule,
TypeOrmModule.forFeature([LibraryItemEntity]),
// Register BullMQ with Redis Sentinel configuration
BullModule.forRootAsync({
imports: [ConfigModule],
inject: [ConfigService],
useFactory: (configService: ConfigService) => {
// Check if we're using Sentinel (production) or standalone Redis (development)
const redisUrl =
configService.get<string>(EnvVariables.REDIS_URL) || REDIS_CONFIG.URL
const useSentinel =
configService.get<string>('NODE_ENV') === 'production' &&
REDIS_CONFIG.SENTINELS
@ -32,19 +34,14 @@ import { LibraryItemEntity } from '../library/entities/library-item.entity'
return {
connection: useSentinel
? {
// Sentinel configuration (production)
sentinels: REDIS_CONFIG.SENTINELS,
name: REDIS_CONFIG.SENTINEL_NAME,
password: REDIS_CONFIG.PASSWORD,
maxRetriesPerRequest: REDIS_CONFIG.MAX_RETRIES_PER_REQUEST,
enableReadyCheck: REDIS_CONFIG.ENABLE_READY_CHECK,
enableOfflineQueue: REDIS_CONFIG.ENABLE_OFFLINE_QUEUE,
}
: {
// Standalone Redis configuration (development)
host: REDIS_CONFIG.HOST,
port: REDIS_CONFIG.PORT,
password: REDIS_CONFIG.PASSWORD,
url: redisUrl,
maxRetriesPerRequest: REDIS_CONFIG.MAX_RETRIES_PER_REQUEST,
enableReadyCheck: REDIS_CONFIG.ENABLE_READY_CHECK,
enableOfflineQueue: REDIS_CONFIG.ENABLE_OFFLINE_QUEUE,
@ -64,7 +61,6 @@ import { LibraryItemEntity } from '../library/entities/library-item.entity'
},
}),
// Register individual queues
BullModule.registerQueue(
{
name: QUEUE_NAMES.CONTENT_PROCESSING,
@ -95,7 +91,7 @@ import { LibraryItemEntity } from '../library/entities/library-item.entity'
delay: 3000,
},
},
}
},
),
],
providers: [EventBusService, QueueHealthIndicator, ContentProcessorService],

182
packages/web-vite/src/pages/LibraryPage.tsx
View file

@ -67,7 +67,8 @@ const LibraryPage: React.FC = () => {
const [activeFolder, setActiveFolder] = useState<string>('all')
const [sortBy, setSortBy] = useState<string>('SAVED_AT')
const [sortOrder, setSortOrder] = useState<'ASC' | 'DESC'>('DESC')
const [toast, setToast] = useState<{ message: string; type: 'success' | 'error' } | null>(null)
const [toast, setToast] =
useState<{ message: string; type: 'success' | 'error' } | null>(null)
const [processingItemId, setProcessingItemId] = useState<string | null>(null)
const [selectedItems, setSelectedItems] = useState<Set<string>>(new Set())
const [isMultiSelectMode, setIsMultiSelectMode] = useState(false)
@ -114,13 +115,13 @@ const LibraryPage: React.FC = () => {
searchParams.sortBy = sortBy
searchParams.sortOrder = sortOrder
const data = await graphqlRequest<{ libraryItems: LibraryItemsConnection }>(
LIBRARY_ITEMS_QUERY,
{
first: INITIAL_PAGE_SIZE,
search: Object.keys(searchParams).length > 0 ? searchParams : undefined
}
)
const data = await graphqlRequest<{
libraryItems: LibraryItemsConnection
}>(LIBRARY_ITEMS_QUERY, {
first: INITIAL_PAGE_SIZE,
search:
Object.keys(searchParams).length > 0 ? searchParams : undefined,
})
setItems(data.libraryItems.items)
setError(null)
@ -137,7 +138,15 @@ const LibraryPage: React.FC = () => {
// Debounce search query - shorter for better UX
const debounceTimer = setTimeout(fetchItems, searchQuery ? 300 : 0)
return () => clearTimeout(debounceTimer)
}, [user, searchQuery, activeFolder, sortBy, sortOrder, selectedLabelFilters])
}, [
user,
searchQuery,
activeFolder,
sortBy,
sortOrder,
selectedLabelFilters,
items.length,
])
// No client-side filtering needed - using server-side search
const filteredItems = items
@ -185,7 +194,10 @@ const LibraryPage: React.FC = () => {
}
}
const showToast = (message: string, type: 'success' | 'error' = 'success') => {
const showToast = (
message: string,
type: 'success' | 'error' = 'success'
) => {
setToast({ message, type })
setTimeout(() => setToast(null), 3000)
}
@ -194,7 +206,10 @@ const LibraryPage: React.FC = () => {
navigate(`/reader/${itemId}`)
}
const handleArchive = async (itemId: string, currentState: LibraryItemState) => {
const handleArchive = async (
itemId: string,
currentState: LibraryItemState
) => {
const isArchived = currentState === 'ARCHIVED'
try {
@ -204,7 +219,11 @@ const LibraryPage: React.FC = () => {
setItems((prevItems) =>
prevItems.map((item) =>
item.id === itemId
? { ...item, state: isArchived ? 'SUCCEEDED' : 'ARCHIVED', folder: isArchived ? 'inbox' : 'archive' }
? {
...item,
state: isArchived ? 'SUCCEEDED' : 'ARCHIVED',
folder: isArchived ? 'inbox' : 'archive',
}
: item
)
)
@ -215,9 +234,7 @@ const LibraryPage: React.FC = () => {
// Revert optimistic update on error
setItems((prevItems) =>
prevItems.map((item) =>
item.id === itemId
? { ...item, state: currentState }
: item
item.id === itemId ? { ...item, state: currentState } : item
)
)
showToast(err instanceof Error ? err.message : 'Action failed', 'error')
@ -280,13 +297,23 @@ const LibraryPage: React.FC = () => {
setItems((prevItems) =>
prevItems.map((item) =>
selectedItems.has(item.id)
? { ...item, state: archived ? 'ARCHIVED' : 'SUCCEEDED', folder: archived ? 'archive' : 'inbox' }
? {
...item,
state: archived ? 'ARCHIVED' : 'SUCCEEDED',
folder: archived ? 'archive' : 'inbox',
}
: item
)
)
const result = await bulkArchive(itemIds, archived)
showToast(result.message || `${result.successCount} items ${archived ? 'archived' : 'unarchived'}`, 'success')
showToast(
result.message ||
`${result.successCount} items ${
archived ? 'archived' : 'unarchived'
}`,
'success'
)
if (result.failureCount > 0 && result.errors) {
console.error('Bulk archive errors:', result.errors)
@ -294,7 +321,10 @@ const LibraryPage: React.FC = () => {
deselectAll()
} catch (err) {
showToast(err instanceof Error ? err.message : 'Bulk archive failed', 'error')
showToast(
err instanceof Error ? err.message : 'Bulk archive failed',
'error'
)
// Refetch to restore correct state
window.location.reload()
}
@ -303,7 +333,9 @@ const LibraryPage: React.FC = () => {
const handleBulkDelete = async () => {
if (selectedItems.size === 0) return
if (!confirm(`Are you sure you want to delete ${selectedItems.size} item(s)?`)) {
if (
!confirm(`Are you sure you want to delete ${selectedItems.size} item(s)?`)
) {
return
}
@ -311,10 +343,15 @@ const LibraryPage: React.FC = () => {
const itemIds = Array.from(selectedItems)
// Optimistic update - remove from list
setItems((prevItems) => prevItems.filter((item) => !selectedItems.has(item.id)))
setItems((prevItems) =>
prevItems.filter((item) => !selectedItems.has(item.id))
)
const result = await bulkDelete(itemIds)
showToast(result.message || `${result.successCount} items deleted`, 'success')
showToast(
result.message || `${result.successCount} items deleted`,
'success'
)
if (result.failureCount > 0 && result.errors) {
console.error('Bulk delete errors:', result.errors)
@ -322,7 +359,10 @@ const LibraryPage: React.FC = () => {
deselectAll()
} catch (err) {
showToast(err instanceof Error ? err.message : 'Bulk delete failed', 'error')
showToast(
err instanceof Error ? err.message : 'Bulk delete failed',
'error'
)
window.location.reload()
}
}
@ -348,7 +388,10 @@ const LibraryPage: React.FC = () => {
)
const result = await bulkMoveToFolder(itemIds, folder)
showToast(result.message || `${result.successCount} items moved to ${folder}`, 'success')
showToast(
result.message || `${result.successCount} items moved to ${folder}`,
'success'
)
if (result.failureCount > 0 && result.errors) {
console.error('Bulk move errors:', result.errors)
@ -356,7 +399,10 @@ const LibraryPage: React.FC = () => {
deselectAll()
} catch (err) {
showToast(err instanceof Error ? err.message : 'Bulk move failed', 'error')
showToast(
err instanceof Error ? err.message : 'Bulk move failed',
'error'
)
window.location.reload()
}
}
@ -377,7 +423,10 @@ const LibraryPage: React.FC = () => {
)
const result = await bulkMarkAsRead(itemIds)
showToast(result.message || `${result.successCount} items marked as read`, 'success')
showToast(
result.message || `${result.successCount} items marked as read`,
'success'
)
if (result.failureCount > 0 && result.errors) {
console.error('Bulk mark as read errors:', result.errors)
@ -385,19 +434,25 @@ const LibraryPage: React.FC = () => {
deselectAll()
} catch (err) {
showToast(err instanceof Error ? err.message : 'Bulk mark as read failed', 'error')
showToast(
err instanceof Error ? err.message : 'Bulk mark as read failed',
'error'
)
window.location.reload()
}
}
const handleLabelsUpdate = async (itemId: string, newLabelNames: string[]) => {
const handleLabelsUpdate = async (
itemId: string,
newLabelNames: string[]
) => {
// Optimistic update - convert label names to Label objects
setItems((prevItems) =>
prevItems.map((item) => {
if (item.id === itemId && allLabels) {
// Map label names to full Label objects from allLabels
const updatedLabels = newLabelNames
.map(name => allLabels.find(l => l.name === name))
.map((name) => allLabels.find((l) => l.name === name))
.filter((l): l is NonNullable<typeof l> => l !== undefined)
return { ...item, labels: updatedLabels }
}
@ -422,13 +477,12 @@ const LibraryPage: React.FC = () => {
searchParams.sortBy = sortBy
searchParams.sortOrder = sortOrder
const data = await graphqlRequest<{ libraryItems: LibraryItemsConnection }>(
LIBRARY_ITEMS_QUERY,
{
first: INITIAL_PAGE_SIZE,
search: Object.keys(searchParams).length > 0 ? searchParams : undefined
}
)
const data = await graphqlRequest<{
libraryItems: LibraryItemsConnection
}>(LIBRARY_ITEMS_QUERY, {
first: INITIAL_PAGE_SIZE,
search: Object.keys(searchParams).length > 0 ? searchParams : undefined,
})
setItems(data.libraryItems.items)
} catch (err) {
@ -468,13 +522,12 @@ const LibraryPage: React.FC = () => {
searchParams.sortBy = sortBy
searchParams.sortOrder = sortOrder
const data = await graphqlRequest<{ libraryItems: LibraryItemsConnection }>(
LIBRARY_ITEMS_QUERY,
{
first: INITIAL_PAGE_SIZE,
search: Object.keys(searchParams).length > 0 ? searchParams : undefined
}
)
const data = await graphqlRequest<{
libraryItems: LibraryItemsConnection
}>(LIBRARY_ITEMS_QUERY, {
first: INITIAL_PAGE_SIZE,
search: Object.keys(searchParams).length > 0 ? searchParams : undefined,
})
setItems(data.libraryItems.items)
} catch (err) {
@ -504,16 +557,19 @@ const LibraryPage: React.FC = () => {
return (
<ErrorBoundary>
{toast && (
<div className={`toast toast-${toast.type}`}>
{toast.message}
</div>
<div className={`toast toast-${toast.type}`}>{toast.message}</div>
)}
<div className="library-page">
<div className="library-header">
<h1>
Your Library {searching && <span className="searching-indicator">Searching...</span>}
Your Library{' '}
{searching && (
<span className="searching-indicator">Searching...</span>
)}
{selectedItems.size > 0 && (
<span className="selection-count">({selectedItems.size} selected)</span>
<span className="selection-count">
({selectedItems.size} selected)
</span>
)}
</h1>
<div className="library-controls">
@ -532,7 +588,9 @@ const LibraryPage: React.FC = () => {
className="label-filter-toggle-btn"
onClick={() => setShowLabelFilter(!showLabelFilter)}
>
🏷️ Filter by Labels {selectedLabelFilters.length > 0 && `(${selectedLabelFilters.length})`}
🏷️ Filter by Labels{' '}
{selectedLabelFilters.length > 0 &&
`(${selectedLabelFilters.length})`}
</button>
{showLabelFilter && (
<div className="label-filter-dropdown">
@ -663,19 +721,25 @@ const LibraryPage: React.FC = () => {
All
</button>
<button
className={`folder-tab ${activeFolder === 'inbox' ? 'active' : ''}`}
className={`folder-tab ${
activeFolder === 'inbox' ? 'active' : ''
}`}
onClick={() => setActiveFolder('inbox')}
>
Inbox
</button>
<button
className={`folder-tab ${activeFolder === 'archive' ? 'active' : ''}`}
className={`folder-tab ${
activeFolder === 'archive' ? 'active' : ''
}`}
onClick={() => setActiveFolder('archive')}
>
Archive
</button>
<button
className={`folder-tab ${activeFolder === 'trash' ? 'active' : ''}`}
className={`folder-tab ${
activeFolder === 'trash' ? 'active' : ''
}`}
onClick={() => setActiveFolder('trash')}
>
Trash
@ -698,7 +762,9 @@ const LibraryPage: React.FC = () => {
</select>
<button
className="sort-order-btn"
onClick={() => setSortOrder(sortOrder === 'DESC' ? 'ASC' : 'DESC')}
onClick={() =>
setSortOrder(sortOrder === 'DESC' ? 'ASC' : 'DESC')
}
title={sortOrder === 'DESC' ? 'Descending' : 'Ascending'}
>
{sortOrder === 'DESC' ? '↓' : '↑'}
@ -747,7 +813,9 @@ const LibraryPage: React.FC = () => {
{filteredItems.map((item) => (
<div
key={item.id}
className={`article-card ${selectedItems.has(item.id) ? 'selected' : ''}`}
className={`article-card ${
selectedItems.has(item.id) ? 'selected' : ''
}`}
>
{isMultiSelectMode && (
<div className="article-checkbox">
@ -797,7 +865,7 @@ const LibraryPage: React.FC = () => {
padding: '0.25rem 0.5rem',
borderRadius: '0.25rem',
fontSize: '0.75rem',
marginRight: '0.25rem'
marginRight: '0.25rem',
}}
>
{label.name}
@ -823,8 +891,10 @@ const LibraryPage: React.FC = () => {
</button>
<LabelPicker
itemId={item.id}
currentLabels={item.labels?.map(l => l.name) || []}
onUpdate={(labelNames) => handleLabelsUpdate(item.id, labelNames)}
currentLabels={item.labels?.map((l) => l.name) || []}
onUpdate={(labelNames) =>
handleLabelsUpdate(item.id, labelNames)
}
/>
<button
className="action-btn action-btn-danger"