diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index ed8bc47ff..a0e3718fd 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -219,4 +219,7 @@ export class LibraryItem { @Column('float') score?: number + + @Column('text') + previewContent?: string } diff --git a/packages/api/src/jobs/find_thumbnail.ts b/packages/api/src/jobs/find_thumbnail.ts index b446e462a..4f0f1005e 100644 --- a/packages/api/src/jobs/find_thumbnail.ts +++ b/packages/api/src/jobs/find_thumbnail.ts @@ -22,7 +22,7 @@ interface ImageSize { export const THUMBNAIL_JOB = 'find-thumbnail' const fetchImage = async (url: string): Promise => { - logger.info('fetching image', url) + logger.info('fetching image', { url }) try { // get image file by url return await axios.get(url, { diff --git a/packages/api/src/jobs/generate_preview_content.ts b/packages/api/src/jobs/generate_preview_content.ts new file mode 100644 index 000000000..c458131eb --- /dev/null +++ b/packages/api/src/jobs/generate_preview_content.ts @@ -0,0 +1,75 @@ +import { parseHTML } from 'linkedom' +import { + findLibraryItemById, + updateLibraryItem, +} from '../services/library_item' +import { findActiveUser } from '../services/user' +import { logger } from '../utils/logger' + +export const GENERATE_PREVIEW_CONTENT_JOB = 'generate-preview' + +interface GeneratePreviewContentData { + libraryItemId: string + userId: string +} + +export const generatePreviewContent = async ( + job: GeneratePreviewContentData +) => { + const { libraryItemId, userId } = job + const user = await findActiveUser(userId) + if (!user) { + logger.error(`User not found: ${userId}`) + return + } + + const libraryItem = await findLibraryItemById(libraryItemId, userId, { + select: ['id', 'readableContent', 'previewContent'], + }) + if (!libraryItem) { + logger.error(`Library item not found: ${libraryItemId}`) + return + } + + if (libraryItem.previewContent && libraryItem.previewContent.length >= 180) { + logger.info(`Preview content already exists: ${libraryItemId}`) + return + } + + const content = libraryItem.readableContent + if (!content) { + logger.error(`Library item has no content: ${libraryItemId}`) + return + } + + // Generate preview content + logger.info(`Generating preview for library item: ${libraryItemId}`) + // the preview content should be within 600 characters + const document = parseHTML(content).document + const previewContent = document.documentElement.textContent + ?.trim() + ?.replace(/\s+/g, ' ') + ?.replace(/\n/g, ' ') + ?.slice(0, 600) + if (!previewContent) { + logger.error( + `Failed to generate preview for library item: ${libraryItemId}` + ) + return + } + + logger.info('Generated preview for library item', { + libraryItemId, + previewContent, + }) + + await updateLibraryItem( + libraryItemId, + { + previewContent, + }, + userId, + undefined, + true + ) +} diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index 3e6fea7fd..ae510b241 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -4,8 +4,10 @@ import { RuleEventType } from './entity/rule' import { env } from './env' import { ReportType } from './generated/graphql' import { + enqueueGeneratePreviewContentJob, enqueueProcessYouTubeVideo, enqueueScoreJob, + enqueueThumbnailJob, enqueueTriggerRuleJob, } from './utils/createTask' import { logger } from './utils/logger' @@ -80,6 +82,42 @@ export const createPubSubClient = (): PubsubClient => { userId, libraryItemId: data.id, }) + + const hasThumbnail = ( + data: any + ): data is { thumbnail: string | null } => { + return 'thumbnail' in data + } + + // we don't want to create thumbnail for imported pages and pages that already have thumbnail + if (!hasThumbnail(data) || !data.thumbnail) { + try { + // create a task to update thumbnail and pre-cache all images + const job = await enqueueThumbnailJob(userId, data.id) + logger.info('Thumbnail job created', { id: job?.id }) + } catch (e) { + logger.error('Failed to enqueue thumbnail job', e) + } + } + + const hasPreviewContent = ( + data: any + ): data is { previewContent: string | null } => { + return 'previewContent' in data + } + + // generate preview content if it is less than 180 characters + if ( + !hasPreviewContent(data) || + (data.previewContent && data.previewContent.length < 180) + ) { + try { + const job = await enqueueGeneratePreviewContentJob(data.id, userId) + logger.info('Generate preview job created', { id: job?.id }) + } catch (e) { + logger.error('Failed to enqueue generate preview job', e) + } + } } }, entityUpdated: async ( diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 188c37efb..f77f92a55 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -31,6 +31,10 @@ import { } from './jobs/email/inbound_emails' import { sendEmailJob, SEND_EMAIL_JOB } from './jobs/email/send_email' import { findThumbnail, THUMBNAIL_JOB } from './jobs/find_thumbnail' +import { + generatePreviewContent, + GENERATE_PREVIEW_CONTENT_JOB, +} from './jobs/generate_preview_content' import { exportAllItems, EXPORT_ALL_ITEMS_JOB_NAME, @@ -194,6 +198,8 @@ export const createWorker = (connection: ConnectionOptions) => return updateHome(job.data) case SCORE_LIBRARY_ITEM_JOB: return scoreLibraryItem(job.data) + case GENERATE_PREVIEW_CONTENT_JOB: + return generatePreviewContent(job.data) default: logger.warning(`[queue-processor] unhandled job: ${job.name}`) } diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 6ed9de486..01391c7da 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -686,7 +686,8 @@ export const functionResolvers = { canComment: false, canShare: true, dir: libraryItem.directionality, - previewContent: libraryItem.description, + previewContent: + libraryItem.previewContent || libraryItem.description, subscription: libraryItem.subscription, siteName: libraryItem.siteName, siteIcon: libraryItem.siteIcon, diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index c6ecbfc9f..fb5a5c2cf 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -150,6 +150,7 @@ export const batchGetLibraryItems = async (ids: readonly string[]) => { 'archivedAt', 'deletedAt', 'slug', + 'previewContent', ], }) diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 699db6919..eb75616ac 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -3,7 +3,6 @@ import { LibraryItem, LibraryItemState, } from '../entity/library_item' -import { enqueueThumbnailJob } from '../utils/createTask' import { cleanUrl, generateSlug, @@ -12,7 +11,6 @@ import { validatedDate, wordsCount, } from '../utils/helpers' -import { logger } from '../utils/logger' import { FAKE_URL_PREFIX, fetchFavicon, @@ -143,15 +141,5 @@ export const saveEmail = async ( await updateReceivedEmail(input.receivedEmailId, 'article', input.userId) - if (!newLibraryItem.thumbnail) { - // create a task to update thumbnail and pre-cache all images - try { - const job = await enqueueThumbnailJob(input.userId, newLibraryItem.id) - logger.info('Created thumbnail job', { taskId: job }) - } catch (e) { - logger.error('Failed to create thumbnail job', e) - } - } - return newLibraryItem } diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index cf1a1e69f..6d7a16be0 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -16,7 +16,6 @@ import { SaveResult, } from '../generated/graphql' import { Merge } from '../util' -import { enqueueThumbnailJob } from '../utils/createTask' import { cleanUrl, generateSlug, @@ -163,17 +162,6 @@ export const savePage = async ( input.rssFeedUrl ) - // we don't want to create thumbnail for imported pages and pages that already have thumbnail - if (!isImported && !parseResult.parsedContent?.previewImage) { - try { - // create a task to update thumbnail and pre-cache all images - const job = await enqueueThumbnailJob(user.id, clientRequestId) - logger.info('Created thumbnail job', { job }) - } catch (e) { - logger.error('Failed to enqueue thumbnail job', e) - } - } - if (parseResult.highlightData) { const highlight: DeepPartial = { ...parseResult.highlightData, @@ -255,6 +243,7 @@ export const parsedContentToLibraryItem = ({ originalContent: originalHtml, readableContent: parsedContent?.content || '', description: parsedContent?.excerpt, + previewContent: parsedContent?.excerpt, title: title || parsedContent?.title || diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 26ef6362b..212628d38 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -29,6 +29,7 @@ import { BulkActionData, BULK_ACTION_JOB_NAME } from '../jobs/bulk_action' import { CallWebhookJobData, CALL_WEBHOOK_JOB_NAME } from '../jobs/call_webhook' import { SendEmailJobData, SEND_EMAIL_JOB } from '../jobs/email/send_email' import { THUMBNAIL_JOB } from '../jobs/find_thumbnail' +import { GENERATE_PREVIEW_CONTENT_JOB } from '../jobs/generate_preview_content' import { EXPORT_ALL_ITEMS_JOB_NAME } from '../jobs/integration/export_all_items' import { ExportItemJobData, @@ -110,6 +111,7 @@ export const getJobPriority = (jobName: string): number => { case EXPORT_ALL_ITEMS_JOB_NAME: case REFRESH_ALL_FEEDS_JOB_NAME: case THUMBNAIL_JOB: + case GENERATE_PREVIEW_CONTENT_JOB: return 100 default: @@ -1024,4 +1026,29 @@ export const enqueueScoreJob = async (data: ScoreLibraryItemJobData) => { }) } +export const enqueueGeneratePreviewContentJob = async ( + libraryItemId: string, + userId: string +) => { + const queue = await getBackendQueue() + if (!queue) { + return undefined + } + + return queue.add( + GENERATE_PREVIEW_CONTENT_JOB, + { + libraryItemId, + userId, + }, + { + jobId: `${GENERATE_PREVIEW_CONTENT_JOB}_${libraryItemId}_${JOB_VERSION}`, + removeOnComplete: true, + removeOnFail: true, + priority: getJobPriority(GENERATE_PREVIEW_CONTENT_JOB), + attempts: 3, + } + ) +} + export default createHttpTaskWithToken diff --git a/packages/db/migrations/0179.do.add_preview_content_to_library_item.sql b/packages/db/migrations/0179.do.add_preview_content_to_library_item.sql new file mode 100755 index 000000000..56e6be51d --- /dev/null +++ b/packages/db/migrations/0179.do.add_preview_content_to_library_item.sql @@ -0,0 +1,10 @@ +-- Type: DO +-- Name: add_preview_content_to_library_item +-- Description: Add preview_content column to library_item table + +BEGIN; + +ALTER TABLE omnivore.library_item + ADD COLUMN preview_content TEXT; + +COMMIT; diff --git a/packages/db/migrations/0179.undo.add_preview_content_to_library_item.sql b/packages/db/migrations/0179.undo.add_preview_content_to_library_item.sql new file mode 100755 index 000000000..fba4a41c2 --- /dev/null +++ b/packages/db/migrations/0179.undo.add_preview_content_to_library_item.sql @@ -0,0 +1,10 @@ +-- Type: UNDO +-- Name: add_preview_content_to_library_item +-- Description: Add preview_content column to library_item table + +BEGIN; + +ALTER TABLE omnivore.library_item + DROP COLUMN preview_content; + +COMMIT;