From 76383d8a338a6fe3d6594db8b52c4aa134f2c1b2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 6 Jun 2024 19:08:41 +0800 Subject: [PATCH 1/2] feat: generate preview content if excerpt less than 180 chars --- packages/api/src/entity/library_item.ts | 3 + .../api/src/jobs/generate_preview_content.ts | 59 +++++++++++++++++++ packages/api/src/queue-processor.ts | 6 ++ .../api/src/resolvers/function_resolvers.ts | 3 +- packages/api/src/services/save_page.ts | 16 ++++- packages/api/src/utils/createTask.ts | 27 +++++++++ ...do.add_preview_content_to_library_item.sql | 10 ++++ ...do.add_preview_content_to_library_item.sql | 10 ++++ 8 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 packages/api/src/jobs/generate_preview_content.ts create mode 100755 packages/db/migrations/0179.do.add_preview_content_to_library_item.sql create mode 100755 packages/db/migrations/0179.undo.add_preview_content_to_library_item.sql diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index ed8bc47ff..a0e3718fd 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -219,4 +219,7 @@ export class LibraryItem { @Column('float') score?: number + + @Column('text') + previewContent?: string } diff --git a/packages/api/src/jobs/generate_preview_content.ts b/packages/api/src/jobs/generate_preview_content.ts new file mode 100644 index 000000000..6b4c9aadd --- /dev/null +++ b/packages/api/src/jobs/generate_preview_content.ts @@ -0,0 +1,59 @@ +import { parseHTML } from 'linkedom' +import { + findLibraryItemById, + updateLibraryItem, +} from '../services/library_item' +import { findActiveUser } from '../services/user' +import { logger } from '../utils/logger' + +export const GENERATE_PREVIEW_CONTENT_JOB = 'generate-preview' + +interface GeneratePreviewContentData { + libraryItemId: string + userId: string +} + +export const generatePreviewContent = async ( + job: GeneratePreviewContentData +) => { + const { libraryItemId, userId } = job + const user = await findActiveUser(userId) + if (!user) { + logger.error(`User not found: ${userId}`) + return + } + + const libraryItem = await findLibraryItemById(libraryItemId, userId) + if (!libraryItem) { + logger.error(`Library item not found: ${libraryItemId}`) + return + } + + const content = libraryItem.readableContent + if (!content) { + logger.error(`Library item has no content: ${libraryItemId}`) + return + } + + // Generate preview content + logger.info(`Generating preview for library item: ${libraryItemId}`) + // the preview content should be within 600 characters + const document = parseHTML(content).document + const previewContent = document.documentElement.textContent?.slice(0, 600) + if (!previewContent) { + logger.error( + `Failed to generate preview for library item: ${libraryItemId}` + ) + return + } + + await updateLibraryItem( + libraryItemId, + { + previewContent, + }, + userId, + undefined, + true + ) +} diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 188c37efb..f77f92a55 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -31,6 +31,10 @@ import { } from './jobs/email/inbound_emails' import { sendEmailJob, SEND_EMAIL_JOB } from './jobs/email/send_email' import { findThumbnail, THUMBNAIL_JOB } from './jobs/find_thumbnail' +import { + generatePreviewContent, + GENERATE_PREVIEW_CONTENT_JOB, +} from './jobs/generate_preview_content' import { exportAllItems, EXPORT_ALL_ITEMS_JOB_NAME, @@ -194,6 +198,8 @@ export const createWorker = (connection: ConnectionOptions) => return updateHome(job.data) case SCORE_LIBRARY_ITEM_JOB: return scoreLibraryItem(job.data) + case GENERATE_PREVIEW_CONTENT_JOB: + return generatePreviewContent(job.data) default: logger.warning(`[queue-processor] unhandled job: ${job.name}`) } diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 6ed9de486..01391c7da 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -686,7 +686,8 @@ export const functionResolvers = { canComment: false, canShare: true, dir: libraryItem.directionality, - previewContent: libraryItem.description, + previewContent: + libraryItem.previewContent || libraryItem.description, subscription: libraryItem.subscription, siteName: libraryItem.siteName, siteIcon: libraryItem.siteIcon, diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index cf1a1e69f..677ff3733 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -16,7 +16,10 @@ import { SaveResult, } from '../generated/graphql' import { Merge } from '../util' -import { enqueueThumbnailJob } from '../utils/createTask' +import { + enqueueGeneratePreviewContentJob, + enqueueThumbnailJob, +} from '../utils/createTask' import { cleanUrl, generateSlug, @@ -174,6 +177,16 @@ export const savePage = async ( } } + const excerpt = parseResult.parsedContent?.excerpt + // generate preview content if excerpt is less than 180 characters + if (!excerpt || excerpt.length < 180) { + try { + await enqueueGeneratePreviewContentJob(clientRequestId, user.id) + } catch (e) { + logger.error('Failed to enqueue generate preview job', e) + } + } + if (parseResult.highlightData) { const highlight: DeepPartial = { ...parseResult.highlightData, @@ -255,6 +268,7 @@ export const parsedContentToLibraryItem = ({ originalContent: originalHtml, readableContent: parsedContent?.content || '', description: parsedContent?.excerpt, + previewContent: parsedContent?.excerpt, title: title || parsedContent?.title || diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 26ef6362b..212628d38 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -29,6 +29,7 @@ import { BulkActionData, BULK_ACTION_JOB_NAME } from '../jobs/bulk_action' import { CallWebhookJobData, CALL_WEBHOOK_JOB_NAME } from '../jobs/call_webhook' import { SendEmailJobData, SEND_EMAIL_JOB } from '../jobs/email/send_email' import { THUMBNAIL_JOB } from '../jobs/find_thumbnail' +import { GENERATE_PREVIEW_CONTENT_JOB } from '../jobs/generate_preview_content' import { EXPORT_ALL_ITEMS_JOB_NAME } from '../jobs/integration/export_all_items' import { ExportItemJobData, @@ -110,6 +111,7 @@ export const getJobPriority = (jobName: string): number => { case EXPORT_ALL_ITEMS_JOB_NAME: case REFRESH_ALL_FEEDS_JOB_NAME: case THUMBNAIL_JOB: + case GENERATE_PREVIEW_CONTENT_JOB: return 100 default: @@ -1024,4 +1026,29 @@ export const enqueueScoreJob = async (data: ScoreLibraryItemJobData) => { }) } +export const enqueueGeneratePreviewContentJob = async ( + libraryItemId: string, + userId: string +) => { + const queue = await getBackendQueue() + if (!queue) { + return undefined + } + + return queue.add( + GENERATE_PREVIEW_CONTENT_JOB, + { + libraryItemId, + userId, + }, + { + jobId: `${GENERATE_PREVIEW_CONTENT_JOB}_${libraryItemId}_${JOB_VERSION}`, + removeOnComplete: true, + removeOnFail: true, + priority: getJobPriority(GENERATE_PREVIEW_CONTENT_JOB), + attempts: 3, + } + ) +} + export default createHttpTaskWithToken diff --git a/packages/db/migrations/0179.do.add_preview_content_to_library_item.sql b/packages/db/migrations/0179.do.add_preview_content_to_library_item.sql new file mode 100755 index 000000000..56e6be51d --- /dev/null +++ b/packages/db/migrations/0179.do.add_preview_content_to_library_item.sql @@ -0,0 +1,10 @@ +-- Type: DO +-- Name: add_preview_content_to_library_item +-- Description: Add preview_content column to library_item table + +BEGIN; + +ALTER TABLE omnivore.library_item + ADD COLUMN preview_content TEXT; + +COMMIT; diff --git a/packages/db/migrations/0179.undo.add_preview_content_to_library_item.sql b/packages/db/migrations/0179.undo.add_preview_content_to_library_item.sql new file mode 100755 index 000000000..fba4a41c2 --- /dev/null +++ b/packages/db/migrations/0179.undo.add_preview_content_to_library_item.sql @@ -0,0 +1,10 @@ +-- Type: UNDO +-- Name: add_preview_content_to_library_item +-- Description: Add preview_content column to library_item table + +BEGIN; + +ALTER TABLE omnivore.library_item + DROP COLUMN preview_content; + +COMMIT; From 0db1108675b10f61258a0636ea85939867c5109e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 7 Jun 2024 13:29:38 +0800 Subject: [PATCH 2/2] replace continuous spaces and new lines with space in preview content --- packages/api/src/jobs/find_thumbnail.ts | 2 +- .../api/src/jobs/generate_preview_content.ts | 20 +++++++++- packages/api/src/pubsub.ts | 38 +++++++++++++++++++ packages/api/src/services/library_item.ts | 1 + packages/api/src/services/save_email.ts | 12 ------ packages/api/src/services/save_page.ts | 25 ------------ 6 files changed, 58 insertions(+), 40 deletions(-) diff --git a/packages/api/src/jobs/find_thumbnail.ts b/packages/api/src/jobs/find_thumbnail.ts index b446e462a..4f0f1005e 100644 --- a/packages/api/src/jobs/find_thumbnail.ts +++ b/packages/api/src/jobs/find_thumbnail.ts @@ -22,7 +22,7 @@ interface ImageSize { export const THUMBNAIL_JOB = 'find-thumbnail' const fetchImage = async (url: string): Promise => { - logger.info('fetching image', url) + logger.info('fetching image', { url }) try { // get image file by url return await axios.get(url, { diff --git a/packages/api/src/jobs/generate_preview_content.ts b/packages/api/src/jobs/generate_preview_content.ts index 6b4c9aadd..c458131eb 100644 --- a/packages/api/src/jobs/generate_preview_content.ts +++ b/packages/api/src/jobs/generate_preview_content.ts @@ -23,12 +23,19 @@ export const generatePreviewContent = async ( return } - const libraryItem = await findLibraryItemById(libraryItemId, userId) + const libraryItem = await findLibraryItemById(libraryItemId, userId, { + select: ['id', 'readableContent', 'previewContent'], + }) if (!libraryItem) { logger.error(`Library item not found: ${libraryItemId}`) return } + if (libraryItem.previewContent && libraryItem.previewContent.length >= 180) { + logger.info(`Preview content already exists: ${libraryItemId}`) + return + } + const content = libraryItem.readableContent if (!content) { logger.error(`Library item has no content: ${libraryItemId}`) @@ -39,7 +46,11 @@ export const generatePreviewContent = async ( logger.info(`Generating preview for library item: ${libraryItemId}`) // the preview content should be within 600 characters const document = parseHTML(content).document - const previewContent = document.documentElement.textContent?.slice(0, 600) + const previewContent = document.documentElement.textContent + ?.trim() + ?.replace(/\s+/g, ' ') + ?.replace(/\n/g, ' ') + ?.slice(0, 600) if (!previewContent) { logger.error( `Failed to generate preview for library item: ${libraryItemId}` @@ -47,6 +58,11 @@ export const generatePreviewContent = async ( return } + logger.info('Generated preview for library item', { + libraryItemId, + previewContent, + }) + await updateLibraryItem( libraryItemId, { diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index 3e6fea7fd..ae510b241 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -4,8 +4,10 @@ import { RuleEventType } from './entity/rule' import { env } from './env' import { ReportType } from './generated/graphql' import { + enqueueGeneratePreviewContentJob, enqueueProcessYouTubeVideo, enqueueScoreJob, + enqueueThumbnailJob, enqueueTriggerRuleJob, } from './utils/createTask' import { logger } from './utils/logger' @@ -80,6 +82,42 @@ export const createPubSubClient = (): PubsubClient => { userId, libraryItemId: data.id, }) + + const hasThumbnail = ( + data: any + ): data is { thumbnail: string | null } => { + return 'thumbnail' in data + } + + // we don't want to create thumbnail for imported pages and pages that already have thumbnail + if (!hasThumbnail(data) || !data.thumbnail) { + try { + // create a task to update thumbnail and pre-cache all images + const job = await enqueueThumbnailJob(userId, data.id) + logger.info('Thumbnail job created', { id: job?.id }) + } catch (e) { + logger.error('Failed to enqueue thumbnail job', e) + } + } + + const hasPreviewContent = ( + data: any + ): data is { previewContent: string | null } => { + return 'previewContent' in data + } + + // generate preview content if it is less than 180 characters + if ( + !hasPreviewContent(data) || + (data.previewContent && data.previewContent.length < 180) + ) { + try { + const job = await enqueueGeneratePreviewContentJob(data.id, userId) + logger.info('Generate preview job created', { id: job?.id }) + } catch (e) { + logger.error('Failed to enqueue generate preview job', e) + } + } } }, entityUpdated: async ( diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index c6ecbfc9f..fb5a5c2cf 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -150,6 +150,7 @@ export const batchGetLibraryItems = async (ids: readonly string[]) => { 'archivedAt', 'deletedAt', 'slug', + 'previewContent', ], }) diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 699db6919..eb75616ac 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -3,7 +3,6 @@ import { LibraryItem, LibraryItemState, } from '../entity/library_item' -import { enqueueThumbnailJob } from '../utils/createTask' import { cleanUrl, generateSlug, @@ -12,7 +11,6 @@ import { validatedDate, wordsCount, } from '../utils/helpers' -import { logger } from '../utils/logger' import { FAKE_URL_PREFIX, fetchFavicon, @@ -143,15 +141,5 @@ export const saveEmail = async ( await updateReceivedEmail(input.receivedEmailId, 'article', input.userId) - if (!newLibraryItem.thumbnail) { - // create a task to update thumbnail and pre-cache all images - try { - const job = await enqueueThumbnailJob(input.userId, newLibraryItem.id) - logger.info('Created thumbnail job', { taskId: job }) - } catch (e) { - logger.error('Failed to create thumbnail job', e) - } - } - return newLibraryItem } diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 677ff3733..6d7a16be0 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -16,10 +16,6 @@ import { SaveResult, } from '../generated/graphql' import { Merge } from '../util' -import { - enqueueGeneratePreviewContentJob, - enqueueThumbnailJob, -} from '../utils/createTask' import { cleanUrl, generateSlug, @@ -166,27 +162,6 @@ export const savePage = async ( input.rssFeedUrl ) - // we don't want to create thumbnail for imported pages and pages that already have thumbnail - if (!isImported && !parseResult.parsedContent?.previewImage) { - try { - // create a task to update thumbnail and pre-cache all images - const job = await enqueueThumbnailJob(user.id, clientRequestId) - logger.info('Created thumbnail job', { job }) - } catch (e) { - logger.error('Failed to enqueue thumbnail job', e) - } - } - - const excerpt = parseResult.parsedContent?.excerpt - // generate preview content if excerpt is less than 180 characters - if (!excerpt || excerpt.length < 180) { - try { - await enqueueGeneratePreviewContentJob(clientRequestId, user.id) - } catch (e) { - logger.error('Failed to enqueue generate preview job', e) - } - } - if (parseResult.highlightData) { const highlight: DeepPartial = { ...parseResult.highlightData,