From 0db1108675b10f61258a0636ea85939867c5109e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 7 Jun 2024 13:29:38 +0800 Subject: [PATCH] replace continuous spaces and new lines with space in preview content --- packages/api/src/jobs/find_thumbnail.ts | 2 +- .../api/src/jobs/generate_preview_content.ts | 20 +++++++++- packages/api/src/pubsub.ts | 38 +++++++++++++++++++ packages/api/src/services/library_item.ts | 1 + packages/api/src/services/save_email.ts | 12 ------ packages/api/src/services/save_page.ts | 25 ------------ 6 files changed, 58 insertions(+), 40 deletions(-) diff --git a/packages/api/src/jobs/find_thumbnail.ts b/packages/api/src/jobs/find_thumbnail.ts index b446e462a..4f0f1005e 100644 --- a/packages/api/src/jobs/find_thumbnail.ts +++ b/packages/api/src/jobs/find_thumbnail.ts @@ -22,7 +22,7 @@ interface ImageSize { export const THUMBNAIL_JOB = 'find-thumbnail' const fetchImage = async (url: string): Promise => { - logger.info('fetching image', url) + logger.info('fetching image', { url }) try { // get image file by url return await axios.get(url, { diff --git a/packages/api/src/jobs/generate_preview_content.ts b/packages/api/src/jobs/generate_preview_content.ts index 6b4c9aadd..c458131eb 100644 --- a/packages/api/src/jobs/generate_preview_content.ts +++ b/packages/api/src/jobs/generate_preview_content.ts @@ -23,12 +23,19 @@ export const generatePreviewContent = async ( return } - const libraryItem = await findLibraryItemById(libraryItemId, userId) + const libraryItem = await findLibraryItemById(libraryItemId, userId, { + select: ['id', 'readableContent', 'previewContent'], + }) if (!libraryItem) { logger.error(`Library item not found: ${libraryItemId}`) return } + if (libraryItem.previewContent && libraryItem.previewContent.length >= 180) { + logger.info(`Preview content already exists: ${libraryItemId}`) + return + } + const content = libraryItem.readableContent if (!content) { logger.error(`Library item has no content: ${libraryItemId}`) @@ -39,7 +46,11 @@ export const generatePreviewContent = async ( logger.info(`Generating preview for library item: ${libraryItemId}`) // the preview content should be within 600 characters const document = parseHTML(content).document - const previewContent = document.documentElement.textContent?.slice(0, 600) + const previewContent = document.documentElement.textContent + ?.trim() + ?.replace(/\s+/g, ' ') + ?.replace(/\n/g, ' ') + ?.slice(0, 600) if (!previewContent) { logger.error( `Failed to generate preview for library item: ${libraryItemId}` @@ -47,6 +58,11 @@ export const generatePreviewContent = async ( return } + logger.info('Generated preview for library item', { + libraryItemId, + previewContent, + }) + await updateLibraryItem( libraryItemId, { diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index 3e6fea7fd..ae510b241 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -4,8 +4,10 @@ import { RuleEventType } from './entity/rule' import { env } from './env' import { ReportType } from './generated/graphql' import { + enqueueGeneratePreviewContentJob, enqueueProcessYouTubeVideo, enqueueScoreJob, + enqueueThumbnailJob, enqueueTriggerRuleJob, } from './utils/createTask' import { logger } from './utils/logger' @@ -80,6 +82,42 @@ export const createPubSubClient = (): PubsubClient => { userId, libraryItemId: data.id, }) + + const hasThumbnail = ( + data: any + ): data is { thumbnail: string | null } => { + return 'thumbnail' in data + } + + // we don't want to create thumbnail for imported pages and pages that already have thumbnail + if (!hasThumbnail(data) || !data.thumbnail) { + try { + // create a task to update thumbnail and pre-cache all images + const job = await enqueueThumbnailJob(userId, data.id) + logger.info('Thumbnail job created', { id: job?.id }) + } catch (e) { + logger.error('Failed to enqueue thumbnail job', e) + } + } + + const hasPreviewContent = ( + data: any + ): data is { previewContent: string | null } => { + return 'previewContent' in data + } + + // generate preview content if it is less than 180 characters + if ( + !hasPreviewContent(data) || + (data.previewContent && data.previewContent.length < 180) + ) { + try { + const job = await enqueueGeneratePreviewContentJob(data.id, userId) + logger.info('Generate preview job created', { id: job?.id }) + } catch (e) { + logger.error('Failed to enqueue generate preview job', e) + } + } } }, entityUpdated: async ( diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index c6ecbfc9f..fb5a5c2cf 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -150,6 +150,7 @@ export const batchGetLibraryItems = async (ids: readonly string[]) => { 'archivedAt', 'deletedAt', 'slug', + 'previewContent', ], }) diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 699db6919..eb75616ac 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -3,7 +3,6 @@ import { LibraryItem, LibraryItemState, } from '../entity/library_item' -import { enqueueThumbnailJob } from '../utils/createTask' import { cleanUrl, generateSlug, @@ -12,7 +11,6 @@ import { validatedDate, wordsCount, } from '../utils/helpers' -import { logger } from '../utils/logger' import { FAKE_URL_PREFIX, fetchFavicon, @@ -143,15 +141,5 @@ export const saveEmail = async ( await updateReceivedEmail(input.receivedEmailId, 'article', input.userId) - if (!newLibraryItem.thumbnail) { - // create a task to update thumbnail and pre-cache all images - try { - const job = await enqueueThumbnailJob(input.userId, newLibraryItem.id) - logger.info('Created thumbnail job', { taskId: job }) - } catch (e) { - logger.error('Failed to create thumbnail job', e) - } - } - return newLibraryItem } diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 677ff3733..6d7a16be0 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -16,10 +16,6 @@ import { SaveResult, } from '../generated/graphql' import { Merge } from '../util' -import { - enqueueGeneratePreviewContentJob, - enqueueThumbnailJob, -} from '../utils/createTask' import { cleanUrl, generateSlug, @@ -166,27 +162,6 @@ export const savePage = async ( input.rssFeedUrl ) - // we don't want to create thumbnail for imported pages and pages that already have thumbnail - if (!isImported && !parseResult.parsedContent?.previewImage) { - try { - // create a task to update thumbnail and pre-cache all images - const job = await enqueueThumbnailJob(user.id, clientRequestId) - logger.info('Created thumbnail job', { job }) - } catch (e) { - logger.error('Failed to enqueue thumbnail job', e) - } - } - - const excerpt = parseResult.parsedContent?.excerpt - // generate preview content if excerpt is less than 180 characters - if (!excerpt || excerpt.length < 180) { - try { - await enqueueGeneratePreviewContentJob(clientRequestId, user.id) - } catch (e) { - logger.error('Failed to enqueue generate preview job', e) - } - } - if (parseResult.highlightData) { const highlight: DeepPartial = { ...parseResult.highlightData,