replace continuous spaces and new lines with space in preview content

packages/api/src/jobs/find_thumbnail.ts

@@ -22,7 +22,7 @@ interface ImageSize {
 export const THUMBNAIL_JOB = 'find-thumbnail'
 
 const fetchImage = async (url: string): Promise<AxiosResponse | null> => {
-  logger.info('fetching image', url)
+  logger.info('fetching image', { url })
   try {
     // get image file by url
     return await axios.get(url, {

packages/api/src/jobs/generate_preview_content.ts

@@ -23,12 +23,19 @@ export const generatePreviewContent = async (
     return
   }
 
-  const libraryItem = await findLibraryItemById(libraryItemId, userId)
+  const libraryItem = await findLibraryItemById(libraryItemId, userId, {
+    select: ['id', 'readableContent', 'previewContent'],
+  })
   if (!libraryItem) {
     logger.error(`Library item not found: ${libraryItemId}`)
     return
   }
 
+  if (libraryItem.previewContent && libraryItem.previewContent.length >= 180) {
+    logger.info(`Preview content already exists: ${libraryItemId}`)
+    return
+  }
+
   const content = libraryItem.readableContent
   if (!content) {
     logger.error(`Library item has no content: ${libraryItemId}`)
@@ -39,7 +46,11 @@ export const generatePreviewContent = async (
   logger.info(`Generating preview for library item: ${libraryItemId}`)
   // the preview content should be within 600 characters
   const document = parseHTML(content).document
-  const previewContent = document.documentElement.textContent?.slice(0, 600)
+  const previewContent = document.documentElement.textContent
+    ?.trim()
+    ?.replace(/\s+/g, ' ')
+    ?.replace(/\n/g, ' ')
+    ?.slice(0, 600)
   if (!previewContent) {
     logger.error(
       `Failed to generate preview for library item: ${libraryItemId}`
@@ -47,6 +58,11 @@ export const generatePreviewContent = async (
     return
   }
 
+  logger.info('Generated preview for library item', {
+    libraryItemId,
+    previewContent,
+  })
+
   await updateLibraryItem(
     libraryItemId,
     {

packages/api/src/pubsub.ts

@@ -4,8 +4,10 @@ import { RuleEventType } from './entity/rule'
 import { env } from './env'
 import { ReportType } from './generated/graphql'
 import {
+  enqueueGeneratePreviewContentJob,
   enqueueProcessYouTubeVideo,
   enqueueScoreJob,
+  enqueueThumbnailJob,
   enqueueTriggerRuleJob,
 } from './utils/createTask'
 import { logger } from './utils/logger'
@@ -80,6 +82,42 @@ export const createPubSubClient = (): PubsubClient => {
           userId,
           libraryItemId: data.id,
         })
+
+        const hasThumbnail = (
+          data: any
+        ): data is { thumbnail: string | null } => {
+          return 'thumbnail' in data
+        }
+
+        // we don't want to create thumbnail for imported pages and pages that already have thumbnail
+        if (!hasThumbnail(data) || !data.thumbnail) {
+          try {
+            // create a task to update thumbnail and pre-cache all images
+            const job = await enqueueThumbnailJob(userId, data.id)
+            logger.info('Thumbnail job created', { id: job?.id })
+          } catch (e) {
+            logger.error('Failed to enqueue thumbnail job', e)
+          }
+        }
+
+        const hasPreviewContent = (
+          data: any
+        ): data is { previewContent: string | null } => {
+          return 'previewContent' in data
+        }
+
+        // generate preview content if it is less than 180 characters
+        if (
+          !hasPreviewContent(data) ||
+          (data.previewContent && data.previewContent.length < 180)
+        ) {
+          try {
+            const job = await enqueueGeneratePreviewContentJob(data.id, userId)
+            logger.info('Generate preview job created', { id: job?.id })
+          } catch (e) {
+            logger.error('Failed to enqueue generate preview job', e)
+          }
+        }
       }
     },
     entityUpdated: async <T extends EntityEvent>(

packages/api/src/services/library_item.ts

@@ -150,6 +150,7 @@ export const batchGetLibraryItems = async (ids: readonly string[]) => {
       'archivedAt',
       'deletedAt',
       'slug',
+      'previewContent',
     ],
   })
 

packages/api/src/services/save_email.ts

@@ -3,7 +3,6 @@ import {
   LibraryItem,
   LibraryItemState,
 } from '../entity/library_item'
-import { enqueueThumbnailJob } from '../utils/createTask'
 import {
   cleanUrl,
   generateSlug,
@@ -12,7 +11,6 @@ import {
   validatedDate,
   wordsCount,
 } from '../utils/helpers'
-import { logger } from '../utils/logger'
 import {
   FAKE_URL_PREFIX,
   fetchFavicon,
@@ -143,15 +141,5 @@ export const saveEmail = async (
 
   await updateReceivedEmail(input.receivedEmailId, 'article', input.userId)
 
-  if (!newLibraryItem.thumbnail) {
-    // create a task to update thumbnail and pre-cache all images
-    try {
-      const job = await enqueueThumbnailJob(input.userId, newLibraryItem.id)
-      logger.info('Created thumbnail job', { taskId: job })
-    } catch (e) {
-      logger.error('Failed to create thumbnail job', e)
-    }
-  }
-
   return newLibraryItem
 }

packages/api/src/services/save_page.ts

@@ -16,10 +16,6 @@ import {
   SaveResult,
 } from '../generated/graphql'
 import { Merge } from '../util'
-import {
-  enqueueGeneratePreviewContentJob,
-  enqueueThumbnailJob,
-} from '../utils/createTask'
 import {
   cleanUrl,
   generateSlug,
@@ -166,27 +162,6 @@ export const savePage = async (
     input.rssFeedUrl
   )
 
-  // we don't want to create thumbnail for imported pages and pages that already have thumbnail
-  if (!isImported && !parseResult.parsedContent?.previewImage) {
-    try {
-      // create a task to update thumbnail and pre-cache all images
-      const job = await enqueueThumbnailJob(user.id, clientRequestId)
-      logger.info('Created thumbnail job', { job })
-    } catch (e) {
-      logger.error('Failed to enqueue thumbnail job', e)
-    }
-  }
-
-  const excerpt = parseResult.parsedContent?.excerpt
-  // generate preview content if excerpt is less than 180 characters
-  if (!excerpt || excerpt.length < 180) {
-    try {
-      await enqueueGeneratePreviewContentJob(clientRequestId, user.id)
-    } catch (e) {
-      logger.error('Failed to enqueue generate preview job', e)
-    }
-  }
-
   if (parseResult.highlightData) {
     const highlight: DeepPartial<Highlight> = {
       ...parseResult.highlightData,