From 505e3666b6641f0f9dc2de26fbe9b4894d20d374 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 May 2024 17:38:24 +0800 Subject: [PATCH 1/4] fix: add author to the chapter in digest --- packages/api/src/jobs/ai/create_digest.ts | 1 + packages/api/src/services/digest.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts index 3184fcff4..7df048c53 100644 --- a/packages/api/src/jobs/ai/create_digest.ts +++ b/packages/api/src/jobs/ai/create_digest.ts @@ -832,6 +832,7 @@ export const createDigest = async (jobData: CreateDigestData) => { thumbnail: item.libraryItem.thumbnail ?? undefined, wordCount: speechFiles[index].wordCount, html: summariesInHtml[index], + author: item.libraryItem.author ?? '', })), createdAt: new Date(), description: '', diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts index 6e0e95d59..d1d5fa1c1 100644 --- a/packages/api/src/services/digest.ts +++ b/packages/api/src/services/digest.ts @@ -10,6 +10,7 @@ export interface Chapter { wordCount: number thumbnail?: string html: string + author: string } export interface Digest { From 9f1babf787fb2607fee9c0fdf7d8863aa8df0732 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 May 2024 17:42:53 +0800 Subject: [PATCH 2/4] select used columns --- packages/api/src/jobs/ai/create_digest.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts index 7df048c53..ae9affa13 100644 --- a/packages/api/src/jobs/ai/create_digest.ts +++ b/packages/api/src/jobs/ai/create_digest.ts @@ -27,6 +27,7 @@ import { findUserAndPersonalization, sendPushNotifications, } from '../../services/user' +import { ANTHROPIC_MODEL, OPENAI_MODEL } from '../../utils/ai' import { analytics } from '../../utils/analytics' import { enqueueSendEmail } from '../../utils/createTask' import { wordsCount } from '../../utils/helpers' @@ -34,7 +35,6 @@ import { logger } from '../../utils/logger' import { htmlToMarkdown, markdownToHtml } from '../../utils/parser' import { uploadToBucket } from '../../utils/uploads' import { getImageSize, _findThumbnail } from '../find_thumbnail' -import { ANTHROPIC_MODEL, OPENAI_MODEL } from '../../utils/ai' export type CreateDigestJobSchedule = 'daily' | 'weekly' @@ -174,7 +174,9 @@ const getCandidatesList = async ( // reason: "most recent 100 items saved over 500 words if (selectedLibraryItemIds) { - return findLibraryItemsByIds(selectedLibraryItemIds, userId) + return findLibraryItemsByIds(selectedLibraryItemIds, userId, { + select: ['id', 'title', 'readableContent', 'author', 'thumbnail'], + }) } // // get the existing candidate ids from cache From 396ada28b88974bd77410d75804187dff88821bc Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 May 2024 17:49:31 +0800 Subject: [PATCH 3/4] pre-cache thumbnail --- packages/api/src/jobs/ai/create_digest.ts | 5 ++++- packages/api/src/jobs/find_thumbnail.ts | 4 ++-- packages/api/src/jobs/rss/refreshFeed.ts | 4 ++-- packages/api/src/routers/svc/following.ts | 4 ++-- packages/api/src/utils/imageproxy.ts | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts index ae9affa13..929b88581 100644 --- a/packages/api/src/jobs/ai/create_digest.ts +++ b/packages/api/src/jobs/ai/create_digest.ts @@ -31,6 +31,7 @@ import { ANTHROPIC_MODEL, OPENAI_MODEL } from '../../utils/ai' import { analytics } from '../../utils/analytics' import { enqueueSendEmail } from '../../utils/createTask' import { wordsCount } from '../../utils/helpers' +import { createThumbnailProxyUrl } from '../../utils/imageproxy' import { logger } from '../../utils/logger' import { htmlToMarkdown, markdownToHtml } from '../../utils/parser' import { uploadToBucket } from '../../utils/uploads' @@ -649,7 +650,9 @@ const findThumbnail = async ( try { for (const thumbnail of thumbnails) { - const size = await getImageSize(thumbnail) + const proxyUrl = createThumbnailProxyUrl(thumbnail) + // pre-cache thumbnail first if exists + const size = await getImageSize(proxyUrl) if (!size) { continue } diff --git a/packages/api/src/jobs/find_thumbnail.ts b/packages/api/src/jobs/find_thumbnail.ts index 3a81a2d6b..b446e462a 100644 --- a/packages/api/src/jobs/find_thumbnail.ts +++ b/packages/api/src/jobs/find_thumbnail.ts @@ -5,7 +5,7 @@ import { findLibraryItemById, updateLibraryItem, } from '../services/library_item' -import { createThumbnailUrl } from '../utils/imageproxy' +import { createThumbnailProxyUrl } from '../utils/imageproxy' import { logger } from '../utils/logger' interface Data { @@ -137,7 +137,7 @@ export const findThumbnail = async (data: Data) => { const thumbnail = item.thumbnail if (thumbnail) { - const proxyUrl = createThumbnailUrl(thumbnail) + const proxyUrl = createThumbnailProxyUrl(thumbnail) // pre-cache thumbnail first if exists const image = await fetchImage(proxyUrl) if (!image) { diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 3bd1d2624..532c7de6f 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -16,7 +16,7 @@ import { import { findActiveUser } from '../../services/user' import createHttpTaskWithToken from '../../utils/createTask' import { cleanUrl } from '../../utils/helpers' -import { createThumbnailUrl } from '../../utils/imageproxy' +import { createThumbnailProxyUrl } from '../../utils/imageproxy' import { logger } from '../../utils/logger' import { RSSRefreshContext } from './refreshAllFeeds' @@ -367,7 +367,7 @@ const createItemWithFeedContent = async ( }) const thumbnail = getThumbnail(item) - const previewImage = thumbnail && createThumbnailUrl(thumbnail) + const previewImage = thumbnail && createThumbnailProxyUrl(thumbnail) const url = cleanUrl(item.link) const user = await findActiveUser(userId) diff --git a/packages/api/src/routers/svc/following.ts b/packages/api/src/routers/svc/following.ts index 8d171af6d..5e29688f3 100644 --- a/packages/api/src/routers/svc/following.ts +++ b/packages/api/src/routers/svc/following.ts @@ -9,7 +9,7 @@ import { createAndSaveLabelsInLibraryItem } from '../../services/labels' import { createOrUpdateLibraryItem } from '../../services/library_item' import { parsedContentToLibraryItem } from '../../services/save_page' import { cleanUrl, generateSlug } from '../../utils/helpers' -import { createThumbnailUrl } from '../../utils/imageproxy' +import { createThumbnailProxyUrl } from '../../utils/imageproxy' import { logger } from '../../utils/logger' import { ParsedContentPuppeteer, @@ -72,7 +72,7 @@ export function followingServiceRouter() { const feedUrl = req.body.addedToFollowingBy const thumbnail = - req.body.thumbnail && createThumbnailUrl(req.body.thumbnail) + req.body.thumbnail && createThumbnailProxyUrl(req.body.thumbnail) const url = cleanUrl(req.body.url) const preparedDocument: PreparedDocumentInput = { diff --git a/packages/api/src/utils/imageproxy.ts b/packages/api/src/utils/imageproxy.ts index f449ca871..00b576215 100644 --- a/packages/api/src/utils/imageproxy.ts +++ b/packages/api/src/utils/imageproxy.ts @@ -28,5 +28,5 @@ export function createImageProxyUrl( return `${env.imageProxy.url}/${width}x${height},s${signature}/${url}` } -export const createThumbnailUrl = (url: string): string => +export const createThumbnailProxyUrl = (url: string): string => createImageProxyUrl(url, 320, 320) From 1a973faadc4ddfb8d9abd74fa6e70c530c9cedcc Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 May 2024 18:12:54 +0800 Subject: [PATCH 4/4] make author optional --- packages/api/src/jobs/ai/create_digest.ts | 6 ++++-- packages/api/src/services/digest.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts index 929b88581..d499dbf57 100644 --- a/packages/api/src/jobs/ai/create_digest.ts +++ b/packages/api/src/jobs/ai/create_digest.ts @@ -834,10 +834,12 @@ export const createDigest = async (jobData: CreateDigestData) => { title: item.libraryItem.title, id: item.libraryItem.id, url: getItemUrl(item.libraryItem.id), - thumbnail: item.libraryItem.thumbnail ?? undefined, + thumbnail: item.libraryItem.thumbnail + ? createThumbnailProxyUrl(item.libraryItem.thumbnail) + : undefined, wordCount: speechFiles[index].wordCount, html: summariesInHtml[index], - author: item.libraryItem.author ?? '', + author: item.libraryItem.author ?? undefined, })), createdAt: new Date(), description: '', diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts index d1d5fa1c1..c44671a90 100644 --- a/packages/api/src/services/digest.ts +++ b/packages/api/src/services/digest.ts @@ -10,7 +10,7 @@ export interface Chapter { wordCount: number thumbnail?: string html: string - author: string + author?: string } export interface Digest {