From 4e60396d174a47679798118093f308a2ee38c37c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 20 Nov 2023 17:28:34 +0800 Subject: [PATCH] save thumbnail of the following item --- packages/api/src/routers/svc/following.ts | 1 + packages/api/src/services/library_item.ts | 4 ++ packages/api/src/utils/imageproxy.ts | 3 + packages/rss-handler/src/index.ts | 73 ++++++++++++++--------- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/packages/api/src/routers/svc/following.ts b/packages/api/src/routers/svc/following.ts index 1bbbdab6d..ef962f7b4 100644 --- a/packages/api/src/routers/svc/following.ts +++ b/packages/api/src/routers/svc/following.ts @@ -23,6 +23,7 @@ export interface SaveFollowingItemRequest { previewContentType?: string publishedAt?: Date savedAt?: Date + thumbnail?: string } function isSaveFollowingItemRequest( diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index acb84fa6e..3511020ab 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -10,6 +10,7 @@ import { authTrx, getColumns } from '../repository' import { libraryItemRepository } from '../repository/library_item' import { SaveFollowingItemRequest } from '../routers/svc/following' import { generateSlug, wordsCount } from '../utils/helpers' +import { createThumbnailUrl } from '../utils/imageproxy' import { DateFilter, FieldFilter, @@ -554,6 +555,8 @@ export const saveFeedItemInFollowing = ( input: SaveFollowingItemRequest, userId: string ) => { + const thumbnail = input.thumbnail && createThumbnailUrl(input.thumbnail) + return authTrx( async (tx) => { const itemToSave: QueryDeepPartialEntity = { @@ -563,6 +566,7 @@ export const saveFeedItemInFollowing = ( subscription: input.addedToFollowingBy, folder: InFilter.FOLLOWING, slug: generateSlug(input.title), + thumbnail, } return tx diff --git a/packages/api/src/utils/imageproxy.ts b/packages/api/src/utils/imageproxy.ts index 104c9b811..f449ca871 100644 --- a/packages/api/src/utils/imageproxy.ts +++ b/packages/api/src/utils/imageproxy.ts @@ -27,3 +27,6 @@ export function createImageProxyUrl( return `${env.imageProxy.url}/${width}x${height},s${signature}/${url}` } + +export const createThumbnailUrl = (url: string): string => + createImageProxyUrl(url, 320, 320) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 2c4461c99..75a621ee5 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -19,6 +19,35 @@ interface RssFeedRequest { // link can be a string or an object type RssFeedItemLink = string | { $: { rel?: string; href: string } } +type RssFeed = Parser.Output<{ + published?: string + updated?: string + created?: string + link?: RssFeedItemLink + links?: RssFeedItemLink[] +}> & { + lastBuildDate?: string + 'syn:updatePeriod'?: string + 'syn:updateFrequency'?: string + 'sy:updatePeriod'?: string + 'sy:updateFrequency'?: string +} +type RssFeedItemMedia = { + $: { url: string; width?: string; height?: string; medium?: string } +} +type RssFeedItem = Item & { + 'media:thumbnail'?: RssFeedItemMedia + 'media:content'?: RssFeedItemMedia[] +} + +const getThumbnail = (item: RssFeedItem) => { + if (item['media:thumbnail']) { + return item['media:thumbnail'].$.url + } + + return item['media:content']?.find((media) => media.$.medium === 'image')?.$ + .url +} function isRssFeedRequest(body: any): body is RssFeedRequest { return ( @@ -125,7 +154,7 @@ const sendUpdateSubscriptionMutation = async ( const createTask = async ( userId: string, feedUrl: string, - item: Item, + item: RssFeedItem, autoAddToLibrary: boolean ) => { if (autoAddToLibrary) { @@ -138,7 +167,7 @@ const createTask = async ( const createSavingItemTask = async ( userId: string, feedUrl: string, - item: Item + item: RssFeedItem ) => { const input = { userId, @@ -167,7 +196,7 @@ const createSavingItemTask = async ( const createFollowingTask = async ( userId: string, feedUrl: string, - item: Item + item: RssFeedItem ) => { const input = { userIds: [userId], @@ -181,6 +210,7 @@ const createFollowingTask = async ( savedAt: item.isoDate, publishedAt: item.isoDate, previewContentType: 'text/html', // TODO: get content type from feed + thumbnail: getThumbnail(item), } try { @@ -217,6 +247,8 @@ const parser = new Parser({ 'published', 'updated', 'created', + ['media:content', 'media:content', { keepArray: true }], + ['media:thumbnail'], ], feed: [ 'lastBuildDate', @@ -228,9 +260,9 @@ const parser = new Parser({ }, }) -const getUpdateFrequency = (feed: any) => { - const updateFrequency = (feed['syn:updateFrequency'] || - feed['sy:updateFrequency']) as string | undefined +const getUpdateFrequency = (feed: RssFeed) => { + const updateFrequency = + feed['syn:updateFrequency'] || feed['sy:updateFrequency'] if (!updateFrequency) { return 1 @@ -244,10 +276,8 @@ const getUpdateFrequency = (feed: any) => { return frequency } -const getUpdatePeriodInHours = (feed: any) => { - const updatePeriod = (feed['syn:updatePeriod'] || feed['sy:updatePeriod']) as - | string - | undefined +const getUpdatePeriodInHours = (feed: RssFeed) => { + const updatePeriod = feed['syn:updatePeriod'] || feed['sy:updatePeriod'] switch (updatePeriod) { case 'hourly': @@ -301,19 +331,7 @@ const processSubscription = async ( scheduledAt: number, lastFetchedChecksum: string, autoAddToLibrary: boolean, - feed: { - lastBuildDate: any - 'syn:updatePeriod': any - 'syn:updateFrequency': any - 'sy:updatePeriod': any - 'sy:updateFrequency': any - } & Parser.Output<{ - published: any - updated: any - created: any - link: any - links: any[] - }> + feed: RssFeed ) => { let lastItemFetchedAt: Date | null = null let lastValidItem: Item | null = null @@ -327,7 +345,7 @@ const processSubscription = async ( // fetch feed let itemCount = 0 - const feedLastBuildDate = feed.lastBuildDate as string | undefined + const feedLastBuildDate = feed.lastBuildDate console.log('Feed last build date', feedLastBuildDate) if ( feedLastBuildDate && @@ -341,10 +359,7 @@ const processSubscription = async ( for (const item of feed.items) { // use published or updated if isoDate is not available for atom feeds item.isoDate = - item.isoDate || - (item.published as string) || - (item.updated as string) || - (item.created as string) + item.isoDate || item.published || item.updated || item.created console.log('Processing feed item', item.links, item.isoDate) if (!item.links || item.links.length === 0) { @@ -352,7 +367,7 @@ const processSubscription = async ( continue } - item.link = getLink(item.links as RssFeedItemLink[]) + item.link = getLink(item.links) if (!item.link) { console.log('Invalid feed item links', item.links) continue