From 08e4bed146d4226517bdf75072ea911a3da4a32e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Feb 2024 12:04:43 +0800 Subject: [PATCH 1/8] rename preview_content to feed_content in library_item table and gql schema --- packages/api/src/entity/library_item.ts | 2 +- packages/api/src/generated/graphql.ts | 6 ++++-- packages/api/src/generated/schema.graphql | 3 ++- packages/api/src/jobs/rss/refreshFeed.ts | 6 +++--- packages/api/src/routers/svc/following.ts | 8 ++++---- packages/api/src/schema.ts | 3 ++- .../0162.do.rename_preview_content_in_library_item.sql | 9 +++++++++ .../0162.undo.rename_preview_content_in_library_item.sql | 9 +++++++++ 8 files changed, 34 insertions(+), 12 deletions(-) create mode 100755 packages/db/migrations/0162.do.rename_preview_content_in_library_item.sql create mode 100755 packages/db/migrations/0162.undo.rename_preview_content_in_library_item.sql diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index f127f52b5..fb92fa618 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -191,7 +191,7 @@ export class LibraryItem { links?: any | null @Column('text') - previewContent?: string | null + feedContent?: string | null @Column('text') previewContentType?: string | null diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 9d908a941..e50946158 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -94,6 +94,7 @@ export type Article = { contentReader: ContentReader; createdAt: Scalars['Date']; description?: Maybe; + feedContent?: Maybe; folder: Scalars['String']; hasContent?: Maybe; hash: Scalars['String']; @@ -2396,6 +2397,7 @@ export type SearchItem = { contentReader: ContentReader; createdAt: Scalars['Date']; description?: Maybe; + feedContent?: Maybe; folder: Scalars['String']; highlights?: Maybe>; id: Scalars['ID']; @@ -2408,7 +2410,6 @@ export type SearchItem = { ownedByViewer?: Maybe; pageId?: Maybe; pageType: PageType; - previewContent?: Maybe; previewContentType?: Maybe; publishedAt?: Maybe; quote?: Maybe; @@ -4584,6 +4585,7 @@ export type ArticleResolvers; createdAt?: Resolver; description?: Resolver, ParentType, ContextType>; + feedContent?: Resolver, ParentType, ContextType>; folder?: Resolver; hasContent?: Resolver, ParentType, ContextType>; hash?: Resolver; @@ -5930,6 +5932,7 @@ export type SearchItemResolvers; createdAt?: Resolver; description?: Resolver, ParentType, ContextType>; + feedContent?: Resolver, ParentType, ContextType>; folder?: Resolver; highlights?: Resolver>, ParentType, ContextType>; id?: Resolver; @@ -5942,7 +5945,6 @@ export type SearchItemResolvers, ParentType, ContextType>; pageId?: Resolver, ParentType, ContextType>; pageType?: Resolver; - previewContent?: Resolver, ParentType, ContextType>; previewContentType?: Resolver, ParentType, ContextType>; publishedAt?: Resolver, ParentType, ContextType>; quote?: Resolver, ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 2df4f681e..633b66562 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -69,6 +69,7 @@ type Article { contentReader: ContentReader! createdAt: Date! description: String + feedContent: String folder: String! hasContent: Boolean hash: String! @@ -1824,6 +1825,7 @@ type SearchItem { contentReader: ContentReader! createdAt: Date! description: String + feedContent: String folder: String! highlights: [Highlight!] id: ID! @@ -1836,7 +1838,6 @@ type SearchItem { ownedByViewer: Boolean pageId: ID pageType: PageType! - previewContent: String previewContentType: String publishedAt: Date quote: String diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 121331038..8cf66ad28 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -273,7 +273,7 @@ const createTask = async ( } if (folder === 'following' && !fetchContent) { - return createItemWithPreviewContent(userId, feedUrl, item) + return createItemWithFeedContent(userId, feedUrl, item) } logger.info(`adding fetch content task ${userId} ${item.link.trim()}`) @@ -309,7 +309,7 @@ const fetchContentAndCreateItem = async ( } } -const createItemWithPreviewContent = async ( +const createItemWithFeedContent = async ( userId: string, feedUrl: string, item: RssFeedItem @@ -321,7 +321,7 @@ const createItemWithPreviewContent = async ( author: item.creator, description: item.summary, addedToFollowingFrom: 'feed', - previewContent: item.content || item.contentSnippet || item.summary, + feedContent: item.content || item.contentSnippet || item.summary, addedToFollowingBy: feedUrl, savedAt: item.isoDate, publishedAt: item.isoDate, diff --git a/packages/api/src/routers/svc/following.ts b/packages/api/src/routers/svc/following.ts index 18e912fda..26dadeee0 100644 --- a/packages/api/src/routers/svc/following.ts +++ b/packages/api/src/routers/svc/following.ts @@ -28,7 +28,7 @@ export interface SaveFollowingItemRequest { author?: string description?: string links?: any - previewContent?: string + feedContent?: string previewContentType?: string publishedAt?: Date savedAt?: Date @@ -76,7 +76,7 @@ export function followingServiceRouter() { const url = cleanUrl(req.body.url) const preparedDocument: PreparedDocumentInput = { - document: req.body.previewContent || '', + document: req.body.feedContent || '', pageInfo: { title: req.body.title, author: req.body.author, @@ -89,7 +89,7 @@ export function followingServiceRouter() { let parsedResult: ParsedContentPuppeteer | undefined // parse the content if we have a preview content - if (req.body.previewContent) { + if (req.body.feedContent) { parsedResult = await parsePreparedContent(url, preparedDocument) } @@ -112,7 +112,7 @@ export function followingServiceRouter() { userId, slug, croppedPathname, - originalHtml: req.body.previewContent, + originalHtml: req.body.feedContent, itemType: parsedResult?.pageType || PageType.Unknown, canonicalUrl: url, folder: FOLDER, diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index d18b91012..2d142dc20 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -397,6 +397,7 @@ const schema = gql` recommendations: [Recommendation!] wordsCount: Int folder: String! + feedContent: String } # Query: article @@ -1634,7 +1635,7 @@ const schema = gql` wordsCount: Int content: String archivedAt: Date - previewContent: String + feedContent: String previewContentType: String links: JSON folder: String! diff --git a/packages/db/migrations/0162.do.rename_preview_content_in_library_item.sql b/packages/db/migrations/0162.do.rename_preview_content_in_library_item.sql new file mode 100755 index 000000000..932baaed3 --- /dev/null +++ b/packages/db/migrations/0162.do.rename_preview_content_in_library_item.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: rename_preview_content_in_library_item +-- Description: Rename preview_content column in library_item table to feed_content + +BEGIN; + +ALTER TABLE omnivore.library_item RENAME COLUMN preview_content TO feed_content; + +COMMIT; diff --git a/packages/db/migrations/0162.undo.rename_preview_content_in_library_item.sql b/packages/db/migrations/0162.undo.rename_preview_content_in_library_item.sql new file mode 100755 index 000000000..77a78d2da --- /dev/null +++ b/packages/db/migrations/0162.undo.rename_preview_content_in_library_item.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: rename_preview_content_in_library_item +-- Description: Rename preview_content column in library_item table to feed_content + +BEGIN; + +ALTER TABLE omnivore.library_item RENAME COLUMN feed_content TO preview_content; + +COMMIT; From f4b6e470f2e42364a3b4714cd102f3dfee1427ce Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Feb 2024 14:19:22 +0800 Subject: [PATCH 2/8] replace fetch_content with fetch_content_type as a enum: ALWAYS, NEVER, WHEN_EMPTY --- packages/api/src/entity/subscription.ts | 12 ++ packages/api/src/generated/graphql.ts | 11 ++ packages/api/src/generated/schema.graphql | 9 ++ packages/api/src/jobs/rss/refreshAllFeeds.ts | 4 +- packages/api/src/jobs/rss/refreshFeed.ts | 109 +++++++++++------- .../api/src/resolvers/subscriptions/index.ts | 13 ++- packages/api/src/schema.ts | 9 ++ packages/api/src/services/save_page.ts | 16 ++- .../api/src/services/update_subscription.ts | 12 +- packages/api/src/utils/createTask.ts | 5 +- ...ename_preview_content_in_library_item.sql} | 4 + ...ename_preview_content_in_library_item.sql} | 4 + 12 files changed, 152 insertions(+), 56 deletions(-) rename packages/db/migrations/{0162.do.rename_preview_content_in_library_item.sql => 0165.do.rename_preview_content_in_library_item.sql} (54%) rename packages/db/migrations/{0162.undo.rename_preview_content_in_library_item.sql => 0165.undo.rename_preview_content_in_library_item.sql} (71%) diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index fd17d8d73..bb9802f7b 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -23,6 +23,12 @@ export enum SubscriptionType { Rss = 'RSS', } +export enum FetchContentType { + Always = 'ALWAYS', + Never = 'NEVER', + WhenEmpty = 'WHEN_EMPTY', +} + @Entity({ name: 'subscriptions' }) export class Subscription { @PrimaryGeneratedColumn('uuid') @@ -98,6 +104,12 @@ export class Subscription { @Column('boolean') fetchContent!: boolean + @Column('enum', { + enum: FetchContentType, + default: FetchContentType.Always, + }) + fetchContentType!: FetchContentType + @Column('text') folder?: string | null } diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index e50946158..de12eefd7 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -854,6 +854,12 @@ export type FetchContentSuccess = { success: Scalars['Boolean']; }; +export enum FetchContentType { + Always = 'ALWAYS', + Never = 'NEVER', + WhenEmpty = 'WHEN_EMPTY' +} + export type Filter = { __typename?: 'Filter'; category?: Maybe; @@ -2818,6 +2824,7 @@ export enum SubscribeErrorCode { export type SubscribeInput = { autoAddToLibrary?: InputMaybe; fetchContent?: InputMaybe; + fetchContentType?: InputMaybe; folder?: InputMaybe; isPrivate?: InputMaybe; subscriptionType?: InputMaybe; @@ -2839,6 +2846,7 @@ export type Subscription = { description?: Maybe; failedAt?: Maybe; fetchContent: Scalars['Boolean']; + fetchContentType: FetchContentType; folder: Scalars['String']; icon?: Maybe; id: Scalars['ID']; @@ -3214,6 +3222,7 @@ export type UpdateSubscriptionInput = { description?: InputMaybe; failedAt?: InputMaybe; fetchContent?: InputMaybe; + fetchContentType?: InputMaybe; folder?: InputMaybe; id: Scalars['ID']; isPrivate?: InputMaybe; @@ -3711,6 +3720,7 @@ export type ResolversTypes = { FetchContentErrorCode: FetchContentErrorCode; FetchContentResult: ResolversTypes['FetchContentError'] | ResolversTypes['FetchContentSuccess']; FetchContentSuccess: ResolverTypeWrapper; + FetchContentType: FetchContentType; Filter: ResolverTypeWrapper; FiltersError: ResolverTypeWrapper; FiltersErrorCode: FiltersErrorCode; @@ -6198,6 +6208,7 @@ export type SubscriptionResolvers, "description", ParentType, ContextType>; failedAt?: SubscriptionResolver, "failedAt", ParentType, ContextType>; fetchContent?: SubscriptionResolver; + fetchContentType?: SubscriptionResolver; folder?: SubscriptionResolver; icon?: SubscriptionResolver, "icon", ParentType, ContextType>; id?: SubscriptionResolver; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 633b66562..e7f5e748a 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -758,6 +758,12 @@ type FetchContentSuccess { success: Boolean! } +enum FetchContentType { + ALWAYS + NEVER + WHEN_EMPTY +} + type Filter { category: String createdAt: Date! @@ -2216,6 +2222,7 @@ enum SubscribeErrorCode { input SubscribeInput { autoAddToLibrary: Boolean fetchContent: Boolean + fetchContentType: FetchContentType folder: String isPrivate: Boolean subscriptionType: SubscriptionType @@ -2235,6 +2242,7 @@ type Subscription { description: String failedAt: Date fetchContent: Boolean! + fetchContentType: FetchContentType! folder: String! icon: String id: ID! @@ -2581,6 +2589,7 @@ input UpdateSubscriptionInput { description: String failedAt: Date fetchContent: Boolean + fetchContentType: FetchContentType folder: String id: ID! isPrivate: Boolean diff --git a/packages/api/src/jobs/rss/refreshAllFeeds.ts b/packages/api/src/jobs/rss/refreshAllFeeds.ts index 26f0882e4..b4b020c96 100644 --- a/packages/api/src/jobs/rss/refreshAllFeeds.ts +++ b/packages/api/src/jobs/rss/refreshAllFeeds.ts @@ -31,7 +31,7 @@ export const refreshAllFeeds = async (db: DataSource): Promise => { ARRAY_AGG(s.most_recent_item_date) AS "mostRecentItemDates", ARRAY_AGG(coalesce(s.scheduled_at, NOW())) AS "scheduledDates", ARRAY_AGG(s.last_fetched_checksum) AS checksums, - ARRAY_AGG(s.fetch_content) AS "fetchContents", + ARRAY_AGG(s.fetch_content_type) AS "fetchContentTypes", ARRAY_AGG(coalesce(s.folder, $3)) AS folders FROM omnivore.subscriptions s @@ -106,7 +106,7 @@ const updateSubscriptionGroup = async ( timestamp.getTime() ), // unix timestamp in milliseconds userIds: group.userIds, - fetchContents: group.fetchContents, + fetchContents: group.fetchContentTypes, folders: group.folders, } diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 8cf66ad28..fc7d2f203 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -2,14 +2,20 @@ import axios from 'axios' import crypto from 'crypto' import { parseHTML } from 'linkedom' import Parser, { Item } from 'rss-parser' +import { FetchContentType } from '../../entity/subscription' import { env } from '../../env' +import { ArticleSavingRequestStatus } from '../../generated/graphql' import { redisDataSource } from '../../redis_data_source' import { validateUrl } from '../../services/create_page_save_request' +import { savePage } from '../../services/save_page' import { updateSubscription, updateSubscriptions, } from '../../services/update_subscription' +import { findActiveUser } from '../../services/user' import createHttpTaskWithToken from '../../utils/createTask' +import { cleanUrl } from '../../utils/helpers' +import { createThumbnailUrl } from '../../utils/imageproxy' import { logger } from '../../utils/logger' import { RSSRefreshContext } from './refreshAllFeeds' @@ -22,7 +28,7 @@ interface RefreshFeedRequest { scheduledTimestamps: number[] // unix timestamp in milliseconds lastFetchedChecksums: string[] userIds: string[] - fetchContents: boolean[] + fetchContentTypes: FetchContentType[] folders: FolderType[] refreshContext?: RSSRefreshContext } @@ -35,7 +41,7 @@ export const isRefreshFeedRequest = (data: any): data is RefreshFeedRequest => { 'scheduledTimestamps' in data && 'userIds' in data && 'lastFetchedChecksums' in data && - 'fetchContents' in data && + 'fetchContentTypes' in data && 'folders' in data ) } @@ -263,7 +269,7 @@ const createTask = async ( userId: string, feedUrl: string, item: RssFeedItem, - fetchContent: boolean, + fetchContentType: FetchContentType, folder: FolderType ) => { const isRecentlySaved = await isItemRecentlySaved(userId, item.link) @@ -272,8 +278,12 @@ const createTask = async ( return true } - if (folder === 'following' && !fetchContent) { - return createItemWithFeedContent(userId, feedUrl, item) + const feedContent = item.content || item.contentSnippet || item.summary + if ( + fetchContentType === FetchContentType.Never || + (fetchContentType === FetchContentType.WhenEmpty && !feedContent) + ) { + return createItemWithFeedContent(userId, feedUrl, item, folder, feedContent) } logger.info(`adding fetch content task ${userId} ${item.link.trim()}`) @@ -312,41 +322,57 @@ const fetchContentAndCreateItem = async ( const createItemWithFeedContent = async ( userId: string, feedUrl: string, - item: RssFeedItem + item: RssFeedItem, + folder: FolderType, + feedContent?: string ) => { - const input = { - userIds: [userId], - url: item.link, - title: item.title, - author: item.creator, - description: item.summary, - addedToFollowingFrom: 'feed', - feedContent: item.content || item.contentSnippet || item.summary, - addedToFollowingBy: feedUrl, - savedAt: item.isoDate, - publishedAt: item.isoDate, - previewContentType: 'text/html', // TODO: get content type from feed - thumbnail: getThumbnail(item), - } - try { - const serviceBaseUrl = process.env.INTERNAL_API_URL - const token = process.env.PUBSUB_VERIFICATION_TOKEN - if (!serviceBaseUrl || !token) { - throw 'Environment not configured correctly' + logger.info('saving feed item with feed content', { + userId, + feedUrl, + item, + folder, + }) + + const thumbnail = getThumbnail(item) + const previewImage = thumbnail && createThumbnailUrl(thumbnail) + const url = cleanUrl(item.link) + + const user = await findActiveUser(userId) + if (!user) { + logger.error('User not found', { userId }) + return false } - // save page - const taskHandlerUrl = `${serviceBaseUrl}/svc/following/save?token=${token}` - const task = await createHttpTaskWithToken({ - queue: env.queue.name, - priority: 'low', - taskHandlerUrl: taskHandlerUrl, - payload: input, - }) - return !!task + const result = await savePage( + { + url, + feedContent, + title: item.title, + folder, + rssFeedUrl: feedUrl, + savedAt: item.isoDate, + publishedAt: item.isoDate, + originalContent: feedContent || '', + source: 'rss-feeder', + state: ArticleSavingRequestStatus.ContentNotFetched, + clientRequestId: '', + author: item.creator, + previewImage, + }, + user + ) + + if (result.__typename === 'SaveError') { + logger.error( + `Error while saving feed item with feed content: ${result.errorCodes[0]}` + ) + return false + } + + return true } catch (error) { - logger.error('Error while creating task', error) + logger.error('Error while saving feed item with feed content', error) return false } } @@ -456,7 +482,7 @@ const processSubscription = async ( mostRecentItemDate: number, scheduledAt: number, lastFetchedChecksum: string, - fetchContent: boolean, + fetchContentType: FetchContentType, folder: FolderType, feed: RssFeed ) => { @@ -547,7 +573,7 @@ const processSubscription = async ( userId, feedUrl, feedItem, - fetchContent, + fetchContentType, folder ) if (!created) { @@ -580,7 +606,7 @@ const processSubscription = async ( userId, feedUrl, lastValidItem, - fetchContent, + fetchContentType, folder ) if (!created) { @@ -626,7 +652,7 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { scheduledTimestamps, userIds, lastFetchedChecksums, - fetchContents, + fetchContentTypes, folders, refreshContext, } = request @@ -666,6 +692,9 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { // process each subscription sequentially for (let i = 0; i < subscriptionIds.length; i++) { const subscriptionId = subscriptionIds[i] + const fetchContentType = allowFetchContent + ? fetchContentTypes[i] + : FetchContentType.Never try { await processSubscription( @@ -677,7 +706,7 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { mostRecentItemDates[i], scheduledTimestamps[i], lastFetchedChecksums[i], - fetchContents[i] && allowFetchContent, + fetchContentType, folders[i], feed ) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index a13529325..bc10867fb 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -3,6 +3,7 @@ import { parseHTML } from 'linkedom' import { Brackets, In } from 'typeorm' import { DEFAULT_SUBSCRIPTION_FOLDER, + FetchContentType, Subscription, SubscriptionStatus, SubscriptionType, @@ -226,7 +227,9 @@ export const subscribeResolver = authorized< // re-subscribe const updatedSubscription = await getRepository(Subscription).save({ ...existingSubscription, - fetchContent: input.fetchContent ?? undefined, + fetchContentType: input.fetchContentType + ? (input.fetchContentType as FetchContentType) + : undefined, folder: input.folder ?? undefined, isPrivate: input.isPrivate, status: SubscriptionStatus.Active, @@ -240,7 +243,7 @@ export const subscribeResolver = authorized< scheduledDates: [new Date()], // fetch immediately mostRecentItemDates: [updatedSubscription.mostRecentItemDate || null], checksums: [updatedSubscription.lastFetchedChecksum || null], - fetchContents: [updatedSubscription.fetchContent], + fetchContentTypes: [updatedSubscription.fetchContentType], folders: [updatedSubscription.folder || DEFAULT_SUBSCRIPTION_FOLDER], }) @@ -254,7 +257,7 @@ export const subscribeResolver = authorized< // limit number of rss subscriptions to max const results = (await getRepository(Subscription).query( - `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, is_private, fetch_content, folder) + `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, is_private, fetch_content_type, folder) select $1, $2, $3, $4, $5, $6, $7, $8, $9 from omnivore.subscriptions where user_id = $5 and type = 'RSS' and status = 'ACTIVE' having count(*) < $10 @@ -267,7 +270,7 @@ export const subscribeResolver = authorized< uid, feed.thumbnail, input.isPrivate, - input.fetchContent ?? true, + input.fetchContentType ?? FetchContentType.Always, input.folder ?? 'following', MAX_RSS_SUBSCRIPTIONS, ] @@ -290,7 +293,7 @@ export const subscribeResolver = authorized< scheduledDates: [new Date()], // fetch immediately mostRecentItemDates: [null], checksums: [null], - fetchContents: [newSubscription.fetchContent], + fetchContentTypes: [newSubscription.fetchContentType], folders: [newSubscription.folder || DEFAULT_SUBSCRIPTION_FOLDER], }) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 2d142dc20..e3421f20a 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1671,6 +1671,12 @@ const schema = gql` NEWSLETTER } + enum FetchContentType { + ALWAYS + NEVER + WHEN_EMPTY + } + type Subscription { id: ID! name: String! @@ -1689,6 +1695,7 @@ const schema = gql` isPrivate: Boolean autoAddToLibrary: Boolean fetchContent: Boolean! + fetchContentType: FetchContentType! folder: String! mostRecentItemDate: Date refreshedAt: Date @@ -2597,6 +2604,7 @@ const schema = gql` isPrivate: Boolean autoAddToLibrary: Boolean fetchContent: Boolean + fetchContentType: FetchContentType folder: String } @@ -2610,6 +2618,7 @@ const schema = gql` isPrivate: Boolean autoAddToLibrary: Boolean fetchContent: Boolean + fetchContentType: FetchContentType folder: String refreshedAt: Date mostRecentItemDate: Date diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index f79b28b52..f8d68bc30 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -11,6 +11,7 @@ import { SavePageInput, SaveResult, } from '../generated/graphql' +import { Merge } from '../util' import { enqueueThumbnailJob } from '../utils/createTask' import { cleanUrl, @@ -61,10 +62,13 @@ const shouldParseInBackend = (input: SavePageInput): boolean => { ) } +export type SavePageArgs = Merge< + SavePageInput, + { feedContent?: string; previewImage?: string; author?: string } +> + export const savePage = async ( - input: SavePageInput & { - finalUrl?: string - }, + input: SavePageArgs, user: User ): Promise => { const [slug, croppedPathname] = createSlug(input.url, input.title) @@ -100,6 +104,8 @@ export const savePage = async ( pageInfo: { title: input.title, canonicalUrl: input.url, + previewImage: input.previewImage, + author: input.author, }, }) @@ -119,6 +125,7 @@ export const savePage = async ( state: input.state || undefined, rssFeedUrl: input.rssFeedUrl, folder: input.folder, + feedContent: input.feedContent, }) const isImported = input.source === 'csv-importer' || input.source === 'pocket' @@ -196,6 +203,7 @@ export const parsedContentToLibraryItem = ({ state, rssFeedUrl, folder, + feedContent, }: { url: string userId: string @@ -215,6 +223,7 @@ export const parsedContentToLibraryItem = ({ state?: ArticleSavingRequestStatus | null rssFeedUrl?: string | null folder?: string | null + feedContent?: string | null }): DeepPartial & { originalUrl: string } => { logger.info('save_page', { url, state, itemId }) return { @@ -257,5 +266,6 @@ export const parsedContentToLibraryItem = ({ archivedAt: state === ArticleSavingRequestStatus.Archived ? new Date() : null, deletedAt: state === ArticleSavingRequestStatus.Deleted ? new Date() : null, + feedContent, } } diff --git a/packages/api/src/services/update_subscription.ts b/packages/api/src/services/update_subscription.ts index bcc077291..a956163ef 100644 --- a/packages/api/src/services/update_subscription.ts +++ b/packages/api/src/services/update_subscription.ts @@ -1,4 +1,8 @@ -import { Subscription, SubscriptionStatus } from '../entity/subscription' +import { + FetchContentType, + Subscription, + SubscriptionStatus, +} from '../entity/subscription' import { getRepository } from '../repository' const ensureOwns = async (userId: string, subscriptionId: string) => { @@ -16,7 +20,7 @@ const ensureOwns = async (userId: string, subscriptionId: string) => { type UpdateSubscriptionData = { autoAddToLibrary?: boolean | null description?: string | null - fetchContent?: boolean | null + fetchContentType?: FetchContentType | null folder?: string | null isPrivate?: boolean | null mostRecentItemDate?: Date | null @@ -48,7 +52,7 @@ export const updateSubscription = async ( failedAt: newData.failedAt || undefined, autoAddToLibrary: newData.autoAddToLibrary ?? undefined, isPrivate: newData.isPrivate ?? undefined, - fetchContent: newData.fetchContent ?? undefined, + fetchContentType: newData.fetchContentType ?? undefined, folder: newData.folder ?? undefined, }) @@ -75,7 +79,7 @@ export const updateSubscriptions = async ( failedAt: newData.failedAt || undefined, autoAddToLibrary: newData.autoAddToLibrary ?? undefined, isPrivate: newData.isPrivate ?? undefined, - fetchContent: newData.fetchContent ?? undefined, + fetchContentType: newData.fetchContentType ?? undefined, folder: newData.folder ?? undefined, })) ) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 50ef58f45..95777a240 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -9,6 +9,7 @@ import { DeepPartial } from 'typeorm' import { v4 as uuid } from 'uuid' import { ImportItemState } from '../entity/integration' import { Recommendation } from '../entity/recommendation' +import { FetchContentType } from '../entity/subscription' import { env } from '../env' import { ArticleSavingRequestStatus, @@ -625,7 +626,7 @@ export interface RssSubscriptionGroup { mostRecentItemDates: (Date | null)[] scheduledDates: Date[] checksums: (string | null)[] - fetchContents: boolean[] + fetchContentTypes: FetchContentType[] folders: string[] } @@ -648,7 +649,7 @@ export const enqueueRssFeedFetch = async ( timestamp.getTime() ), // unix timestamp in milliseconds userIds: subscriptionGroup.userIds, - fetchContents: subscriptionGroup.fetchContents, + fetchContentTypes: subscriptionGroup.fetchContentTypes, folders: subscriptionGroup.folders, } diff --git a/packages/db/migrations/0162.do.rename_preview_content_in_library_item.sql b/packages/db/migrations/0165.do.rename_preview_content_in_library_item.sql similarity index 54% rename from packages/db/migrations/0162.do.rename_preview_content_in_library_item.sql rename to packages/db/migrations/0165.do.rename_preview_content_in_library_item.sql index 932baaed3..098e76e49 100755 --- a/packages/db/migrations/0162.do.rename_preview_content_in_library_item.sql +++ b/packages/db/migrations/0165.do.rename_preview_content_in_library_item.sql @@ -6,4 +6,8 @@ BEGIN; ALTER TABLE omnivore.library_item RENAME COLUMN preview_content TO feed_content; +CREATE TYPE fetch_content_enum AS ENUM ('ALWAYS', 'NEVER', 'WHEN_EMPTY'); + +ALTER TABLE omnivore.subscriptions ADD COLUMN fetch_content_type fetch_content_enum NOT NULL DEFAULT 'ALWAYS'::fetch_content_enum; + COMMIT; diff --git a/packages/db/migrations/0162.undo.rename_preview_content_in_library_item.sql b/packages/db/migrations/0165.undo.rename_preview_content_in_library_item.sql similarity index 71% rename from packages/db/migrations/0162.undo.rename_preview_content_in_library_item.sql rename to packages/db/migrations/0165.undo.rename_preview_content_in_library_item.sql index 77a78d2da..b2aa4a628 100755 --- a/packages/db/migrations/0162.undo.rename_preview_content_in_library_item.sql +++ b/packages/db/migrations/0165.undo.rename_preview_content_in_library_item.sql @@ -4,6 +4,10 @@ BEGIN; +ALTER TABLE omnivore.subscriptions DROP COLUMN fetch_content_type; + +DROP TYPE fetch_content_enum; + ALTER TABLE omnivore.library_item RENAME COLUMN feed_content TO preview_content; COMMIT; From df0bda9cb25eceaa837b7991584573bbf99f310b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Feb 2024 14:21:34 +0800 Subject: [PATCH 3/8] fix a typo --- packages/api/src/jobs/rss/refreshFeed.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index fc7d2f203..b9d8ba85e 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -281,7 +281,7 @@ const createTask = async ( const feedContent = item.content || item.contentSnippet || item.summary if ( fetchContentType === FetchContentType.Never || - (fetchContentType === FetchContentType.WhenEmpty && !feedContent) + (fetchContentType === FetchContentType.WhenEmpty && feedContent) ) { return createItemWithFeedContent(userId, feedUrl, item, folder, feedContent) } From 510cd18b177433120e80c2c71fc80e96730b4b86 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Feb 2024 14:54:37 +0800 Subject: [PATCH 4/8] use json_agg to aggregate enum in db --- packages/api/src/jobs/rss/refreshAllFeeds.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/jobs/rss/refreshAllFeeds.ts b/packages/api/src/jobs/rss/refreshAllFeeds.ts index b4b020c96..d68263400 100644 --- a/packages/api/src/jobs/rss/refreshAllFeeds.ts +++ b/packages/api/src/jobs/rss/refreshAllFeeds.ts @@ -31,7 +31,7 @@ export const refreshAllFeeds = async (db: DataSource): Promise => { ARRAY_AGG(s.most_recent_item_date) AS "mostRecentItemDates", ARRAY_AGG(coalesce(s.scheduled_at, NOW())) AS "scheduledDates", ARRAY_AGG(s.last_fetched_checksum) AS checksums, - ARRAY_AGG(s.fetch_content_type) AS "fetchContentTypes", + JSON_AGG(s.fetch_content_type) AS "fetchContentTypes", ARRAY_AGG(coalesce(s.folder, $3)) AS folders FROM omnivore.subscriptions s @@ -106,7 +106,7 @@ const updateSubscriptionGroup = async ( timestamp.getTime() ), // unix timestamp in milliseconds userIds: group.userIds, - fetchContents: group.fetchContentTypes, + fetchContentTypes: group.fetchContentTypes, folders: group.folders, } From 49d1ae503711ba9b25173fa0ce9e8ca81509d3bb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Feb 2024 15:32:41 +0800 Subject: [PATCH 5/8] fix bug --- packages/api/src/services/update_subscription.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/services/update_subscription.ts b/packages/api/src/services/update_subscription.ts index a956163ef..9d0970ba5 100644 --- a/packages/api/src/services/update_subscription.ts +++ b/packages/api/src/services/update_subscription.ts @@ -52,7 +52,7 @@ export const updateSubscription = async ( failedAt: newData.failedAt || undefined, autoAddToLibrary: newData.autoAddToLibrary ?? undefined, isPrivate: newData.isPrivate ?? undefined, - fetchContentType: newData.fetchContentType ?? undefined, + fetchContentType: newData.fetchContentType || undefined, folder: newData.folder ?? undefined, }) From 67f54a137292db27b5b9f783ea7a6f9d07438a0b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Feb 2024 15:44:53 +0800 Subject: [PATCH 6/8] add fetchContent resolver to the function list --- packages/api/src/resolvers/function_resolvers.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 4815da662..b2084b3b7 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -38,7 +38,7 @@ import { generateDownloadSignedUrl, generateUploadFilePathName, } from '../utils/uploads' -import { emptyTrashResolver } from './article' +import { emptyTrashResolver, fetchContentResolver } from './article' import { optInFeatureResolver } from './features' import { uploadImportFileResolver } from './importers/uploadImportFileResolver' import { @@ -296,6 +296,7 @@ export const functionResolvers = { moveToFolder: moveToFolderResolver, updateNewsletterEmail: updateNewsletterEmailResolver, emptyTrash: emptyTrashResolver, + fetchContent: fetchContentResolver, }, Query: { me: getMeUserResolver, @@ -623,4 +624,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('MoveToFolder'), ...resultResolveTypeResolver('UpdateNewsletterEmail'), ...resultResolveTypeResolver('EmptyTrash'), + ...resultResolveTypeResolver('FetchContent'), } From 6b0cefe190da3c9643b06c841880f22f02c9adad Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Feb 2024 17:14:52 +0800 Subject: [PATCH 7/8] add representation column to the highlight table and gql schema which is either CONTENT or FEED_CONTENT --- packages/api/src/entity/highlight.ts | 11 +++++++++++ packages/api/src/generated/graphql.ts | 10 ++++++++++ packages/api/src/generated/schema.graphql | 8 ++++++++ packages/api/src/resolvers/highlight/index.ts | 4 +++- packages/api/src/schema.ts | 8 ++++++++ ...0165.do.rename_preview_content_in_library_item.sql | 3 ++- ...65.undo.rename_preview_content_in_library_item.sql | 4 +++- 7 files changed, 45 insertions(+), 3 deletions(-) diff --git a/packages/api/src/entity/highlight.ts b/packages/api/src/entity/highlight.ts index f34ba69e8..602c9f959 100644 --- a/packages/api/src/entity/highlight.ts +++ b/packages/api/src/entity/highlight.ts @@ -19,6 +19,11 @@ export enum HighlightType { Note = 'NOTE', // to be deleted in favor of note on library item } +export enum RepresentationType { + Content = 'CONTENT', + FeedContent = 'FEED_CONTENT', +} + @Entity({ name: 'highlight' }) export class Highlight { @PrimaryGeneratedColumn('uuid') @@ -87,4 +92,10 @@ export class Highlight { inverseJoinColumn: { name: 'label_id' }, }) labels?: Label[] + + @Column('enum', { + enum: RepresentationType, + default: RepresentationType.Content, + }) + representation!: RepresentationType } diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index de12eefd7..028155767 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -368,6 +368,7 @@ export type CreateHighlightInput = { patch?: InputMaybe; prefix?: InputMaybe; quote?: InputMaybe; + representation?: InputMaybe; sharedAt?: InputMaybe; shortId: Scalars['String']; suffix?: InputMaybe; @@ -1024,6 +1025,7 @@ export type Highlight = { quote?: Maybe; reactions: Array; replies: Array; + representation: RepresentationType; sharedAt?: Maybe; shortId: Scalars['String']; suffix?: Maybe; @@ -1281,6 +1283,7 @@ export type MergeHighlightInput = { patch: Scalars['String']; prefix?: InputMaybe; quote: Scalars['String']; + representation?: InputMaybe; shortId: Scalars['ID']; suffix?: InputMaybe; }; @@ -2178,6 +2181,11 @@ export enum ReportType { Spam = 'SPAM' } +export enum RepresentationType { + Content = 'CONTENT', + FeedContent = 'FEED_CONTENT' +} + export type RevokeApiKeyError = { __typename?: 'RevokeApiKeyError'; errorCodes: Array; @@ -3870,6 +3878,7 @@ export type ResolversTypes = { ReportItemInput: ReportItemInput; ReportItemResult: ResolverTypeWrapper; ReportType: ReportType; + RepresentationType: RepresentationType; RevokeApiKeyError: ResolverTypeWrapper; RevokeApiKeyErrorCode: RevokeApiKeyErrorCode; RevokeApiKeyResult: ResolversTypes['RevokeApiKeyError'] | ResolversTypes['RevokeApiKeySuccess']; @@ -5256,6 +5265,7 @@ export type HighlightResolvers, ParentType, ContextType>; reactions?: Resolver, ParentType, ContextType>; replies?: Resolver, ParentType, ContextType>; + representation?: Resolver; sharedAt?: Resolver, ParentType, ContextType>; shortId?: Resolver; suffix?: Resolver, ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index e7f5e748a..776a19d59 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -319,6 +319,7 @@ input CreateHighlightInput { patch: String prefix: String quote: String + representation: RepresentationType sharedAt: Date shortId: String! suffix: String @@ -912,6 +913,7 @@ type Highlight { quote: String reactions: [Reaction!]! replies: [HighlightReply!]! + representation: RepresentationType! sharedAt: Date shortId: String! suffix: String @@ -1148,6 +1150,7 @@ input MergeHighlightInput { patch: String! prefix: String quote: String! + representation: RepresentationType shortId: ID! suffix: String } @@ -1622,6 +1625,11 @@ enum ReportType { SPAM } +enum RepresentationType { + CONTENT + FEED_CONTENT +} + type RevokeApiKeyError { errorCodes: [RevokeApiKeyErrorCode!]! } diff --git a/packages/api/src/resolvers/highlight/index.ts b/packages/api/src/resolvers/highlight/index.ts index 9853ef7e4..e68ac948b 100644 --- a/packages/api/src/resolvers/highlight/index.ts +++ b/packages/api/src/resolvers/highlight/index.ts @@ -5,6 +5,7 @@ import { DeepPartial } from 'typeorm' import { Highlight as HighlightData, HighlightType, + RepresentationType, } from '../../entity/highlight' import { Label } from '../../entity/label' import { env } from '../../env' @@ -34,8 +35,8 @@ import { updateHighlight, } from '../../services/highlights' import { analytics } from '../../utils/analytics' -import { highlightDataToHighlight } from '../../utils/helpers' import { authorized } from '../../utils/gql-utils' +import { highlightDataToHighlight } from '../../utils/helpers' export const createHighlightResolver = authorized< CreateHighlightSuccess, @@ -51,6 +52,7 @@ export const createHighlightResolver = authorized< highlightType: input.type || HighlightType.Highlight, highlightPositionAnchorIndex: input.highlightPositionAnchorIndex || 0, highlightPositionPercent: input.highlightPositionPercent || 0, + representation: input.representation || RepresentationType.Content, }, input.articleId, uid, diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e3421f20a..5566c9a02 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -704,6 +704,11 @@ const schema = gql` NOTE } + enum RepresentationType { + CONTENT + FEED_CONTENT + } + # Highlight type Highlight { id: ID! @@ -729,6 +734,7 @@ const schema = gql` type: HighlightType! html: String color: String + representation: RepresentationType! } input CreateHighlightInput { @@ -746,6 +752,7 @@ const schema = gql` type: HighlightType html: String color: String + representation: RepresentationType } type CreateHighlightSuccess { @@ -780,6 +787,7 @@ const schema = gql` highlightPositionAnchorIndex: Int html: String color: String + representation: RepresentationType } type MergeHighlightSuccess { diff --git a/packages/db/migrations/0165.do.rename_preview_content_in_library_item.sql b/packages/db/migrations/0165.do.rename_preview_content_in_library_item.sql index 098e76e49..69fe27eb1 100755 --- a/packages/db/migrations/0165.do.rename_preview_content_in_library_item.sql +++ b/packages/db/migrations/0165.do.rename_preview_content_in_library_item.sql @@ -7,7 +7,8 @@ BEGIN; ALTER TABLE omnivore.library_item RENAME COLUMN preview_content TO feed_content; CREATE TYPE fetch_content_enum AS ENUM ('ALWAYS', 'NEVER', 'WHEN_EMPTY'); - ALTER TABLE omnivore.subscriptions ADD COLUMN fetch_content_type fetch_content_enum NOT NULL DEFAULT 'ALWAYS'::fetch_content_enum; +CREATE TYPE representation_type AS ENUM ('CONTENT', 'FEED_CONTENT'); +ALTER TABLE omnivore.highlight ADD COLUMN representation representation_type NOT NULL DEFAULT 'CONTENT'::representation_type; COMMIT; diff --git a/packages/db/migrations/0165.undo.rename_preview_content_in_library_item.sql b/packages/db/migrations/0165.undo.rename_preview_content_in_library_item.sql index b2aa4a628..29da95a98 100755 --- a/packages/db/migrations/0165.undo.rename_preview_content_in_library_item.sql +++ b/packages/db/migrations/0165.undo.rename_preview_content_in_library_item.sql @@ -4,8 +4,10 @@ BEGIN; -ALTER TABLE omnivore.subscriptions DROP COLUMN fetch_content_type; +ALTER TABLE omnivore.highlight DROP COLUMN representation; +DROP TYPE representation_type; +ALTER TABLE omnivore.subscriptions DROP COLUMN fetch_content_type; DROP TYPE fetch_content_enum; ALTER TABLE omnivore.library_item RENAME COLUMN feed_content TO preview_content; From 7db70998228d118c74d5808270d194b4a27f185a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 23 Feb 2024 19:43:43 +0800 Subject: [PATCH 8/8] fix tests --- packages/api/src/resolvers/highlight/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/resolvers/highlight/index.ts b/packages/api/src/resolvers/highlight/index.ts index e68ac948b..c384298d4 100644 --- a/packages/api/src/resolvers/highlight/index.ts +++ b/packages/api/src/resolvers/highlight/index.ts @@ -131,6 +131,7 @@ export const mergeHighlightResolver = authorized< libraryItem: { id: input.articleId }, highlightPositionAnchorIndex: input.highlightPositionAnchorIndex || 0, highlightPositionPercent: input.highlightPositionPercent || 0, + representation: input.representation || RepresentationType.Content, } const newHighlight = await mergeHighlights(