From 83054cfebae2570f6894a6bddfc3c1610c33adc0 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 27 Oct 2023 17:26:41 +0800 Subject: [PATCH 01/30] create feed, feed_item and user_feed_item tables in db --- packages/db/migrations/0144.do.feed.sql | 72 +++++++++++++++++++++++ packages/db/migrations/0144.undo.feed.sql | 13 ++++ 2 files changed, 85 insertions(+) create mode 100755 packages/db/migrations/0144.do.feed.sql create mode 100755 packages/db/migrations/0144.undo.feed.sql diff --git a/packages/db/migrations/0144.do.feed.sql b/packages/db/migrations/0144.do.feed.sql new file mode 100755 index 000000000..a97e176a6 --- /dev/null +++ b/packages/db/migrations/0144.do.feed.sql @@ -0,0 +1,72 @@ +-- Type: DO +-- Name: feed +-- Description: Create feed, feed_item and user_feed_item tables + +BEGIN; + +CREATE TABLE omnivore.feed ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + title text NOT NULL, + url text NOT NULL, + author text, + description text, + image text, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, + published_at timestamptz, + UNIQUE(url) +); + +CREATE INDEX IF NOT EXISTS feed_title_idx ON omnivore.feed(title); + +CREATE TRIGGER update_feed_modtime BEFORE UPDATE ON omnivore.feed FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +GRANT SELECT, INSERT, UPDATE ON omnivore.feed TO omnivore_user; + +CREATE TABLE omnivore.feed_item ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + guid text NOT NULL, + title text NOT NULL, + links text[] NOT NULL, + author text, + summary text, + categories text[], + content text, + preview_content text, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, + published_at timestamptz, + feed_id uuid NOT NULL REFERENCES omnivore.feed(id) ON DELETE CASCADE, + UNIQUE(guid) +); + +CREATE INDEX IF NOT EXISTS feed_item_feed_id_idx ON omnivore.feed_item(feed_id); + +CREATE TRIGGER update_feed_item_modtime BEFORE UPDATE ON omnivore.feed_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +GRANT SELECT, INSERT, UPDATE ON omnivore.feed_item TO omnivore_user; + +CREATE TABLE omnivore.user_feed_item ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + user_id uuid NOT NULL REFERENCES omnivore.user(id) ON DELETE CASCADE, + feed_item_id uuid NOT NULL REFERENCES omnivore.feed_item(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, + hidden_at timestamptz, + saved_at timestamptz +); + +CREATE INDEX IF NOT EXISTS user_feed_item_user_id_idx ON omnivore.user_feed_item(user_id); +CREATE INDEX IF NOT EXISTS user_feed_item_feed_item_id_idx ON omnivore.user_feed_item(feed_item_id); + +CREATE TRIGGER update_user_feed_item_modtime BEFORE UPDATE ON omnivore.user_feed_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +ALTER TABLE omnivore.user_feed_item ENABLE ROW LEVEL SECURITY; + +CREATE POLICY user_feed_item_policy ON omnivore.user_feed_item + USING (user_id = omnivore.get_current_user_id()) + WITH CHECK (user_id = omnivore.get_current_user_id()); + +GRANT SELECT, INSERT, UPDATE ON omnivore.user_feed_item TO omnivore_user; + +COMMIT; diff --git a/packages/db/migrations/0144.undo.feed.sql b/packages/db/migrations/0144.undo.feed.sql new file mode 100755 index 000000000..92770ee98 --- /dev/null +++ b/packages/db/migrations/0144.undo.feed.sql @@ -0,0 +1,13 @@ +-- Type: UNDO +-- Name: feed +-- Description: Create feed, feed_item and user_feed_item tables + +BEGIN; + +DROP TABLE IF EXISTS omnivore.user_feed_item; + +DROP TABLE IF EXISTS omnivore.feed_item; + +DROP TABLE IF EXISTS omnivore.feed; + +COMMIT; From e282eea99f070c7f5ef951438e4ebe0bcc1902bb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 27 Oct 2023 17:54:40 +0800 Subject: [PATCH 02/30] add feeds graphql api to the schema --- packages/api/src/generated/graphql.ts | 97 +++++++++++++++++++++++ packages/api/src/generated/schema.graphql | 34 ++++++++ packages/api/src/schema.ts | 39 +++++++++ 3 files changed, 170 insertions(+) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 773e67137..9b0e4f777 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -725,6 +725,19 @@ export type Feature = { updatedAt?: Maybe; }; +export type Feed = { + __typename?: 'Feed'; + author?: Maybe; + createdAt: Scalars['Date']; + description?: Maybe; + id: Scalars['ID']; + image?: Maybe; + publishedAt?: Maybe; + title: Scalars['String']; + updatedAt: Scalars['Date']; + url: Scalars['String']; +}; + export type FeedArticle = { __typename?: 'FeedArticle'; annotationsCount?: Maybe; @@ -762,6 +775,30 @@ export type FeedArticlesSuccess = { pageInfo: PageInfo; }; +export type FeedEdge = { + __typename?: 'FeedEdge'; + cursor: Scalars['String']; + node: Feed; +}; + +export type FeedsError = { + __typename?: 'FeedsError'; + errorCodes: Array; +}; + +export enum FeedsErrorCode { + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED' +} + +export type FeedsResult = FeedsError | FeedsSuccess; + +export type FeedsSuccess = { + __typename?: 'FeedsSuccess'; + edges: Array; + pageInfo: PageInfo; +}; + export type Filter = { __typename?: 'Filter'; category: Scalars['String']; @@ -1725,6 +1762,7 @@ export type Query = { article: ArticleResult; articleSavingRequest: ArticleSavingRequestResult; deviceTokens: DeviceTokensResult; + feeds: FeedsResult; filters: FiltersResult; getUserPersonalization: GetUserPersonalizationResult; groups: GroupsResult; @@ -1762,6 +1800,14 @@ export type QueryArticleSavingRequestArgs = { }; +export type QueryFeedsArgs = { + after?: InputMaybe; + first?: InputMaybe; + query: Scalars['String']; + sort?: InputMaybe; +}; + + export type QueryRulesArgs = { enabled?: InputMaybe; }; @@ -3459,12 +3505,18 @@ export type ResolversTypes = { DeviceTokensResult: ResolversTypes['DeviceTokensError'] | ResolversTypes['DeviceTokensSuccess']; DeviceTokensSuccess: ResolverTypeWrapper; Feature: ResolverTypeWrapper; + Feed: ResolverTypeWrapper; FeedArticle: ResolverTypeWrapper; FeedArticleEdge: ResolverTypeWrapper; FeedArticlesError: ResolverTypeWrapper; FeedArticlesErrorCode: FeedArticlesErrorCode; FeedArticlesResult: ResolversTypes['FeedArticlesError'] | ResolversTypes['FeedArticlesSuccess']; FeedArticlesSuccess: ResolverTypeWrapper; + FeedEdge: ResolverTypeWrapper; + FeedsError: ResolverTypeWrapper; + FeedsErrorCode: FeedsErrorCode; + FeedsResult: ResolversTypes['FeedsError'] | ResolversTypes['FeedsSuccess']; + FeedsSuccess: ResolverTypeWrapper; Filter: ResolverTypeWrapper; FiltersError: ResolverTypeWrapper; FiltersErrorCode: FiltersErrorCode; @@ -3939,11 +3991,16 @@ export type ResolversParentTypes = { DeviceTokensResult: ResolversParentTypes['DeviceTokensError'] | ResolversParentTypes['DeviceTokensSuccess']; DeviceTokensSuccess: DeviceTokensSuccess; Feature: Feature; + Feed: Feed; FeedArticle: FeedArticle; FeedArticleEdge: FeedArticleEdge; FeedArticlesError: FeedArticlesError; FeedArticlesResult: ResolversParentTypes['FeedArticlesError'] | ResolversParentTypes['FeedArticlesSuccess']; FeedArticlesSuccess: FeedArticlesSuccess; + FeedEdge: FeedEdge; + FeedsError: FeedsError; + FeedsResult: ResolversParentTypes['FeedsError'] | ResolversParentTypes['FeedsSuccess']; + FeedsSuccess: FeedsSuccess; Filter: Filter; FiltersError: FiltersError; FiltersResult: ResolversParentTypes['FiltersError'] | ResolversParentTypes['FiltersSuccess']; @@ -4734,6 +4791,19 @@ export type FeatureResolvers; }; +export type FeedResolvers = { + author?: Resolver, ParentType, ContextType>; + createdAt?: Resolver; + description?: Resolver, ParentType, ContextType>; + id?: Resolver; + image?: Resolver, ParentType, ContextType>; + publishedAt?: Resolver, ParentType, ContextType>; + title?: Resolver; + updatedAt?: Resolver; + url?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type FeedArticleResolvers = { annotationsCount?: Resolver, ParentType, ContextType>; article?: Resolver; @@ -4769,6 +4839,27 @@ export type FeedArticlesSuccessResolvers; }; +export type FeedEdgeResolvers = { + cursor?: Resolver; + node?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type FeedsErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type FeedsResultResolvers = { + __resolveType: TypeResolveFn<'FeedsError' | 'FeedsSuccess', ParentType, ContextType>; +}; + +export type FeedsSuccessResolvers = { + edges?: Resolver, ParentType, ContextType>; + pageInfo?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type FilterResolvers = { category?: Resolver; createdAt?: Resolver; @@ -5266,6 +5357,7 @@ export type QueryResolvers>; articleSavingRequest?: Resolver>; deviceTokens?: Resolver; + feeds?: Resolver>; filters?: Resolver; getUserPersonalization?: Resolver; groups?: Resolver; @@ -6287,11 +6379,16 @@ export type Resolvers = { DeviceTokensResult?: DeviceTokensResultResolvers; DeviceTokensSuccess?: DeviceTokensSuccessResolvers; Feature?: FeatureResolvers; + Feed?: FeedResolvers; FeedArticle?: FeedArticleResolvers; FeedArticleEdge?: FeedArticleEdgeResolvers; FeedArticlesError?: FeedArticlesErrorResolvers; FeedArticlesResult?: FeedArticlesResultResolvers; FeedArticlesSuccess?: FeedArticlesSuccessResolvers; + FeedEdge?: FeedEdgeResolvers; + FeedsError?: FeedsErrorResolvers; + FeedsResult?: FeedsResultResolvers; + FeedsSuccess?: FeedsSuccessResolvers; Filter?: FilterResolvers; FiltersError?: FiltersErrorResolvers; FiltersResult?: FiltersResultResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 09150be0c..8455bd80c 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -641,6 +641,18 @@ type Feature { updatedAt: Date } +type Feed { + author: String + createdAt: Date! + description: String + id: ID! + image: String + publishedAt: Date + title: String! + updatedAt: Date! + url: String! +} + type FeedArticle { annotationsCount: Int article: Article! @@ -674,6 +686,27 @@ type FeedArticlesSuccess { pageInfo: PageInfo! } +type FeedEdge { + cursor: String! + node: Feed! +} + +type FeedsError { + errorCodes: [FeedsErrorCode!]! +} + +enum FeedsErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union FeedsResult = FeedsError | FeedsSuccess + +type FeedsSuccess { + edges: [FeedEdge!]! + pageInfo: PageInfo! +} + type Filter { category: String! createdAt: Date! @@ -1294,6 +1327,7 @@ type Query { article(format: String, slug: String!, username: String!): ArticleResult! articleSavingRequest(id: ID, url: String): ArticleSavingRequestResult! deviceTokens: DeviceTokensResult! + feeds(after: String, first: Int, query: String!, sort: SortParams): FeedsResult! filters: FiltersResult! getUserPersonalization: GetUserPersonalizationResult! groups: GroupsResult! diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 4e0692bcc..24ef1ae9a 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2605,6 +2605,39 @@ const schema = gql` email: String! } + union FeedsResult = FeedsSuccess | FeedsError + + type FeedsSuccess { + edges: [FeedEdge!]! + pageInfo: PageInfo! + } + + type FeedEdge { + cursor: String! + node: Feed! + } + + type FeedsError { + errorCodes: [FeedsErrorCode!]! + } + + enum FeedsErrorCode { + UNAUTHORIZED + BAD_REQUEST + } + + type Feed { + id: ID! + title: String! + url: String! + description: String + image: String + createdAt: Date! + updatedAt: Date! + publishedAt: Date + author: String + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2766,6 +2799,12 @@ const schema = gql` filters: FiltersResult! groups: GroupsResult! recentEmails: RecentEmailsResult! + feeds( + after: String + first: Int + query: String! + sort: SortParams + ): FeedsResult! } ` From 878d881e0586bcb4ad41e639487e8da81d99d035 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 27 Oct 2023 18:04:08 +0800 Subject: [PATCH 03/30] add feed entity --- packages/api/src/entity/feed.ts | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 packages/api/src/entity/feed.ts diff --git a/packages/api/src/entity/feed.ts b/packages/api/src/entity/feed.ts new file mode 100644 index 000000000..ab6412602 --- /dev/null +++ b/packages/api/src/entity/feed.ts @@ -0,0 +1,37 @@ +import { + Column, + CreateDateColumn, + Entity, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' + +@Entity() +export class Feed { + @PrimaryGeneratedColumn('uuid') + id!: string + + @Column('text') + title!: string + + @Column('text') + url!: string + + @Column('text') + author?: string + + @Column('text') + description?: string + + @Column('text') + image?: string + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + createdAt!: Date + + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + updatedAt!: Date + + @Column('timestamptz') + publishedAt?: Date | null +} From fc783f01ecd81f417ff77770c7b5e7079aac9abb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 31 Oct 2023 14:26:04 +0800 Subject: [PATCH 04/30] add feeds graphql api for searching feed by title or url --- packages/api/src/generated/graphql.ts | 16 +++-- packages/api/src/generated/schema.graphql | 9 ++- packages/api/src/repository/feed.ts | 29 +++++++++ packages/api/src/resolvers/feeds/index.ts | 61 +++++++++++++++++++ packages/api/src/schema.ts | 14 +++-- packages/api/src/services/feeds.ts | 0 .../{0144.do.feed.sql => 0145.do.feed.sql} | 0 ...{0144.undo.feed.sql => 0145.undo.feed.sql} | 0 8 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 packages/api/src/repository/feed.ts create mode 100644 packages/api/src/resolvers/feeds/index.ts create mode 100644 packages/api/src/services/feeds.ts rename packages/db/migrations/{0144.do.feed.sql => 0145.do.feed.sql} (100%) rename packages/db/migrations/{0144.undo.feed.sql => 0145.undo.feed.sql} (100%) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 9b0e4f777..3d2524102 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -791,6 +791,13 @@ export enum FeedsErrorCode { Unauthorized = 'UNAUTHORIZED' } +export type FeedsInput = { + after?: InputMaybe; + first?: InputMaybe; + query: Scalars['String']; + sort?: InputMaybe; +}; + export type FeedsResult = FeedsError | FeedsSuccess; export type FeedsSuccess = { @@ -1801,10 +1808,7 @@ export type QueryArticleSavingRequestArgs = { export type QueryFeedsArgs = { - after?: InputMaybe; - first?: InputMaybe; - query: Scalars['String']; - sort?: InputMaybe; + input: FeedsInput; }; @@ -3515,6 +3519,7 @@ export type ResolversTypes = { FeedEdge: ResolverTypeWrapper; FeedsError: ResolverTypeWrapper; FeedsErrorCode: FeedsErrorCode; + FeedsInput: FeedsInput; FeedsResult: ResolversTypes['FeedsError'] | ResolversTypes['FeedsSuccess']; FeedsSuccess: ResolverTypeWrapper; Filter: ResolverTypeWrapper; @@ -3999,6 +4004,7 @@ export type ResolversParentTypes = { FeedArticlesSuccess: FeedArticlesSuccess; FeedEdge: FeedEdge; FeedsError: FeedsError; + FeedsInput: FeedsInput; FeedsResult: ResolversParentTypes['FeedsError'] | ResolversParentTypes['FeedsSuccess']; FeedsSuccess: FeedsSuccess; Filter: Filter; @@ -5357,7 +5363,7 @@ export type QueryResolvers>; articleSavingRequest?: Resolver>; deviceTokens?: Resolver; - feeds?: Resolver>; + feeds?: Resolver>; filters?: Resolver; getUserPersonalization?: Resolver; groups?: Resolver; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 8455bd80c..719c59e02 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -700,6 +700,13 @@ enum FeedsErrorCode { UNAUTHORIZED } +input FeedsInput { + after: String + first: Int + query: String! + sort: SortParams +} + union FeedsResult = FeedsError | FeedsSuccess type FeedsSuccess { @@ -1327,7 +1334,7 @@ type Query { article(format: String, slug: String!, username: String!): ArticleResult! articleSavingRequest(id: ID, url: String): ArticleSavingRequestResult! deviceTokens: DeviceTokensResult! - feeds(after: String, first: Int, query: String!, sort: SortParams): FeedsResult! + feeds(input: FeedsInput!): FeedsResult! filters: FiltersResult! getUserPersonalization: GetUserPersonalizationResult! groups: GroupsResult! diff --git a/packages/api/src/repository/feed.ts b/packages/api/src/repository/feed.ts new file mode 100644 index 000000000..99d242de8 --- /dev/null +++ b/packages/api/src/repository/feed.ts @@ -0,0 +1,29 @@ +import { ILike } from 'typeorm' +import { appDataSource } from '../data_source' +import { Feed } from '../entity/feed' + +export const feedRepository = appDataSource.getRepository(Feed).extend({ + async searchFeeds( + query: string, + take = 10, + skip = 0, + orderBy = 'createdAt', + order = 'DESC' + ) { + query = `%${query}%` + const where = [{ title: ILike(query) }, { url: ILike(query) }] + const feeds = await this.find({ + where, + order: { [orderBy]: order }, + take, + skip, + }) + + const count = await this.countBy(where) + + return { + feeds, + count, + } + }, +}) diff --git a/packages/api/src/resolvers/feeds/index.ts b/packages/api/src/resolvers/feeds/index.ts new file mode 100644 index 000000000..f3ef53b51 --- /dev/null +++ b/packages/api/src/resolvers/feeds/index.ts @@ -0,0 +1,61 @@ +import { + FeedEdge, + FeedsError, + FeedsErrorCode, + FeedsSuccess, + QueryFeedsArgs, +} from '../../generated/graphql' +import { feedRepository } from '../../repository/feed' +import { authorized } from '../../utils/helpers' + +export const feedsResolve = authorized< + FeedsSuccess, + FeedsError, + QueryFeedsArgs +>(async (_, { input }, { log }) => { + try { + const startCursor = input.after || '' + const start = + startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 + const first = input.first || 10 + + const { feeds, count } = await feedRepository.searchFeeds( + input.query, + first + 1, // fetch one extra to check if there is a next page + start, + input.sort?.by, + input.sort?.order || undefined + ) + + const hasNextPage = feeds.length > first + const endCursor = String(start + feeds.length - (hasNextPage ? 1 : 0)) + + if (hasNextPage) { + // remove an extra if exists + feeds.pop() + } + + const edges: FeedEdge[] = feeds.map((feed) => ({ + node: feed, + cursor: endCursor, + })) + + return { + __typename: 'FeedsSuccess', + edges, + pageInfo: { + hasPreviousPage: start > 0, + hasNextPage, + startCursor, + endCursor, + totalCount: count, + }, + } + } catch (error) { + log.error('Error fetching feeds', error) + + return { + errorCodes: [FeedsErrorCode.BadRequest], + } + } +}) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 24ef1ae9a..bd1b2df8f 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2605,6 +2605,13 @@ const schema = gql` email: String! } + input FeedsInput { + after: String + first: Int @sanitize(minLength: 1, maxLength: 100) + query: String! @sanitize(maxLength: 255) + sort: SortParams + } + union FeedsResult = FeedsSuccess | FeedsError type FeedsSuccess { @@ -2799,12 +2806,7 @@ const schema = gql` filters: FiltersResult! groups: GroupsResult! recentEmails: RecentEmailsResult! - feeds( - after: String - first: Int - query: String! - sort: SortParams - ): FeedsResult! + feeds(input: FeedsInput!): FeedsResult! } ` diff --git a/packages/api/src/services/feeds.ts b/packages/api/src/services/feeds.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/db/migrations/0144.do.feed.sql b/packages/db/migrations/0145.do.feed.sql similarity index 100% rename from packages/db/migrations/0144.do.feed.sql rename to packages/db/migrations/0145.do.feed.sql diff --git a/packages/db/migrations/0144.undo.feed.sql b/packages/db/migrations/0145.undo.feed.sql similarity index 100% rename from packages/db/migrations/0144.undo.feed.sql rename to packages/db/migrations/0145.undo.feed.sql From da0d1705b50d234f5ea52651c9b4e5dd04259a59 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 31 Oct 2023 20:34:50 +0800 Subject: [PATCH 05/30] add following graphql api to fetch user following --- packages/api/src/entity/feed_item.ts | 53 ++++++++ packages/api/src/entity/user_feed_item.ts | 37 ++++++ packages/api/src/generated/graphql.ts | 113 +++++++++++++++++- packages/api/src/generated/schema.graphql | 43 ++++++- packages/api/src/repository/feed.ts | 14 ++- packages/api/src/repository/user_feed_item.ts | 36 ++++++ packages/api/src/resolvers/feeds/index.ts | 73 ++++++++++- packages/api/src/routers/feed_router.ts | 7 ++ packages/api/src/schema.ts | 50 +++++++- packages/api/src/services/feeds.ts | 9 ++ 10 files changed, 424 insertions(+), 11 deletions(-) create mode 100644 packages/api/src/entity/feed_item.ts create mode 100644 packages/api/src/entity/user_feed_item.ts create mode 100644 packages/api/src/repository/user_feed_item.ts create mode 100644 packages/api/src/routers/feed_router.ts diff --git a/packages/api/src/entity/feed_item.ts b/packages/api/src/entity/feed_item.ts new file mode 100644 index 000000000..541e6e194 --- /dev/null +++ b/packages/api/src/entity/feed_item.ts @@ -0,0 +1,53 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' +import { Feed } from './feed' + +@Entity() +export class FeedItem { + @PrimaryGeneratedColumn('uuid') + id!: string + + @Column('text') + guid!: string + + @Column('text') + title!: string + + @Column('array') + links!: string[] + + @Column('text') + author?: string | null + + @Column('text') + summary?: string | null + + @Column('array') + categories?: string[] | null + + @Column('text') + content?: string | null + + @Column('text') + previewContent?: string | null + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + createdAt!: Date + + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + updatedAt!: Date + + @Column('timestamptz') + publishedAt?: Date | null + + @ManyToOne(() => Feed, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'feed_id' }) + feed!: Feed +} diff --git a/packages/api/src/entity/user_feed_item.ts b/packages/api/src/entity/user_feed_item.ts new file mode 100644 index 000000000..46ea8f8bc --- /dev/null +++ b/packages/api/src/entity/user_feed_item.ts @@ -0,0 +1,37 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' +import { FeedItem } from './feed_item' +import { User } from './user' + +@Entity() +export class UserFeedItem { + @PrimaryGeneratedColumn('uuid') + id!: string + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + createdAt!: Date + + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + updatedAt!: Date + + @Column('timestamptz') + hiddenAt?: Date | null + + @Column('timestamptz') + savedAt?: Date | null + + @ManyToOne(() => FeedItem, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'feed_item_id' }) + feedItem!: FeedItem + + @ManyToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'user_id' }) + user!: User +} diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 3d2524102..8dfbe5a08 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -794,7 +794,7 @@ export enum FeedsErrorCode { export type FeedsInput = { after?: InputMaybe; first?: InputMaybe; - query: Scalars['String']; + query?: InputMaybe; sort?: InputMaybe; }; @@ -837,6 +837,50 @@ export type FiltersSuccess = { filters: Array; }; +export type Following = { + __typename?: 'Following'; + author?: Maybe; + categories?: Maybe>; + content?: Maybe; + createdAt: Scalars['Date']; + feedItemId: Scalars['ID']; + hiddenAt?: Maybe; + id: Scalars['ID']; + isHidden: Scalars['Boolean']; + isSaved: Scalars['Boolean']; + links: Array; + previewContent?: Maybe; + publishedAt?: Maybe; + savedAt?: Maybe; + summary?: Maybe; + title: Scalars['String']; + updatedAt: Scalars['Date']; +}; + +export type FollowingEdge = { + __typename?: 'FollowingEdge'; + cursor: Scalars['String']; + node: Following; +}; + +export type FollowingError = { + __typename?: 'FollowingError'; + errorCodes: Array; +}; + +export enum FollowingErrorCode { + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED' +} + +export type FollowingResult = FollowingError | FollowingSuccess; + +export type FollowingSuccess = { + __typename?: 'FollowingSuccess'; + edges: Array; + pageInfo: PageInfo; +}; + export type GenerateApiKeyError = { __typename?: 'GenerateApiKeyError'; errorCodes: Array; @@ -1771,6 +1815,7 @@ export type Query = { deviceTokens: DeviceTokensResult; feeds: FeedsResult; filters: FiltersResult; + following: FollowingResult; getUserPersonalization: GetUserPersonalizationResult; groups: GroupsResult; hello?: Maybe; @@ -1812,6 +1857,14 @@ export type QueryFeedsArgs = { }; +export type QueryFollowingArgs = { + after?: InputMaybe; + first?: InputMaybe; + since?: InputMaybe; + until?: InputMaybe; +}; + + export type QueryRulesArgs = { enabled?: InputMaybe; }; @@ -3528,6 +3581,12 @@ export type ResolversTypes = { FiltersResult: ResolversTypes['FiltersError'] | ResolversTypes['FiltersSuccess']; FiltersSuccess: ResolverTypeWrapper; Float: ResolverTypeWrapper; + Following: ResolverTypeWrapper; + FollowingEdge: ResolverTypeWrapper; + FollowingError: ResolverTypeWrapper; + FollowingErrorCode: FollowingErrorCode; + FollowingResult: ResolversTypes['FollowingError'] | ResolversTypes['FollowingSuccess']; + FollowingSuccess: ResolverTypeWrapper; GenerateApiKeyError: ResolverTypeWrapper; GenerateApiKeyErrorCode: GenerateApiKeyErrorCode; GenerateApiKeyInput: GenerateApiKeyInput; @@ -4012,6 +4071,11 @@ export type ResolversParentTypes = { FiltersResult: ResolversParentTypes['FiltersError'] | ResolversParentTypes['FiltersSuccess']; FiltersSuccess: FiltersSuccess; Float: Scalars['Float']; + Following: Following; + FollowingEdge: FollowingEdge; + FollowingError: FollowingError; + FollowingResult: ResolversParentTypes['FollowingError'] | ResolversParentTypes['FollowingSuccess']; + FollowingSuccess: FollowingSuccess; GenerateApiKeyError: GenerateApiKeyError; GenerateApiKeyInput: GenerateApiKeyInput; GenerateApiKeyResult: ResolversParentTypes['GenerateApiKeyError'] | ResolversParentTypes['GenerateApiKeySuccess']; @@ -4894,6 +4958,47 @@ export type FiltersSuccessResolvers; }; +export type FollowingResolvers = { + author?: Resolver, ParentType, ContextType>; + categories?: Resolver>, ParentType, ContextType>; + content?: Resolver, ParentType, ContextType>; + createdAt?: Resolver; + feedItemId?: Resolver; + hiddenAt?: Resolver, ParentType, ContextType>; + id?: Resolver; + isHidden?: Resolver; + isSaved?: Resolver; + links?: Resolver, ParentType, ContextType>; + previewContent?: Resolver, ParentType, ContextType>; + publishedAt?: Resolver, ParentType, ContextType>; + savedAt?: Resolver, ParentType, ContextType>; + summary?: Resolver, ParentType, ContextType>; + title?: Resolver; + updatedAt?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type FollowingEdgeResolvers = { + cursor?: Resolver; + node?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type FollowingErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type FollowingResultResolvers = { + __resolveType: TypeResolveFn<'FollowingError' | 'FollowingSuccess', ParentType, ContextType>; +}; + +export type FollowingSuccessResolvers = { + edges?: Resolver, ParentType, ContextType>; + pageInfo?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type GenerateApiKeyErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -5365,6 +5470,7 @@ export type QueryResolvers; feeds?: Resolver>; filters?: Resolver; + following?: Resolver>; getUserPersonalization?: Resolver; groups?: Resolver; hello?: Resolver, ParentType, ContextType>; @@ -6399,6 +6505,11 @@ export type Resolvers = { FiltersError?: FiltersErrorResolvers; FiltersResult?: FiltersResultResolvers; FiltersSuccess?: FiltersSuccessResolvers; + Following?: FollowingResolvers; + FollowingEdge?: FollowingEdgeResolvers; + FollowingError?: FollowingErrorResolvers; + FollowingResult?: FollowingResultResolvers; + FollowingSuccess?: FollowingSuccessResolvers; GenerateApiKeyError?: GenerateApiKeyErrorResolvers; GenerateApiKeyResult?: GenerateApiKeyResultResolvers; GenerateApiKeySuccess?: GenerateApiKeySuccessResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 719c59e02..b071c75df 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -703,7 +703,7 @@ enum FeedsErrorCode { input FeedsInput { after: String first: Int - query: String! + query: String sort: SortParams } @@ -742,6 +742,46 @@ type FiltersSuccess { filters: [Filter!]! } +type Following { + author: String + categories: [String!] + content: String + createdAt: Date! + feedItemId: ID! + hiddenAt: Date + id: ID! + isHidden: Boolean! + isSaved: Boolean! + links: [String!]! + previewContent: String + publishedAt: Date + savedAt: Date + summary: String + title: String! + updatedAt: Date! +} + +type FollowingEdge { + cursor: String! + node: Following! +} + +type FollowingError { + errorCodes: [FollowingErrorCode!]! +} + +enum FollowingErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union FollowingResult = FollowingError | FollowingSuccess + +type FollowingSuccess { + edges: [FollowingEdge!]! + pageInfo: PageInfo! +} + type GenerateApiKeyError { errorCodes: [GenerateApiKeyErrorCode!]! } @@ -1336,6 +1376,7 @@ type Query { deviceTokens: DeviceTokensResult! feeds(input: FeedsInput!): FeedsResult! filters: FiltersResult! + following(after: String, first: Int, since: Date, until: Date): FollowingResult! getUserPersonalization: GetUserPersonalizationResult! groups: GroupsResult! hello: String diff --git a/packages/api/src/repository/feed.ts b/packages/api/src/repository/feed.ts index 99d242de8..72f337d16 100644 --- a/packages/api/src/repository/feed.ts +++ b/packages/api/src/repository/feed.ts @@ -4,14 +4,18 @@ import { Feed } from '../entity/feed' export const feedRepository = appDataSource.getRepository(Feed).extend({ async searchFeeds( - query: string, + query = '', take = 10, skip = 0, - orderBy = 'createdAt', - order = 'DESC' + orderBy = 'title', + order = 'ASC' ) { - query = `%${query}%` - const where = [{ title: ILike(query) }, { url: ILike(query) }] + const where = [] + if (query !== '') { + query = `%${query}%` + where.push({ title: ILike(query) }, { url: ILike(query) }) + } + const feeds = await this.find({ where, order: { [orderBy]: order }, diff --git a/packages/api/src/repository/user_feed_item.ts b/packages/api/src/repository/user_feed_item.ts new file mode 100644 index 000000000..dcbf488b8 --- /dev/null +++ b/packages/api/src/repository/user_feed_item.ts @@ -0,0 +1,36 @@ +import { Between, FindOptionsWhere } from 'typeorm' +import { appDataSource } from '../data_source' +import { UserFeedItem } from '../entity/user_feed_item' + +export const userFeedItemRepository = appDataSource + .getRepository(UserFeedItem) + .extend({ + async searchUserFeedItems( + take = 10, + skip = 0, + since = new Date(0), + until = new Date() + ) { + const where: FindOptionsWhere = { + updatedAt: Between(since, until), + } + + const userFeedItems = await this.find({ + where, + order: { updatedAt: 'DESC' }, + take, + skip, + relations: { + user: true, + feedItem: true, + }, + }) + + const count = await this.countBy(where) + + return { + userFeedItems, + count, + } + }, + }) diff --git a/packages/api/src/resolvers/feeds/index.ts b/packages/api/src/resolvers/feeds/index.ts index f3ef53b51..20c503464 100644 --- a/packages/api/src/resolvers/feeds/index.ts +++ b/packages/api/src/resolvers/feeds/index.ts @@ -3,9 +3,15 @@ import { FeedsError, FeedsErrorCode, FeedsSuccess, + FollowingEdge, + FollowingError, + FollowingErrorCode, + FollowingSuccess, QueryFeedsArgs, + QueryFollowingArgs, } from '../../generated/graphql' import { feedRepository } from '../../repository/feed' +import { userFeedItemRepository } from '../../repository/user_feed_item' import { authorized } from '../../utils/helpers' export const feedsResolve = authorized< @@ -17,10 +23,10 @@ export const feedsResolve = authorized< const startCursor = input.after || '' const start = startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 - const first = input.first || 10 + const first = Math.min(input.first || 10, 100) // cap at 100 const { feeds, count } = await feedRepository.searchFeeds( - input.query, + input.query || '', first + 1, // fetch one extra to check if there is a next page start, input.sort?.by, @@ -59,3 +65,66 @@ export const feedsResolve = authorized< } } }) + +export const followingResolver = authorized< + FollowingSuccess, + FollowingError, + QueryFollowingArgs +>(async (_, args, { authTrx, log }) => { + try { + const startCursor = args.after || '' + const start = + startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 + const first = Math.min(args.first || 10, 100) // cap at 100 + const since = args.since ? new Date(args.since) : undefined + const until = args.until ? new Date(args.until) : undefined + + const { userFeedItems, count } = await authTrx((tx) => + tx.withRepository(userFeedItemRepository).searchUserFeedItems( + first + 1, // fetch one extra to check if there is a next page + start, + since, + until + ) + ) + + const hasNextPage = userFeedItems.length > first + const endCursor = String( + start + userFeedItems.length - (hasNextPage ? 1 : 0) + ) + + if (hasNextPage) { + // remove an extra if exists + userFeedItems.pop() + } + + const edges: FollowingEdge[] = userFeedItems.map((item) => ({ + node: { + ...item.feedItem, + ...item, + isHidden: !!item.hiddenAt, + isSaved: !!item.savedAt, + feedItemId: item.feedItem.id, + }, + cursor: endCursor, + })) + + return { + __typename: 'FollowingSuccess', + edges, + pageInfo: { + hasPreviousPage: start > 0, + hasNextPage, + startCursor, + endCursor, + totalCount: count, + }, + } + } catch (error) { + log.error('Error fetching following', error) + + return { + errorCodes: [FollowingErrorCode.Unauthorized], + } + } +}) diff --git a/packages/api/src/routers/feed_router.ts b/packages/api/src/routers/feed_router.ts new file mode 100644 index 000000000..9480a29da --- /dev/null +++ b/packages/api/src/routers/feed_router.ts @@ -0,0 +1,7 @@ +import express from "express"; + +export function feedRouter() { + const router = express.Router() + + router.post('/') +} diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index bd1b2df8f..fd961d197 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2607,8 +2607,8 @@ const schema = gql` input FeedsInput { after: String - first: Int @sanitize(minLength: 1, maxLength: 100) - query: String! @sanitize(maxLength: 255) + first: Int + query: String @sanitize(maxLength: 255) sort: SortParams } @@ -2645,6 +2645,46 @@ const schema = gql` author: String } + union FollowingResult = FollowingSuccess | FollowingError + + type FollowingSuccess { + edges: [FollowingEdge!]! + pageInfo: PageInfo! + } + + type FollowingEdge { + cursor: String! + node: Following! + } + + type FollowingError { + errorCodes: [FollowingErrorCode!]! + } + + enum FollowingErrorCode { + UNAUTHORIZED + BAD_REQUEST + } + + type Following { + id: ID! + feedItemId: ID! + title: String! + links: [String!]! + author: String + summary: String + categories: [String!] + content: String + previewContent: String + createdAt: Date! + updatedAt: Date! + savedAt: Date + hiddenAt: Date + publishedAt: Date + isHidden: Boolean! + isSaved: Boolean! + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2807,6 +2847,12 @@ const schema = gql` groups: GroupsResult! recentEmails: RecentEmailsResult! feeds(input: FeedsInput!): FeedsResult! + following( + after: String + first: Int + since: Date + until: Date + ): FollowingResult! } ` diff --git a/packages/api/src/services/feeds.ts b/packages/api/src/services/feeds.ts index e69de29bb..725d59139 100644 --- a/packages/api/src/services/feeds.ts +++ b/packages/api/src/services/feeds.ts @@ -0,0 +1,9 @@ +export const searchFollowing = async ( + userId: string, + first = 10, + after = '', + since?: Date, + until?: Date +) => { + +} From 1da8dc65450398b6482e803967950c496b3692e1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Nov 2023 14:53:46 +0800 Subject: [PATCH 06/30] update library_item table --- packages/api/src/entity/feed_item.ts | 53 --------- packages/api/src/generated/graphql.ts | 111 ------------------ packages/api/src/generated/schema.graphql | 41 ------- packages/api/src/resolvers/following/index.ts | 61 ++++++++++ packages/api/src/routers/feed_router.ts | 7 -- packages/api/src/schema.ts | 46 -------- packages/db/migrations/0145.do.feed.sql | 72 ------------ packages/db/migrations/0145.do.following.sql | 39 ++++++ packages/db/migrations/0145.undo.feed.sql | 13 -- .../db/migrations/0145.undo.following.sql | 22 ++++ 10 files changed, 122 insertions(+), 343 deletions(-) delete mode 100644 packages/api/src/entity/feed_item.ts create mode 100644 packages/api/src/resolvers/following/index.ts delete mode 100644 packages/api/src/routers/feed_router.ts delete mode 100755 packages/db/migrations/0145.do.feed.sql create mode 100755 packages/db/migrations/0145.do.following.sql delete mode 100755 packages/db/migrations/0145.undo.feed.sql create mode 100755 packages/db/migrations/0145.undo.following.sql diff --git a/packages/api/src/entity/feed_item.ts b/packages/api/src/entity/feed_item.ts deleted file mode 100644 index 541e6e194..000000000 --- a/packages/api/src/entity/feed_item.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - JoinColumn, - ManyToOne, - PrimaryGeneratedColumn, - UpdateDateColumn, -} from 'typeorm' -import { Feed } from './feed' - -@Entity() -export class FeedItem { - @PrimaryGeneratedColumn('uuid') - id!: string - - @Column('text') - guid!: string - - @Column('text') - title!: string - - @Column('array') - links!: string[] - - @Column('text') - author?: string | null - - @Column('text') - summary?: string | null - - @Column('array') - categories?: string[] | null - - @Column('text') - content?: string | null - - @Column('text') - previewContent?: string | null - - @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) - createdAt!: Date - - @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) - updatedAt!: Date - - @Column('timestamptz') - publishedAt?: Date | null - - @ManyToOne(() => Feed, { onDelete: 'CASCADE' }) - @JoinColumn({ name: 'feed_id' }) - feed!: Feed -} diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 8dfbe5a08..849afba50 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -837,50 +837,6 @@ export type FiltersSuccess = { filters: Array; }; -export type Following = { - __typename?: 'Following'; - author?: Maybe; - categories?: Maybe>; - content?: Maybe; - createdAt: Scalars['Date']; - feedItemId: Scalars['ID']; - hiddenAt?: Maybe; - id: Scalars['ID']; - isHidden: Scalars['Boolean']; - isSaved: Scalars['Boolean']; - links: Array; - previewContent?: Maybe; - publishedAt?: Maybe; - savedAt?: Maybe; - summary?: Maybe; - title: Scalars['String']; - updatedAt: Scalars['Date']; -}; - -export type FollowingEdge = { - __typename?: 'FollowingEdge'; - cursor: Scalars['String']; - node: Following; -}; - -export type FollowingError = { - __typename?: 'FollowingError'; - errorCodes: Array; -}; - -export enum FollowingErrorCode { - BadRequest = 'BAD_REQUEST', - Unauthorized = 'UNAUTHORIZED' -} - -export type FollowingResult = FollowingError | FollowingSuccess; - -export type FollowingSuccess = { - __typename?: 'FollowingSuccess'; - edges: Array; - pageInfo: PageInfo; -}; - export type GenerateApiKeyError = { __typename?: 'GenerateApiKeyError'; errorCodes: Array; @@ -1815,7 +1771,6 @@ export type Query = { deviceTokens: DeviceTokensResult; feeds: FeedsResult; filters: FiltersResult; - following: FollowingResult; getUserPersonalization: GetUserPersonalizationResult; groups: GroupsResult; hello?: Maybe; @@ -1857,14 +1812,6 @@ export type QueryFeedsArgs = { }; -export type QueryFollowingArgs = { - after?: InputMaybe; - first?: InputMaybe; - since?: InputMaybe; - until?: InputMaybe; -}; - - export type QueryRulesArgs = { enabled?: InputMaybe; }; @@ -3581,12 +3528,6 @@ export type ResolversTypes = { FiltersResult: ResolversTypes['FiltersError'] | ResolversTypes['FiltersSuccess']; FiltersSuccess: ResolverTypeWrapper; Float: ResolverTypeWrapper; - Following: ResolverTypeWrapper; - FollowingEdge: ResolverTypeWrapper; - FollowingError: ResolverTypeWrapper; - FollowingErrorCode: FollowingErrorCode; - FollowingResult: ResolversTypes['FollowingError'] | ResolversTypes['FollowingSuccess']; - FollowingSuccess: ResolverTypeWrapper; GenerateApiKeyError: ResolverTypeWrapper; GenerateApiKeyErrorCode: GenerateApiKeyErrorCode; GenerateApiKeyInput: GenerateApiKeyInput; @@ -4071,11 +4012,6 @@ export type ResolversParentTypes = { FiltersResult: ResolversParentTypes['FiltersError'] | ResolversParentTypes['FiltersSuccess']; FiltersSuccess: FiltersSuccess; Float: Scalars['Float']; - Following: Following; - FollowingEdge: FollowingEdge; - FollowingError: FollowingError; - FollowingResult: ResolversParentTypes['FollowingError'] | ResolversParentTypes['FollowingSuccess']; - FollowingSuccess: FollowingSuccess; GenerateApiKeyError: GenerateApiKeyError; GenerateApiKeyInput: GenerateApiKeyInput; GenerateApiKeyResult: ResolversParentTypes['GenerateApiKeyError'] | ResolversParentTypes['GenerateApiKeySuccess']; @@ -4958,47 +4894,6 @@ export type FiltersSuccessResolvers; }; -export type FollowingResolvers = { - author?: Resolver, ParentType, ContextType>; - categories?: Resolver>, ParentType, ContextType>; - content?: Resolver, ParentType, ContextType>; - createdAt?: Resolver; - feedItemId?: Resolver; - hiddenAt?: Resolver, ParentType, ContextType>; - id?: Resolver; - isHidden?: Resolver; - isSaved?: Resolver; - links?: Resolver, ParentType, ContextType>; - previewContent?: Resolver, ParentType, ContextType>; - publishedAt?: Resolver, ParentType, ContextType>; - savedAt?: Resolver, ParentType, ContextType>; - summary?: Resolver, ParentType, ContextType>; - title?: Resolver; - updatedAt?: Resolver; - __isTypeOf?: IsTypeOfResolverFn; -}; - -export type FollowingEdgeResolvers = { - cursor?: Resolver; - node?: Resolver; - __isTypeOf?: IsTypeOfResolverFn; -}; - -export type FollowingErrorResolvers = { - errorCodes?: Resolver, ParentType, ContextType>; - __isTypeOf?: IsTypeOfResolverFn; -}; - -export type FollowingResultResolvers = { - __resolveType: TypeResolveFn<'FollowingError' | 'FollowingSuccess', ParentType, ContextType>; -}; - -export type FollowingSuccessResolvers = { - edges?: Resolver, ParentType, ContextType>; - pageInfo?: Resolver; - __isTypeOf?: IsTypeOfResolverFn; -}; - export type GenerateApiKeyErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -5470,7 +5365,6 @@ export type QueryResolvers; feeds?: Resolver>; filters?: Resolver; - following?: Resolver>; getUserPersonalization?: Resolver; groups?: Resolver; hello?: Resolver, ParentType, ContextType>; @@ -6505,11 +6399,6 @@ export type Resolvers = { FiltersError?: FiltersErrorResolvers; FiltersResult?: FiltersResultResolvers; FiltersSuccess?: FiltersSuccessResolvers; - Following?: FollowingResolvers; - FollowingEdge?: FollowingEdgeResolvers; - FollowingError?: FollowingErrorResolvers; - FollowingResult?: FollowingResultResolvers; - FollowingSuccess?: FollowingSuccessResolvers; GenerateApiKeyError?: GenerateApiKeyErrorResolvers; GenerateApiKeyResult?: GenerateApiKeyResultResolvers; GenerateApiKeySuccess?: GenerateApiKeySuccessResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index b071c75df..8b704c249 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -742,46 +742,6 @@ type FiltersSuccess { filters: [Filter!]! } -type Following { - author: String - categories: [String!] - content: String - createdAt: Date! - feedItemId: ID! - hiddenAt: Date - id: ID! - isHidden: Boolean! - isSaved: Boolean! - links: [String!]! - previewContent: String - publishedAt: Date - savedAt: Date - summary: String - title: String! - updatedAt: Date! -} - -type FollowingEdge { - cursor: String! - node: Following! -} - -type FollowingError { - errorCodes: [FollowingErrorCode!]! -} - -enum FollowingErrorCode { - BAD_REQUEST - UNAUTHORIZED -} - -union FollowingResult = FollowingError | FollowingSuccess - -type FollowingSuccess { - edges: [FollowingEdge!]! - pageInfo: PageInfo! -} - type GenerateApiKeyError { errorCodes: [GenerateApiKeyErrorCode!]! } @@ -1376,7 +1336,6 @@ type Query { deviceTokens: DeviceTokensResult! feeds(input: FeedsInput!): FeedsResult! filters: FiltersResult! - following(after: String, first: Int, since: Date, until: Date): FollowingResult! getUserPersonalization: GetUserPersonalizationResult! groups: GroupsResult! hello: String diff --git a/packages/api/src/resolvers/following/index.ts b/packages/api/src/resolvers/following/index.ts new file mode 100644 index 000000000..08c219436 --- /dev/null +++ b/packages/api/src/resolvers/following/index.ts @@ -0,0 +1,61 @@ +import { + FeedEdge, + FeedsError, + FeedsErrorCode, + FeedsSuccess, + QueryFeedsArgs, +} from '../../generated/graphql' +import { feedRepository } from '../../repository/feed' +import { authorized } from '../../utils/helpers' + +export const feedsResolve = authorized< + FeedsSuccess, + FeedsError, + QueryFeedsArgs +>(async (_, { input }, { log }) => { + try { + const startCursor = input.after || '' + const start = + startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 + const first = Math.min(input.first || 10, 100) // cap at 100 + + const { feeds, count } = await feedRepository.searchFeeds( + input.query || '', + first + 1, // fetch one extra to check if there is a next page + start, + input.sort?.by, + input.sort?.order || undefined + ) + + const hasNextPage = feeds.length > first + const endCursor = String(start + feeds.length - (hasNextPage ? 1 : 0)) + + if (hasNextPage) { + // remove an extra if exists + feeds.pop() + } + + const edges: FeedEdge[] = feeds.map((feed) => ({ + node: feed, + cursor: endCursor, + })) + + return { + __typename: 'FeedsSuccess', + edges, + pageInfo: { + hasPreviousPage: start > 0, + hasNextPage, + startCursor, + endCursor, + totalCount: count, + }, + } + } catch (error) { + log.error('Error fetching feeds', error) + + return { + errorCodes: [FeedsErrorCode.BadRequest], + } + } +}) diff --git a/packages/api/src/routers/feed_router.ts b/packages/api/src/routers/feed_router.ts deleted file mode 100644 index 9480a29da..000000000 --- a/packages/api/src/routers/feed_router.ts +++ /dev/null @@ -1,7 +0,0 @@ -import express from "express"; - -export function feedRouter() { - const router = express.Router() - - router.post('/') -} diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index fd961d197..4cd073b8f 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2645,46 +2645,6 @@ const schema = gql` author: String } - union FollowingResult = FollowingSuccess | FollowingError - - type FollowingSuccess { - edges: [FollowingEdge!]! - pageInfo: PageInfo! - } - - type FollowingEdge { - cursor: String! - node: Following! - } - - type FollowingError { - errorCodes: [FollowingErrorCode!]! - } - - enum FollowingErrorCode { - UNAUTHORIZED - BAD_REQUEST - } - - type Following { - id: ID! - feedItemId: ID! - title: String! - links: [String!]! - author: String - summary: String - categories: [String!] - content: String - previewContent: String - createdAt: Date! - updatedAt: Date! - savedAt: Date - hiddenAt: Date - publishedAt: Date - isHidden: Boolean! - isSaved: Boolean! - } - # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2847,12 +2807,6 @@ const schema = gql` groups: GroupsResult! recentEmails: RecentEmailsResult! feeds(input: FeedsInput!): FeedsResult! - following( - after: String - first: Int - since: Date - until: Date - ): FollowingResult! } ` diff --git a/packages/db/migrations/0145.do.feed.sql b/packages/db/migrations/0145.do.feed.sql deleted file mode 100755 index a97e176a6..000000000 --- a/packages/db/migrations/0145.do.feed.sql +++ /dev/null @@ -1,72 +0,0 @@ --- Type: DO --- Name: feed --- Description: Create feed, feed_item and user_feed_item tables - -BEGIN; - -CREATE TABLE omnivore.feed ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), - title text NOT NULL, - url text NOT NULL, - author text, - description text, - image text, - created_at timestamptz NOT NULL DEFAULT current_timestamp, - updated_at timestamptz NOT NULL DEFAULT current_timestamp, - published_at timestamptz, - UNIQUE(url) -); - -CREATE INDEX IF NOT EXISTS feed_title_idx ON omnivore.feed(title); - -CREATE TRIGGER update_feed_modtime BEFORE UPDATE ON omnivore.feed FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); - -GRANT SELECT, INSERT, UPDATE ON omnivore.feed TO omnivore_user; - -CREATE TABLE omnivore.feed_item ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), - guid text NOT NULL, - title text NOT NULL, - links text[] NOT NULL, - author text, - summary text, - categories text[], - content text, - preview_content text, - created_at timestamptz NOT NULL DEFAULT current_timestamp, - updated_at timestamptz NOT NULL DEFAULT current_timestamp, - published_at timestamptz, - feed_id uuid NOT NULL REFERENCES omnivore.feed(id) ON DELETE CASCADE, - UNIQUE(guid) -); - -CREATE INDEX IF NOT EXISTS feed_item_feed_id_idx ON omnivore.feed_item(feed_id); - -CREATE TRIGGER update_feed_item_modtime BEFORE UPDATE ON omnivore.feed_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); - -GRANT SELECT, INSERT, UPDATE ON omnivore.feed_item TO omnivore_user; - -CREATE TABLE omnivore.user_feed_item ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), - user_id uuid NOT NULL REFERENCES omnivore.user(id) ON DELETE CASCADE, - feed_item_id uuid NOT NULL REFERENCES omnivore.feed_item(id) ON DELETE CASCADE, - created_at timestamptz NOT NULL DEFAULT current_timestamp, - updated_at timestamptz NOT NULL DEFAULT current_timestamp, - hidden_at timestamptz, - saved_at timestamptz -); - -CREATE INDEX IF NOT EXISTS user_feed_item_user_id_idx ON omnivore.user_feed_item(user_id); -CREATE INDEX IF NOT EXISTS user_feed_item_feed_item_id_idx ON omnivore.user_feed_item(feed_item_id); - -CREATE TRIGGER update_user_feed_item_modtime BEFORE UPDATE ON omnivore.user_feed_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); - -ALTER TABLE omnivore.user_feed_item ENABLE ROW LEVEL SECURITY; - -CREATE POLICY user_feed_item_policy ON omnivore.user_feed_item - USING (user_id = omnivore.get_current_user_id()) - WITH CHECK (user_id = omnivore.get_current_user_id()); - -GRANT SELECT, INSERT, UPDATE ON omnivore.user_feed_item TO omnivore_user; - -COMMIT; diff --git a/packages/db/migrations/0145.do.following.sql b/packages/db/migrations/0145.do.following.sql new file mode 100755 index 000000000..dcf3ab65d --- /dev/null +++ b/packages/db/migrations/0145.do.following.sql @@ -0,0 +1,39 @@ +-- Type: DO +-- Name: following +-- Description: Create tables for following feature + +BEGIN; + +ALTER TABLE omnivore.subscriptions + ADD COLUMN is_public boolean, + ADD COLUMN is_fetching_content boolean; + +ALTER TABLE omnivore.library_item + ADD COLUMN hidden_at timestamptz, + ADD COLUMN shared_at timestamptz, + ADD COLUMN shared_by text, + ADD COLUMN links jsonb, + ADD COLUMN preview_content text, + ADD COLUMN seen_at timestamptz, + ALTER COLUMN saved_at DROP NOT NULL; + +CREATE TABLE omnivore.feed ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + title text NOT NULL, + url text NOT NULL, + author text, + description text, + image text, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, + published_at timestamptz, + UNIQUE(url) +); + +CREATE INDEX feed_title_idx ON omnivore.feed(title); + +CREATE TRIGGER update_feed_modtime BEFORE UPDATE ON omnivore.feed FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +GRANT SELECT, INSERT, UPDATE ON omnivore.feed TO omnivore_user; + +COMMIT; diff --git a/packages/db/migrations/0145.undo.feed.sql b/packages/db/migrations/0145.undo.feed.sql deleted file mode 100755 index 92770ee98..000000000 --- a/packages/db/migrations/0145.undo.feed.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Type: UNDO --- Name: feed --- Description: Create feed, feed_item and user_feed_item tables - -BEGIN; - -DROP TABLE IF EXISTS omnivore.user_feed_item; - -DROP TABLE IF EXISTS omnivore.feed_item; - -DROP TABLE IF EXISTS omnivore.feed; - -COMMIT; diff --git a/packages/db/migrations/0145.undo.following.sql b/packages/db/migrations/0145.undo.following.sql new file mode 100755 index 000000000..1b1e13de2 --- /dev/null +++ b/packages/db/migrations/0145.undo.following.sql @@ -0,0 +1,22 @@ +-- Type: UNDO +-- Name: following +-- Description: Create tables for following feature + +BEGIN; + +DROP TABLE omnivore.feed; + +ALTER TABLE omnivore.library_item + DROP COLUMN hidden_at, + DROP COLUMN shared_at, + DROP COLUMN shared_by, + DROP COLUMN links, + DROP COLUMN preview_content, + DROP COLUMN seen_at, + ALTER COLUMN saved_at SET NOT NULL; + +ALTER TABLE omnivore.subscriptions + DROP COLUMN is_public, + DROP COLUMN is_fetching_content; + +COMMIT; From 679512c9d1275a6400c9f323c3c13acb88356f77 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Nov 2023 16:04:08 +0800 Subject: [PATCH 07/30] add columns to library_item --- packages/api/src/entity/user_feed_item.ts | 37 ----- packages/api/src/repository/user_feed_item.ts | 36 ----- packages/api/src/resolvers/feeds/index.ts | 130 ------------------ packages/api/src/services/feeds.ts | 9 -- packages/api/src/services/library_item.ts | 10 +- packages/api/src/utils/search.ts | 1 + packages/db/migrations/0145.do.following.sql | 2 +- .../db/migrations/0145.undo.following.sql | 2 +- 8 files changed, 11 insertions(+), 216 deletions(-) delete mode 100644 packages/api/src/entity/user_feed_item.ts delete mode 100644 packages/api/src/repository/user_feed_item.ts delete mode 100644 packages/api/src/resolvers/feeds/index.ts delete mode 100644 packages/api/src/services/feeds.ts diff --git a/packages/api/src/entity/user_feed_item.ts b/packages/api/src/entity/user_feed_item.ts deleted file mode 100644 index 46ea8f8bc..000000000 --- a/packages/api/src/entity/user_feed_item.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - JoinColumn, - ManyToOne, - PrimaryGeneratedColumn, - UpdateDateColumn, -} from 'typeorm' -import { FeedItem } from './feed_item' -import { User } from './user' - -@Entity() -export class UserFeedItem { - @PrimaryGeneratedColumn('uuid') - id!: string - - @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) - createdAt!: Date - - @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) - updatedAt!: Date - - @Column('timestamptz') - hiddenAt?: Date | null - - @Column('timestamptz') - savedAt?: Date | null - - @ManyToOne(() => FeedItem, { onDelete: 'CASCADE' }) - @JoinColumn({ name: 'feed_item_id' }) - feedItem!: FeedItem - - @ManyToOne(() => User, { onDelete: 'CASCADE' }) - @JoinColumn({ name: 'user_id' }) - user!: User -} diff --git a/packages/api/src/repository/user_feed_item.ts b/packages/api/src/repository/user_feed_item.ts deleted file mode 100644 index dcbf488b8..000000000 --- a/packages/api/src/repository/user_feed_item.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Between, FindOptionsWhere } from 'typeorm' -import { appDataSource } from '../data_source' -import { UserFeedItem } from '../entity/user_feed_item' - -export const userFeedItemRepository = appDataSource - .getRepository(UserFeedItem) - .extend({ - async searchUserFeedItems( - take = 10, - skip = 0, - since = new Date(0), - until = new Date() - ) { - const where: FindOptionsWhere = { - updatedAt: Between(since, until), - } - - const userFeedItems = await this.find({ - where, - order: { updatedAt: 'DESC' }, - take, - skip, - relations: { - user: true, - feedItem: true, - }, - }) - - const count = await this.countBy(where) - - return { - userFeedItems, - count, - } - }, - }) diff --git a/packages/api/src/resolvers/feeds/index.ts b/packages/api/src/resolvers/feeds/index.ts deleted file mode 100644 index 20c503464..000000000 --- a/packages/api/src/resolvers/feeds/index.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { - FeedEdge, - FeedsError, - FeedsErrorCode, - FeedsSuccess, - FollowingEdge, - FollowingError, - FollowingErrorCode, - FollowingSuccess, - QueryFeedsArgs, - QueryFollowingArgs, -} from '../../generated/graphql' -import { feedRepository } from '../../repository/feed' -import { userFeedItemRepository } from '../../repository/user_feed_item' -import { authorized } from '../../utils/helpers' - -export const feedsResolve = authorized< - FeedsSuccess, - FeedsError, - QueryFeedsArgs ->(async (_, { input }, { log }) => { - try { - const startCursor = input.after || '' - const start = - startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 - const first = Math.min(input.first || 10, 100) // cap at 100 - - const { feeds, count } = await feedRepository.searchFeeds( - input.query || '', - first + 1, // fetch one extra to check if there is a next page - start, - input.sort?.by, - input.sort?.order || undefined - ) - - const hasNextPage = feeds.length > first - const endCursor = String(start + feeds.length - (hasNextPage ? 1 : 0)) - - if (hasNextPage) { - // remove an extra if exists - feeds.pop() - } - - const edges: FeedEdge[] = feeds.map((feed) => ({ - node: feed, - cursor: endCursor, - })) - - return { - __typename: 'FeedsSuccess', - edges, - pageInfo: { - hasPreviousPage: start > 0, - hasNextPage, - startCursor, - endCursor, - totalCount: count, - }, - } - } catch (error) { - log.error('Error fetching feeds', error) - - return { - errorCodes: [FeedsErrorCode.BadRequest], - } - } -}) - -export const followingResolver = authorized< - FollowingSuccess, - FollowingError, - QueryFollowingArgs ->(async (_, args, { authTrx, log }) => { - try { - const startCursor = args.after || '' - const start = - startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 - const first = Math.min(args.first || 10, 100) // cap at 100 - const since = args.since ? new Date(args.since) : undefined - const until = args.until ? new Date(args.until) : undefined - - const { userFeedItems, count } = await authTrx((tx) => - tx.withRepository(userFeedItemRepository).searchUserFeedItems( - first + 1, // fetch one extra to check if there is a next page - start, - since, - until - ) - ) - - const hasNextPage = userFeedItems.length > first - const endCursor = String( - start + userFeedItems.length - (hasNextPage ? 1 : 0) - ) - - if (hasNextPage) { - // remove an extra if exists - userFeedItems.pop() - } - - const edges: FollowingEdge[] = userFeedItems.map((item) => ({ - node: { - ...item.feedItem, - ...item, - isHidden: !!item.hiddenAt, - isSaved: !!item.savedAt, - feedItemId: item.feedItem.id, - }, - cursor: endCursor, - })) - - return { - __typename: 'FollowingSuccess', - edges, - pageInfo: { - hasPreviousPage: start > 0, - hasNextPage, - startCursor, - endCursor, - totalCount: count, - }, - } - } catch (error) { - log.error('Error fetching following', error) - - return { - errorCodes: [FollowingErrorCode.Unauthorized], - } - } -}) diff --git a/packages/api/src/services/feeds.ts b/packages/api/src/services/feeds.ts deleted file mode 100644 index 725d59139..000000000 --- a/packages/api/src/services/feeds.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const searchFollowing = async ( - userId: string, - first = 10, - after = '', - since?: Date, - until?: Date -) => { - -} diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 7fec735e0..4448a5964 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -103,7 +103,9 @@ const buildWhereClause = ( }) } - if (args.inFilter !== InFilter.ALL) { + if (args.inFilter !== InFilter.FOLLOWING) { + queryBuilder.andWhere('library_item.is_in_library = true') + switch (args.inFilter) { case InFilter.INBOX: queryBuilder.andWhere('library_item.archived_at IS NULL') @@ -131,6 +133,10 @@ const buildWhereClause = ( .andWhere('library_item.archived_at IS NULL') break } + } else { + queryBuilder + .andWhere('library_item.shared_by IS NOT NULL') + .andWhere('library_item.hidden_at IS NULL') } if (args.readFilter !== ReadFilter.ALL) { @@ -318,7 +324,7 @@ export const searchLibraryItems = async ( const queryBuilder = tx .createQueryBuilder(LibraryItem, 'library_item') .select(selectColumns) - .where({ user: { id: userId } }) + .where('library_item.user_id = :userId', { userId }) // build the where clause buildWhereClause(queryBuilder, args) diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index 39d9975d0..c83237500 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -25,6 +25,7 @@ export enum InFilter { TRASH, SUBSCRIPTION, LIBRARY, + FOLLOWING, } export interface SearchFilter { diff --git a/packages/db/migrations/0145.do.following.sql b/packages/db/migrations/0145.do.following.sql index dcf3ab65d..ba024fe7c 100755 --- a/packages/db/migrations/0145.do.following.sql +++ b/packages/db/migrations/0145.do.following.sql @@ -15,7 +15,7 @@ ALTER TABLE omnivore.library_item ADD COLUMN links jsonb, ADD COLUMN preview_content text, ADD COLUMN seen_at timestamptz, - ALTER COLUMN saved_at DROP NOT NULL; + ADD COLUMN is_in_library boolean NOT NULL DEFAULT true; CREATE TABLE omnivore.feed ( id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), diff --git a/packages/db/migrations/0145.undo.following.sql b/packages/db/migrations/0145.undo.following.sql index 1b1e13de2..2c323ec4b 100755 --- a/packages/db/migrations/0145.undo.following.sql +++ b/packages/db/migrations/0145.undo.following.sql @@ -13,7 +13,7 @@ ALTER TABLE omnivore.library_item DROP COLUMN links, DROP COLUMN preview_content, DROP COLUMN seen_at, - ALTER COLUMN saved_at SET NOT NULL; + DROP COLUMN is_in_library; ALTER TABLE omnivore.subscriptions DROP COLUMN is_public, From 9b0b378c3905fc28ef0ad12de600db00f438e45d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Nov 2023 16:26:03 +0800 Subject: [PATCH 08/30] check is_fetching_content and only fetch content if required for rss feed items --- packages/api/src/entity/library_item.ts | 21 +++++++++++++++++++ packages/api/src/entity/subscription.ts | 6 ++++++ packages/api/src/generated/graphql.ts | 4 ++++ packages/api/src/generated/schema.graphql | 4 ++++ .../api/src/resolvers/subscriptions/index.ts | 5 ++++- packages/api/src/routers/svc/rss_feed.ts | 3 ++- packages/api/src/schema.ts | 4 ++++ packages/api/src/utils/createTask.ts | 2 ++ packages/rss-handler/src/index.ts | 4 +++- 9 files changed, 50 insertions(+), 3 deletions(-) diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 5689c83d6..07cc2b879 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -196,4 +196,25 @@ export class LibraryItem { @Column('text', { nullable: true }) recommenderNames?: string[] | null + + @Column('timestamptz') + hiddenAt?: Date | null + + @Column('timestamptz') + sharedAt?: Date | null + + @Column('text') + sharedBy?: string | null + + @Column('jsonb') + links?: Record[] | null + + @Column('text') + previewContent?: string | null + + @Column('timestamptz') + seenAt?: Date | null + + @Column('boolean') + isInLibrary!: boolean } diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index fea63d890..af5c54fa4 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -70,4 +70,10 @@ export class Subscription { @Column('timestamp', { nullable: true }) scheduledAt?: Date | null + + @Column('boolean') + isPublic?: boolean | null + + @Column('boolean') + isFetchingContent?: boolean | null } diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 849afba50..d3f3147c8 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2678,6 +2678,8 @@ export enum SubscribeErrorCode { } export type SubscribeInput = { + isFetchingContent?: InputMaybe; + isPublic?: InputMaybe; subscriptionType?: InputMaybe; url: Scalars['String']; }; @@ -3038,6 +3040,8 @@ export enum UpdateSubscriptionErrorCode { export type UpdateSubscriptionInput = { description?: InputMaybe; id: Scalars['ID']; + isFetchingContent?: InputMaybe; + isPublic?: InputMaybe; lastFetchedAt?: InputMaybe; lastFetchedChecksum?: InputMaybe; name?: InputMaybe; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 8b704c249..5e1ae27af 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2112,6 +2112,8 @@ enum SubscribeErrorCode { } input SubscribeInput { + isFetchingContent: Boolean + isPublic: Boolean subscriptionType: SubscriptionType url: String! } @@ -2443,6 +2445,8 @@ enum UpdateSubscriptionErrorCode { input UpdateSubscriptionInput { description: String id: ID! + isFetchingContent: Boolean + isPublic: Boolean lastFetchedAt: Date lastFetchedChecksum: String name: String diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 8d5b74a83..20d9825c1 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -213,6 +213,7 @@ export const subscribeResolver = authorized< scheduledDates: [new Date()], // fetch immediately fetchedDates: [updatedSubscription.lastFetchedAt || null], checksums: [updatedSubscription.lastFetchedChecksum || null], + isFetchingContents: [!!updatedSubscription.isFetchingContent], }) return { @@ -228,7 +229,7 @@ export const subscribeResolver = authorized< // limit number of rss subscriptions to 150 const results = (await getRepository(Subscription).query( `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) - select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions + select $1, $2, $3, $4, $5, $6, $8 from omnivore.subscriptions where user_id = $5 and type = 'RSS' and status = 'ACTIVE' having count(*) < $7 returning *;`, @@ -240,6 +241,7 @@ export const subscribeResolver = authorized< uid, feed.image?.url || null, MAX_RSS_SUBSCRIPTIONS, + !!input.isFetchingContent, ] )) as Subscription[] @@ -259,6 +261,7 @@ export const subscribeResolver = authorized< scheduledDates: [new Date()], // fetch immediately fetchedDates: [null], checksums: [null], + isFetchingContents: [!!newSubscription.isFetchingContent], }) return { diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 3b6a21247..793fa8843 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -34,7 +34,8 @@ export function rssFeedRouter() { ARRAY_AGG(user_id) AS "userIds", ARRAY_AGG(last_fetched_at) AS "fetchedDates", ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", - ARRAY_AGG(last_fetched_checksum) AS checksums + ARRAY_AGG(last_fetched_checksum) AS checksums, + ARRAY_AGG(coalesce(is_fetching_content, false)) AS "isFetchingContents", FROM omnivore.subscriptions WHERE diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 4cd073b8f..e272c9d76 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2554,6 +2554,8 @@ const schema = gql` input SubscribeInput { url: String! subscriptionType: SubscriptionType + isPublic: Boolean + isFetchingContent: Boolean } input UpdateSubscriptionInput { @@ -2564,6 +2566,8 @@ const schema = gql` lastFetchedChecksum: String status: SubscriptionStatus scheduledAt: Date + isPublic: Boolean + isFetchingContent: Boolean } union UpdateSubscriptionResult = diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 643772116..8764f4495 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -615,6 +615,7 @@ export interface RssSubscriptionGroup { fetchedDates: (Date | null)[] scheduledDates: Date[] checksums: (string | null)[] + isFetchingContents: boolean[] } export const enqueueRssFeedFetch = async ( @@ -632,6 +633,7 @@ export const enqueueRssFeedFetch = async ( timestamp.getTime() ), // unix timestamp in milliseconds userIds: subscriptionGroup.userIds, + isFetchingContents: subscriptionGroup.isFetchingContents, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 04abddecb..e5b9f68f2 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -14,6 +14,7 @@ interface RssFeedRequest { scheduledTimestamps: number[] // unix timestamp in milliseconds lastFetchedChecksums: string[] userIds: string[] + isFetchingContents: boolean[] } // link can be a string or an object @@ -26,7 +27,8 @@ function isRssFeedRequest(body: any): body is RssFeedRequest { 'lastFetchedTimestamps' in body && 'scheduledTimestamps' in body && 'userIds' in body && - 'lastFetchedChecksums' in body + 'lastFetchedChecksums' in body && + 'isFetchingContents' in body ) } From 2c9ec454f585be7268c77081db4b3793a834bbd5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Nov 2023 20:13:09 +0800 Subject: [PATCH 09/30] fix tests --- packages/api/src/resolvers/subscriptions/index.ts | 8 ++++---- packages/api/src/routers/svc/rss_feed.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 20d9825c1..a2fa5e8c4 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -228,10 +228,10 @@ export const subscribeResolver = authorized< // limit number of rss subscriptions to 150 const results = (await getRepository(Subscription).query( - `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) - select $1, $2, $3, $4, $5, $6, $8 from omnivore.subscriptions + `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, is_fetching_content) + select $1, $2, $3, $4, $5, $6, $7 from omnivore.subscriptions where user_id = $5 and type = 'RSS' and status = 'ACTIVE' - having count(*) < $7 + having count(*) < $8 returning *;`, [ feed.title, @@ -240,8 +240,8 @@ export const subscribeResolver = authorized< SubscriptionType.Rss, uid, feed.image?.url || null, - MAX_RSS_SUBSCRIPTIONS, !!input.isFetchingContent, + MAX_RSS_SUBSCRIPTIONS, ] )) as Subscription[] diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 793fa8843..597a49e7c 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -35,7 +35,7 @@ export function rssFeedRouter() { ARRAY_AGG(last_fetched_at) AS "fetchedDates", ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", ARRAY_AGG(last_fetched_checksum) AS checksums, - ARRAY_AGG(coalesce(is_fetching_content, false)) AS "isFetchingContents", + ARRAY_AGG(coalesce(is_fetching_content, false)) AS "isFetchingContents" FROM omnivore.subscriptions WHERE From 98faefa6444b01b3e5dfe2da4df62d29ec02bc3d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 6 Nov 2023 18:38:04 +0800 Subject: [PATCH 10/30] add date variables to date search --- packages/api/src/utils/search.ts | 51 +++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index c83237500..d9bd61e5a 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -3,6 +3,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-call */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { DateTime } from 'luxon' import { ISearchParserDictionary, parse, @@ -244,21 +245,49 @@ const parseDateFilter = ( return undefined } + switch (field.toLowerCase()) { + case 'published': + field = 'published_at' + break + case 'saved': + field = 'saved_at' + break + case 'updated': + field = 'updated_at' + } + + // check for special date filters + switch (str.toLowerCase()) { + case 'today': + return { + field, + startDate: DateTime.local().startOf('day').toJSDate(), + } + case 'yesterday': { + const yesterday = DateTime.local().minus({ days: 1 }) + return { + field, + startDate: yesterday.startOf('day').toJSDate(), + endDate: yesterday.endOf('day').toJSDate(), + } + } + case 'this week': + return { + field, + startDate: DateTime.local().startOf('week').toJSDate(), + } + case 'this month': + return { + field, + startDate: DateTime.local().startOf('month').toJSDate(), + } + } + + // check for date ranges const [start, end] = str.split('..') const startDate = start && start !== '*' ? new Date(start) : undefined const endDate = end && end !== '*' ? new Date(end) : undefined - switch (field.toUpperCase()) { - case 'PUBLISHED': - field = 'published_at' - break - case 'SAVED': - field = 'saved_at' - break - case 'UPDATED': - field = 'updated_at' - } - return { field, startDate, From a00ce8893ec2d65ee6fa32a9bb3d66fd77efd15d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 6 Nov 2023 20:49:43 +0800 Subject: [PATCH 11/30] reduce feed parsing --- packages/rss-handler/src/index.ts | 48 +++++++++++++++++++------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index e5b9f68f2..dfa7f3294 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -247,7 +247,20 @@ const processSubscription = async ( fetchResult: { content: string; checksum: string }, lastFetchedAt: number, scheduledAt: number, - lastFetchedChecksum: string + lastFetchedChecksum: string, + feed: { + lastBuildDate: any + 'syn:updatePeriod': any + 'syn:updateFrequency': any + 'sy:updatePeriod': any + 'sy:updateFrequency': any + } & Parser.Output<{ + published: any + updated: any + created: any + link: any + links: any[] + }> ) => { let lastItemFetchedAt: Date | null = null let lastValidItem: Item | null = null @@ -260,8 +273,6 @@ const processSubscription = async ( // fetch feed let itemCount = 0 - const feed = await parser.parseString(fetchResult.content) - console.log('Fetched feed', feed.title, new Date()) const feedLastBuildDate = feed.lastBuildDate as string | undefined console.log('Feed last build date', feedLastBuildDate) @@ -392,24 +403,23 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( console.log('Processing feed', feedUrl) const fetchResult = await fetchAndChecksum(feedUrl) + const feed = await parser.parseString(fetchResult.content) + console.log('Fetched feed', feed.title, new Date()) - for (let i = 0; i < subscriptionIds.length; i++) { - const subscriptionId = subscriptionIds[i] - const lastFetchedAt = lastFetchedTimestamps[i] - const scheduledAt = scheduledTimestamps[i] - const userId = userIds[i] - const lastFetchedChecksum = lastFetchedChecksums[i] - - await processSubscription( - subscriptionId, - userId, - feedUrl, - fetchResult, - lastFetchedAt, - scheduledAt, - lastFetchedChecksum + await Promise.all( + subscriptionIds.map((_, i) => + processSubscription( + subscriptionIds[i], + userIds[i], + feedUrl, + fetchResult, + lastFetchedTimestamps[i], + scheduledTimestamps[i], + lastFetchedChecksums[i], + feed + ) ) - } + ) res.send('ok') } catch (e) { From 3ff9ce3a097ab5ef89065a2428ab98b1d97c74fc Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 7 Nov 2023 16:48:55 +0800 Subject: [PATCH 12/30] add in:following search filter --- packages/api/src/resolvers/article/index.ts | 8 ++- packages/api/src/services/library_item.ts | 63 ++++++++++--------- packages/api/src/utils/search.ts | 16 +---- ...do.following.sql => 0146.do.following.sql} | 0 ....following.sql => 0146.undo.following.sql} | 0 5 files changed, 41 insertions(+), 46 deletions(-) rename packages/db/migrations/{0145.do.following.sql => 0146.do.following.sql} (100%) rename packages/db/migrations/{0145.undo.following.sql => 0146.undo.following.sql} (100%) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 8a4c8c8c3..6a2d03e55 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -95,7 +95,11 @@ import { ParsedContentPuppeteer, parsePreparedContent, } from '../../utils/parser' -import { parseSearchQuery, sortParamsToSort } from '../../utils/search' +import { + InFilter, + parseSearchQuery, + sortParamsToSort, +} from '../../utils/search' import { getStorageFileDetails } from '../../utils/uploads' import { itemTypeForContentType } from '../upload_files' @@ -736,7 +740,7 @@ export const updatesSinceResolver = authorized< const sort = sortParamsToSort(sortParams) const startCursor = after || '' - const size = first || 10 + const size = Math.min(first || 10, 100) // limit to 100 items let startDate = new Date(since) if (isNaN(startDate.getTime())) { // for android app compatibility diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 4448a5964..fd79746ef 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -103,40 +103,41 @@ const buildWhereClause = ( }) } - if (args.inFilter !== InFilter.FOLLOWING) { - queryBuilder.andWhere('library_item.is_in_library = true') + if (args.inFilter !== InFilter.ALL) { + if (args.inFilter === InFilter.FOLLOWING) { + queryBuilder + .andWhere('library_item.shared_by IS NOT NULL') + .andWhere('library_item.hidden_at IS NULL') + } else { + queryBuilder.andWhere('library_item.is_in_library = true') - switch (args.inFilter) { - case InFilter.INBOX: - queryBuilder.andWhere('library_item.archived_at IS NULL') - break - case InFilter.ARCHIVE: - queryBuilder.andWhere('library_item.archived_at IS NOT NULL') - break - case InFilter.TRASH: - // return only deleted pages within 14 days - queryBuilder.andWhere( - "library_item.deleted_at >= now() - interval '14 days'" - ) - break - case InFilter.SUBSCRIPTION: - queryBuilder - .andWhere("NOT ('library' ILIKE ANY (library_item.label_names))") - .andWhere('library_item.archived_at IS NULL') - .andWhere('library_item.subscription IS NOT NULL') - break - case InFilter.LIBRARY: - queryBuilder - .andWhere( - "(library_item.subscription IS NULL OR 'library' ILIKE ANY (library_item.label_names))" + switch (args.inFilter) { + case InFilter.INBOX: + queryBuilder.andWhere('library_item.archived_at IS NULL') + break + case InFilter.ARCHIVE: + queryBuilder.andWhere('library_item.archived_at IS NOT NULL') + break + case InFilter.TRASH: + // return only deleted pages within 14 days + queryBuilder.andWhere( + "library_item.deleted_at >= now() - interval '14 days'" ) - .andWhere('library_item.archived_at IS NULL') - break + break + case InFilter.SUBSCRIPTION: + queryBuilder + .andWhere("NOT ('library' ILIKE ANY (library_item.label_names))") + .andWhere('library_item.archived_at IS NULL') + .andWhere('library_item.subscription IS NOT NULL') + break + case InFilter.LIBRARY: + queryBuilder + .andWhere( + "(library_item.subscription IS NULL OR 'library' ILIKE ANY (library_item.label_names))" + ) + .andWhere('library_item.archived_at IS NULL') + } } - } else { - queryBuilder - .andWhere('library_item.shared_by IS NOT NULL') - .andWhere('library_item.hidden_at IS NULL') } if (args.readFilter !== ReadFilter.ALL) { diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index d9bd61e5a..60f6d64e0 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -138,6 +138,8 @@ const parseInFilter = ( return InFilter.SUBSCRIPTION case 'LIBRARY': return InFilter.LIBRARY + case 'FOLLOWING': + return InFilter.FOLLOWING } return query ? InFilter.ALL : InFilter.INBOX @@ -409,19 +411,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { } if (!searchQuery) { - return { - query: undefined, - inFilter: InFilter.INBOX, - readFilter: ReadFilter.ALL, - labelFilters: [], - hasFilters: [], - dateFilters: [], - termFilters: [], - matchFilters: [], - ids: [], - noFilters: [], - rangeFilters: [], - } + return result } const parsed = parse(searchQuery, { diff --git a/packages/db/migrations/0145.do.following.sql b/packages/db/migrations/0146.do.following.sql similarity index 100% rename from packages/db/migrations/0145.do.following.sql rename to packages/db/migrations/0146.do.following.sql diff --git a/packages/db/migrations/0145.undo.following.sql b/packages/db/migrations/0146.undo.following.sql similarity index 100% rename from packages/db/migrations/0145.undo.following.sql rename to packages/db/migrations/0146.undo.following.sql From 99616501cfbf41982182dc646dcbc61818e32392 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 8 Nov 2023 09:40:39 +0800 Subject: [PATCH 13/30] add saveFollowing API --- packages/api/src/entity/library_item.ts | 5 +- packages/api/src/generated/graphql.ts | 120 ++++++++++++++++-- packages/api/src/generated/schema.graphql | 55 +++++++- packages/api/src/resolvers/following/index.ts | 23 ++++ packages/api/src/schema.ts | 57 +++++++-- packages/api/src/services/library_item.ts | 24 +++- packages/db/migrations/0146.do.following.sql | 1 + .../db/migrations/0146.undo.following.sql | 1 + 8 files changed, 254 insertions(+), 32 deletions(-) diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 07cc2b879..e2cf946d6 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -207,7 +207,7 @@ export class LibraryItem { sharedBy?: string | null @Column('jsonb') - links?: Record[] | null + links?: any | null @Column('text') previewContent?: string | null @@ -217,4 +217,7 @@ export class LibraryItem { @Column('boolean') isInLibrary!: boolean + + @Column('text') + sharedSource?: string | null } diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index d3f3147c8..86039cb1a 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -837,6 +837,26 @@ export type FiltersSuccess = { filters: Array; }; +export type Following = { + __typename?: 'Following'; + SharedAt: Scalars['Date']; + author?: Maybe; + createdAt: Scalars['Date']; + description?: Maybe; + hiddenAt?: Maybe; + id: Scalars['ID']; + image?: Maybe; + links?: Maybe; + previewContent?: Maybe; + publishedAt?: Maybe; + seenAt?: Maybe; + sharedBy: Scalars['String']; + sharedSource: Scalars['String']; + title: Scalars['String']; + updatedAt: Scalars['Date']; + url: Scalars['String']; +}; + export type GenerateApiKeyError = { __typename?: 'GenerateApiKeyError'; errorCodes: Array; @@ -1320,6 +1340,7 @@ export type Mutation = { saveArticleReadingProgress: SaveArticleReadingProgressResult; saveFile: SaveResult; saveFilter: SaveFilterResult; + saveFollowing: SaveFollowingResult; savePage: SaveResult; saveUrl: SaveResult; setBookmarkArticle: SetBookmarkArticleResult; @@ -1516,6 +1537,11 @@ export type MutationSaveFilterArgs = { }; +export type MutationSaveFollowingArgs = { + input: SaveFollowingInput; +}; + + export type MutationSavePageArgs = { input: SavePageInput; }; @@ -2211,6 +2237,36 @@ export type SaveFilterSuccess = { filter: Filter; }; +export type SaveFollowingError = { + __typename?: 'SaveFollowingError'; + errorCodes: Array; +}; + +export enum SaveFollowingErrorCode { + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED' +} + +export type SaveFollowingInput = { + author?: InputMaybe; + description?: InputMaybe; + links?: InputMaybe; + previewContent?: InputMaybe; + publishedAt?: InputMaybe; + sharedAt: Scalars['Date']; + sharedBy: Scalars['String']; + sharedSource: Scalars['String']; + title: Scalars['String']; + url: Scalars['String']; +}; + +export type SaveFollowingResult = SaveFollowingError | SaveFollowingSuccess; + +export type SaveFollowingSuccess = { + __typename?: 'SaveFollowingSuccess'; + following: Following; +}; + export type SavePageInput = { clientRequestId: Scalars['ID']; labels?: InputMaybe>; @@ -2257,10 +2313,8 @@ export enum SearchErrorCode { export type SearchItem = { __typename?: 'SearchItem'; - annotation?: Maybe; archivedAt?: Maybe; author?: Maybe; - color?: Maybe; content?: Maybe; contentReader: ContentReader; createdAt: Scalars['Date']; @@ -2273,25 +2327,20 @@ export type SearchItem = { language?: Maybe; originalArticleUrl?: Maybe; ownedByViewer?: Maybe; - pageId?: Maybe; pageType: PageType; publishedAt?: Maybe; - quote?: Maybe; readAt?: Maybe; readingProgressAnchorIndex: Scalars['Int']; readingProgressPercent: Scalars['Float']; readingProgressTopPercent?: Maybe; recommendations?: Maybe>; savedAt: Scalars['Date']; - shortId?: Maybe; siteIcon?: Maybe; siteName?: Maybe; slug: Scalars['String']; state?: Maybe; subscription?: Maybe; title: Scalars['String']; - unsubHttpUrl?: Maybe; - unsubMailTo?: Maybe; updatedAt?: Maybe; uploadFileId?: Maybe; url: Scalars['String']; @@ -3532,6 +3581,7 @@ export type ResolversTypes = { FiltersResult: ResolversTypes['FiltersError'] | ResolversTypes['FiltersSuccess']; FiltersSuccess: ResolverTypeWrapper; Float: ResolverTypeWrapper; + Following: ResolverTypeWrapper; GenerateApiKeyError: ResolverTypeWrapper; GenerateApiKeyErrorCode: GenerateApiKeyErrorCode; GenerateApiKeyInput: GenerateApiKeyInput; @@ -3697,6 +3747,11 @@ export type ResolversTypes = { SaveFilterInput: SaveFilterInput; SaveFilterResult: ResolversTypes['SaveFilterError'] | ResolversTypes['SaveFilterSuccess']; SaveFilterSuccess: ResolverTypeWrapper; + SaveFollowingError: ResolverTypeWrapper; + SaveFollowingErrorCode: SaveFollowingErrorCode; + SaveFollowingInput: SaveFollowingInput; + SaveFollowingResult: ResolversTypes['SaveFollowingError'] | ResolversTypes['SaveFollowingSuccess']; + SaveFollowingSuccess: ResolverTypeWrapper; SavePageInput: SavePageInput; SaveResult: ResolversTypes['SaveError'] | ResolversTypes['SaveSuccess']; SaveSuccess: ResolverTypeWrapper; @@ -4016,6 +4071,7 @@ export type ResolversParentTypes = { FiltersResult: ResolversParentTypes['FiltersError'] | ResolversParentTypes['FiltersSuccess']; FiltersSuccess: FiltersSuccess; Float: Scalars['Float']; + Following: Following; GenerateApiKeyError: GenerateApiKeyError; GenerateApiKeyInput: GenerateApiKeyInput; GenerateApiKeyResult: ResolversParentTypes['GenerateApiKeyError'] | ResolversParentTypes['GenerateApiKeySuccess']; @@ -4145,6 +4201,10 @@ export type ResolversParentTypes = { SaveFilterInput: SaveFilterInput; SaveFilterResult: ResolversParentTypes['SaveFilterError'] | ResolversParentTypes['SaveFilterSuccess']; SaveFilterSuccess: SaveFilterSuccess; + SaveFollowingError: SaveFollowingError; + SaveFollowingInput: SaveFollowingInput; + SaveFollowingResult: ResolversParentTypes['SaveFollowingError'] | ResolversParentTypes['SaveFollowingSuccess']; + SaveFollowingSuccess: SaveFollowingSuccess; SavePageInput: SavePageInput; SaveResult: ResolversParentTypes['SaveError'] | ResolversParentTypes['SaveSuccess']; SaveSuccess: SaveSuccess; @@ -4898,6 +4958,26 @@ export type FiltersSuccessResolvers; }; +export type FollowingResolvers = { + SharedAt?: Resolver; + author?: Resolver, ParentType, ContextType>; + createdAt?: Resolver; + description?: Resolver, ParentType, ContextType>; + hiddenAt?: Resolver, ParentType, ContextType>; + id?: Resolver; + image?: Resolver, ParentType, ContextType>; + links?: Resolver, ParentType, ContextType>; + previewContent?: Resolver, ParentType, ContextType>; + publishedAt?: Resolver, ParentType, ContextType>; + seenAt?: Resolver, ParentType, ContextType>; + sharedBy?: Resolver; + sharedSource?: Resolver; + title?: Resolver; + updatedAt?: Resolver; + url?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type GenerateApiKeyErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -5263,6 +5343,7 @@ export type MutationResolvers>; saveFile?: Resolver>; saveFilter?: Resolver>; + saveFollowing?: Resolver>; savePage?: Resolver>; saveUrl?: Resolver>; setBookmarkArticle?: Resolver>; @@ -5622,6 +5703,20 @@ export type SaveFilterSuccessResolvers; }; +export type SaveFollowingErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type SaveFollowingResultResolvers = { + __resolveType: TypeResolveFn<'SaveFollowingError' | 'SaveFollowingSuccess', ParentType, ContextType>; +}; + +export type SaveFollowingSuccessResolvers = { + following?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type SaveResultResolvers = { __resolveType: TypeResolveFn<'SaveError' | 'SaveSuccess', ParentType, ContextType>; }; @@ -5638,10 +5733,8 @@ export type SearchErrorResolvers = { - annotation?: Resolver, ParentType, ContextType>; archivedAt?: Resolver, ParentType, ContextType>; author?: Resolver, ParentType, ContextType>; - color?: Resolver, ParentType, ContextType>; content?: Resolver, ParentType, ContextType>; contentReader?: Resolver; createdAt?: Resolver; @@ -5654,25 +5747,20 @@ export type SearchItemResolvers, ParentType, ContextType>; originalArticleUrl?: Resolver, ParentType, ContextType>; ownedByViewer?: Resolver, ParentType, ContextType>; - pageId?: Resolver, ParentType, ContextType>; pageType?: Resolver; publishedAt?: Resolver, ParentType, ContextType>; - quote?: Resolver, ParentType, ContextType>; readAt?: Resolver, ParentType, ContextType>; readingProgressAnchorIndex?: Resolver; readingProgressPercent?: Resolver; readingProgressTopPercent?: Resolver, ParentType, ContextType>; recommendations?: Resolver>, ParentType, ContextType>; savedAt?: Resolver; - shortId?: Resolver, ParentType, ContextType>; siteIcon?: Resolver, ParentType, ContextType>; siteName?: Resolver, ParentType, ContextType>; slug?: Resolver; state?: Resolver, ParentType, ContextType>; subscription?: Resolver, ParentType, ContextType>; title?: Resolver; - unsubHttpUrl?: Resolver, ParentType, ContextType>; - unsubMailTo?: Resolver, ParentType, ContextType>; updatedAt?: Resolver, ParentType, ContextType>; uploadFileId?: Resolver, ParentType, ContextType>; url?: Resolver; @@ -6403,6 +6491,7 @@ export type Resolvers = { FiltersError?: FiltersErrorResolvers; FiltersResult?: FiltersResultResolvers; FiltersSuccess?: FiltersSuccessResolvers; + Following?: FollowingResolvers; GenerateApiKeyError?: GenerateApiKeyErrorResolvers; GenerateApiKeyResult?: GenerateApiKeyResultResolvers; GenerateApiKeySuccess?: GenerateApiKeySuccessResolvers; @@ -6513,6 +6602,9 @@ export type Resolvers = { SaveFilterError?: SaveFilterErrorResolvers; SaveFilterResult?: SaveFilterResultResolvers; SaveFilterSuccess?: SaveFilterSuccessResolvers; + SaveFollowingError?: SaveFollowingErrorResolvers; + SaveFollowingResult?: SaveFollowingResultResolvers; + SaveFollowingSuccess?: SaveFollowingSuccessResolvers; SaveResult?: SaveResultResolvers; SaveSuccess?: SaveSuccessResolvers; SearchError?: SearchErrorResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 5e1ae27af..120c9d275 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -742,6 +742,25 @@ type FiltersSuccess { filters: [Filter!]! } +type Following { + SharedAt: Date! + author: String + createdAt: Date! + description: String + hiddenAt: Date + id: ID! + image: String + links: JSON + previewContent: String + publishedAt: Date + seenAt: Date + sharedBy: String! + sharedSource: String! + title: String! + updatedAt: Date! + url: String! +} + type GenerateApiKeyError { errorCodes: [GenerateApiKeyErrorCode!]! } @@ -1185,6 +1204,7 @@ type Mutation { saveArticleReadingProgress(input: SaveArticleReadingProgressInput!): SaveArticleReadingProgressResult! saveFile(input: SaveFileInput!): SaveResult! saveFilter(input: SaveFilterInput!): SaveFilterResult! + saveFollowing(input: SaveFollowingInput!): SaveFollowingResult! savePage(input: SavePageInput!): SaveResult! saveUrl(input: SaveUrlInput!): SaveResult! setBookmarkArticle(input: SetBookmarkArticleInput!): SetBookmarkArticleResult! @@ -1678,6 +1698,34 @@ type SaveFilterSuccess { filter: Filter! } +type SaveFollowingError { + errorCodes: [SaveFollowingErrorCode!]! +} + +enum SaveFollowingErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input SaveFollowingInput { + author: String + description: String + links: JSON + previewContent: String + publishedAt: Date + sharedAt: Date! + sharedBy: String! + sharedSource: String! + title: String! + url: String! +} + +union SaveFollowingResult = SaveFollowingError | SaveFollowingSuccess + +type SaveFollowingSuccess { + following: Following! +} + input SavePageInput { clientRequestId: ID! labels: [CreateLabelInput!] @@ -1721,10 +1769,8 @@ enum SearchErrorCode { } type SearchItem { - annotation: String archivedAt: Date author: String - color: String content: String contentReader: ContentReader! createdAt: Date! @@ -1737,25 +1783,20 @@ type SearchItem { language: String originalArticleUrl: String ownedByViewer: Boolean - pageId: ID pageType: PageType! publishedAt: Date - quote: String readAt: Date readingProgressAnchorIndex: Int! readingProgressPercent: Float! readingProgressTopPercent: Float recommendations: [Recommendation!] savedAt: Date! - shortId: String siteIcon: String siteName: String slug: String! state: ArticleSavingRequestStatus subscription: String title: String! - unsubHttpUrl: String - unsubMailTo: String updatedAt: Date uploadFileId: ID url: String! diff --git a/packages/api/src/resolvers/following/index.ts b/packages/api/src/resolvers/following/index.ts index 08c219436..b97951e54 100644 --- a/packages/api/src/resolvers/following/index.ts +++ b/packages/api/src/resolvers/following/index.ts @@ -3,9 +3,13 @@ import { FeedsError, FeedsErrorCode, FeedsSuccess, + MutationSaveFollowingArgs, QueryFeedsArgs, + SaveFollowingError, + SaveFollowingSuccess, } from '../../generated/graphql' import { feedRepository } from '../../repository/feed' +import { createFollowing } from '../../services/library_item' import { authorized } from '../../utils/helpers' export const feedsResolve = authorized< @@ -59,3 +63,22 @@ export const feedsResolve = authorized< } } }) + +export const saveFollowingResolver = authorized< + SaveFollowingSuccess, + SaveFollowingError, + MutationSaveFollowingArgs +>(async (_, { input }, { uid }) => { + const newItem = await createFollowing(input, uid) + + return { + __typename: 'SaveFollowingSuccess', + following: { + ...newItem, + url: newItem.originalUrl, + SharedAt: new Date(input.sharedAt), + sharedBy: input.sharedBy, + sharedSource: input.sharedSource, + }, + } +}) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e272c9d76..f5420fc29 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1568,7 +1568,6 @@ const schema = gql` union SearchResult = SearchSuccess | SearchError type SearchItem { - # used for pages id: ID! title: String! slug: String! @@ -1590,16 +1589,8 @@ const schema = gql` # for uploaded file articles (PDFs), we track the original article URL separately! originalArticleUrl: String uploadFileId: ID - # used for highlights - pageId: ID - shortId: String - quote: String - annotation: String - color: String labels: [Label!] subscription: String - unsubMailTo: String - unsubHttpUrl: String state: ArticleSavingRequestStatus siteName: String language: String @@ -2649,6 +2640,53 @@ const schema = gql` author: String } + union SaveFollowingResult = SaveFollowingSuccess | SaveFollowingError + + type SaveFollowingSuccess { + following: Following! + } + + type Following { + id: ID! + title: String! + url: String! + author: String + image: String + description: String + seenAt: Date + createdAt: Date! + updatedAt: Date! + publishedAt: Date + hiddenAt: Date + SharedAt: Date! + sharedBy: String! + links: JSON + previewContent: String + sharedSource: String! + } + + type SaveFollowingError { + errorCodes: [SaveFollowingErrorCode!]! + } + + enum SaveFollowingErrorCode { + UNAUTHORIZED + BAD_REQUEST + } + + input SaveFollowingInput { + url: String! + title: String! + author: String + description: String + publishedAt: Date + sharedSource: String! + links: JSON + previewContent: String + sharedBy: String! + sharedAt: Date! + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2752,6 +2790,7 @@ const schema = gql` updateSubscription( input: UpdateSubscriptionInput! ): UpdateSubscriptionResult! + saveFollowing(input: SaveFollowingInput!): SaveFollowingResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index fd79746ef..8f462849c 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -4,7 +4,7 @@ import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' import { LibraryItem, LibraryItemState } from '../entity/library_item' -import { BulkActionType } from '../generated/graphql' +import { BulkActionType, SaveFollowingInput } from '../generated/graphql' import { createPubSubClient, EntityType } from '../pubsub' import { authTrx, getColumns } from '../repository' import { libraryItemRepository } from '../repository/library_item' @@ -567,6 +567,28 @@ export const createLibraryItem = async ( return newLibraryItem } +export const createFollowing = async ( + input: SaveFollowingInput, + userId: string +): Promise => { + return createLibraryItem( + { + ...input, + originalUrl: input.url, + isInLibrary: false, + state: LibraryItemState.Succeeded, + wordCount: 0, + user: { id: userId }, + sharedAt: new Date(input.sharedAt), + sharedSource: input.sharedSource, + sharedBy: input.sharedBy, + }, + userId, + undefined, + true + ) +} + export const findLibraryItemsByPrefix = async ( prefix: string, userId: string, diff --git a/packages/db/migrations/0146.do.following.sql b/packages/db/migrations/0146.do.following.sql index ba024fe7c..318aa3b59 100755 --- a/packages/db/migrations/0146.do.following.sql +++ b/packages/db/migrations/0146.do.following.sql @@ -15,6 +15,7 @@ ALTER TABLE omnivore.library_item ADD COLUMN links jsonb, ADD COLUMN preview_content text, ADD COLUMN seen_at timestamptz, + ADD COLUMN shared_source text, ADD COLUMN is_in_library boolean NOT NULL DEFAULT true; CREATE TABLE omnivore.feed ( diff --git a/packages/db/migrations/0146.undo.following.sql b/packages/db/migrations/0146.undo.following.sql index 2c323ec4b..e134e80a2 100755 --- a/packages/db/migrations/0146.undo.following.sql +++ b/packages/db/migrations/0146.undo.following.sql @@ -13,6 +13,7 @@ ALTER TABLE omnivore.library_item DROP COLUMN links, DROP COLUMN preview_content, DROP COLUMN seen_at, + DROP COLUMN shared_source, DROP COLUMN is_in_library; ALTER TABLE omnivore.subscriptions From cc53992b9a182115399cbefdced334bdc5326764 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 8 Nov 2023 11:43:11 +0800 Subject: [PATCH 14/30] add addFollowingToLibrary API --- packages/api/src/generated/graphql.ts | 49 +++++++++++++++ packages/api/src/generated/schema.graphql | 17 +++++ packages/api/src/resolvers/following/index.ts | 62 +++++++++++++++++++ packages/api/src/schema.ts | 19 ++++++ 4 files changed, 147 insertions(+) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 86039cb1a..b52e38e24 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -17,6 +17,24 @@ export type Scalars = { JSON: any; }; +export type AddFollowingToLibraryError = { + __typename?: 'AddFollowingToLibraryError'; + errorCodes: Array; +}; + +export enum AddFollowingToLibraryErrorCode { + BadRequest = 'BAD_REQUEST', + NotFound = 'NOT_FOUND', + Unauthorized = 'UNAUTHORIZED' +} + +export type AddFollowingToLibraryResult = AddFollowingToLibraryError | AddFollowingToLibrarySuccess; + +export type AddFollowingToLibrarySuccess = { + __typename?: 'AddFollowingToLibrarySuccess'; + articleSavingRequest: ArticleSavingRequest; +}; + export type AddPopularReadError = { __typename?: 'AddPopularReadError'; errorCodes: Array; @@ -1305,6 +1323,7 @@ export type MoveLabelSuccess = { export type Mutation = { __typename?: 'Mutation'; + addFollowingToLibrary: AddFollowingToLibraryResult; addPopularRead: AddPopularReadResult; bulkAction: BulkActionResult; createArticle: CreateArticleResult; @@ -1368,6 +1387,11 @@ export type Mutation = { }; +export type MutationAddFollowingToLibraryArgs = { + id: Scalars['ID']; +}; + + export type MutationAddPopularReadArgs = { name: Scalars['String']; }; @@ -3429,6 +3453,10 @@ export type DirectiveResolverFn; + AddFollowingToLibraryErrorCode: AddFollowingToLibraryErrorCode; + AddFollowingToLibraryResult: ResolversTypes['AddFollowingToLibraryError'] | ResolversTypes['AddFollowingToLibrarySuccess']; + AddFollowingToLibrarySuccess: ResolverTypeWrapper; AddPopularReadError: ResolverTypeWrapper; AddPopularReadErrorCode: AddPopularReadErrorCode; AddPopularReadResult: ResolversTypes['AddPopularReadError'] | ResolversTypes['AddPopularReadSuccess']; @@ -3953,6 +3981,9 @@ export type ResolversTypes = { /** Mapping between all available schema types and the resolvers parents */ export type ResolversParentTypes = { + AddFollowingToLibraryError: AddFollowingToLibraryError; + AddFollowingToLibraryResult: ResolversParentTypes['AddFollowingToLibraryError'] | ResolversParentTypes['AddFollowingToLibrarySuccess']; + AddFollowingToLibrarySuccess: AddFollowingToLibrarySuccess; AddPopularReadError: AddPopularReadError; AddPopularReadResult: ResolversParentTypes['AddPopularReadError'] | ResolversParentTypes['AddPopularReadSuccess']; AddPopularReadSuccess: AddPopularReadSuccess; @@ -4367,6 +4398,20 @@ export type SanitizeDirectiveArgs = { export type SanitizeDirectiveResolver = DirectiveResolverFn; +export type AddFollowingToLibraryErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type AddFollowingToLibraryResultResolvers = { + __resolveType: TypeResolveFn<'AddFollowingToLibraryError' | 'AddFollowingToLibrarySuccess', ParentType, ContextType>; +}; + +export type AddFollowingToLibrarySuccessResolvers = { + articleSavingRequest?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type AddPopularReadErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -5308,6 +5353,7 @@ export type MoveLabelSuccessResolvers = { + addFollowingToLibrary?: Resolver>; addPopularRead?: Resolver>; bulkAction?: Resolver>; createArticle?: Resolver>; @@ -6386,6 +6432,9 @@ export type WebhooksSuccessResolvers = { + AddFollowingToLibraryError?: AddFollowingToLibraryErrorResolvers; + AddFollowingToLibraryResult?: AddFollowingToLibraryResultResolvers; + AddFollowingToLibrarySuccess?: AddFollowingToLibrarySuccessResolvers; AddPopularReadError?: AddPopularReadErrorResolvers; AddPopularReadResult?: AddPopularReadResultResolvers; AddPopularReadSuccess?: AddPopularReadSuccessResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 120c9d275..848ef48d8 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1,5 +1,21 @@ directive @sanitize(allowedTags: [String], maxLength: Int, minLength: Int, pattern: String) on INPUT_FIELD_DEFINITION +type AddFollowingToLibraryError { + errorCodes: [AddFollowingToLibraryErrorCode!]! +} + +enum AddFollowingToLibraryErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union AddFollowingToLibraryResult = AddFollowingToLibraryError | AddFollowingToLibrarySuccess + +type AddFollowingToLibrarySuccess { + articleSavingRequest: ArticleSavingRequest! +} + type AddPopularReadError { errorCodes: [AddPopularReadErrorCode!]! } @@ -1169,6 +1185,7 @@ type MoveLabelSuccess { } type Mutation { + addFollowingToLibrary(id: ID!): AddFollowingToLibraryResult! addPopularRead(name: String!): AddPopularReadResult! bulkAction(action: BulkActionType!, async: Boolean, expectedCount: Int, labelIds: [ID!], query: String!): BulkActionResult! createArticle(input: CreateArticleInput!): CreateArticleResult! diff --git a/packages/api/src/resolvers/following/index.ts b/packages/api/src/resolvers/following/index.ts index b97951e54..d7ea0129d 100644 --- a/packages/api/src/resolvers/following/index.ts +++ b/packages/api/src/resolvers/following/index.ts @@ -1,15 +1,23 @@ +import { IsNull, Not } from 'typeorm' +import { LibraryItem } from '../../entity/library_item' import { + AddFollowingToLibraryError, + AddFollowingToLibraryErrorCode, + AddFollowingToLibrarySuccess, FeedEdge, FeedsError, FeedsErrorCode, FeedsSuccess, + MutationAddFollowingToLibraryArgs, MutationSaveFollowingArgs, QueryFeedsArgs, SaveFollowingError, SaveFollowingSuccess, } from '../../generated/graphql' import { feedRepository } from '../../repository/feed' +import { createPageSaveRequest } from '../../services/create_page_save_request' import { createFollowing } from '../../services/library_item' +import { analytics } from '../../utils/analytics' import { authorized } from '../../utils/helpers' export const feedsResolve = authorized< @@ -69,6 +77,14 @@ export const saveFollowingResolver = authorized< SaveFollowingError, MutationSaveFollowingArgs >(async (_, { input }, { uid }) => { + analytics.track({ + userId: uid, + event: 'save_following', + properties: { + url: input.url, + }, + }) + const newItem = await createFollowing(input, uid) return { @@ -82,3 +98,49 @@ export const saveFollowingResolver = authorized< }, } }) + +export const addFollowingToLibraryResolver = authorized< + AddFollowingToLibrarySuccess, + AddFollowingToLibraryError, + MutationAddFollowingToLibraryArgs +>(async (_, { id }, { authTrx, pubsub, uid }) => { + analytics.track({ + userId: uid, + event: 'add_following_to_library', + properties: { + id, + }, + }) + + const item = await authTrx((tx) => + tx.getRepository(LibraryItem).findOne({ + where: { + id, + sharedAt: Not(IsNull()), + isInLibrary: false, + }, + relations: ['user'], + }) + ) + + if (!item) { + return { + errorCodes: [AddFollowingToLibraryErrorCode.NotFound], + } + } + + const articleSavingRequest = await createPageSaveRequest({ + userId: uid, + url: item.originalUrl, + articleSavingRequestId: id, + priority: 'high', + publishedAt: item.publishedAt || undefined, + savedAt: item.savedAt || undefined, + pubsub, + }) + + return { + __typename: 'AddFollowingToLibrarySuccess', + articleSavingRequest, + } +}) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index f5420fc29..f1c4ed27c 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2687,6 +2687,24 @@ const schema = gql` sharedAt: Date! } + union AddFollowingToLibraryResult = + AddFollowingToLibrarySuccess + | AddFollowingToLibraryError + + type AddFollowingToLibrarySuccess { + articleSavingRequest: ArticleSavingRequest! + } + + type AddFollowingToLibraryError { + errorCodes: [AddFollowingToLibraryErrorCode!]! + } + + enum AddFollowingToLibraryErrorCode { + UNAUTHORIZED + BAD_REQUEST + NOT_FOUND + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2791,6 +2809,7 @@ const schema = gql` input: UpdateSubscriptionInput! ): UpdateSubscriptionResult! saveFollowing(input: SaveFollowingInput!): SaveFollowingResult! + addFollowingToLibrary(id: ID!): AddFollowingToLibraryResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed From cbb5a0a9216b79eeda7da582170ace40bdd39ca4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 8 Nov 2023 12:54:16 +0800 Subject: [PATCH 15/30] revert searchItem changes --- packages/api/src/generated/graphql.ts | 14 ++++++++++++++ packages/api/src/generated/schema.graphql | 7 +++++++ packages/api/src/schema.ts | 9 +++++++++ 3 files changed, 30 insertions(+) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index b52e38e24..a28aebb33 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2337,8 +2337,10 @@ export enum SearchErrorCode { export type SearchItem = { __typename?: 'SearchItem'; + annotation?: Maybe; archivedAt?: Maybe; author?: Maybe; + color?: Maybe; content?: Maybe; contentReader: ContentReader; createdAt: Scalars['Date']; @@ -2351,20 +2353,25 @@ export type SearchItem = { language?: Maybe; originalArticleUrl?: Maybe; ownedByViewer?: Maybe; + pageId?: Maybe; pageType: PageType; publishedAt?: Maybe; + quote?: Maybe; readAt?: Maybe; readingProgressAnchorIndex: Scalars['Int']; readingProgressPercent: Scalars['Float']; readingProgressTopPercent?: Maybe; recommendations?: Maybe>; savedAt: Scalars['Date']; + shortId?: Maybe; siteIcon?: Maybe; siteName?: Maybe; slug: Scalars['String']; state?: Maybe; subscription?: Maybe; title: Scalars['String']; + unsubHttpUrl?: Maybe; + unsubMailTo?: Maybe; updatedAt?: Maybe; uploadFileId?: Maybe; url: Scalars['String']; @@ -5779,8 +5786,10 @@ export type SearchErrorResolvers = { + annotation?: Resolver, ParentType, ContextType>; archivedAt?: Resolver, ParentType, ContextType>; author?: Resolver, ParentType, ContextType>; + color?: Resolver, ParentType, ContextType>; content?: Resolver, ParentType, ContextType>; contentReader?: Resolver; createdAt?: Resolver; @@ -5793,20 +5802,25 @@ export type SearchItemResolvers, ParentType, ContextType>; originalArticleUrl?: Resolver, ParentType, ContextType>; ownedByViewer?: Resolver, ParentType, ContextType>; + pageId?: Resolver, ParentType, ContextType>; pageType?: Resolver; publishedAt?: Resolver, ParentType, ContextType>; + quote?: Resolver, ParentType, ContextType>; readAt?: Resolver, ParentType, ContextType>; readingProgressAnchorIndex?: Resolver; readingProgressPercent?: Resolver; readingProgressTopPercent?: Resolver, ParentType, ContextType>; recommendations?: Resolver>, ParentType, ContextType>; savedAt?: Resolver; + shortId?: Resolver, ParentType, ContextType>; siteIcon?: Resolver, ParentType, ContextType>; siteName?: Resolver, ParentType, ContextType>; slug?: Resolver; state?: Resolver, ParentType, ContextType>; subscription?: Resolver, ParentType, ContextType>; title?: Resolver; + unsubHttpUrl?: Resolver, ParentType, ContextType>; + unsubMailTo?: Resolver, ParentType, ContextType>; updatedAt?: Resolver, ParentType, ContextType>; uploadFileId?: Resolver, ParentType, ContextType>; url?: Resolver; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 848ef48d8..fcb09a256 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1786,8 +1786,10 @@ enum SearchErrorCode { } type SearchItem { + annotation: String archivedAt: Date author: String + color: String content: String contentReader: ContentReader! createdAt: Date! @@ -1800,20 +1802,25 @@ type SearchItem { language: String originalArticleUrl: String ownedByViewer: Boolean + pageId: ID pageType: PageType! publishedAt: Date + quote: String readAt: Date readingProgressAnchorIndex: Int! readingProgressPercent: Float! readingProgressTopPercent: Float recommendations: [Recommendation!] savedAt: Date! + shortId: String siteIcon: String siteName: String slug: String! state: ArticleSavingRequestStatus subscription: String title: String! + unsubHttpUrl: String + unsubMailTo: String updatedAt: Date uploadFileId: ID url: String! diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index f1c4ed27c..c794d0ad0 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1568,6 +1568,7 @@ const schema = gql` union SearchResult = SearchSuccess | SearchError type SearchItem { + # used for pages id: ID! title: String! slug: String! @@ -1589,8 +1590,16 @@ const schema = gql` # for uploaded file articles (PDFs), we track the original article URL separately! originalArticleUrl: String uploadFileId: ID + # used for highlights + pageId: ID + shortId: String + quote: String + annotation: String + color: String labels: [Label!] subscription: String + unsubMailTo: String + unsubHttpUrl: String state: ArticleSavingRequestStatus siteName: String language: String From 4bef0a006aa9024529982b3db06964f13993de57 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 8 Nov 2023 15:23:49 +0800 Subject: [PATCH 16/30] add following handler cloud run service --- packages/following-handler/.dockerignore | 5 +++ packages/following-handler/.eslintignore | 2 ++ packages/following-handler/.eslintrc | 6 ++++ packages/following-handler/.gcloudignore | 16 ++++++++++ packages/following-handler/Dockerfile | 26 ++++++++++++++++ packages/following-handler/mocha-config.json | 5 +++ packages/following-handler/package.json | 31 +++++++++++++++++++ packages/following-handler/src/index.ts | 19 ++++++++++++ .../following-handler/test/babel-register.js | 3 ++ packages/following-handler/test/stub.test.ts | 8 +++++ packages/following-handler/tsconfig.json | 8 +++++ 11 files changed, 129 insertions(+) create mode 100644 packages/following-handler/.dockerignore create mode 100644 packages/following-handler/.eslintignore create mode 100644 packages/following-handler/.eslintrc create mode 100644 packages/following-handler/.gcloudignore create mode 100644 packages/following-handler/Dockerfile create mode 100644 packages/following-handler/mocha-config.json create mode 100644 packages/following-handler/package.json create mode 100644 packages/following-handler/src/index.ts create mode 100644 packages/following-handler/test/babel-register.js create mode 100644 packages/following-handler/test/stub.test.ts create mode 100644 packages/following-handler/tsconfig.json diff --git a/packages/following-handler/.dockerignore b/packages/following-handler/.dockerignore new file mode 100644 index 000000000..d8aea4ee6 --- /dev/null +++ b/packages/following-handler/.dockerignore @@ -0,0 +1,5 @@ +node_modules +build +.env* +Dockerfile +.dockerignore diff --git a/packages/following-handler/.eslintignore b/packages/following-handler/.eslintignore new file mode 100644 index 000000000..b38db2f29 --- /dev/null +++ b/packages/following-handler/.eslintignore @@ -0,0 +1,2 @@ +node_modules/ +build/ diff --git a/packages/following-handler/.eslintrc b/packages/following-handler/.eslintrc new file mode 100644 index 000000000..e006282a6 --- /dev/null +++ b/packages/following-handler/.eslintrc @@ -0,0 +1,6 @@ +{ + "extends": "../../.eslintrc", + "parserOptions": { + "project": "tsconfig.json" + } +} \ No newline at end of file diff --git a/packages/following-handler/.gcloudignore b/packages/following-handler/.gcloudignore new file mode 100644 index 000000000..ccc4eb240 --- /dev/null +++ b/packages/following-handler/.gcloudignore @@ -0,0 +1,16 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +node_modules diff --git a/packages/following-handler/Dockerfile b/packages/following-handler/Dockerfile new file mode 100644 index 000000000..85974823e --- /dev/null +++ b/packages/following-handler/Dockerfile @@ -0,0 +1,26 @@ +FROM node:18.16-alpine + +# Run everything after as non-privileged user. +WORKDIR /app + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .eslintrc . + +COPY /packages/following-handler/package.json ./packages/following-handler/package.json + +RUN yarn install --pure-lockfile + +COPY /packages/following-handler ./packages/following-handler +RUN yarn workspace @omnivore/rss-handler build + +# After building, fetch the production dependencies +RUN rm -rf /app/packages/following-handler/node_modules +RUN rm -rf /app/node_modules +RUN yarn install --pure-lockfile --production + +EXPOSE 8080 + +CMD ["yarn", "workspace", "@omnivore/following-handler", "start"] + diff --git a/packages/following-handler/mocha-config.json b/packages/following-handler/mocha-config.json new file mode 100644 index 000000000..44d1d24c1 --- /dev/null +++ b/packages/following-handler/mocha-config.json @@ -0,0 +1,5 @@ +{ + "extension": ["ts"], + "spec": "test/**/*.test.ts", + "require": "test/babel-register.js" + } \ No newline at end of file diff --git a/packages/following-handler/package.json b/packages/following-handler/package.json new file mode 100644 index 000000000..895476eea --- /dev/null +++ b/packages/following-handler/package.json @@ -0,0 +1,31 @@ +{ + "name": "@omnivore/following-handler", + "version": "1.0.0", + "main": "build/src/index.js", + "files": [ + "build/src" + ], + "license": "Apache-2.0", + "scripts": { + "test": "yarn mocha -r ts-node/register --config mocha-config.json", + "test:typecheck": "tsc --noEmit", + "lint": "eslint src --ext ts,js,tsx,jsx", + "compile": "tsc", + "build": "tsc", + "start": "functions-framework --target=followingHandler", + "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"" + }, + "devDependencies": { + "chai": "^4.3.6", + "eslint-plugin-prettier": "^4.0.0", + "mocha": "^10.0.0" + }, + "dependencies": { + "@google-cloud/functions-framework": "3.1.2", + "@sentry/serverless": "^7.77.0", + "dotenv": "^16.0.1" + }, + "volta": { + "extends": "../../package.json" + } +} diff --git a/packages/following-handler/src/index.ts b/packages/following-handler/src/index.ts new file mode 100644 index 000000000..cbc8e307c --- /dev/null +++ b/packages/following-handler/src/index.ts @@ -0,0 +1,19 @@ +import * as Sentry from '@sentry/serverless' +import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import + +dotenv.config() +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}) + +export const followingHandler = Sentry.GCPFunction.wrapHttpFunction( + (req, res) => { + if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) { + console.log('query does not include valid token') + return res.sendStatus(403) + } + + res.send('ok') + } +) diff --git a/packages/following-handler/test/babel-register.js b/packages/following-handler/test/babel-register.js new file mode 100644 index 000000000..a6f65f60a --- /dev/null +++ b/packages/following-handler/test/babel-register.js @@ -0,0 +1,3 @@ +const register = require('@babel/register').default + +register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/following-handler/test/stub.test.ts b/packages/following-handler/test/stub.test.ts new file mode 100644 index 000000000..935d1e499 --- /dev/null +++ b/packages/following-handler/test/stub.test.ts @@ -0,0 +1,8 @@ +import { expect } from 'chai' +import 'mocha' + +describe('stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) diff --git a/packages/following-handler/tsconfig.json b/packages/following-handler/tsconfig.json new file mode 100644 index 000000000..7ebe093f6 --- /dev/null +++ b/packages/following-handler/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./../../tsconfig.json", + "compilerOptions": { + "outDir": "build", + "rootDir": "." + }, + "include": ["src"] +} From a71181982f2010a575115224dbde07ba70b7494d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 9 Nov 2023 13:18:24 +0800 Subject: [PATCH 17/30] create a following task for each rss item fetched --- packages/following-handler/.dockerignore | 5 --- packages/following-handler/.eslintignore | 2 - packages/following-handler/.eslintrc | 6 --- packages/following-handler/.gcloudignore | 16 ------- packages/following-handler/Dockerfile | 26 ------------ packages/following-handler/mocha-config.json | 5 --- packages/following-handler/package.json | 31 -------------- packages/following-handler/src/index.ts | 19 --------- .../following-handler/test/babel-register.js | 3 -- packages/following-handler/test/stub.test.ts | 8 ---- packages/following-handler/tsconfig.json | 8 ---- packages/rss-handler/src/index.ts | 42 +++++++++++++++++-- packages/rss-handler/src/task.ts | 1 + 13 files changed, 40 insertions(+), 132 deletions(-) delete mode 100644 packages/following-handler/.dockerignore delete mode 100644 packages/following-handler/.eslintignore delete mode 100644 packages/following-handler/.eslintrc delete mode 100644 packages/following-handler/.gcloudignore delete mode 100644 packages/following-handler/Dockerfile delete mode 100644 packages/following-handler/mocha-config.json delete mode 100644 packages/following-handler/package.json delete mode 100644 packages/following-handler/src/index.ts delete mode 100644 packages/following-handler/test/babel-register.js delete mode 100644 packages/following-handler/test/stub.test.ts delete mode 100644 packages/following-handler/tsconfig.json diff --git a/packages/following-handler/.dockerignore b/packages/following-handler/.dockerignore deleted file mode 100644 index d8aea4ee6..000000000 --- a/packages/following-handler/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -node_modules -build -.env* -Dockerfile -.dockerignore diff --git a/packages/following-handler/.eslintignore b/packages/following-handler/.eslintignore deleted file mode 100644 index b38db2f29..000000000 --- a/packages/following-handler/.eslintignore +++ /dev/null @@ -1,2 +0,0 @@ -node_modules/ -build/ diff --git a/packages/following-handler/.eslintrc b/packages/following-handler/.eslintrc deleted file mode 100644 index e006282a6..000000000 --- a/packages/following-handler/.eslintrc +++ /dev/null @@ -1,6 +0,0 @@ -{ - "extends": "../../.eslintrc", - "parserOptions": { - "project": "tsconfig.json" - } -} \ No newline at end of file diff --git a/packages/following-handler/.gcloudignore b/packages/following-handler/.gcloudignore deleted file mode 100644 index ccc4eb240..000000000 --- a/packages/following-handler/.gcloudignore +++ /dev/null @@ -1,16 +0,0 @@ -# This file specifies files that are *not* uploaded to Google Cloud Platform -# using gcloud. It follows the same syntax as .gitignore, with the addition of -# "#!include" directives (which insert the entries of the given .gitignore-style -# file at that point). -# -# For more information, run: -# $ gcloud topic gcloudignore -# -.gcloudignore -# If you would like to upload your .git directory, .gitignore file or files -# from your .gitignore file, remove the corresponding line -# below: -.git -.gitignore - -node_modules diff --git a/packages/following-handler/Dockerfile b/packages/following-handler/Dockerfile deleted file mode 100644 index 85974823e..000000000 --- a/packages/following-handler/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM node:18.16-alpine - -# Run everything after as non-privileged user. -WORKDIR /app - -COPY package.json . -COPY yarn.lock . -COPY tsconfig.json . -COPY .eslintrc . - -COPY /packages/following-handler/package.json ./packages/following-handler/package.json - -RUN yarn install --pure-lockfile - -COPY /packages/following-handler ./packages/following-handler -RUN yarn workspace @omnivore/rss-handler build - -# After building, fetch the production dependencies -RUN rm -rf /app/packages/following-handler/node_modules -RUN rm -rf /app/node_modules -RUN yarn install --pure-lockfile --production - -EXPOSE 8080 - -CMD ["yarn", "workspace", "@omnivore/following-handler", "start"] - diff --git a/packages/following-handler/mocha-config.json b/packages/following-handler/mocha-config.json deleted file mode 100644 index 44d1d24c1..000000000 --- a/packages/following-handler/mocha-config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "extension": ["ts"], - "spec": "test/**/*.test.ts", - "require": "test/babel-register.js" - } \ No newline at end of file diff --git a/packages/following-handler/package.json b/packages/following-handler/package.json deleted file mode 100644 index 895476eea..000000000 --- a/packages/following-handler/package.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "@omnivore/following-handler", - "version": "1.0.0", - "main": "build/src/index.js", - "files": [ - "build/src" - ], - "license": "Apache-2.0", - "scripts": { - "test": "yarn mocha -r ts-node/register --config mocha-config.json", - "test:typecheck": "tsc --noEmit", - "lint": "eslint src --ext ts,js,tsx,jsx", - "compile": "tsc", - "build": "tsc", - "start": "functions-framework --target=followingHandler", - "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"" - }, - "devDependencies": { - "chai": "^4.3.6", - "eslint-plugin-prettier": "^4.0.0", - "mocha": "^10.0.0" - }, - "dependencies": { - "@google-cloud/functions-framework": "3.1.2", - "@sentry/serverless": "^7.77.0", - "dotenv": "^16.0.1" - }, - "volta": { - "extends": "../../package.json" - } -} diff --git a/packages/following-handler/src/index.ts b/packages/following-handler/src/index.ts deleted file mode 100644 index cbc8e307c..000000000 --- a/packages/following-handler/src/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -import * as Sentry from '@sentry/serverless' -import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import - -dotenv.config() -Sentry.GCPFunction.init({ - dsn: process.env.SENTRY_DSN, - tracesSampleRate: 0, -}) - -export const followingHandler = Sentry.GCPFunction.wrapHttpFunction( - (req, res) => { - if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) { - console.log('query does not include valid token') - return res.sendStatus(403) - } - - res.send('ok') - } -) diff --git a/packages/following-handler/test/babel-register.js b/packages/following-handler/test/babel-register.js deleted file mode 100644 index a6f65f60a..000000000 --- a/packages/following-handler/test/babel-register.js +++ /dev/null @@ -1,3 +0,0 @@ -const register = require('@babel/register').default - -register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/following-handler/test/stub.test.ts b/packages/following-handler/test/stub.test.ts deleted file mode 100644 index 935d1e499..000000000 --- a/packages/following-handler/test/stub.test.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { expect } from 'chai' -import 'mocha' - -describe('stub test', () => { - it('should pass', () => { - expect(true).to.be.true - }) -}) diff --git a/packages/following-handler/tsconfig.json b/packages/following-handler/tsconfig.json deleted file mode 100644 index 7ebe093f6..000000000 --- a/packages/following-handler/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "./../../tsconfig.json", - "compilerOptions": { - "outDir": "build", - "rootDir": "." - }, - "include": ["src"] -} diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index dfa7f3294..68a6f9e6a 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -5,7 +5,11 @@ import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-d import * as jwt from 'jsonwebtoken' import Parser, { Item } from 'rss-parser' import { promisify } from 'util' -import { CONTENT_FETCH_URL, createCloudTask } from './task' +import { + CONTENT_FETCH_URL, + createCloudTask, + FOLLOWING_HANDLER_URL, +} from './task' interface RssFeedRequest { subscriptionIds: string[] @@ -151,6 +155,38 @@ const createSavingItemTask = async ( } } +const createFollowingTask = async ( + userId: string, + feedUrl: string, + item: Item +) => { + const input = { + userId, + url: item.link, + title: item.title, + author: item.creator, + description: item.summary, + sharedSource: 'rss-feeder', + previewContent: item.content, + sharedBy: feedUrl, + savedAt: item.isoDate, + publishedAt: item.isoDate, + sharedAt: item.isoDate, + } + + try { + console.log('Creating task', input.url) + // save page + const task = await createCloudTask(FOLLOWING_HANDLER_URL, input) + console.log('Created task', task) + + return !!task + } catch (error) { + console.error('Error while creating task', error) + return false + } +} + dotenv.config() Sentry.GCPFunction.init({ dsn: process.env.SENTRY_DSN, @@ -330,7 +366,7 @@ const processSubscription = async ( continue } - const created = await createSavingItemTask(userId, feedUrl, item) + const created = await createFollowingTask(userId, feedUrl, item) if (!created) { console.error('Failed to create task for feed item', item.link) continue @@ -353,7 +389,7 @@ const processSubscription = async ( } // the feed has never been fetched, save at least the last valid item - const created = await createSavingItemTask(userId, feedUrl, lastValidItem) + const created = await createFollowingTask(userId, feedUrl, lastValidItem) if (!created) { console.error('Failed to create task for feed item', lastValidItem.link) throw new Error('Failed to create task for feed item') diff --git a/packages/rss-handler/src/task.ts b/packages/rss-handler/src/task.ts index 8a89fbd43..9fe02fca7 100644 --- a/packages/rss-handler/src/task.ts +++ b/packages/rss-handler/src/task.ts @@ -4,6 +4,7 @@ import { CloudTasksClient, protos } from '@google-cloud/tasks' const cloudTask = new CloudTasksClient() export const CONTENT_FETCH_URL = process.env.CONTENT_FETCH_GCF_URL +export const FOLLOWING_HANDLER_URL = process.env.FOLLOWING_HANDLER_URL export const createCloudTask = async ( taskHandlerUrl: string | undefined, From b7496db56c855ecd1e50d650cecbfa3ff09c6767 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 9 Nov 2023 15:53:44 +0800 Subject: [PATCH 18/30] add following handler to save following item --- packages/api/src/generated/graphql.ts | 108 +----------------- packages/api/src/generated/schema.graphql | 50 +------- packages/api/src/resolvers/following/index.ts | 86 +++++++------- packages/api/src/routers/svc/following.ts | 71 ++++++++++++ packages/api/src/schema.ts | 50 +------- packages/api/src/server.ts | 2 + .../src/services/create_page_save_request.ts | 7 +- packages/api/src/services/library_item.ts | 43 ++++--- packages/api/src/services/save_page.ts | 1 + packages/db/migrations/0146.do.following.sql | 5 + .../db/migrations/0146.undo.following.sql | 2 + packages/rss-handler/src/index.ts | 4 +- 12 files changed, 158 insertions(+), 271 deletions(-) create mode 100644 packages/api/src/routers/svc/following.ts diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index a28aebb33..6208561bd 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -23,8 +23,8 @@ export type AddFollowingToLibraryError = { }; export enum AddFollowingToLibraryErrorCode { + AlreadyExists = 'ALREADY_EXISTS', BadRequest = 'BAD_REQUEST', - NotFound = 'NOT_FOUND', Unauthorized = 'UNAUTHORIZED' } @@ -855,26 +855,6 @@ export type FiltersSuccess = { filters: Array; }; -export type Following = { - __typename?: 'Following'; - SharedAt: Scalars['Date']; - author?: Maybe; - createdAt: Scalars['Date']; - description?: Maybe; - hiddenAt?: Maybe; - id: Scalars['ID']; - image?: Maybe; - links?: Maybe; - previewContent?: Maybe; - publishedAt?: Maybe; - seenAt?: Maybe; - sharedBy: Scalars['String']; - sharedSource: Scalars['String']; - title: Scalars['String']; - updatedAt: Scalars['Date']; - url: Scalars['String']; -}; - export type GenerateApiKeyError = { __typename?: 'GenerateApiKeyError'; errorCodes: Array; @@ -1359,7 +1339,6 @@ export type Mutation = { saveArticleReadingProgress: SaveArticleReadingProgressResult; saveFile: SaveResult; saveFilter: SaveFilterResult; - saveFollowing: SaveFollowingResult; savePage: SaveResult; saveUrl: SaveResult; setBookmarkArticle: SetBookmarkArticleResult; @@ -1561,11 +1540,6 @@ export type MutationSaveFilterArgs = { }; -export type MutationSaveFollowingArgs = { - input: SaveFollowingInput; -}; - - export type MutationSavePageArgs = { input: SavePageInput; }; @@ -2261,36 +2235,6 @@ export type SaveFilterSuccess = { filter: Filter; }; -export type SaveFollowingError = { - __typename?: 'SaveFollowingError'; - errorCodes: Array; -}; - -export enum SaveFollowingErrorCode { - BadRequest = 'BAD_REQUEST', - Unauthorized = 'UNAUTHORIZED' -} - -export type SaveFollowingInput = { - author?: InputMaybe; - description?: InputMaybe; - links?: InputMaybe; - previewContent?: InputMaybe; - publishedAt?: InputMaybe; - sharedAt: Scalars['Date']; - sharedBy: Scalars['String']; - sharedSource: Scalars['String']; - title: Scalars['String']; - url: Scalars['String']; -}; - -export type SaveFollowingResult = SaveFollowingError | SaveFollowingSuccess; - -export type SaveFollowingSuccess = { - __typename?: 'SaveFollowingSuccess'; - following: Following; -}; - export type SavePageInput = { clientRequestId: Scalars['ID']; labels?: InputMaybe>; @@ -3616,7 +3560,6 @@ export type ResolversTypes = { FiltersResult: ResolversTypes['FiltersError'] | ResolversTypes['FiltersSuccess']; FiltersSuccess: ResolverTypeWrapper; Float: ResolverTypeWrapper; - Following: ResolverTypeWrapper; GenerateApiKeyError: ResolverTypeWrapper; GenerateApiKeyErrorCode: GenerateApiKeyErrorCode; GenerateApiKeyInput: GenerateApiKeyInput; @@ -3782,11 +3725,6 @@ export type ResolversTypes = { SaveFilterInput: SaveFilterInput; SaveFilterResult: ResolversTypes['SaveFilterError'] | ResolversTypes['SaveFilterSuccess']; SaveFilterSuccess: ResolverTypeWrapper; - SaveFollowingError: ResolverTypeWrapper; - SaveFollowingErrorCode: SaveFollowingErrorCode; - SaveFollowingInput: SaveFollowingInput; - SaveFollowingResult: ResolversTypes['SaveFollowingError'] | ResolversTypes['SaveFollowingSuccess']; - SaveFollowingSuccess: ResolverTypeWrapper; SavePageInput: SavePageInput; SaveResult: ResolversTypes['SaveError'] | ResolversTypes['SaveSuccess']; SaveSuccess: ResolverTypeWrapper; @@ -4109,7 +4047,6 @@ export type ResolversParentTypes = { FiltersResult: ResolversParentTypes['FiltersError'] | ResolversParentTypes['FiltersSuccess']; FiltersSuccess: FiltersSuccess; Float: Scalars['Float']; - Following: Following; GenerateApiKeyError: GenerateApiKeyError; GenerateApiKeyInput: GenerateApiKeyInput; GenerateApiKeyResult: ResolversParentTypes['GenerateApiKeyError'] | ResolversParentTypes['GenerateApiKeySuccess']; @@ -4239,10 +4176,6 @@ export type ResolversParentTypes = { SaveFilterInput: SaveFilterInput; SaveFilterResult: ResolversParentTypes['SaveFilterError'] | ResolversParentTypes['SaveFilterSuccess']; SaveFilterSuccess: SaveFilterSuccess; - SaveFollowingError: SaveFollowingError; - SaveFollowingInput: SaveFollowingInput; - SaveFollowingResult: ResolversParentTypes['SaveFollowingError'] | ResolversParentTypes['SaveFollowingSuccess']; - SaveFollowingSuccess: SaveFollowingSuccess; SavePageInput: SavePageInput; SaveResult: ResolversParentTypes['SaveError'] | ResolversParentTypes['SaveSuccess']; SaveSuccess: SaveSuccess; @@ -5010,26 +4943,6 @@ export type FiltersSuccessResolvers; }; -export type FollowingResolvers = { - SharedAt?: Resolver; - author?: Resolver, ParentType, ContextType>; - createdAt?: Resolver; - description?: Resolver, ParentType, ContextType>; - hiddenAt?: Resolver, ParentType, ContextType>; - id?: Resolver; - image?: Resolver, ParentType, ContextType>; - links?: Resolver, ParentType, ContextType>; - previewContent?: Resolver, ParentType, ContextType>; - publishedAt?: Resolver, ParentType, ContextType>; - seenAt?: Resolver, ParentType, ContextType>; - sharedBy?: Resolver; - sharedSource?: Resolver; - title?: Resolver; - updatedAt?: Resolver; - url?: Resolver; - __isTypeOf?: IsTypeOfResolverFn; -}; - export type GenerateApiKeyErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -5396,7 +5309,6 @@ export type MutationResolvers>; saveFile?: Resolver>; saveFilter?: Resolver>; - saveFollowing?: Resolver>; savePage?: Resolver>; saveUrl?: Resolver>; setBookmarkArticle?: Resolver>; @@ -5756,20 +5668,6 @@ export type SaveFilterSuccessResolvers; }; -export type SaveFollowingErrorResolvers = { - errorCodes?: Resolver, ParentType, ContextType>; - __isTypeOf?: IsTypeOfResolverFn; -}; - -export type SaveFollowingResultResolvers = { - __resolveType: TypeResolveFn<'SaveFollowingError' | 'SaveFollowingSuccess', ParentType, ContextType>; -}; - -export type SaveFollowingSuccessResolvers = { - following?: Resolver; - __isTypeOf?: IsTypeOfResolverFn; -}; - export type SaveResultResolvers = { __resolveType: TypeResolveFn<'SaveError' | 'SaveSuccess', ParentType, ContextType>; }; @@ -6554,7 +6452,6 @@ export type Resolvers = { FiltersError?: FiltersErrorResolvers; FiltersResult?: FiltersResultResolvers; FiltersSuccess?: FiltersSuccessResolvers; - Following?: FollowingResolvers; GenerateApiKeyError?: GenerateApiKeyErrorResolvers; GenerateApiKeyResult?: GenerateApiKeyResultResolvers; GenerateApiKeySuccess?: GenerateApiKeySuccessResolvers; @@ -6665,9 +6562,6 @@ export type Resolvers = { SaveFilterError?: SaveFilterErrorResolvers; SaveFilterResult?: SaveFilterResultResolvers; SaveFilterSuccess?: SaveFilterSuccessResolvers; - SaveFollowingError?: SaveFollowingErrorResolvers; - SaveFollowingResult?: SaveFollowingResultResolvers; - SaveFollowingSuccess?: SaveFollowingSuccessResolvers; SaveResult?: SaveResultResolvers; SaveSuccess?: SaveSuccessResolvers; SearchError?: SearchErrorResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index fcb09a256..c987d21e0 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -5,8 +5,8 @@ type AddFollowingToLibraryError { } enum AddFollowingToLibraryErrorCode { + ALREADY_EXISTS BAD_REQUEST - NOT_FOUND UNAUTHORIZED } @@ -758,25 +758,6 @@ type FiltersSuccess { filters: [Filter!]! } -type Following { - SharedAt: Date! - author: String - createdAt: Date! - description: String - hiddenAt: Date - id: ID! - image: String - links: JSON - previewContent: String - publishedAt: Date - seenAt: Date - sharedBy: String! - sharedSource: String! - title: String! - updatedAt: Date! - url: String! -} - type GenerateApiKeyError { errorCodes: [GenerateApiKeyErrorCode!]! } @@ -1221,7 +1202,6 @@ type Mutation { saveArticleReadingProgress(input: SaveArticleReadingProgressInput!): SaveArticleReadingProgressResult! saveFile(input: SaveFileInput!): SaveResult! saveFilter(input: SaveFilterInput!): SaveFilterResult! - saveFollowing(input: SaveFollowingInput!): SaveFollowingResult! savePage(input: SavePageInput!): SaveResult! saveUrl(input: SaveUrlInput!): SaveResult! setBookmarkArticle(input: SetBookmarkArticleInput!): SetBookmarkArticleResult! @@ -1715,34 +1695,6 @@ type SaveFilterSuccess { filter: Filter! } -type SaveFollowingError { - errorCodes: [SaveFollowingErrorCode!]! -} - -enum SaveFollowingErrorCode { - BAD_REQUEST - UNAUTHORIZED -} - -input SaveFollowingInput { - author: String - description: String - links: JSON - previewContent: String - publishedAt: Date - sharedAt: Date! - sharedBy: String! - sharedSource: String! - title: String! - url: String! -} - -union SaveFollowingResult = SaveFollowingError | SaveFollowingSuccess - -type SaveFollowingSuccess { - following: Following! -} - input SavePageInput { clientRequestId: ID! labels: [CreateLabelInput!] diff --git a/packages/api/src/resolvers/following/index.ts b/packages/api/src/resolvers/following/index.ts index d7ea0129d..b4f3073bd 100644 --- a/packages/api/src/resolvers/following/index.ts +++ b/packages/api/src/resolvers/following/index.ts @@ -9,16 +9,16 @@ import { FeedsErrorCode, FeedsSuccess, MutationAddFollowingToLibraryArgs, - MutationSaveFollowingArgs, QueryFeedsArgs, - SaveFollowingError, - SaveFollowingSuccess, } from '../../generated/graphql' import { feedRepository } from '../../repository/feed' import { createPageSaveRequest } from '../../services/create_page_save_request' -import { createFollowing } from '../../services/library_item' +import { updateLibraryItem } from '../../services/library_item' import { analytics } from '../../utils/analytics' -import { authorized } from '../../utils/helpers' +import { + authorized, + libraryItemToArticleSavingRequest, +} from '../../utils/helpers' export const feedsResolve = authorized< FeedsSuccess, @@ -72,33 +72,6 @@ export const feedsResolve = authorized< } }) -export const saveFollowingResolver = authorized< - SaveFollowingSuccess, - SaveFollowingError, - MutationSaveFollowingArgs ->(async (_, { input }, { uid }) => { - analytics.track({ - userId: uid, - event: 'save_following', - properties: { - url: input.url, - }, - }) - - const newItem = await createFollowing(input, uid) - - return { - __typename: 'SaveFollowingSuccess', - following: { - ...newItem, - url: newItem.originalUrl, - SharedAt: new Date(input.sharedAt), - sharedBy: input.sharedBy, - sharedSource: input.sharedSource, - }, - } -}) - export const addFollowingToLibraryResolver = authorized< AddFollowingToLibrarySuccess, AddFollowingToLibraryError, @@ -117,7 +90,6 @@ export const addFollowingToLibraryResolver = authorized< where: { id, sharedAt: Not(IsNull()), - isInLibrary: false, }, relations: ['user'], }) @@ -125,22 +97,48 @@ export const addFollowingToLibraryResolver = authorized< if (!item) { return { - errorCodes: [AddFollowingToLibraryErrorCode.NotFound], + errorCodes: [AddFollowingToLibraryErrorCode.Unauthorized], } } - const articleSavingRequest = await createPageSaveRequest({ - userId: uid, - url: item.originalUrl, - articleSavingRequestId: id, - priority: 'high', - publishedAt: item.publishedAt || undefined, - savedAt: item.savedAt || undefined, - pubsub, - }) + if (item.isInLibrary) { + return { + errorCodes: [AddFollowingToLibraryErrorCode.AlreadyExists], + } + } + + // if the content is not fetched yet, create a page save request + if (!item.readableContent) { + const articleSavingRequest = await createPageSaveRequest({ + userId: uid, + url: item.originalUrl, + articleSavingRequestId: id, + priority: 'high', + publishedAt: item.publishedAt || undefined, + pubsub, + }) + + return { + __typename: 'AddFollowingToLibrarySuccess', + articleSavingRequest, + } + } + + const updatedItem = await updateLibraryItem( + item.id, + { + isInLibrary: true, + savedAt: new Date(), + }, + uid, + pubsub + ) return { __typename: 'AddFollowingToLibrarySuccess', - articleSavingRequest, + articleSavingRequest: libraryItemToArticleSavingRequest( + item.user, + updatedItem + ), } }) diff --git a/packages/api/src/routers/svc/following.ts b/packages/api/src/routers/svc/following.ts new file mode 100644 index 000000000..2bcba9f86 --- /dev/null +++ b/packages/api/src/routers/svc/following.ts @@ -0,0 +1,71 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import express from 'express' +import { saveFeedItemInFollowing } from '../../services/library_item' +import { logger } from '../../utils/logger' + +type SharedSource = 'feed' | 'newsletter' | 'user' + +export interface SaveFollowingItemRequest { + userIds: string[] + title: string + url: string + itemId: string + sharedAt: Date + sharedBy: string + sharedSource: SharedSource + author?: string + description?: string + links?: any + previewContent?: string + publishedAt?: Date + savedAt?: Date +} + +function isSaveFollowingItemRequest( + body: any +): body is SaveFollowingItemRequest { + return ( + 'userIds' in body && + 'sharedAt' in body && + 'sharedBy' in body && + 'sharedSource' in body && + 'url' in body && + 'itemId' in body && + 'title' in body + ) +} + +export function followingServiceRouter() { + const router = express.Router() + + router.post('/save', async (req, res) => { + logger.info('save following item request', req.body) + + if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) { + console.log('query does not include valid token') + return res.sendStatus(403) + } + + if (!isSaveFollowingItemRequest(req.body)) { + console.error('Invalid request body', req.body) + return res.status(400).send('INVALID_REQUEST_BODY') + } + + if (req.body.sharedSource === 'feed') { + logger.info('saving feed item') + + const result = await saveFeedItemInFollowing(req.body) + if (result.identifiers.length === 0) { + logger.error('error saving feed item in following') + return res.status(500).send('ERROR_SAVING_FEED_ITEM') + } + + logger.info('feed item saved in following') + return res.sendStatus(200) + } + + res.sendStatus(200) + }) + + return router +} diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index c794d0ad0..07e107448 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2649,53 +2649,6 @@ const schema = gql` author: String } - union SaveFollowingResult = SaveFollowingSuccess | SaveFollowingError - - type SaveFollowingSuccess { - following: Following! - } - - type Following { - id: ID! - title: String! - url: String! - author: String - image: String - description: String - seenAt: Date - createdAt: Date! - updatedAt: Date! - publishedAt: Date - hiddenAt: Date - SharedAt: Date! - sharedBy: String! - links: JSON - previewContent: String - sharedSource: String! - } - - type SaveFollowingError { - errorCodes: [SaveFollowingErrorCode!]! - } - - enum SaveFollowingErrorCode { - UNAUTHORIZED - BAD_REQUEST - } - - input SaveFollowingInput { - url: String! - title: String! - author: String - description: String - publishedAt: Date - sharedSource: String! - links: JSON - previewContent: String - sharedBy: String! - sharedAt: Date! - } - union AddFollowingToLibraryResult = AddFollowingToLibrarySuccess | AddFollowingToLibraryError @@ -2711,7 +2664,7 @@ const schema = gql` enum AddFollowingToLibraryErrorCode { UNAUTHORIZED BAD_REQUEST - NOT_FOUND + ALREADY_EXISTS } # Mutations @@ -2817,7 +2770,6 @@ const schema = gql` updateSubscription( input: UpdateSubscriptionInput! ): UpdateSubscriptionResult! - saveFollowing(input: SaveFollowingInput!): SaveFollowingResult! addFollowingToLibrary(id: ID!): AddFollowingToLibraryResult! } diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 1209deb9e..888383e66 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -25,6 +25,7 @@ import { pageRouter } from './routers/page_router' import { contentServiceRouter } from './routers/svc/content' import { emailsServiceRouter } from './routers/svc/emails' import { emailAttachmentRouter } from './routers/svc/email_attachment' +import { followingServiceRouter } from './routers/svc/following' import { integrationsServiceRouter } from './routers/svc/integrations' import { linkServiceRouter } from './routers/svc/links' import { newsletterServiceRouter } from './routers/svc/newsletters' @@ -125,6 +126,7 @@ export const createApp = (): { app.use('/svc/pubsub/user', userServiceRouter()) // app.use('/svc/reminders', remindersServiceRouter()) app.use('/svc/email-attachment', emailAttachmentRouter()) + app.use('/svc/following', followingServiceRouter()) if (env.dev.isLocal) { app.use('/local/debug', localDebugRouter()) diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index bec2f2329..9553165b4 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -130,8 +130,11 @@ export const createPageSaveRequest = async ({ pubsub ) } - // reset state to processing - if (libraryItem.state !== LibraryItemState.Processing) { + // reset state to processing if in following + if ( + libraryItem.state !== LibraryItemState.Processing && + !libraryItem.sharedAt + ) { libraryItem = await updateLibraryItem( libraryItem.id, { diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 8f462849c..38178c340 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -4,10 +4,12 @@ import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' import { LibraryItem, LibraryItemState } from '../entity/library_item' -import { BulkActionType, SaveFollowingInput } from '../generated/graphql' +import { BulkActionType } from '../generated/graphql' import { createPubSubClient, EntityType } from '../pubsub' import { authTrx, getColumns } from '../repository' import { libraryItemRepository } from '../repository/library_item' +import { SaveFollowingItemRequest } from '../routers/svc/following' +import { SetClaimsRole } from '../utils/dictionary' import { wordsCount } from '../utils/helpers' import { DateFilter, @@ -567,25 +569,30 @@ export const createLibraryItem = async ( return newLibraryItem } -export const createFollowing = async ( - input: SaveFollowingInput, - userId: string -): Promise => { - return createLibraryItem( - { - ...input, - originalUrl: input.url, - isInLibrary: false, - state: LibraryItemState.Succeeded, - wordCount: 0, - user: { id: userId }, - sharedAt: new Date(input.sharedAt), - sharedSource: input.sharedSource, - sharedBy: input.sharedBy, +export const saveFeedItemInFollowing = (input: SaveFollowingItemRequest) => { + return authTrx( + async (tx) => { + const libraryItems: QueryDeepPartialEntity[] = + input.userIds.map((userId) => ({ + ...input, + user: { id: userId }, + isInLibrary: false, + originalUrl: input.url, + subscription: input.sharedBy, + })) + + return tx + .getRepository(LibraryItem) + .createQueryBuilder() + .insert() + .values(libraryItems) + .orIgnore() // ignore if the item already exists + .returning('*') + .execute() }, - userId, undefined, - true + undefined, + SetClaimsRole.ADMIN ) } diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 6e6b7b954..a70d34693 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -144,6 +144,7 @@ export const savePage = async ( ...itemToSave, id: undefined, slug: undefined, + isInLibrary: true, } as QueryDeepPartialEntity, user.id ) diff --git a/packages/db/migrations/0146.do.following.sql b/packages/db/migrations/0146.do.following.sql index 318aa3b59..54d877fd4 100755 --- a/packages/db/migrations/0146.do.following.sql +++ b/packages/db/migrations/0146.do.following.sql @@ -18,6 +18,11 @@ ALTER TABLE omnivore.library_item ADD COLUMN shared_source text, ADD COLUMN is_in_library boolean NOT NULL DEFAULT true; +CREATE POLICY library_item_admin_policy on omnivore.library_item + FOR ALL + TO omnivore_admin + USING (true); + CREATE TABLE omnivore.feed ( id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), title text NOT NULL, diff --git a/packages/db/migrations/0146.undo.following.sql b/packages/db/migrations/0146.undo.following.sql index e134e80a2..3716f588b 100755 --- a/packages/db/migrations/0146.undo.following.sql +++ b/packages/db/migrations/0146.undo.following.sql @@ -6,6 +6,8 @@ BEGIN; DROP TABLE omnivore.feed; +DROP policy library_item_admin_policy ON omnivore.library_item; + ALTER TABLE omnivore.library_item DROP COLUMN hidden_at, DROP COLUMN shared_at, diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 68a6f9e6a..2d3efa45a 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -161,12 +161,12 @@ const createFollowingTask = async ( item: Item ) => { const input = { - userId, + userIds: [userId], url: item.link, title: item.title, author: item.creator, description: item.summary, - sharedSource: 'rss-feeder', + sharedSource: 'feed', previewContent: item.content, sharedBy: feedUrl, savedAt: item.isoDate, From f96ba17c9243efac37e16e0532677969b17fa8ab Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 9 Nov 2023 16:09:23 +0800 Subject: [PATCH 19/30] fetch rss content if required --- packages/api/src/resolvers/following/index.ts | 2 +- packages/rss-handler/src/index.ts | 25 +++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/packages/api/src/resolvers/following/index.ts b/packages/api/src/resolvers/following/index.ts index b4f3073bd..45d79eba5 100644 --- a/packages/api/src/resolvers/following/index.ts +++ b/packages/api/src/resolvers/following/index.ts @@ -137,7 +137,7 @@ export const addFollowingToLibraryResolver = authorized< return { __typename: 'AddFollowingToLibrarySuccess', articleSavingRequest: libraryItemToArticleSavingRequest( - item.user, + updatedItem.user, updatedItem ), } diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 2d3efa45a..3eece1030 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -126,6 +126,19 @@ const sendUpdateSubscriptionMutation = async ( } } +const createTask = async ( + userId: string, + feedUrl: string, + item: Item, + isFetchingContent: boolean +) => { + if (isFetchingContent) { + return createSavingItemTask(userId, feedUrl, item) + } + + return createFollowingTask(userId, feedUrl, item) +} + const createSavingItemTask = async ( userId: string, feedUrl: string, @@ -284,6 +297,7 @@ const processSubscription = async ( lastFetchedAt: number, scheduledAt: number, lastFetchedChecksum: string, + isFetchingContent: boolean, feed: { lastBuildDate: any 'syn:updatePeriod': any @@ -366,7 +380,7 @@ const processSubscription = async ( continue } - const created = await createFollowingTask(userId, feedUrl, item) + const created = await createTask(userId, feedUrl, item, isFetchingContent) if (!created) { console.error('Failed to create task for feed item', item.link) continue @@ -389,7 +403,12 @@ const processSubscription = async ( } // the feed has never been fetched, save at least the last valid item - const created = await createFollowingTask(userId, feedUrl, lastValidItem) + const created = await createTask( + userId, + feedUrl, + lastValidItem, + isFetchingContent + ) if (!created) { console.error('Failed to create task for feed item', lastValidItem.link) throw new Error('Failed to create task for feed item') @@ -435,6 +454,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( scheduledTimestamps, userIds, lastFetchedChecksums, + isFetchingContents, } = req.body console.log('Processing feed', feedUrl) @@ -452,6 +472,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( lastFetchedTimestamps[i], scheduledTimestamps[i], lastFetchedChecksums[i], + isFetchingContents[i], feed ) ) From ffac5c195d514713d077da0a9760e305d009ea38 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 9 Nov 2023 16:13:31 +0800 Subject: [PATCH 20/30] update comment --- packages/api/src/services/create_page_save_request.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 9553165b4..c2b60c8d2 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -130,7 +130,7 @@ export const createPageSaveRequest = async ({ pubsub ) } - // reset state to processing if in following + // reset state to processing if not in following if ( libraryItem.state !== LibraryItemState.Processing && !libraryItem.sharedAt From 9dfc5f856df35807c3505f5134da053709b797e7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 9 Nov 2023 19:00:57 +0800 Subject: [PATCH 21/30] rename variables --- packages/api/src/entity/library_item.ts | 16 +-- packages/api/src/entity/subscription.ts | 2 +- packages/api/src/generated/graphql.ts | 98 +++++++++---------- packages/api/src/generated/schema.graphql | 34 +++---- packages/api/src/resolvers/following/index.ts | 39 ++++---- .../api/src/resolvers/function_resolvers.ts | 3 + .../api/src/resolvers/subscriptions/index.ts | 4 +- packages/api/src/routers/svc/following.ts | 17 ++-- packages/api/src/routers/svc/rss_feed.ts | 2 +- packages/api/src/schema.ts | 16 +-- .../src/services/create_page_save_request.ts | 2 +- packages/api/src/services/library_item.ts | 4 +- packages/api/src/services/save_page.ts | 2 +- packages/api/src/utils/createTask.ts | 4 +- packages/db/migrations/0146.do.following.sql | 12 +-- .../db/migrations/0146.undo.following.sql | 12 +-- packages/rss-handler/src/index.ts | 27 ++--- 17 files changed, 151 insertions(+), 143 deletions(-) diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index e2cf946d6..4ef28cb19 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -201,10 +201,10 @@ export class LibraryItem { hiddenAt?: Date | null @Column('timestamptz') - sharedAt?: Date | null + addedToFollowingAt?: Date | null @Column('text') - sharedBy?: string | null + addedToFollowingBy?: string | null @Column('jsonb') links?: any | null @@ -212,12 +212,12 @@ export class LibraryItem { @Column('text') previewContent?: string | null - @Column('timestamptz') - seenAt?: Date | null - - @Column('boolean') - isInLibrary!: boolean + @Column('text') + previewContentType?: string | null @Column('text') - sharedSource?: string | null + addedToFollowingFrom?: string | null + + @Column('timestamptz') + addedToLibraryAt?: Date | null } diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index af5c54fa4..b144234c4 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -75,5 +75,5 @@ export class Subscription { isPublic?: boolean | null @Column('boolean') - isFetchingContent?: boolean | null + autoAddToLibrary?: boolean | null } diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 6208561bd..2c1cef363 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -17,24 +17,6 @@ export type Scalars = { JSON: any; }; -export type AddFollowingToLibraryError = { - __typename?: 'AddFollowingToLibraryError'; - errorCodes: Array; -}; - -export enum AddFollowingToLibraryErrorCode { - AlreadyExists = 'ALREADY_EXISTS', - BadRequest = 'BAD_REQUEST', - Unauthorized = 'UNAUTHORIZED' -} - -export type AddFollowingToLibraryResult = AddFollowingToLibraryError | AddFollowingToLibrarySuccess; - -export type AddFollowingToLibrarySuccess = { - __typename?: 'AddFollowingToLibrarySuccess'; - articleSavingRequest: ArticleSavingRequest; -}; - export type AddPopularReadError = { __typename?: 'AddPopularReadError'; errorCodes: Array; @@ -271,6 +253,24 @@ export enum ContentReader { Web = 'WEB' } +export type CopyFromFollowingToLibraryError = { + __typename?: 'CopyFromFollowingToLibraryError'; + errorCodes: Array; +}; + +export enum CopyFromFollowingToLibraryErrorCode { + AlreadyExists = 'ALREADY_EXISTS', + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED' +} + +export type CopyFromFollowingToLibraryResult = CopyFromFollowingToLibraryError | CopyFromFollowingToLibrarySuccess; + +export type CopyFromFollowingToLibrarySuccess = { + __typename?: 'CopyFromFollowingToLibrarySuccess'; + articleSavingRequest: ArticleSavingRequest; +}; + export type CreateArticleError = { __typename?: 'CreateArticleError'; errorCodes: Array; @@ -1303,9 +1303,9 @@ export type MoveLabelSuccess = { export type Mutation = { __typename?: 'Mutation'; - addFollowingToLibrary: AddFollowingToLibraryResult; addPopularRead: AddPopularReadResult; bulkAction: BulkActionResult; + copyFromFollowingToLibrary: CopyFromFollowingToLibraryResult; createArticle: CreateArticleResult; createArticleSavingRequest: CreateArticleSavingRequestResult; createGroup: CreateGroupResult; @@ -1366,11 +1366,6 @@ export type Mutation = { }; -export type MutationAddFollowingToLibraryArgs = { - id: Scalars['ID']; -}; - - export type MutationAddPopularReadArgs = { name: Scalars['String']; }; @@ -1385,6 +1380,11 @@ export type MutationBulkActionArgs = { }; +export type MutationCopyFromFollowingToLibraryArgs = { + id: Scalars['ID']; +}; + + export type MutationCreateArticleArgs = { input: CreateArticleInput; }; @@ -3404,10 +3404,6 @@ export type DirectiveResolverFn; - AddFollowingToLibraryErrorCode: AddFollowingToLibraryErrorCode; - AddFollowingToLibraryResult: ResolversTypes['AddFollowingToLibraryError'] | ResolversTypes['AddFollowingToLibrarySuccess']; - AddFollowingToLibrarySuccess: ResolverTypeWrapper; AddPopularReadError: ResolverTypeWrapper; AddPopularReadErrorCode: AddPopularReadErrorCode; AddPopularReadResult: ResolversTypes['AddPopularReadError'] | ResolversTypes['AddPopularReadSuccess']; @@ -3446,6 +3442,10 @@ export type ResolversTypes = { BulkActionSuccess: ResolverTypeWrapper; BulkActionType: BulkActionType; ContentReader: ContentReader; + CopyFromFollowingToLibraryError: ResolverTypeWrapper; + CopyFromFollowingToLibraryErrorCode: CopyFromFollowingToLibraryErrorCode; + CopyFromFollowingToLibraryResult: ResolversTypes['CopyFromFollowingToLibraryError'] | ResolversTypes['CopyFromFollowingToLibrarySuccess']; + CopyFromFollowingToLibrarySuccess: ResolverTypeWrapper; CreateArticleError: ResolverTypeWrapper; CreateArticleErrorCode: CreateArticleErrorCode; CreateArticleInput: CreateArticleInput; @@ -3926,9 +3926,6 @@ export type ResolversTypes = { /** Mapping between all available schema types and the resolvers parents */ export type ResolversParentTypes = { - AddFollowingToLibraryError: AddFollowingToLibraryError; - AddFollowingToLibraryResult: ResolversParentTypes['AddFollowingToLibraryError'] | ResolversParentTypes['AddFollowingToLibrarySuccess']; - AddFollowingToLibrarySuccess: AddFollowingToLibrarySuccess; AddPopularReadError: AddPopularReadError; AddPopularReadResult: ResolversParentTypes['AddPopularReadError'] | ResolversParentTypes['AddPopularReadSuccess']; AddPopularReadSuccess: AddPopularReadSuccess; @@ -3957,6 +3954,9 @@ export type ResolversParentTypes = { BulkActionError: BulkActionError; BulkActionResult: ResolversParentTypes['BulkActionError'] | ResolversParentTypes['BulkActionSuccess']; BulkActionSuccess: BulkActionSuccess; + CopyFromFollowingToLibraryError: CopyFromFollowingToLibraryError; + CopyFromFollowingToLibraryResult: ResolversParentTypes['CopyFromFollowingToLibraryError'] | ResolversParentTypes['CopyFromFollowingToLibrarySuccess']; + CopyFromFollowingToLibrarySuccess: CopyFromFollowingToLibrarySuccess; CreateArticleError: CreateArticleError; CreateArticleInput: CreateArticleInput; CreateArticleResult: ResolversParentTypes['CreateArticleError'] | ResolversParentTypes['CreateArticleSuccess']; @@ -4338,20 +4338,6 @@ export type SanitizeDirectiveArgs = { export type SanitizeDirectiveResolver = DirectiveResolverFn; -export type AddFollowingToLibraryErrorResolvers = { - errorCodes?: Resolver, ParentType, ContextType>; - __isTypeOf?: IsTypeOfResolverFn; -}; - -export type AddFollowingToLibraryResultResolvers = { - __resolveType: TypeResolveFn<'AddFollowingToLibraryError' | 'AddFollowingToLibrarySuccess', ParentType, ContextType>; -}; - -export type AddFollowingToLibrarySuccessResolvers = { - articleSavingRequest?: Resolver; - __isTypeOf?: IsTypeOfResolverFn; -}; - export type AddPopularReadErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -4528,6 +4514,20 @@ export type BulkActionSuccessResolvers; }; +export type CopyFromFollowingToLibraryErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type CopyFromFollowingToLibraryResultResolvers = { + __resolveType: TypeResolveFn<'CopyFromFollowingToLibraryError' | 'CopyFromFollowingToLibrarySuccess', ParentType, ContextType>; +}; + +export type CopyFromFollowingToLibrarySuccessResolvers = { + articleSavingRequest?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type CreateArticleErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -5273,9 +5273,9 @@ export type MoveLabelSuccessResolvers = { - addFollowingToLibrary?: Resolver>; addPopularRead?: Resolver>; bulkAction?: Resolver>; + copyFromFollowingToLibrary?: Resolver>; createArticle?: Resolver>; createArticleSavingRequest?: Resolver>; createGroup?: Resolver>; @@ -6344,9 +6344,6 @@ export type WebhooksSuccessResolvers = { - AddFollowingToLibraryError?: AddFollowingToLibraryErrorResolvers; - AddFollowingToLibraryResult?: AddFollowingToLibraryResultResolvers; - AddFollowingToLibrarySuccess?: AddFollowingToLibrarySuccessResolvers; AddPopularReadError?: AddPopularReadErrorResolvers; AddPopularReadResult?: AddPopularReadResultResolvers; AddPopularReadSuccess?: AddPopularReadSuccessResolvers; @@ -6372,6 +6369,9 @@ export type Resolvers = { BulkActionError?: BulkActionErrorResolvers; BulkActionResult?: BulkActionResultResolvers; BulkActionSuccess?: BulkActionSuccessResolvers; + CopyFromFollowingToLibraryError?: CopyFromFollowingToLibraryErrorResolvers; + CopyFromFollowingToLibraryResult?: CopyFromFollowingToLibraryResultResolvers; + CopyFromFollowingToLibrarySuccess?: CopyFromFollowingToLibrarySuccessResolvers; CreateArticleError?: CreateArticleErrorResolvers; CreateArticleResult?: CreateArticleResultResolvers; CreateArticleSavingRequestError?: CreateArticleSavingRequestErrorResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index c987d21e0..21aecd91d 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1,21 +1,5 @@ directive @sanitize(allowedTags: [String], maxLength: Int, minLength: Int, pattern: String) on INPUT_FIELD_DEFINITION -type AddFollowingToLibraryError { - errorCodes: [AddFollowingToLibraryErrorCode!]! -} - -enum AddFollowingToLibraryErrorCode { - ALREADY_EXISTS - BAD_REQUEST - UNAUTHORIZED -} - -union AddFollowingToLibraryResult = AddFollowingToLibraryError | AddFollowingToLibrarySuccess - -type AddFollowingToLibrarySuccess { - articleSavingRequest: ArticleSavingRequest! -} - type AddPopularReadError { errorCodes: [AddPopularReadErrorCode!]! } @@ -227,6 +211,22 @@ enum ContentReader { WEB } +type CopyFromFollowingToLibraryError { + errorCodes: [CopyFromFollowingToLibraryErrorCode!]! +} + +enum CopyFromFollowingToLibraryErrorCode { + ALREADY_EXISTS + BAD_REQUEST + UNAUTHORIZED +} + +union CopyFromFollowingToLibraryResult = CopyFromFollowingToLibraryError | CopyFromFollowingToLibrarySuccess + +type CopyFromFollowingToLibrarySuccess { + articleSavingRequest: ArticleSavingRequest! +} + type CreateArticleError { errorCodes: [CreateArticleErrorCode!]! } @@ -1166,9 +1166,9 @@ type MoveLabelSuccess { } type Mutation { - addFollowingToLibrary(id: ID!): AddFollowingToLibraryResult! addPopularRead(name: String!): AddPopularReadResult! bulkAction(action: BulkActionType!, async: Boolean, expectedCount: Int, labelIds: [ID!], query: String!): BulkActionResult! + copyFromFollowingToLibrary(id: ID!): CopyFromFollowingToLibraryResult! createArticle(input: CreateArticleInput!): CreateArticleResult! createArticleSavingRequest(input: CreateArticleSavingRequestInput!): CreateArticleSavingRequestResult! createGroup(input: CreateGroupInput!): CreateGroupResult! diff --git a/packages/api/src/resolvers/following/index.ts b/packages/api/src/resolvers/following/index.ts index 45d79eba5..439ee7bd0 100644 --- a/packages/api/src/resolvers/following/index.ts +++ b/packages/api/src/resolvers/following/index.ts @@ -1,14 +1,14 @@ import { IsNull, Not } from 'typeorm' import { LibraryItem } from '../../entity/library_item' import { - AddFollowingToLibraryError, - AddFollowingToLibraryErrorCode, - AddFollowingToLibrarySuccess, + CopyFromFollowingToLibraryError, + CopyFromFollowingToLibraryErrorCode, + CopyFromFollowingToLibrarySuccess, FeedEdge, FeedsError, FeedsErrorCode, FeedsSuccess, - MutationAddFollowingToLibraryArgs, + MutationCopyFromFollowingToLibraryArgs, QueryFeedsArgs, } from '../../generated/graphql' import { feedRepository } from '../../repository/feed' @@ -20,7 +20,7 @@ import { libraryItemToArticleSavingRequest, } from '../../utils/helpers' -export const feedsResolve = authorized< +export const feedsResolver = authorized< FeedsSuccess, FeedsError, QueryFeedsArgs @@ -72,14 +72,14 @@ export const feedsResolve = authorized< } }) -export const addFollowingToLibraryResolver = authorized< - AddFollowingToLibrarySuccess, - AddFollowingToLibraryError, - MutationAddFollowingToLibraryArgs +export const copyFromFollowingToLibraryResolver = authorized< + CopyFromFollowingToLibrarySuccess, + CopyFromFollowingToLibraryError, + MutationCopyFromFollowingToLibraryArgs >(async (_, { id }, { authTrx, pubsub, uid }) => { analytics.track({ userId: uid, - event: 'add_following_to_library', + event: 'copy_from_following_to_library', properties: { id, }, @@ -89,7 +89,7 @@ export const addFollowingToLibraryResolver = authorized< tx.getRepository(LibraryItem).findOne({ where: { id, - sharedAt: Not(IsNull()), + addedToFollowingAt: Not(IsNull()), }, relations: ['user'], }) @@ -97,16 +97,18 @@ export const addFollowingToLibraryResolver = authorized< if (!item) { return { - errorCodes: [AddFollowingToLibraryErrorCode.Unauthorized], + errorCodes: [CopyFromFollowingToLibraryErrorCode.Unauthorized], } } - if (item.isInLibrary) { + if (item.addedToLibraryAt) { return { - errorCodes: [AddFollowingToLibraryErrorCode.AlreadyExists], + errorCodes: [CopyFromFollowingToLibraryErrorCode.AlreadyExists], } } + const addedToLibraryAt = new Date() + // if the content is not fetched yet, create a page save request if (!item.readableContent) { const articleSavingRequest = await createPageSaveRequest({ @@ -115,11 +117,12 @@ export const addFollowingToLibraryResolver = authorized< articleSavingRequestId: id, priority: 'high', publishedAt: item.publishedAt || undefined, + savedAt: addedToLibraryAt, pubsub, }) return { - __typename: 'AddFollowingToLibrarySuccess', + __typename: 'CopyFromFollowingToLibrarySuccess', articleSavingRequest, } } @@ -127,15 +130,15 @@ export const addFollowingToLibraryResolver = authorized< const updatedItem = await updateLibraryItem( item.id, { - isInLibrary: true, - savedAt: new Date(), + savedAt: addedToLibraryAt, + addedToLibraryAt, }, uid, pubsub ) return { - __typename: 'AddFollowingToLibrarySuccess', + __typename: 'CopyFromFollowingToLibrarySuccess', articleSavingRequest: libraryItemToArticleSavingRequest( updatedItem.user, updatedItem diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 365a32162..57200c42e 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -29,6 +29,7 @@ import { generateUploadFilePathName, } from '../utils/uploads' import { optInFeatureResolver } from './features' +import { copyFromFollowingToLibraryResolver, feedsResolver } from './following' import { uploadImportFileResolver } from './importers/uploadImportFileResolver' import { addPopularReadResolver, @@ -215,6 +216,7 @@ export const functionResolvers = { updateSubscription: updateSubscriptionResolver, updateFilter: updateFilterResolver, updateEmail: updateEmailResolver, + copyFromFollowingToLibrary: copyFromFollowingToLibraryResolver, }, Query: { me: getMeUserResolver, @@ -246,6 +248,7 @@ export const functionResolvers = { filters: filtersResolver, groups: groupsResolver, recentEmails: recentEmailsResolver, + feeds: feedsResolver, }, User: { async intercomHash( diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index a2fa5e8c4..e213017ca 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -213,7 +213,7 @@ export const subscribeResolver = authorized< scheduledDates: [new Date()], // fetch immediately fetchedDates: [updatedSubscription.lastFetchedAt || null], checksums: [updatedSubscription.lastFetchedChecksum || null], - isFetchingContents: [!!updatedSubscription.isFetchingContent], + addToLibraryFlags: [!!updatedSubscription.autoAddToLibrary], }) return { @@ -261,7 +261,7 @@ export const subscribeResolver = authorized< scheduledDates: [new Date()], // fetch immediately fetchedDates: [null], checksums: [null], - isFetchingContents: [!!newSubscription.isFetchingContent], + addToLibraryFlags: [!!newSubscription.autoAddToLibrary], }) return { diff --git a/packages/api/src/routers/svc/following.ts b/packages/api/src/routers/svc/following.ts index 2bcba9f86..21c26d4de 100644 --- a/packages/api/src/routers/svc/following.ts +++ b/packages/api/src/routers/svc/following.ts @@ -3,20 +3,21 @@ import express from 'express' import { saveFeedItemInFollowing } from '../../services/library_item' import { logger } from '../../utils/logger' -type SharedSource = 'feed' | 'newsletter' | 'user' +type SourceOfFollowing = 'feed' | 'newsletter' | 'user' export interface SaveFollowingItemRequest { userIds: string[] title: string url: string itemId: string - sharedAt: Date - sharedBy: string - sharedSource: SharedSource + addedToFollowingAt: Date + addedToFollowingBy: string + addedToFollowingFrom: SourceOfFollowing author?: string description?: string links?: any previewContent?: string + previewContentType?: string publishedAt?: Date savedAt?: Date } @@ -26,9 +27,9 @@ function isSaveFollowingItemRequest( ): body is SaveFollowingItemRequest { return ( 'userIds' in body && - 'sharedAt' in body && - 'sharedBy' in body && - 'sharedSource' in body && + 'addedToFollowingAt' in body && + 'addedToFollowingBy' in body && + 'addedToFollowingFrom' in body && 'url' in body && 'itemId' in body && 'title' in body @@ -51,7 +52,7 @@ export function followingServiceRouter() { return res.status(400).send('INVALID_REQUEST_BODY') } - if (req.body.sharedSource === 'feed') { + if (req.body.addedToFollowingFrom === 'feed') { logger.info('saving feed item') const result = await saveFeedItemInFollowing(req.body) diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 597a49e7c..595a233ad 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -35,7 +35,7 @@ export function rssFeedRouter() { ARRAY_AGG(last_fetched_at) AS "fetchedDates", ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", ARRAY_AGG(last_fetched_checksum) AS checksums, - ARRAY_AGG(coalesce(is_fetching_content, false)) AS "isFetchingContents" + ARRAY_AGG(coalesce(auto_add_to_library, false)) AS "addToLibraryFlags" FROM omnivore.subscriptions WHERE diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 07e107448..b42470ec9 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2649,19 +2649,19 @@ const schema = gql` author: String } - union AddFollowingToLibraryResult = - AddFollowingToLibrarySuccess - | AddFollowingToLibraryError + union CopyFromFollowingToLibraryResult = + CopyFromFollowingToLibrarySuccess + | CopyFromFollowingToLibraryError - type AddFollowingToLibrarySuccess { + type CopyFromFollowingToLibrarySuccess { articleSavingRequest: ArticleSavingRequest! } - type AddFollowingToLibraryError { - errorCodes: [AddFollowingToLibraryErrorCode!]! + type CopyFromFollowingToLibraryError { + errorCodes: [CopyFromFollowingToLibraryErrorCode!]! } - enum AddFollowingToLibraryErrorCode { + enum CopyFromFollowingToLibraryErrorCode { UNAUTHORIZED BAD_REQUEST ALREADY_EXISTS @@ -2770,7 +2770,7 @@ const schema = gql` updateSubscription( input: UpdateSubscriptionInput! ): UpdateSubscriptionResult! - addFollowingToLibrary(id: ID!): AddFollowingToLibraryResult! + copyFromFollowingToLibrary(id: ID!): CopyFromFollowingToLibraryResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index c2b60c8d2..5e8beca29 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -133,7 +133,7 @@ export const createPageSaveRequest = async ({ // reset state to processing if not in following if ( libraryItem.state !== LibraryItemState.Processing && - !libraryItem.sharedAt + !libraryItem.addedToFollowingAt ) { libraryItem = await updateLibraryItem( libraryItem.id, diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 38178c340..fdd4df1ce 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -576,9 +576,9 @@ export const saveFeedItemInFollowing = (input: SaveFollowingItemRequest) => { input.userIds.map((userId) => ({ ...input, user: { id: userId }, - isInLibrary: false, originalUrl: input.url, - subscription: input.sharedBy, + subscription: input.addedToFollowingBy, + addedToLibraryAt: null, })) return tx diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index a70d34693..ef8c91882 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -144,7 +144,6 @@ export const savePage = async ( ...itemToSave, id: undefined, slug: undefined, - isInLibrary: true, } as QueryDeepPartialEntity, user.id ) @@ -275,5 +274,6 @@ export const parsedContentToLibraryItem = ({ subscription: rssFeedUrl, archivedAt: state === ArticleSavingRequestStatus.Archived ? new Date() : undefined, + addedToLibraryAt: validatedDate(saveTime), } } diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 8764f4495..3c11e5028 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -615,7 +615,7 @@ export interface RssSubscriptionGroup { fetchedDates: (Date | null)[] scheduledDates: Date[] checksums: (string | null)[] - isFetchingContents: boolean[] + addToLibraryFlags: boolean[] } export const enqueueRssFeedFetch = async ( @@ -633,7 +633,7 @@ export const enqueueRssFeedFetch = async ( timestamp.getTime() ), // unix timestamp in milliseconds userIds: subscriptionGroup.userIds, - isFetchingContents: subscriptionGroup.isFetchingContents, + addToLibraryFlags: subscriptionGroup.addToLibraryFlags, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/db/migrations/0146.do.following.sql b/packages/db/migrations/0146.do.following.sql index 54d877fd4..b99889b09 100755 --- a/packages/db/migrations/0146.do.following.sql +++ b/packages/db/migrations/0146.do.following.sql @@ -6,17 +6,17 @@ BEGIN; ALTER TABLE omnivore.subscriptions ADD COLUMN is_public boolean, - ADD COLUMN is_fetching_content boolean; + ADD COLUMN auto_add_to_library boolean; ALTER TABLE omnivore.library_item ADD COLUMN hidden_at timestamptz, - ADD COLUMN shared_at timestamptz, - ADD COLUMN shared_by text, + ADD COLUMN added_to_following_at timestamptz, + ADD COLUMN added_to_following_by text, ADD COLUMN links jsonb, ADD COLUMN preview_content text, - ADD COLUMN seen_at timestamptz, - ADD COLUMN shared_source text, - ADD COLUMN is_in_library boolean NOT NULL DEFAULT true; + ADD COLUMN preview_content_type text, + ADD COLUMN added_to_following_from text, + ADD COLUMN added_to_library_at timestamptz DEFAULT current_timestamp; CREATE POLICY library_item_admin_policy on omnivore.library_item FOR ALL diff --git a/packages/db/migrations/0146.undo.following.sql b/packages/db/migrations/0146.undo.following.sql index 3716f588b..169d4dbf5 100755 --- a/packages/db/migrations/0146.undo.following.sql +++ b/packages/db/migrations/0146.undo.following.sql @@ -10,16 +10,16 @@ DROP policy library_item_admin_policy ON omnivore.library_item; ALTER TABLE omnivore.library_item DROP COLUMN hidden_at, - DROP COLUMN shared_at, - DROP COLUMN shared_by, + DROP COLUMN added_to_following_at, + DROP COLUMN added_to_following_by, DROP COLUMN links, + DROP COLUMN preview_content_type, DROP COLUMN preview_content, - DROP COLUMN seen_at, - DROP COLUMN shared_source, - DROP COLUMN is_in_library; + DROP COLUMN added_to_following_from, + DROP COLUMN added_to_library_at; ALTER TABLE omnivore.subscriptions DROP COLUMN is_public, - DROP COLUMN is_fetching_content; + DROP COLUMN auto_add_to_library; COMMIT; diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 3eece1030..1648cdefc 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -18,7 +18,7 @@ interface RssFeedRequest { scheduledTimestamps: number[] // unix timestamp in milliseconds lastFetchedChecksums: string[] userIds: string[] - isFetchingContents: boolean[] + addToLibraryFlags: boolean[] } // link can be a string or an object @@ -32,7 +32,7 @@ function isRssFeedRequest(body: any): body is RssFeedRequest { 'scheduledTimestamps' in body && 'userIds' in body && 'lastFetchedChecksums' in body && - 'isFetchingContents' in body + 'addToLibraryFlags' in body ) } @@ -130,9 +130,9 @@ const createTask = async ( userId: string, feedUrl: string, item: Item, - isFetchingContent: boolean + autoAddToLibrary: boolean ) => { - if (isFetchingContent) { + if (autoAddToLibrary) { return createSavingItemTask(userId, feedUrl, item) } @@ -179,12 +179,13 @@ const createFollowingTask = async ( title: item.title, author: item.creator, description: item.summary, - sharedSource: 'feed', - previewContent: item.content, - sharedBy: feedUrl, + addedToFollowingFrom: 'feed', + previewContent: item.content || item.contentSnippet, + addedToFollowingBy: feedUrl, savedAt: item.isoDate, publishedAt: item.isoDate, - sharedAt: item.isoDate, + addedToFollowingAt: item.isoDate, + previewContentType: 'text/html', // TODO: get content type from feed } try { @@ -297,7 +298,7 @@ const processSubscription = async ( lastFetchedAt: number, scheduledAt: number, lastFetchedChecksum: string, - isFetchingContent: boolean, + autoAddToLibrary: boolean, feed: { lastBuildDate: any 'syn:updatePeriod': any @@ -380,7 +381,7 @@ const processSubscription = async ( continue } - const created = await createTask(userId, feedUrl, item, isFetchingContent) + const created = await createTask(userId, feedUrl, item, autoAddToLibrary) if (!created) { console.error('Failed to create task for feed item', item.link) continue @@ -407,7 +408,7 @@ const processSubscription = async ( userId, feedUrl, lastValidItem, - isFetchingContent + autoAddToLibrary ) if (!created) { console.error('Failed to create task for feed item', lastValidItem.link) @@ -454,7 +455,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( scheduledTimestamps, userIds, lastFetchedChecksums, - isFetchingContents, + addToLibraryFlags, } = req.body console.log('Processing feed', feedUrl) @@ -472,7 +473,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( lastFetchedTimestamps[i], scheduledTimestamps[i], lastFetchedChecksums[i], - isFetchingContents[i], + addToLibraryFlags[i], feed ) ) From 490b40e467e714ae7c3d7fee99cfd86afd7eba38 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 9 Nov 2023 19:08:52 +0800 Subject: [PATCH 22/30] fix tests --- packages/api/src/generated/graphql.ts | 4 ++-- packages/api/src/generated/schema.graphql | 4 ++-- packages/api/src/resolvers/subscriptions/index.ts | 5 +++-- packages/api/src/schema.ts | 4 ++-- packages/api/src/services/library_item.ts | 4 ++-- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 2c1cef363..11e30a4db 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2702,7 +2702,7 @@ export enum SubscribeErrorCode { } export type SubscribeInput = { - isFetchingContent?: InputMaybe; + autoAddToLibrary?: InputMaybe; isPublic?: InputMaybe; subscriptionType?: InputMaybe; url: Scalars['String']; @@ -3062,9 +3062,9 @@ export enum UpdateSubscriptionErrorCode { } export type UpdateSubscriptionInput = { + autoAddToLibrary?: InputMaybe; description?: InputMaybe; id: Scalars['ID']; - isFetchingContent?: InputMaybe; isPublic?: InputMaybe; lastFetchedAt?: InputMaybe; lastFetchedChecksum?: InputMaybe; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 21aecd91d..d1c693d02 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2129,7 +2129,7 @@ enum SubscribeErrorCode { } input SubscribeInput { - isFetchingContent: Boolean + autoAddToLibrary: Boolean isPublic: Boolean subscriptionType: SubscriptionType url: String! @@ -2460,9 +2460,9 @@ enum UpdateSubscriptionErrorCode { } input UpdateSubscriptionInput { + autoAddToLibrary: Boolean description: String id: ID! - isFetchingContent: Boolean isPublic: Boolean lastFetchedAt: Date lastFetchedChecksum: String diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index e213017ca..38ac03f87 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -228,7 +228,7 @@ export const subscribeResolver = authorized< // limit number of rss subscriptions to 150 const results = (await getRepository(Subscription).query( - `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, is_fetching_content) + `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, auto_add_to_library) select $1, $2, $3, $4, $5, $6, $7 from omnivore.subscriptions where user_id = $5 and type = 'RSS' and status = 'ACTIVE' having count(*) < $8 @@ -240,7 +240,7 @@ export const subscribeResolver = authorized< SubscriptionType.Rss, uid, feed.image?.url || null, - !!input.isFetchingContent, + !!input.autoAddToLibrary, MAX_RSS_SUBSCRIPTIONS, ] )) as Subscription[] @@ -315,6 +315,7 @@ export const updateSubscriptionResolver = authorized< scheduledAt: input.scheduledAt ? new Date(input.scheduledAt) : undefined, + autoAddToLibrary: input.autoAddToLibrary || undefined, }) return repo.findOneByOrFail({ diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index b42470ec9..3e6ad457d 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2555,7 +2555,7 @@ const schema = gql` url: String! subscriptionType: SubscriptionType isPublic: Boolean - isFetchingContent: Boolean + autoAddToLibrary: Boolean } input UpdateSubscriptionInput { @@ -2567,7 +2567,7 @@ const schema = gql` status: SubscriptionStatus scheduledAt: Date isPublic: Boolean - isFetchingContent: Boolean + autoAddToLibrary: Boolean } union UpdateSubscriptionResult = diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index fdd4df1ce..7e0eba5af 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -108,10 +108,10 @@ const buildWhereClause = ( if (args.inFilter !== InFilter.ALL) { if (args.inFilter === InFilter.FOLLOWING) { queryBuilder - .andWhere('library_item.shared_by IS NOT NULL') + .andWhere('library_item.added_to_following_at IS NOT NULL') .andWhere('library_item.hidden_at IS NULL') } else { - queryBuilder.andWhere('library_item.is_in_library = true') + queryBuilder.andWhere('library_item.added_to_library_at IS NOT NULL') switch (args.inFilter) { case InFilter.INBOX: From 98cc0e2b82bd114590d3c8d35b74311d68f3a3be Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 9 Nov 2023 22:06:30 +0800 Subject: [PATCH 23/30] upsert feed table when subscribing --- .../api/src/resolvers/subscriptions/index.ts | 56 +++++++++++++------ 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 38ac03f87..1a25f482a 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -1,5 +1,6 @@ import Parser from 'rss-parser' import { Brackets } from 'typeorm' +import { Feed } from '../../entity/feed' import { Subscription } from '../../entity/subscription' import { env } from '../../env' import { @@ -175,7 +176,7 @@ export const subscribeResolver = authorized< SubscribeSuccessPartial, SubscribeError, MutationSubscribeArgs ->(async (_, { input }, { uid, log }) => { +>(async (_, { input }, { authTrx, uid, log }) => { try { analytics.track({ userId: uid, @@ -226,24 +227,42 @@ export const subscribeResolver = authorized< // validate rss feed const feed = await parser.parseURL(input.url) - // limit number of rss subscriptions to 150 - const results = (await getRepository(Subscription).query( - `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, auto_add_to_library) - select $1, $2, $3, $4, $5, $6, $7 from omnivore.subscriptions + const results = await authTrx(async (t) => { + await t.getRepository(Feed).upsert( + { + url: feed.feedUrl, + title: feed.title, + description: feed.description, + image: feed.image?.url, + }, + { + conflictPaths: ['url'], + skipUpdateIfNoValuesChanged: true, + } + ) + + // limit number of rss subscriptions to 150 + const results = (await t.getRepository(Subscription).query( + `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, auto_add_to_library, is_public) + select $1, $2, $3, $4, $5, $6, $7, $8 from omnivore.subscriptions where user_id = $5 and type = 'RSS' and status = 'ACTIVE' - having count(*) < $8 + having count(*) < $9 returning *;`, - [ - feed.title, - input.url, - feed.description || null, - SubscriptionType.Rss, - uid, - feed.image?.url || null, - !!input.autoAddToLibrary, - MAX_RSS_SUBSCRIPTIONS, - ] - )) as Subscription[] + [ + feed.title, + input.url, + feed.description || null, + SubscriptionType.Rss, + uid, + feed.image?.url || null, + input.autoAddToLibrary ?? null, + input.isPublic ?? null, + MAX_RSS_SUBSCRIPTIONS, + ] + )) as Subscription[] + + return results + }) if (results.length === 0) { return { @@ -315,7 +334,8 @@ export const updateSubscriptionResolver = authorized< scheduledAt: input.scheduledAt ? new Date(input.scheduledAt) : undefined, - autoAddToLibrary: input.autoAddToLibrary || undefined, + autoAddToLibrary: input.autoAddToLibrary ?? undefined, + isPublic: input.isPublic ?? undefined, }) return repo.findOneByOrFail({ From 69dc1ba47a5f4450f13b66d86862016cf1acad2f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 9 Nov 2023 22:39:51 +0800 Subject: [PATCH 24/30] fix tests --- packages/api/src/entity/subscription.ts | 2 +- packages/api/src/generated/graphql.ts | 8 +++-- packages/api/src/generated/schema.graphql | 6 ++-- .../api/src/resolvers/subscriptions/index.ts | 34 ++++++++++--------- packages/api/src/schema.ts | 6 ++-- .../api/test/resolvers/subscriptions.test.ts | 27 +++++++++------ packages/db/migrations/0146.do.following.sql | 4 +-- .../db/migrations/0146.undo.following.sql | 2 +- 8 files changed, 53 insertions(+), 36 deletions(-) diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index b144234c4..10416a551 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -72,7 +72,7 @@ export class Subscription { scheduledAt?: Date | null @Column('boolean') - isPublic?: boolean | null + isPrivate?: boolean | null @Column('boolean') autoAddToLibrary?: boolean | null diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 11e30a4db..c9ba27c77 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2703,7 +2703,7 @@ export enum SubscribeErrorCode { export type SubscribeInput = { autoAddToLibrary?: InputMaybe; - isPublic?: InputMaybe; + isPrivate?: InputMaybe; subscriptionType?: InputMaybe; url: Scalars['String']; }; @@ -2717,11 +2717,13 @@ export type SubscribeSuccess = { export type Subscription = { __typename?: 'Subscription'; + autoAddToLibrary?: Maybe; count: Scalars['Int']; createdAt: Scalars['Date']; description?: Maybe; icon?: Maybe; id: Scalars['ID']; + isPrivate?: Maybe; lastFetchedAt?: Maybe; name: Scalars['String']; newsletterEmail?: Maybe; @@ -3065,7 +3067,7 @@ export type UpdateSubscriptionInput = { autoAddToLibrary?: InputMaybe; description?: InputMaybe; id: Scalars['ID']; - isPublic?: InputMaybe; + isPrivate?: InputMaybe; lastFetchedAt?: InputMaybe; lastFetchedChecksum?: InputMaybe; name?: InputMaybe; @@ -5948,11 +5950,13 @@ export type SubscribeSuccessResolvers = { + autoAddToLibrary?: SubscriptionResolver, "autoAddToLibrary", ParentType, ContextType>; count?: SubscriptionResolver; createdAt?: SubscriptionResolver; description?: SubscriptionResolver, "description", ParentType, ContextType>; icon?: SubscriptionResolver, "icon", ParentType, ContextType>; id?: SubscriptionResolver; + isPrivate?: SubscriptionResolver, "isPrivate", ParentType, ContextType>; lastFetchedAt?: SubscriptionResolver, "lastFetchedAt", ParentType, ContextType>; name?: SubscriptionResolver; newsletterEmail?: SubscriptionResolver, "newsletterEmail", ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index d1c693d02..4a8dcedc4 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2130,7 +2130,7 @@ enum SubscribeErrorCode { input SubscribeInput { autoAddToLibrary: Boolean - isPublic: Boolean + isPrivate: Boolean subscriptionType: SubscriptionType url: String! } @@ -2142,11 +2142,13 @@ type SubscribeSuccess { } type Subscription { + autoAddToLibrary: Boolean count: Int! createdAt: Date! description: String icon: String id: ID! + isPrivate: Boolean lastFetchedAt: Date name: String! newsletterEmail: String @@ -2463,7 +2465,7 @@ input UpdateSubscriptionInput { autoAddToLibrary: Boolean description: String id: ID! - isPublic: Boolean + isPrivate: Boolean lastFetchedAt: Date lastFetchedChecksum: String name: String diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 1a25f482a..4b4ede2ec 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -228,35 +228,37 @@ export const subscribeResolver = authorized< const feed = await parser.parseURL(input.url) const results = await authTrx(async (t) => { - await t.getRepository(Feed).upsert( - { - url: feed.feedUrl, - title: feed.title, - description: feed.description, - image: feed.image?.url, - }, - { - conflictPaths: ['url'], - skipUpdateIfNoValuesChanged: true, - } - ) + if (!input.isPrivate) { + await t.getRepository(Feed).upsert( + { + url: feed.feedUrl, + title: feed.title, + description: feed.description, + image: feed.image?.url, + }, + { + conflictPaths: ['url'], + skipUpdateIfNoValuesChanged: true, + } + ) + } // limit number of rss subscriptions to 150 const results = (await t.getRepository(Subscription).query( - `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, auto_add_to_library, is_public) + `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, auto_add_to_library, is_private) select $1, $2, $3, $4, $5, $6, $7, $8 from omnivore.subscriptions where user_id = $5 and type = 'RSS' and status = 'ACTIVE' having count(*) < $9 returning *;`, [ feed.title, - input.url, + feed.feedUrl, feed.description || null, SubscriptionType.Rss, uid, feed.image?.url || null, input.autoAddToLibrary ?? null, - input.isPublic ?? null, + input.isPrivate ?? null, MAX_RSS_SUBSCRIPTIONS, ] )) as Subscription[] @@ -335,7 +337,7 @@ export const updateSubscriptionResolver = authorized< ? new Date(input.scheduledAt) : undefined, autoAddToLibrary: input.autoAddToLibrary ?? undefined, - isPublic: input.isPublic ?? undefined, + isPrivate: input.isPrivate ?? undefined, }) return repo.findOneByOrFail({ diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 3e6ad457d..69a95fd79 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1658,6 +1658,8 @@ const schema = gql` lastFetchedAt: Date createdAt: Date! updatedAt: Date + isPrivate: Boolean + autoAddToLibrary: Boolean } enum SubscriptionStatus { @@ -2554,7 +2556,7 @@ const schema = gql` input SubscribeInput { url: String! subscriptionType: SubscriptionType - isPublic: Boolean + isPrivate: Boolean autoAddToLibrary: Boolean } @@ -2566,7 +2568,7 @@ const schema = gql` lastFetchedChecksum: String status: SubscriptionStatus scheduledAt: Date - isPublic: Boolean + isPrivate: Boolean autoAddToLibrary: Boolean } diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 02f7cd1b8..28d96b564 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -3,6 +3,7 @@ import 'mocha' import Parser from 'rss-parser' import sinon from 'sinon' import sinonChai from 'sinon-chai' +import { Feed } from '../../src/entity/feed' import { NewsletterEmail } from '../../src/entity/newsletter_email' import { Subscription } from '../../src/entity/subscription' import { User } from '../../src/entity/user' @@ -332,7 +333,9 @@ describe('Subscriptions API', () => { const updatedSubscription = await getRepository(Subscription).findOneBy({ id: subscription.id, }) - expect(updatedSubscription?.status).to.eql(SubscriptionStatus.Unsubscribed) + expect(updatedSubscription?.status).to.eql( + SubscriptionStatus.Unsubscribed + ) // check if the email was sent expect(fake).to.have.been.calledOnceWith({ @@ -371,10 +374,15 @@ describe('Subscriptions API', () => { before(async () => { // fake rss parser - sinon.replace(Parser.prototype, 'parseURL', sinon.fake.resolves({ - title: 'RSS Feed', - description: 'RSS Feed Description', - })) + sinon.replace( + Parser.prototype, + 'parseURL', + sinon.fake.resolves({ + title: 'RSS Feed', + description: 'RSS Feed Description', + feedUrl: url, + }) + ) }) after(() => { @@ -398,6 +406,7 @@ describe('Subscriptions API', () => { after(async () => { await deleteSubscription(existingSubscription.id) + await getRepository(Feed).delete({ url: existingSubscription.url }) }) it('returns an error', async () => { @@ -439,11 +448,9 @@ describe('Subscriptions API', () => { }) it('creates a rss subscription', async () => { - const res = await graphqlRequest( - query, - authToken, - { input: { url, subscriptionType } }, - ).expect(200) + const res = await graphqlRequest(query, authToken, { + input: { url, subscriptionType }, + }).expect(200) expect(res.body.data.subscribe.subscriptions).to.have.lengthOf(1) expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') diff --git a/packages/db/migrations/0146.do.following.sql b/packages/db/migrations/0146.do.following.sql index b99889b09..e5ea56937 100755 --- a/packages/db/migrations/0146.do.following.sql +++ b/packages/db/migrations/0146.do.following.sql @@ -5,7 +5,7 @@ BEGIN; ALTER TABLE omnivore.subscriptions - ADD COLUMN is_public boolean, + ADD COLUMN is_private boolean, ADD COLUMN auto_add_to_library boolean; ALTER TABLE omnivore.library_item @@ -40,6 +40,6 @@ CREATE INDEX feed_title_idx ON omnivore.feed(title); CREATE TRIGGER update_feed_modtime BEFORE UPDATE ON omnivore.feed FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); -GRANT SELECT, INSERT, UPDATE ON omnivore.feed TO omnivore_user; +GRANT SELECT, INSERT, UPDATE, DELETE ON omnivore.feed TO omnivore_user; COMMIT; diff --git a/packages/db/migrations/0146.undo.following.sql b/packages/db/migrations/0146.undo.following.sql index 169d4dbf5..46d8abbae 100755 --- a/packages/db/migrations/0146.undo.following.sql +++ b/packages/db/migrations/0146.undo.following.sql @@ -19,7 +19,7 @@ ALTER TABLE omnivore.library_item DROP COLUMN added_to_library_at; ALTER TABLE omnivore.subscriptions - DROP COLUMN is_public, + DROP COLUMN is_private, DROP COLUMN auto_add_to_library; COMMIT; From adf13131db1cddf5f8fe61c3b2a31a716908ff3e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 10 Nov 2023 17:34:46 +0800 Subject: [PATCH 25/30] add folder to library_item table --- packages/api/src/entity/library_item.ts | 14 +-- packages/api/src/generated/graphql.ts | 99 ++++++++++--------- packages/api/src/generated/schema.graphql | 34 +++---- packages/api/src/resolvers/article/index.ts | 11 ++- packages/api/src/resolvers/following/index.ts | 41 ++++---- .../api/src/resolvers/function_resolvers.ts | 4 +- packages/api/src/resolvers/links/index.ts | 8 +- packages/api/src/resolvers/update/index.ts | 4 - packages/api/src/routers/svc/following.ts | 7 +- packages/api/src/schema.ts | 14 ++- .../src/services/create_page_save_request.ts | 5 +- packages/api/src/services/library_item.ts | 74 +++----------- packages/api/src/services/save_page.ts | 7 +- packages/api/src/utils/helpers.ts | 5 +- packages/api/src/utils/search.ts | 19 ++-- packages/api/test/resolvers/article.test.ts | 32 +++--- packages/api/test/routers/auth.test.ts | 11 ++- packages/db/migrations/0146.do.following.sql | 6 +- .../db/migrations/0146.undo.following.sql | 8 +- packages/rss-handler/src/index.ts | 1 - 20 files changed, 160 insertions(+), 244 deletions(-) diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 4ef28cb19..447de3bfa 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -197,15 +197,6 @@ export class LibraryItem { @Column('text', { nullable: true }) recommenderNames?: string[] | null - @Column('timestamptz') - hiddenAt?: Date | null - - @Column('timestamptz') - addedToFollowingAt?: Date | null - - @Column('text') - addedToFollowingBy?: string | null - @Column('jsonb') links?: any | null @@ -216,8 +207,5 @@ export class LibraryItem { previewContentType?: string | null @Column('text') - addedToFollowingFrom?: string | null - - @Column('timestamptz') - addedToLibraryAt?: Date | null + folder!: string } diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index c9ba27c77..e6c518eeb 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -253,24 +253,6 @@ export enum ContentReader { Web = 'WEB' } -export type CopyFromFollowingToLibraryError = { - __typename?: 'CopyFromFollowingToLibraryError'; - errorCodes: Array; -}; - -export enum CopyFromFollowingToLibraryErrorCode { - AlreadyExists = 'ALREADY_EXISTS', - BadRequest = 'BAD_REQUEST', - Unauthorized = 'UNAUTHORIZED' -} - -export type CopyFromFollowingToLibraryResult = CopyFromFollowingToLibraryError | CopyFromFollowingToLibrarySuccess; - -export type CopyFromFollowingToLibrarySuccess = { - __typename?: 'CopyFromFollowingToLibrarySuccess'; - articleSavingRequest: ArticleSavingRequest; -}; - export type CreateArticleError = { __typename?: 'CreateArticleError'; errorCodes: Array; @@ -1301,11 +1283,28 @@ export type MoveLabelSuccess = { label: Label; }; +export type MoveToFolderError = { + __typename?: 'MoveToFolderError'; + errorCodes: Array; +}; + +export enum MoveToFolderErrorCode { + AlreadyExists = 'ALREADY_EXISTS', + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED' +} + +export type MoveToFolderResult = MoveToFolderError | MoveToFolderSuccess; + +export type MoveToFolderSuccess = { + __typename?: 'MoveToFolderSuccess'; + articleSavingRequest: ArticleSavingRequest; +}; + export type Mutation = { __typename?: 'Mutation'; addPopularRead: AddPopularReadResult; bulkAction: BulkActionResult; - copyFromFollowingToLibrary: CopyFromFollowingToLibraryResult; createArticle: CreateArticleResult; createArticleSavingRequest: CreateArticleSavingRequestResult; createGroup: CreateGroupResult; @@ -1331,6 +1330,7 @@ export type Mutation = { mergeHighlight: MergeHighlightResult; moveFilter: MoveFilterResult; moveLabel: MoveLabelResult; + moveToFolder: MoveToFolderResult; optInFeature: OptInFeatureResult; recommend: RecommendResult; recommendHighlights: RecommendHighlightsResult; @@ -1380,11 +1380,6 @@ export type MutationBulkActionArgs = { }; -export type MutationCopyFromFollowingToLibraryArgs = { - id: Scalars['ID']; -}; - - export type MutationCreateArticleArgs = { input: CreateArticleInput; }; @@ -1500,6 +1495,12 @@ export type MutationMoveLabelArgs = { }; +export type MutationMoveToFolderArgs = { + folder: Scalars['String']; + id: Scalars['ID']; +}; + + export type MutationOptInFeatureArgs = { input: OptInFeatureInput; }; @@ -3444,10 +3445,6 @@ export type ResolversTypes = { BulkActionSuccess: ResolverTypeWrapper; BulkActionType: BulkActionType; ContentReader: ContentReader; - CopyFromFollowingToLibraryError: ResolverTypeWrapper; - CopyFromFollowingToLibraryErrorCode: CopyFromFollowingToLibraryErrorCode; - CopyFromFollowingToLibraryResult: ResolversTypes['CopyFromFollowingToLibraryError'] | ResolversTypes['CopyFromFollowingToLibrarySuccess']; - CopyFromFollowingToLibrarySuccess: ResolverTypeWrapper; CreateArticleError: ResolverTypeWrapper; CreateArticleErrorCode: CreateArticleErrorCode; CreateArticleInput: CreateArticleInput; @@ -3648,6 +3645,10 @@ export type ResolversTypes = { MoveLabelInput: MoveLabelInput; MoveLabelResult: ResolversTypes['MoveLabelError'] | ResolversTypes['MoveLabelSuccess']; MoveLabelSuccess: ResolverTypeWrapper; + MoveToFolderError: ResolverTypeWrapper; + MoveToFolderErrorCode: MoveToFolderErrorCode; + MoveToFolderResult: ResolversTypes['MoveToFolderError'] | ResolversTypes['MoveToFolderSuccess']; + MoveToFolderSuccess: ResolverTypeWrapper; Mutation: ResolverTypeWrapper<{}>; NewsletterEmail: ResolverTypeWrapper; NewsletterEmailsError: ResolverTypeWrapper; @@ -3956,9 +3957,6 @@ export type ResolversParentTypes = { BulkActionError: BulkActionError; BulkActionResult: ResolversParentTypes['BulkActionError'] | ResolversParentTypes['BulkActionSuccess']; BulkActionSuccess: BulkActionSuccess; - CopyFromFollowingToLibraryError: CopyFromFollowingToLibraryError; - CopyFromFollowingToLibraryResult: ResolversParentTypes['CopyFromFollowingToLibraryError'] | ResolversParentTypes['CopyFromFollowingToLibrarySuccess']; - CopyFromFollowingToLibrarySuccess: CopyFromFollowingToLibrarySuccess; CreateArticleError: CreateArticleError; CreateArticleInput: CreateArticleInput; CreateArticleResult: ResolversParentTypes['CreateArticleError'] | ResolversParentTypes['CreateArticleSuccess']; @@ -4116,6 +4114,9 @@ export type ResolversParentTypes = { MoveLabelInput: MoveLabelInput; MoveLabelResult: ResolversParentTypes['MoveLabelError'] | ResolversParentTypes['MoveLabelSuccess']; MoveLabelSuccess: MoveLabelSuccess; + MoveToFolderError: MoveToFolderError; + MoveToFolderResult: ResolversParentTypes['MoveToFolderError'] | ResolversParentTypes['MoveToFolderSuccess']; + MoveToFolderSuccess: MoveToFolderSuccess; Mutation: {}; NewsletterEmail: NewsletterEmail; NewsletterEmailsError: NewsletterEmailsError; @@ -4516,20 +4517,6 @@ export type BulkActionSuccessResolvers; }; -export type CopyFromFollowingToLibraryErrorResolvers = { - errorCodes?: Resolver, ParentType, ContextType>; - __isTypeOf?: IsTypeOfResolverFn; -}; - -export type CopyFromFollowingToLibraryResultResolvers = { - __resolveType: TypeResolveFn<'CopyFromFollowingToLibraryError' | 'CopyFromFollowingToLibrarySuccess', ParentType, ContextType>; -}; - -export type CopyFromFollowingToLibrarySuccessResolvers = { - articleSavingRequest?: Resolver; - __isTypeOf?: IsTypeOfResolverFn; -}; - export type CreateArticleErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -5274,10 +5261,23 @@ export type MoveLabelSuccessResolvers; }; +export type MoveToFolderErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type MoveToFolderResultResolvers = { + __resolveType: TypeResolveFn<'MoveToFolderError' | 'MoveToFolderSuccess', ParentType, ContextType>; +}; + +export type MoveToFolderSuccessResolvers = { + articleSavingRequest?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type MutationResolvers = { addPopularRead?: Resolver>; bulkAction?: Resolver>; - copyFromFollowingToLibrary?: Resolver>; createArticle?: Resolver>; createArticleSavingRequest?: Resolver>; createGroup?: Resolver>; @@ -5303,6 +5303,7 @@ export type MutationResolvers>; moveFilter?: Resolver>; moveLabel?: Resolver>; + moveToFolder?: Resolver>; optInFeature?: Resolver>; recommend?: Resolver>; recommendHighlights?: Resolver>; @@ -6373,9 +6374,6 @@ export type Resolvers = { BulkActionError?: BulkActionErrorResolvers; BulkActionResult?: BulkActionResultResolvers; BulkActionSuccess?: BulkActionSuccessResolvers; - CopyFromFollowingToLibraryError?: CopyFromFollowingToLibraryErrorResolvers; - CopyFromFollowingToLibraryResult?: CopyFromFollowingToLibraryResultResolvers; - CopyFromFollowingToLibrarySuccess?: CopyFromFollowingToLibrarySuccessResolvers; CreateArticleError?: CreateArticleErrorResolvers; CreateArticleResult?: CreateArticleResultResolvers; CreateArticleSavingRequestError?: CreateArticleSavingRequestErrorResolvers; @@ -6515,6 +6513,9 @@ export type Resolvers = { MoveLabelError?: MoveLabelErrorResolvers; MoveLabelResult?: MoveLabelResultResolvers; MoveLabelSuccess?: MoveLabelSuccessResolvers; + MoveToFolderError?: MoveToFolderErrorResolvers; + MoveToFolderResult?: MoveToFolderResultResolvers; + MoveToFolderSuccess?: MoveToFolderSuccessResolvers; Mutation?: MutationResolvers; NewsletterEmail?: NewsletterEmailResolvers; NewsletterEmailsError?: NewsletterEmailsErrorResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 4a8dcedc4..01ea5b6b3 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -211,22 +211,6 @@ enum ContentReader { WEB } -type CopyFromFollowingToLibraryError { - errorCodes: [CopyFromFollowingToLibraryErrorCode!]! -} - -enum CopyFromFollowingToLibraryErrorCode { - ALREADY_EXISTS - BAD_REQUEST - UNAUTHORIZED -} - -union CopyFromFollowingToLibraryResult = CopyFromFollowingToLibraryError | CopyFromFollowingToLibrarySuccess - -type CopyFromFollowingToLibrarySuccess { - articleSavingRequest: ArticleSavingRequest! -} - type CreateArticleError { errorCodes: [CreateArticleErrorCode!]! } @@ -1165,10 +1149,25 @@ type MoveLabelSuccess { label: Label! } +type MoveToFolderError { + errorCodes: [MoveToFolderErrorCode!]! +} + +enum MoveToFolderErrorCode { + ALREADY_EXISTS + BAD_REQUEST + UNAUTHORIZED +} + +union MoveToFolderResult = MoveToFolderError | MoveToFolderSuccess + +type MoveToFolderSuccess { + articleSavingRequest: ArticleSavingRequest! +} + type Mutation { addPopularRead(name: String!): AddPopularReadResult! bulkAction(action: BulkActionType!, async: Boolean, expectedCount: Int, labelIds: [ID!], query: String!): BulkActionResult! - copyFromFollowingToLibrary(id: ID!): CopyFromFollowingToLibraryResult! createArticle(input: CreateArticleInput!): CreateArticleResult! createArticleSavingRequest(input: CreateArticleSavingRequestInput!): CreateArticleSavingRequestResult! createGroup(input: CreateGroupInput!): CreateGroupResult! @@ -1194,6 +1193,7 @@ type Mutation { mergeHighlight(input: MergeHighlightInput!): MergeHighlightResult! moveFilter(input: MoveFilterInput!): MoveFilterResult! moveLabel(input: MoveLabelInput!): MoveLabelResult! + moveToFolder(folder: String!, id: ID!): MoveToFolderResult! optInFeature(input: OptInFeatureInput!): OptInFeatureResult! recommend(input: RecommendInput!): RecommendResult! recommendHighlights(input: RecommendHighlightsInput!): RecommendHighlightsResult! diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 6a2d03e55..8f9adaedb 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -6,7 +6,7 @@ import { Readability } from '@omnivore/readability' import graphqlFields from 'graphql-fields' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' -import { LibraryItem, LibraryItemState } from '../../entity/library_item' +import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' import { ArticleError, @@ -405,7 +405,7 @@ export const getArticleResolver = authorized< }) ) - if (!libraryItem || libraryItem.state === LibraryItemState.Deleted) { + if (!libraryItem || libraryItem.folder === InFilter.TRASH) { return { errorCodes: [ArticleErrorCode.NotFound] } } @@ -527,8 +527,8 @@ export const setBookmarkArticleResolver = authorized< const deletedLibraryItem = await updateLibraryItem( articleID, { - state: LibraryItemState.Deleted, - deletedAt: new Date(), + folder: InFilter.TRASH, + savedAt: new Date(), }, uid, pubsub @@ -754,6 +754,7 @@ export const updatesSinceResolver = authorized< includeDeleted: true, dateFilters: [{ field: 'updatedAt', startDate }], sort, + inFilter: InFilter.ALL, }, uid ) @@ -865,7 +866,7 @@ export const setFavoriteArticleResolver = authorized< }) const getUpdateReason = (libraryItem: LibraryItem, since: Date) => { - if (libraryItem.state === LibraryItemState.Deleted) { + if (libraryItem.folder === InFilter.TRASH) { return UpdateReason.Deleted } if (libraryItem.createdAt >= since) { diff --git a/packages/api/src/resolvers/following/index.ts b/packages/api/src/resolvers/following/index.ts index 439ee7bd0..6c8f1deb7 100644 --- a/packages/api/src/resolvers/following/index.ts +++ b/packages/api/src/resolvers/following/index.ts @@ -1,14 +1,13 @@ -import { IsNull, Not } from 'typeorm' import { LibraryItem } from '../../entity/library_item' import { - CopyFromFollowingToLibraryError, - CopyFromFollowingToLibraryErrorCode, - CopyFromFollowingToLibrarySuccess, FeedEdge, FeedsError, FeedsErrorCode, FeedsSuccess, - MutationCopyFromFollowingToLibraryArgs, + MoveToFolderError, + MoveToFolderErrorCode, + MoveToFolderSuccess, + MutationMoveToFolderArgs, QueryFeedsArgs, } from '../../generated/graphql' import { feedRepository } from '../../repository/feed' @@ -72,16 +71,17 @@ export const feedsResolver = authorized< } }) -export const copyFromFollowingToLibraryResolver = authorized< - CopyFromFollowingToLibrarySuccess, - CopyFromFollowingToLibraryError, - MutationCopyFromFollowingToLibraryArgs ->(async (_, { id }, { authTrx, pubsub, uid }) => { +export const moveToFolderResolver = authorized< + MoveToFolderSuccess, + MoveToFolderError, + MutationMoveToFolderArgs +>(async (_, { id, folder }, { authTrx, pubsub, uid }) => { analytics.track({ userId: uid, - event: 'copy_from_following_to_library', + event: 'move_to_folder', properties: { id, + folder, }, }) @@ -89,7 +89,6 @@ export const copyFromFollowingToLibraryResolver = authorized< tx.getRepository(LibraryItem).findOne({ where: { id, - addedToFollowingAt: Not(IsNull()), }, relations: ['user'], }) @@ -97,17 +96,17 @@ export const copyFromFollowingToLibraryResolver = authorized< if (!item) { return { - errorCodes: [CopyFromFollowingToLibraryErrorCode.Unauthorized], + errorCodes: [MoveToFolderErrorCode.Unauthorized], } } - if (item.addedToLibraryAt) { + if (item.folder === folder) { return { - errorCodes: [CopyFromFollowingToLibraryErrorCode.AlreadyExists], + errorCodes: [MoveToFolderErrorCode.AlreadyExists], } } - const addedToLibraryAt = new Date() + const savedAt = new Date() // if the content is not fetched yet, create a page save request if (!item.readableContent) { @@ -117,12 +116,12 @@ export const copyFromFollowingToLibraryResolver = authorized< articleSavingRequestId: id, priority: 'high', publishedAt: item.publishedAt || undefined, - savedAt: addedToLibraryAt, + savedAt, pubsub, }) return { - __typename: 'CopyFromFollowingToLibrarySuccess', + __typename: 'MoveToFolderSuccess', articleSavingRequest, } } @@ -130,15 +129,15 @@ export const copyFromFollowingToLibraryResolver = authorized< const updatedItem = await updateLibraryItem( item.id, { - savedAt: addedToLibraryAt, - addedToLibraryAt, + folder, + savedAt, }, uid, pubsub ) return { - __typename: 'CopyFromFollowingToLibrarySuccess', + __typename: 'MoveToFolderSuccess', articleSavingRequest: libraryItemToArticleSavingRequest( updatedItem.user, updatedItem diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 57200c42e..ff363e78f 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -29,7 +29,7 @@ import { generateUploadFilePathName, } from '../utils/uploads' import { optInFeatureResolver } from './features' -import { copyFromFollowingToLibraryResolver, feedsResolver } from './following' +import { feedsResolver, moveToFolderResolver } from './following' import { uploadImportFileResolver } from './importers/uploadImportFileResolver' import { addPopularReadResolver, @@ -216,7 +216,7 @@ export const functionResolvers = { updateSubscription: updateSubscriptionResolver, updateFilter: updateFilterResolver, updateEmail: updateEmailResolver, - copyFromFollowingToLibrary: copyFromFollowingToLibraryResolver, + moveToFolder: moveToFolderResolver, }, Query: { me: getMeUserResolver, diff --git a/packages/api/src/resolvers/links/index.ts b/packages/api/src/resolvers/links/index.ts index 9e2dd3025..d16b9a155 100644 --- a/packages/api/src/resolvers/links/index.ts +++ b/packages/api/src/resolvers/links/index.ts @@ -1,4 +1,3 @@ -import { LibraryItemState } from '../../entity/library_item' import { env } from '../../env' import { ArchiveLinkError, @@ -9,6 +8,7 @@ import { import { updateLibraryItem } from '../../services/library_item' import { analytics } from '../../utils/analytics' import { authorized } from '../../utils/helpers' +import { InFilter } from '../../utils/search' // export const updateLinkShareInfoResolver = authorized< // UpdateLinkShareInfoSuccess, @@ -66,10 +66,8 @@ export const setLinkArchivedResolver = authorized< await updateLibraryItem( args.input.linkId, { - archivedAt: args.input.archived ? new Date() : null, - state: args.input.archived - ? LibraryItemState.Archived - : LibraryItemState.Succeeded, + savedAt: new Date(), + folder: args.input.archived ? InFilter.ARCHIVE : InFilter.INBOX, }, uid ) diff --git a/packages/api/src/resolvers/update/index.ts b/packages/api/src/resolvers/update/index.ts index 255efd2b5..10eb68d32 100644 --- a/packages/api/src/resolvers/update/index.ts +++ b/packages/api/src/resolvers/update/index.ts @@ -1,4 +1,3 @@ -import { LibraryItemState } from '../../entity/library_item' import { MutationUpdatePageArgs, UpdatePageError, @@ -21,9 +20,6 @@ export const updatePageResolver = authorized< savedAt: input.savedAt ? new Date(input.savedAt) : undefined, publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, thumbnail: input.previewImage ?? undefined, - state: input.state - ? (input.state as unknown as LibraryItemState) - : undefined, }, uid ) diff --git a/packages/api/src/routers/svc/following.ts b/packages/api/src/routers/svc/following.ts index 21c26d4de..44a191352 100644 --- a/packages/api/src/routers/svc/following.ts +++ b/packages/api/src/routers/svc/following.ts @@ -10,7 +10,6 @@ export interface SaveFollowingItemRequest { title: string url: string itemId: string - addedToFollowingAt: Date addedToFollowingBy: string addedToFollowingFrom: SourceOfFollowing author?: string @@ -19,7 +18,7 @@ export interface SaveFollowingItemRequest { previewContent?: string previewContentType?: string publishedAt?: Date - savedAt?: Date + savedAt: Date } function isSaveFollowingItemRequest( @@ -27,12 +26,12 @@ function isSaveFollowingItemRequest( ): body is SaveFollowingItemRequest { return ( 'userIds' in body && - 'addedToFollowingAt' in body && 'addedToFollowingBy' in body && 'addedToFollowingFrom' in body && 'url' in body && 'itemId' in body && - 'title' in body + 'title' in body && + 'savedAt' in body ) } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 69a95fd79..1ddb694b9 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2651,19 +2651,17 @@ const schema = gql` author: String } - union CopyFromFollowingToLibraryResult = - CopyFromFollowingToLibrarySuccess - | CopyFromFollowingToLibraryError + union MoveToFolderResult = MoveToFolderSuccess | MoveToFolderError - type CopyFromFollowingToLibrarySuccess { + type MoveToFolderSuccess { articleSavingRequest: ArticleSavingRequest! } - type CopyFromFollowingToLibraryError { - errorCodes: [CopyFromFollowingToLibraryErrorCode!]! + type MoveToFolderError { + errorCodes: [MoveToFolderErrorCode!]! } - enum CopyFromFollowingToLibraryErrorCode { + enum MoveToFolderErrorCode { UNAUTHORIZED BAD_REQUEST ALREADY_EXISTS @@ -2772,7 +2770,7 @@ const schema = gql` updateSubscription( input: UpdateSubscriptionInput! ): UpdateSubscriptionResult! - copyFromFollowingToLibrary(id: ID!): CopyFromFollowingToLibraryResult! + moveToFolder(id: ID!, folder: String!): MoveToFolderResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 5e8beca29..01db62eb2 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -131,10 +131,7 @@ export const createPageSaveRequest = async ({ ) } // reset state to processing if not in following - if ( - libraryItem.state !== LibraryItemState.Processing && - !libraryItem.addedToFollowingAt - ) { + if (libraryItem.state !== LibraryItemState.Processing) { libraryItem = await updateLibraryItem( libraryItem.id, { diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 7e0eba5af..10488c1e4 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -31,7 +31,7 @@ export interface SearchArgs { size?: number sort?: Sort query?: string - inFilter?: InFilter + inFilter: InFilter readFilter?: ReadFilter typeFilter?: string labelFilters?: LabelFilter[] @@ -64,7 +64,6 @@ export interface SearchResultItem { title: string uploadFileId?: string | null url: string - archivedAt?: Date | null readingProgressTopPercent?: number readingProgressPercent: number readingProgressAnchorIndex: number @@ -106,40 +105,9 @@ const buildWhereClause = ( } if (args.inFilter !== InFilter.ALL) { - if (args.inFilter === InFilter.FOLLOWING) { - queryBuilder - .andWhere('library_item.added_to_following_at IS NOT NULL') - .andWhere('library_item.hidden_at IS NULL') - } else { - queryBuilder.andWhere('library_item.added_to_library_at IS NOT NULL') - - switch (args.inFilter) { - case InFilter.INBOX: - queryBuilder.andWhere('library_item.archived_at IS NULL') - break - case InFilter.ARCHIVE: - queryBuilder.andWhere('library_item.archived_at IS NOT NULL') - break - case InFilter.TRASH: - // return only deleted pages within 14 days - queryBuilder.andWhere( - "library_item.deleted_at >= now() - interval '14 days'" - ) - break - case InFilter.SUBSCRIPTION: - queryBuilder - .andWhere("NOT ('library' ILIKE ANY (library_item.label_names))") - .andWhere('library_item.archived_at IS NULL') - .andWhere('library_item.subscription IS NOT NULL') - break - case InFilter.LIBRARY: - queryBuilder - .andWhere( - "(library_item.subscription IS NULL OR 'library' ILIKE ANY (library_item.label_names))" - ) - .andWhere('library_item.archived_at IS NULL') - } - } + queryBuilder.andWhere('library_item.folder = :folder', { + folder: args.inFilter, + }) } if (args.readFilter !== ReadFilter.ALL) { @@ -171,6 +139,8 @@ const buildWhereClause = ( case HasFilter.LABELS: queryBuilder.andWhere("library_item.label_names <> '{}'") break + case HasFilter.SUBSCRIPTIONS: + queryBuilder.andWhere('library_item.subscription is NOT NULL') } }) } @@ -259,8 +229,8 @@ const buildWhereClause = ( } if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) { - queryBuilder.andWhere('library_item.state <> :state', { - state: LibraryItemState.Deleted, + queryBuilder.andWhere('library_item.folder <> :folder', { + folder: InFilter.TRASH, }) } @@ -397,8 +367,6 @@ export const restoreLibraryItem = async ( { state: LibraryItemState.Succeeded, savedAt: new Date(), - archivedAt: null, - deletedAt: null, }, userId, pubsub @@ -414,22 +382,6 @@ export const updateLibraryItem = async ( const updatedLibraryItem = await authTrx( async (tx) => { const itemRepo = tx.withRepository(libraryItemRepository) - - // reset deletedAt and archivedAt - switch (libraryItem.state) { - case LibraryItemState.Archived: - libraryItem.archivedAt = new Date() - break - case LibraryItemState.Deleted: - libraryItem.deletedAt = new Date() - break - case LibraryItemState.Processing: - case LibraryItemState.Succeeded: - libraryItem.archivedAt = null - libraryItem.deletedAt = null - break - } - await itemRepo.update(id, libraryItem) return itemRepo.findOneByOrFail({ id }) @@ -578,7 +530,7 @@ export const saveFeedItemInFollowing = (input: SaveFollowingItemRequest) => { user: { id: userId }, originalUrl: input.url, subscription: input.addedToFollowingBy, - addedToLibraryAt: null, + folder: InFilter.FOLLOWING, })) return tx @@ -649,14 +601,14 @@ export const updateLibraryItems = async ( switch (action) { case BulkActionType.Archive: values = { - archivedAt: new Date(), - state: LibraryItemState.Archived, + folder: InFilter.ARCHIVE, + savedAt: new Date(), } break case BulkActionType.Delete: values = { - deletedAt: new Date(), - state: LibraryItemState.Deleted, + savedAt: new Date(), + folder: InFilter.TRASH, } break case BulkActionType.AddLabels: diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index ef8c91882..811b176a1 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -261,9 +261,6 @@ export const parsedContentToLibraryItem = ({ uploadFileId: uploadFileId || undefined, readingProgressTopPercent: 0, readingProgressHighestReadAnchor: 0, - state: state - ? (state as unknown as LibraryItemState) - : LibraryItemState.Succeeded, createdAt: validatedDate(saveTime), savedAt: validatedDate(saveTime), siteName: parsedContent?.siteName, @@ -272,8 +269,6 @@ export const parsedContentToLibraryItem = ({ wordCount: wordsCount(parsedContent?.textContent || ''), contentReader: contentReaderForLibraryItem(itemType, uploadFileId), subscription: rssFeedUrl, - archivedAt: - state === ArticleSavingRequestStatus.Archived ? new Date() : undefined, - addedToLibraryAt: validatedDate(saveTime), + folder: state === ArticleSavingRequestStatus.Archived ? 'archive' : 'inbox', } } diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index b0ff7cfcf..4a1554536 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -30,6 +30,7 @@ import { validateUrl } from '../services/create_page_save_request' import { updateLibraryItem } from '../services/library_item' import { Merge } from '../util' import { logger } from './logger' +import { InFilter } from './search' interface InputObject { // eslint-disable-next-line @typescript-eslint/no-explicit-any [key: string]: any @@ -239,7 +240,7 @@ export const libraryItemToArticle = (item: LibraryItem): Article => ({ state: item.state as unknown as ArticleSavingRequestStatus, content: item.readableContent, hash: item.textContentHash || '', - isArchived: !!item.archivedAt, + isArchived: item.folder != InFilter.ARCHIVE, recommendations: item.recommendations?.map( recommandationDataToRecommendation ), @@ -258,7 +259,7 @@ export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({ url: item.originalUrl, state: item.state as unknown as ArticleSavingRequestStatus, content: item.readableContent, - isArchived: !!item.archivedAt, + isArchived: item.folder != InFilter.ARCHIVE, pageType: item.itemType as unknown as PageType, readingProgressPercent: item.readingProgressBottomPercent, contentReader: item.contentReader as unknown as ContentReader, diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index 60f6d64e0..af8e26d9f 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -20,13 +20,11 @@ export enum ReadFilter { } export enum InFilter { - ALL, - INBOX, - ARCHIVE, - TRASH, - SUBSCRIPTION, - LIBRARY, - FOLLOWING, + ALL = 'all', + INBOX = 'inbox', + ARCHIVE = 'archive', + TRASH = 'trash', + FOLLOWING = 'following', } export interface SearchFilter { @@ -59,6 +57,7 @@ export type LabelFilter = { export enum HasFilter { HIGHLIGHTS, LABELS, + SUBSCRIPTIONS, } export interface DateFilter { @@ -134,10 +133,6 @@ const parseInFilter = ( return InFilter.ARCHIVE case 'TRASH': return InFilter.TRASH - case 'SUBSCRIPTION': - return InFilter.SUBSCRIPTION - case 'LIBRARY': - return InFilter.LIBRARY case 'FOLLOWING': return InFilter.FOLLOWING } @@ -236,6 +231,8 @@ const parseHasFilter = (str?: string): HasFilter | undefined => { return HasFilter.HIGHLIGHTS case 'LABELS': return HasFilter.LABELS + case 'SUBSCRIPTIONS': + return HasFilter.SUBSCRIPTIONS } } diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 0763a1a49..be0e3d86c 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -410,8 +410,7 @@ describe('Article API', () => { title, user: { id: user.id }, originalUrl: url, - archivedAt: new Date(), - state: LibraryItemState.Archived, + folder: 'archive', }, user.id ) @@ -609,7 +608,7 @@ describe('Article API', () => { ).expect(200) const savedItem = await findLibraryItemByUrl(url, user.id) - expect(savedItem?.archivedAt).to.not.be.null + expect(savedItem?.folder).to.eql('archive') expect(savedItem?.labels?.map((l) => l.name)).to.eql(labels) }) }) @@ -1031,8 +1030,7 @@ describe('Article API', () => { readableContent: '

test 1

', slug: 'test slug 1', originalUrl: `${url}/test1`, - archivedAt: new Date(), - state: LibraryItemState.Archived, + folder: 'archive', }, { user, @@ -1040,8 +1038,7 @@ describe('Article API', () => { readableContent: '

test 2

', slug: 'test slug 2', originalUrl: `${url}/test2`, - archivedAt: new Date(), - state: LibraryItemState.Archived, + folder: 'archive', }, { user, @@ -1153,12 +1150,12 @@ describe('Article API', () => { }) }) - context("when in:library label:test' is in the query", () => { + context("when in:inbox label:test' is in the query", () => { let items: LibraryItem[] = [] let label: Label before(async () => { - keyword = 'in:library label:test' + keyword = 'in:inbox label:test' // Create some test items label = await createLabel('test', '', user.id) items = await createLibraryItems( @@ -1184,8 +1181,7 @@ describe('Article API', () => { readableContent: '

test 3

', slug: 'test slug 3', originalUrl: `${url}/test3`, - archivedAt: new Date(), - state: LibraryItemState.Archived, + folder: 'archive', }, ], user.id @@ -1276,7 +1272,7 @@ describe('Article API', () => { slug: 'test slug 1', originalUrl: `${url}/test1`, itemType: PageType.File, - archivedAt: new Date(), + folder: 'archive', }, { user, @@ -1284,7 +1280,7 @@ describe('Article API', () => { readableContent: '

test 2

', slug: 'test slug 2', originalUrl: `${url}/test2`, - archivedAt: new Date(), + folder: 'archive', readingProgressBottomPercent: 100, }, { @@ -1325,8 +1321,8 @@ describe('Article API', () => { readableContent: '

test 1

', slug: 'test slug 1', originalUrl: `${url}/test1`, - archivedAt: new Date(), subscription: 'feed', + folder: 'archive', }, { user, @@ -1342,7 +1338,7 @@ describe('Article API', () => { readableContent: '

test 3

', slug: 'test slug 3', originalUrl: `${url}/test3`, - archivedAt: new Date(), + folder: 'archive', }, ], user.id @@ -1375,7 +1371,7 @@ describe('Article API', () => { readableContent: '

test 1

', slug: 'test slug 1', originalUrl: `${url}/test1`, - deletedAt: new Date(), + folder: 'trash', }, { user, @@ -1383,8 +1379,8 @@ describe('Article API', () => { readableContent: '

test 2

', slug: 'test slug 2', originalUrl: `${url}/test2`, - deletedAt: new Date(), readingProgressBottomPercent: 100, + folder: 'trash', }, { user, @@ -1746,7 +1742,7 @@ describe('Article API', () => { for (let i = 0; i < 3; i++) { await updateLibraryItem( items[i].id, - { state: LibraryItemState.Deleted, deletedAt: new Date() }, + { folder: 'trash', savedAt: new Date() }, user.id ) deletedItems.push(items[i]) diff --git a/packages/api/test/routers/auth.test.ts b/packages/api/test/routers/auth.test.ts index 47d87737f..5cf40e3e3 100644 --- a/packages/api/test/routers/auth.test.ts +++ b/packages/api/test/routers/auth.test.ts @@ -15,6 +15,7 @@ import { generateVerificationToken, hashPassword, } from '../../src/utils/auth' +import { InFilter } from '../../src/utils/search' import * as util from '../../src/utils/sendEmail' import { createTestUser } from '../db' import { generateFakeUuid, request } from '../util' @@ -607,7 +608,10 @@ describe('auth router', () => { 'web' ).expect(200) const user = await userRepository.findOneByOrFail({ name }) - const { count } = await searchLibraryItems({}, user.id) + const { count } = await searchLibraryItems( + { inFilter: InFilter.ALL }, + user.id + ) expect(count).to.eql(3) }) @@ -628,7 +632,10 @@ describe('auth router', () => { 'ios' ).expect(200) const user = await userRepository.findOneByOrFail({ name }) - const { count } = await searchLibraryItems({}, user.id) + const { count } = await searchLibraryItems( + { inFilter: InFilter.ALL }, + user.id + ) expect(count).to.eql(4) }) diff --git a/packages/db/migrations/0146.do.following.sql b/packages/db/migrations/0146.do.following.sql index e5ea56937..44afe8da5 100755 --- a/packages/db/migrations/0146.do.following.sql +++ b/packages/db/migrations/0146.do.following.sql @@ -9,14 +9,10 @@ ALTER TABLE omnivore.subscriptions ADD COLUMN auto_add_to_library boolean; ALTER TABLE omnivore.library_item - ADD COLUMN hidden_at timestamptz, - ADD COLUMN added_to_following_at timestamptz, - ADD COLUMN added_to_following_by text, ADD COLUMN links jsonb, ADD COLUMN preview_content text, ADD COLUMN preview_content_type text, - ADD COLUMN added_to_following_from text, - ADD COLUMN added_to_library_at timestamptz DEFAULT current_timestamp; + ADD COLUMN folder text NOT NULL DEFAULT 'inbox'; CREATE POLICY library_item_admin_policy on omnivore.library_item FOR ALL diff --git a/packages/db/migrations/0146.undo.following.sql b/packages/db/migrations/0146.undo.following.sql index 46d8abbae..10054874c 100755 --- a/packages/db/migrations/0146.undo.following.sql +++ b/packages/db/migrations/0146.undo.following.sql @@ -9,14 +9,10 @@ DROP TABLE omnivore.feed; DROP policy library_item_admin_policy ON omnivore.library_item; ALTER TABLE omnivore.library_item - DROP COLUMN hidden_at, - DROP COLUMN added_to_following_at, - DROP COLUMN added_to_following_by, DROP COLUMN links, - DROP COLUMN preview_content_type, DROP COLUMN preview_content, - DROP COLUMN added_to_following_from, - DROP COLUMN added_to_library_at; + DROP COLUMN preview_content_type, + DROP COLUMN folder; ALTER TABLE omnivore.subscriptions DROP COLUMN is_private, diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 1648cdefc..34b0a7f9d 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -184,7 +184,6 @@ const createFollowingTask = async ( addedToFollowingBy: feedUrl, savedAt: item.isoDate, publishedAt: item.isoDate, - addedToFollowingAt: item.isoDate, previewContentType: 'text/html', // TODO: get content type from feed } From 59bbc5562b9cab6f358eb4561bb89023f07240ee Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 10 Nov 2023 22:37:14 +0800 Subject: [PATCH 26/30] fix tests --- packages/api/src/services/library_item.ts | 4 +--- packages/api/src/utils/helpers.ts | 4 ++-- packages/api/test/resolvers/article.test.ts | 11 +---------- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 10488c1e4..9eb4b6734 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -229,9 +229,7 @@ const buildWhereClause = ( } if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) { - queryBuilder.andWhere('library_item.folder <> :folder', { - folder: InFilter.TRASH, - }) + queryBuilder.andWhere("library_item.folder <> 'trash'") } if (args.noFilters) { diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 4a1554536..80a883be7 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -240,7 +240,7 @@ export const libraryItemToArticle = (item: LibraryItem): Article => ({ state: item.state as unknown as ArticleSavingRequestStatus, content: item.readableContent, hash: item.textContentHash || '', - isArchived: item.folder != InFilter.ARCHIVE, + isArchived: item.folder === InFilter.ARCHIVE, recommendations: item.recommendations?.map( recommandationDataToRecommendation ), @@ -259,7 +259,7 @@ export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({ url: item.originalUrl, state: item.state as unknown as ArticleSavingRequestStatus, content: item.readableContent, - isArchived: item.folder != InFilter.ARCHIVE, + isArchived: item.folder === InFilter.ARCHIVE, pageType: item.itemType as unknown as PageType, readingProgressPercent: item.readingProgressBottomPercent, contentReader: item.contentReader as unknown as ContentReader, diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index be0e3d86c..6fc8f5f88 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -691,7 +691,7 @@ describe('Article API', () => { 200 ) const item = await findLibraryItemById(itemId, user.id) - expect(item?.state).to.eql(LibraryItemState.Deleted) + expect(item?.folder).to.eql('trash') }) }) @@ -1167,14 +1167,6 @@ describe('Article API', () => { slug: 'test slug 1', originalUrl: `${url}/test1`, }, - { - user, - title: 'test title 2', - readableContent: '

test 2

', - slug: 'test slug 2', - originalUrl: `${url}/test2`, - subscription: 'test subscription', - }, { user, title: 'test title 3', @@ -1188,7 +1180,6 @@ describe('Article API', () => { ) await saveLabelsInLibraryItem([label], items[0].id, user.id) await saveLabelsInLibraryItem([label], items[1].id, user.id) - await saveLabelsInLibraryItem([label], items[2].id, user.id) }) after(async () => { From 2ade0698c503f172d18840d35a8279f8a67e58a4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 10 Nov 2023 23:03:01 +0800 Subject: [PATCH 27/30] remove unused code --- .../api/src/services/integrations/pocket.ts | 128 +----------------- .../api/src/services/integrations/readwise.ts | 27 ---- packages/api/src/services/library_item.ts | 4 +- packages/api/src/services/save_file.ts | 15 +- packages/api/src/services/save_page.ts | 1 + 5 files changed, 14 insertions(+), 161 deletions(-) diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 86a99f8ef..1ea4d6d87 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -1,55 +1,7 @@ import axios from 'axios' -import { LibraryItemState } from '../../entity/library_item' import { env } from '../../env' import { logger } from '../../utils/logger' -import { - IntegrationClient, - RetrievedResult, - RetrieveRequest, -} from './integration' - -interface PocketResponse { - status: number // 1 if success - complete: number // 1 if all items have been returned - list: { - [key: string]: PocketItem - } - since: number // unix timestamp in seconds - search_meta: { - search_type: string - } - error: string -} - -interface PocketItem { - item_id: string - resolved_id: string - given_url: string - resolved_url: string - given_title: string - resolved_title: string - favorite: string - status: string - excerpt: string - word_count: string - tags?: { - [key: string]: Tag - } - authors?: { - [key: string]: Author - } -} - -interface Tag { - item_id: string - tag: string -} - -interface Author { - item_id: string - author_id: string - name: string -} +import { IntegrationClient } from './integration' export class PocketClient implements IntegrationClient { name = 'POCKET' @@ -83,82 +35,4 @@ export class PocketClient implements IntegrationClient { return null } } - - retrievePocketData = async ( - accessToken: string, - since: number, // unix timestamp in seconds - count = 100, - offset = 0 - ): Promise => { - const url = `${this.apiUrl}/get` - try { - const response = await axios.post( - url, - { - consumer_key: env.pocket.consumerKey, - access_token: accessToken, - state: 'all', - detailType: 'complete', - since, - sort: 'oldest', - count, - offset, - }, - { - headers: this.headers, - timeout: 10000, // 10 seconds - } - ) - - return response.data - } catch (error) { - if (axios.isAxiosError(error)) { - logger.error(error.response) - } else { - logger.error(error) - } - - return null - } - } - - retrieve = async ({ - token, - since = 0, - count = 100, - offset = 0, - }: RetrieveRequest): Promise => { - const pocketData = await this.retrievePocketData( - token, - since / 1000, - count, - offset - ) - if (!pocketData) { - throw new Error('Error retrieving pocket data') - } - - const pocketItems = Object.values(pocketData.list) - const statusToState: Record = { - '0': LibraryItemState.Succeeded, - '1': LibraryItemState.Archived, - '2': LibraryItemState.Deleted, - } - const data = pocketItems.map((item) => ({ - url: item.given_url, - labels: item.tags - ? Object.values(item.tags).map((tag) => tag.tag) - : undefined, - state: statusToState[item.status], - })) - - if (pocketData.error) { - throw new Error(`Error retrieving pocket data: ${pocketData.error}`) - } - - return { - data, - since: pocketData.since * 1000, - } - } } diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 42f50f744..7764294d2 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -2,33 +2,6 @@ import axios from 'axios' import { logger } from '../../utils/logger' import { IntegrationClient } from './integration' -interface ReadwiseHighlight { - // The highlight text, (technically the only field required in a highlight object) - text: string - // The title of the page the highlight is on - title?: string - // The author of the page the highlight is on - author?: string - // The URL of the page image - image_url?: string - // The URL of the page - source_url?: string - // A meaningful unique identifier for your app - source_type?: string - // One of: books, articles, tweets or podcasts - category?: string - // Annotation note attached to the specific highlight - note?: string - // Highlight's location in the source text. Used to order the highlights - location?: number - // One of: page, order or time_offset - location_type?: string - // A datetime representing when the highlight was taken in the ISO 8601 format - highlighted_at?: string - // Unique url of the specific highlight - highlight_url?: string -} - export class ReadwiseClient implements IntegrationClient { name = 'READWISE' apiUrl = 'https://readwise.io/api/v2' diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 9eb4b6734..5cbfeb134 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -223,9 +223,7 @@ const buildWhereClause = ( } if (!args.includePending) { - queryBuilder.andWhere('library_item.state <> :state', { - state: LibraryItemState.Processing, - }) + queryBuilder.andWhere("library_item.state <> 'PROCESSING'") } if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) { diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index f428cb77a..bffe06fb8 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -1,7 +1,12 @@ import { LibraryItemState } from '../entity/library_item' import { User } from '../entity/user' import { homePageURL } from '../env' -import { SaveErrorCode, SaveFileInput, SaveResult } from '../generated/graphql' +import { + ArticleSavingRequestStatus, + SaveErrorCode, + SaveFileInput, + SaveResult, +} from '../generated/graphql' import { getStorageFileDetails } from '../utils/uploads' import { findOrCreateLabels, saveLabelsInLibraryItem } from './labels' import { updateLibraryItem } from './library_item' @@ -32,9 +37,11 @@ export const saveFile = async ( await updateLibraryItem( input.clientRequestId, { - state: input.state - ? (input.state as unknown as LibraryItemState) - : LibraryItemState.Succeeded, + state: LibraryItemState.Succeeded, + folder: + input.state === ArticleSavingRequestStatus.Archived + ? 'archive' + : 'inbox', }, user.id ) diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 811b176a1..5ee1b2708 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -270,5 +270,6 @@ export const parsedContentToLibraryItem = ({ contentReader: contentReaderForLibraryItem(itemType, uploadFileId), subscription: rssFeedUrl, folder: state === ArticleSavingRequestStatus.Archived ? 'archive' : 'inbox', + state: LibraryItemState.Succeeded, } } From 05b386ad791a8a8fd950169b4fa3d0ccceec1630 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sat, 11 Nov 2023 11:28:16 +0800 Subject: [PATCH 28/30] migrate existing data --- packages/db/migrations/0146.do.following.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/db/migrations/0146.do.following.sql b/packages/db/migrations/0146.do.following.sql index 44afe8da5..30d92256c 100755 --- a/packages/db/migrations/0146.do.following.sql +++ b/packages/db/migrations/0146.do.following.sql @@ -14,6 +14,9 @@ ALTER TABLE omnivore.library_item ADD COLUMN preview_content_type text, ADD COLUMN folder text NOT NULL DEFAULT 'inbox'; +UPDATE omnivore.library_item SET folder = 'archive' WHERE archived_at IS NOT NULL; +UPDATE omnivore.library_item SET folder = 'trash' WHERE deleted_at IS NOT NULL; + CREATE POLICY library_item_admin_policy on omnivore.library_item FOR ALL TO omnivore_admin From 3d84108202d8231cd2eac730bd87f792c9f52e17 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 14 Nov 2023 15:27:58 +0800 Subject: [PATCH 29/30] remove feeds table --- .../api/src/resolvers/subscriptions/index.ts | 50 ++++++------------- .../api/test/resolvers/subscriptions.test.ts | 2 - packages/db/migrations/0146.do.following.sql | 19 ------- .../db/migrations/0146.undo.following.sql | 2 - 4 files changed, 15 insertions(+), 58 deletions(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 4b4ede2ec..0680339ff 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -1,6 +1,5 @@ import Parser from 'rss-parser' import { Brackets } from 'typeorm' -import { Feed } from '../../entity/feed' import { Subscription } from '../../entity/subscription' import { env } from '../../env' import { @@ -227,44 +226,25 @@ export const subscribeResolver = authorized< // validate rss feed const feed = await parser.parseURL(input.url) - const results = await authTrx(async (t) => { - if (!input.isPrivate) { - await t.getRepository(Feed).upsert( - { - url: feed.feedUrl, - title: feed.title, - description: feed.description, - image: feed.image?.url, - }, - { - conflictPaths: ['url'], - skipUpdateIfNoValuesChanged: true, - } - ) - } - - // limit number of rss subscriptions to 150 - const results = (await t.getRepository(Subscription).query( - `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, auto_add_to_library, is_private) + // limit number of rss subscriptions to 150 + const results = (await getRepository(Subscription).query( + `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, auto_add_to_library, is_private) select $1, $2, $3, $4, $5, $6, $7, $8 from omnivore.subscriptions where user_id = $5 and type = 'RSS' and status = 'ACTIVE' having count(*) < $9 returning *;`, - [ - feed.title, - feed.feedUrl, - feed.description || null, - SubscriptionType.Rss, - uid, - feed.image?.url || null, - input.autoAddToLibrary ?? null, - input.isPrivate ?? null, - MAX_RSS_SUBSCRIPTIONS, - ] - )) as Subscription[] - - return results - }) + [ + feed.title, + feed.feedUrl, + feed.description || null, + SubscriptionType.Rss, + uid, + feed.image?.url || null, + input.autoAddToLibrary ?? null, + input.isPrivate ?? null, + MAX_RSS_SUBSCRIPTIONS, + ] + )) as Subscription[] if (results.length === 0) { return { diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 28d96b564..cfdf6b10e 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -3,7 +3,6 @@ import 'mocha' import Parser from 'rss-parser' import sinon from 'sinon' import sinonChai from 'sinon-chai' -import { Feed } from '../../src/entity/feed' import { NewsletterEmail } from '../../src/entity/newsletter_email' import { Subscription } from '../../src/entity/subscription' import { User } from '../../src/entity/user' @@ -406,7 +405,6 @@ describe('Subscriptions API', () => { after(async () => { await deleteSubscription(existingSubscription.id) - await getRepository(Feed).delete({ url: existingSubscription.url }) }) it('returns an error', async () => { diff --git a/packages/db/migrations/0146.do.following.sql b/packages/db/migrations/0146.do.following.sql index 30d92256c..61e59ccc7 100755 --- a/packages/db/migrations/0146.do.following.sql +++ b/packages/db/migrations/0146.do.following.sql @@ -22,23 +22,4 @@ CREATE POLICY library_item_admin_policy on omnivore.library_item TO omnivore_admin USING (true); -CREATE TABLE omnivore.feed ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), - title text NOT NULL, - url text NOT NULL, - author text, - description text, - image text, - created_at timestamptz NOT NULL DEFAULT current_timestamp, - updated_at timestamptz NOT NULL DEFAULT current_timestamp, - published_at timestamptz, - UNIQUE(url) -); - -CREATE INDEX feed_title_idx ON omnivore.feed(title); - -CREATE TRIGGER update_feed_modtime BEFORE UPDATE ON omnivore.feed FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); - -GRANT SELECT, INSERT, UPDATE, DELETE ON omnivore.feed TO omnivore_user; - COMMIT; diff --git a/packages/db/migrations/0146.undo.following.sql b/packages/db/migrations/0146.undo.following.sql index 10054874c..8d41f08e4 100755 --- a/packages/db/migrations/0146.undo.following.sql +++ b/packages/db/migrations/0146.undo.following.sql @@ -4,8 +4,6 @@ BEGIN; -DROP TABLE omnivore.feed; - DROP policy library_item_admin_policy ON omnivore.library_item; ALTER TABLE omnivore.library_item From 609d30adbb7fc99b0d869d54d0f6ee46f1acfead Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 Nov 2023 10:22:58 +0800 Subject: [PATCH 30/30] compatiblity with old saved searches --- packages/api/src/services/create_user.ts | 2 +- packages/api/src/services/library_item.ts | 4 +++- packages/api/src/utils/search.ts | 9 ++++++++- .../web/components/templates/homeFeed/EmptyLibrary.tsx | 4 ++-- .../components/templates/homeFeed/LibraryFilterMenu.tsx | 2 +- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/api/src/services/create_user.ts b/packages/api/src/services/create_user.ts index 6fd8e2033..a04ff8164 100644 --- a/packages/api/src/services/create_user.ts +++ b/packages/api/src/services/create_user.ts @@ -159,7 +159,7 @@ const createDefaultFiltersForUser = name: 'Continue Reading', filter: 'in:inbox sort:read-desc is:reading', }, - { name: 'Non-Feed Items', filter: 'in:library' }, + { name: 'Non-Feed Items', filter: 'no:subscription' }, { name: 'Highlights', filter: 'has:highlights mode:highlights' }, { name: 'Unlabeled', filter: 'no:label' }, { name: 'Oldest First', filter: 'sort:saved-asc' }, diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 5cbfeb134..6d66184a3 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -232,7 +232,9 @@ const buildWhereClause = ( if (args.noFilters) { args.noFilters.forEach((filter) => { - queryBuilder.andWhere(`library_item.${filter.field} = '{}'`) + queryBuilder.andWhere( + `library_item.${filter.field} = '{}' OR library_item.${filter.field} IS NULL` + ) }) } diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index af8e26d9f..dde0e019d 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -386,13 +386,20 @@ const parseNoFilter = (str?: string): NoFilter | undefined => { return { field: 'highlight_annotations' } case 'label': return { field: 'label_names' } + case 'subscription': + return { field: 'subscription' } } return undefined } export const parseSearchQuery = (query: string | undefined): SearchFilter => { - const searchQuery = query ? query.replace(/\W\s":/g, '') : undefined + const searchQuery = query + ? query + .replace(/\W\s":/g, '') + .replace('in:subscription', 'has:subscriptions') // compatibility with old search + .replace('in:library', 'no:subscription') // compatibility with old search + : undefined const result: SearchFilter = { query: searchQuery, readFilter: ReadFilter.ALL, diff --git a/packages/web/components/templates/homeFeed/EmptyLibrary.tsx b/packages/web/components/templates/homeFeed/EmptyLibrary.tsx index 72da7891b..23dfe3d7b 100644 --- a/packages/web/components/templates/homeFeed/EmptyLibrary.tsx +++ b/packages/web/components/templates/homeFeed/EmptyLibrary.tsx @@ -169,7 +169,7 @@ export const EmptyLibrary = (props: EmptyLibraryProps) => { return 'inbox' case 'in:inbox sort:read-desc is:reading': return 'continue' - case 'in:library': + case 'no:subscription': return 'non-feed' case 'has:highlights mode:highlights': return 'highlight' @@ -181,7 +181,7 @@ export const EmptyLibrary = (props: EmptyLibraryProps) => { return 'archive' case 'label:RSS': return 'feed' - case 'in:subscription': + case 'has:subscriptions': return 'subscription' case 'label:Newsletter': return 'newsletter' diff --git a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx index 13a27dea4..d7bb99f9b 100644 --- a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx +++ b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx @@ -241,7 +241,7 @@ function Subscriptions( > {!collapsed ? ( <> - +