diff --git a/packages/api/src/entity/feed.ts b/packages/api/src/entity/feed.ts new file mode 100644 index 000000000..ab6412602 --- /dev/null +++ b/packages/api/src/entity/feed.ts @@ -0,0 +1,37 @@ +import { + Column, + CreateDateColumn, + Entity, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' + +@Entity() +export class Feed { + @PrimaryGeneratedColumn('uuid') + id!: string + + @Column('text') + title!: string + + @Column('text') + url!: string + + @Column('text') + author?: string + + @Column('text') + description?: string + + @Column('text') + image?: string + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + createdAt!: Date + + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + updatedAt!: Date + + @Column('timestamptz') + publishedAt?: Date | null +} diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 5689c83d6..447de3bfa 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -196,4 +196,16 @@ export class LibraryItem { @Column('text', { nullable: true }) recommenderNames?: string[] | null + + @Column('jsonb') + links?: any | null + + @Column('text') + previewContent?: string | null + + @Column('text') + previewContentType?: string | null + + @Column('text') + folder!: string } diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index fea63d890..10416a551 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -70,4 +70,10 @@ export class Subscription { @Column('timestamp', { nullable: true }) scheduledAt?: Date | null + + @Column('boolean') + isPrivate?: boolean | null + + @Column('boolean') + autoAddToLibrary?: boolean | null } diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 773e67137..e6c518eeb 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -725,6 +725,19 @@ export type Feature = { updatedAt?: Maybe; }; +export type Feed = { + __typename?: 'Feed'; + author?: Maybe; + createdAt: Scalars['Date']; + description?: Maybe; + id: Scalars['ID']; + image?: Maybe; + publishedAt?: Maybe; + title: Scalars['String']; + updatedAt: Scalars['Date']; + url: Scalars['String']; +}; + export type FeedArticle = { __typename?: 'FeedArticle'; annotationsCount?: Maybe; @@ -762,6 +775,37 @@ export type FeedArticlesSuccess = { pageInfo: PageInfo; }; +export type FeedEdge = { + __typename?: 'FeedEdge'; + cursor: Scalars['String']; + node: Feed; +}; + +export type FeedsError = { + __typename?: 'FeedsError'; + errorCodes: Array; +}; + +export enum FeedsErrorCode { + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED' +} + +export type FeedsInput = { + after?: InputMaybe; + first?: InputMaybe; + query?: InputMaybe; + sort?: InputMaybe; +}; + +export type FeedsResult = FeedsError | FeedsSuccess; + +export type FeedsSuccess = { + __typename?: 'FeedsSuccess'; + edges: Array; + pageInfo: PageInfo; +}; + export type Filter = { __typename?: 'Filter'; category: Scalars['String']; @@ -1239,6 +1283,24 @@ export type MoveLabelSuccess = { label: Label; }; +export type MoveToFolderError = { + __typename?: 'MoveToFolderError'; + errorCodes: Array; +}; + +export enum MoveToFolderErrorCode { + AlreadyExists = 'ALREADY_EXISTS', + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED' +} + +export type MoveToFolderResult = MoveToFolderError | MoveToFolderSuccess; + +export type MoveToFolderSuccess = { + __typename?: 'MoveToFolderSuccess'; + articleSavingRequest: ArticleSavingRequest; +}; + export type Mutation = { __typename?: 'Mutation'; addPopularRead: AddPopularReadResult; @@ -1268,6 +1330,7 @@ export type Mutation = { mergeHighlight: MergeHighlightResult; moveFilter: MoveFilterResult; moveLabel: MoveLabelResult; + moveToFolder: MoveToFolderResult; optInFeature: OptInFeatureResult; recommend: RecommendResult; recommendHighlights: RecommendHighlightsResult; @@ -1432,6 +1495,12 @@ export type MutationMoveLabelArgs = { }; +export type MutationMoveToFolderArgs = { + folder: Scalars['String']; + id: Scalars['ID']; +}; + + export type MutationOptInFeatureArgs = { input: OptInFeatureInput; }; @@ -1725,6 +1794,7 @@ export type Query = { article: ArticleResult; articleSavingRequest: ArticleSavingRequestResult; deviceTokens: DeviceTokensResult; + feeds: FeedsResult; filters: FiltersResult; getUserPersonalization: GetUserPersonalizationResult; groups: GroupsResult; @@ -1762,6 +1832,11 @@ export type QueryArticleSavingRequestArgs = { }; +export type QueryFeedsArgs = { + input: FeedsInput; +}; + + export type QueryRulesArgs = { enabled?: InputMaybe; }; @@ -2628,6 +2703,8 @@ export enum SubscribeErrorCode { } export type SubscribeInput = { + autoAddToLibrary?: InputMaybe; + isPrivate?: InputMaybe; subscriptionType?: InputMaybe; url: Scalars['String']; }; @@ -2641,11 +2718,13 @@ export type SubscribeSuccess = { export type Subscription = { __typename?: 'Subscription'; + autoAddToLibrary?: Maybe; count: Scalars['Int']; createdAt: Scalars['Date']; description?: Maybe; icon?: Maybe; id: Scalars['ID']; + isPrivate?: Maybe; lastFetchedAt?: Maybe; name: Scalars['String']; newsletterEmail?: Maybe; @@ -2986,8 +3065,10 @@ export enum UpdateSubscriptionErrorCode { } export type UpdateSubscriptionInput = { + autoAddToLibrary?: InputMaybe; description?: InputMaybe; id: Scalars['ID']; + isPrivate?: InputMaybe; lastFetchedAt?: InputMaybe; lastFetchedChecksum?: InputMaybe; name?: InputMaybe; @@ -3459,12 +3540,19 @@ export type ResolversTypes = { DeviceTokensResult: ResolversTypes['DeviceTokensError'] | ResolversTypes['DeviceTokensSuccess']; DeviceTokensSuccess: ResolverTypeWrapper; Feature: ResolverTypeWrapper; + Feed: ResolverTypeWrapper; FeedArticle: ResolverTypeWrapper; FeedArticleEdge: ResolverTypeWrapper; FeedArticlesError: ResolverTypeWrapper; FeedArticlesErrorCode: FeedArticlesErrorCode; FeedArticlesResult: ResolversTypes['FeedArticlesError'] | ResolversTypes['FeedArticlesSuccess']; FeedArticlesSuccess: ResolverTypeWrapper; + FeedEdge: ResolverTypeWrapper; + FeedsError: ResolverTypeWrapper; + FeedsErrorCode: FeedsErrorCode; + FeedsInput: FeedsInput; + FeedsResult: ResolversTypes['FeedsError'] | ResolversTypes['FeedsSuccess']; + FeedsSuccess: ResolverTypeWrapper; Filter: ResolverTypeWrapper; FiltersError: ResolverTypeWrapper; FiltersErrorCode: FiltersErrorCode; @@ -3557,6 +3645,10 @@ export type ResolversTypes = { MoveLabelInput: MoveLabelInput; MoveLabelResult: ResolversTypes['MoveLabelError'] | ResolversTypes['MoveLabelSuccess']; MoveLabelSuccess: ResolverTypeWrapper; + MoveToFolderError: ResolverTypeWrapper; + MoveToFolderErrorCode: MoveToFolderErrorCode; + MoveToFolderResult: ResolversTypes['MoveToFolderError'] | ResolversTypes['MoveToFolderSuccess']; + MoveToFolderSuccess: ResolverTypeWrapper; Mutation: ResolverTypeWrapper<{}>; NewsletterEmail: ResolverTypeWrapper; NewsletterEmailsError: ResolverTypeWrapper; @@ -3939,11 +4031,17 @@ export type ResolversParentTypes = { DeviceTokensResult: ResolversParentTypes['DeviceTokensError'] | ResolversParentTypes['DeviceTokensSuccess']; DeviceTokensSuccess: DeviceTokensSuccess; Feature: Feature; + Feed: Feed; FeedArticle: FeedArticle; FeedArticleEdge: FeedArticleEdge; FeedArticlesError: FeedArticlesError; FeedArticlesResult: ResolversParentTypes['FeedArticlesError'] | ResolversParentTypes['FeedArticlesSuccess']; FeedArticlesSuccess: FeedArticlesSuccess; + FeedEdge: FeedEdge; + FeedsError: FeedsError; + FeedsInput: FeedsInput; + FeedsResult: ResolversParentTypes['FeedsError'] | ResolversParentTypes['FeedsSuccess']; + FeedsSuccess: FeedsSuccess; Filter: Filter; FiltersError: FiltersError; FiltersResult: ResolversParentTypes['FiltersError'] | ResolversParentTypes['FiltersSuccess']; @@ -4016,6 +4114,9 @@ export type ResolversParentTypes = { MoveLabelInput: MoveLabelInput; MoveLabelResult: ResolversParentTypes['MoveLabelError'] | ResolversParentTypes['MoveLabelSuccess']; MoveLabelSuccess: MoveLabelSuccess; + MoveToFolderError: MoveToFolderError; + MoveToFolderResult: ResolversParentTypes['MoveToFolderError'] | ResolversParentTypes['MoveToFolderSuccess']; + MoveToFolderSuccess: MoveToFolderSuccess; Mutation: {}; NewsletterEmail: NewsletterEmail; NewsletterEmailsError: NewsletterEmailsError; @@ -4734,6 +4835,19 @@ export type FeatureResolvers; }; +export type FeedResolvers = { + author?: Resolver, ParentType, ContextType>; + createdAt?: Resolver; + description?: Resolver, ParentType, ContextType>; + id?: Resolver; + image?: Resolver, ParentType, ContextType>; + publishedAt?: Resolver, ParentType, ContextType>; + title?: Resolver; + updatedAt?: Resolver; + url?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type FeedArticleResolvers = { annotationsCount?: Resolver, ParentType, ContextType>; article?: Resolver; @@ -4769,6 +4883,27 @@ export type FeedArticlesSuccessResolvers; }; +export type FeedEdgeResolvers = { + cursor?: Resolver; + node?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type FeedsErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type FeedsResultResolvers = { + __resolveType: TypeResolveFn<'FeedsError' | 'FeedsSuccess', ParentType, ContextType>; +}; + +export type FeedsSuccessResolvers = { + edges?: Resolver, ParentType, ContextType>; + pageInfo?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type FilterResolvers = { category?: Resolver; createdAt?: Resolver; @@ -5126,6 +5261,20 @@ export type MoveLabelSuccessResolvers; }; +export type MoveToFolderErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type MoveToFolderResultResolvers = { + __resolveType: TypeResolveFn<'MoveToFolderError' | 'MoveToFolderSuccess', ParentType, ContextType>; +}; + +export type MoveToFolderSuccessResolvers = { + articleSavingRequest?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type MutationResolvers = { addPopularRead?: Resolver>; bulkAction?: Resolver>; @@ -5154,6 +5303,7 @@ export type MutationResolvers>; moveFilter?: Resolver>; moveLabel?: Resolver>; + moveToFolder?: Resolver>; optInFeature?: Resolver>; recommend?: Resolver>; recommendHighlights?: Resolver>; @@ -5266,6 +5416,7 @@ export type QueryResolvers>; articleSavingRequest?: Resolver>; deviceTokens?: Resolver; + feeds?: Resolver>; filters?: Resolver; getUserPersonalization?: Resolver; groups?: Resolver; @@ -5800,11 +5951,13 @@ export type SubscribeSuccessResolvers = { + autoAddToLibrary?: SubscriptionResolver, "autoAddToLibrary", ParentType, ContextType>; count?: SubscriptionResolver; createdAt?: SubscriptionResolver; description?: SubscriptionResolver, "description", ParentType, ContextType>; icon?: SubscriptionResolver, "icon", ParentType, ContextType>; id?: SubscriptionResolver; + isPrivate?: SubscriptionResolver, "isPrivate", ParentType, ContextType>; lastFetchedAt?: SubscriptionResolver, "lastFetchedAt", ParentType, ContextType>; name?: SubscriptionResolver; newsletterEmail?: SubscriptionResolver, "newsletterEmail", ParentType, ContextType>; @@ -6287,11 +6440,16 @@ export type Resolvers = { DeviceTokensResult?: DeviceTokensResultResolvers; DeviceTokensSuccess?: DeviceTokensSuccessResolvers; Feature?: FeatureResolvers; + Feed?: FeedResolvers; FeedArticle?: FeedArticleResolvers; FeedArticleEdge?: FeedArticleEdgeResolvers; FeedArticlesError?: FeedArticlesErrorResolvers; FeedArticlesResult?: FeedArticlesResultResolvers; FeedArticlesSuccess?: FeedArticlesSuccessResolvers; + FeedEdge?: FeedEdgeResolvers; + FeedsError?: FeedsErrorResolvers; + FeedsResult?: FeedsResultResolvers; + FeedsSuccess?: FeedsSuccessResolvers; Filter?: FilterResolvers; FiltersError?: FiltersErrorResolvers; FiltersResult?: FiltersResultResolvers; @@ -6355,6 +6513,9 @@ export type Resolvers = { MoveLabelError?: MoveLabelErrorResolvers; MoveLabelResult?: MoveLabelResultResolvers; MoveLabelSuccess?: MoveLabelSuccessResolvers; + MoveToFolderError?: MoveToFolderErrorResolvers; + MoveToFolderResult?: MoveToFolderResultResolvers; + MoveToFolderSuccess?: MoveToFolderSuccessResolvers; Mutation?: MutationResolvers; NewsletterEmail?: NewsletterEmailResolvers; NewsletterEmailsError?: NewsletterEmailsErrorResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 09150be0c..01ea5b6b3 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -641,6 +641,18 @@ type Feature { updatedAt: Date } +type Feed { + author: String + createdAt: Date! + description: String + id: ID! + image: String + publishedAt: Date + title: String! + updatedAt: Date! + url: String! +} + type FeedArticle { annotationsCount: Int article: Article! @@ -674,6 +686,34 @@ type FeedArticlesSuccess { pageInfo: PageInfo! } +type FeedEdge { + cursor: String! + node: Feed! +} + +type FeedsError { + errorCodes: [FeedsErrorCode!]! +} + +enum FeedsErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +input FeedsInput { + after: String + first: Int + query: String + sort: SortParams +} + +union FeedsResult = FeedsError | FeedsSuccess + +type FeedsSuccess { + edges: [FeedEdge!]! + pageInfo: PageInfo! +} + type Filter { category: String! createdAt: Date! @@ -1109,6 +1149,22 @@ type MoveLabelSuccess { label: Label! } +type MoveToFolderError { + errorCodes: [MoveToFolderErrorCode!]! +} + +enum MoveToFolderErrorCode { + ALREADY_EXISTS + BAD_REQUEST + UNAUTHORIZED +} + +union MoveToFolderResult = MoveToFolderError | MoveToFolderSuccess + +type MoveToFolderSuccess { + articleSavingRequest: ArticleSavingRequest! +} + type Mutation { addPopularRead(name: String!): AddPopularReadResult! bulkAction(action: BulkActionType!, async: Boolean, expectedCount: Int, labelIds: [ID!], query: String!): BulkActionResult! @@ -1137,6 +1193,7 @@ type Mutation { mergeHighlight(input: MergeHighlightInput!): MergeHighlightResult! moveFilter(input: MoveFilterInput!): MoveFilterResult! moveLabel(input: MoveLabelInput!): MoveLabelResult! + moveToFolder(folder: String!, id: ID!): MoveToFolderResult! optInFeature(input: OptInFeatureInput!): OptInFeatureResult! recommend(input: RecommendInput!): RecommendResult! recommendHighlights(input: RecommendHighlightsInput!): RecommendHighlightsResult! @@ -1294,6 +1351,7 @@ type Query { article(format: String, slug: String!, username: String!): ArticleResult! articleSavingRequest(id: ID, url: String): ArticleSavingRequestResult! deviceTokens: DeviceTokensResult! + feeds(input: FeedsInput!): FeedsResult! filters: FiltersResult! getUserPersonalization: GetUserPersonalizationResult! groups: GroupsResult! @@ -2071,6 +2129,8 @@ enum SubscribeErrorCode { } input SubscribeInput { + autoAddToLibrary: Boolean + isPrivate: Boolean subscriptionType: SubscriptionType url: String! } @@ -2082,11 +2142,13 @@ type SubscribeSuccess { } type Subscription { + autoAddToLibrary: Boolean count: Int! createdAt: Date! description: String icon: String id: ID! + isPrivate: Boolean lastFetchedAt: Date name: String! newsletterEmail: String @@ -2400,8 +2462,10 @@ enum UpdateSubscriptionErrorCode { } input UpdateSubscriptionInput { + autoAddToLibrary: Boolean description: String id: ID! + isPrivate: Boolean lastFetchedAt: Date lastFetchedChecksum: String name: String diff --git a/packages/api/src/repository/feed.ts b/packages/api/src/repository/feed.ts new file mode 100644 index 000000000..72f337d16 --- /dev/null +++ b/packages/api/src/repository/feed.ts @@ -0,0 +1,33 @@ +import { ILike } from 'typeorm' +import { appDataSource } from '../data_source' +import { Feed } from '../entity/feed' + +export const feedRepository = appDataSource.getRepository(Feed).extend({ + async searchFeeds( + query = '', + take = 10, + skip = 0, + orderBy = 'title', + order = 'ASC' + ) { + const where = [] + if (query !== '') { + query = `%${query}%` + where.push({ title: ILike(query) }, { url: ILike(query) }) + } + + const feeds = await this.find({ + where, + order: { [orderBy]: order }, + take, + skip, + }) + + const count = await this.countBy(where) + + return { + feeds, + count, + } + }, +}) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 8a4c8c8c3..8f9adaedb 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -6,7 +6,7 @@ import { Readability } from '@omnivore/readability' import graphqlFields from 'graphql-fields' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' -import { LibraryItem, LibraryItemState } from '../../entity/library_item' +import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' import { ArticleError, @@ -95,7 +95,11 @@ import { ParsedContentPuppeteer, parsePreparedContent, } from '../../utils/parser' -import { parseSearchQuery, sortParamsToSort } from '../../utils/search' +import { + InFilter, + parseSearchQuery, + sortParamsToSort, +} from '../../utils/search' import { getStorageFileDetails } from '../../utils/uploads' import { itemTypeForContentType } from '../upload_files' @@ -401,7 +405,7 @@ export const getArticleResolver = authorized< }) ) - if (!libraryItem || libraryItem.state === LibraryItemState.Deleted) { + if (!libraryItem || libraryItem.folder === InFilter.TRASH) { return { errorCodes: [ArticleErrorCode.NotFound] } } @@ -523,8 +527,8 @@ export const setBookmarkArticleResolver = authorized< const deletedLibraryItem = await updateLibraryItem( articleID, { - state: LibraryItemState.Deleted, - deletedAt: new Date(), + folder: InFilter.TRASH, + savedAt: new Date(), }, uid, pubsub @@ -736,7 +740,7 @@ export const updatesSinceResolver = authorized< const sort = sortParamsToSort(sortParams) const startCursor = after || '' - const size = first || 10 + const size = Math.min(first || 10, 100) // limit to 100 items let startDate = new Date(since) if (isNaN(startDate.getTime())) { // for android app compatibility @@ -750,6 +754,7 @@ export const updatesSinceResolver = authorized< includeDeleted: true, dateFilters: [{ field: 'updatedAt', startDate }], sort, + inFilter: InFilter.ALL, }, uid ) @@ -861,7 +866,7 @@ export const setFavoriteArticleResolver = authorized< }) const getUpdateReason = (libraryItem: LibraryItem, since: Date) => { - if (libraryItem.state === LibraryItemState.Deleted) { + if (libraryItem.folder === InFilter.TRASH) { return UpdateReason.Deleted } if (libraryItem.createdAt >= since) { diff --git a/packages/api/src/resolvers/following/index.ts b/packages/api/src/resolvers/following/index.ts new file mode 100644 index 000000000..6c8f1deb7 --- /dev/null +++ b/packages/api/src/resolvers/following/index.ts @@ -0,0 +1,146 @@ +import { LibraryItem } from '../../entity/library_item' +import { + FeedEdge, + FeedsError, + FeedsErrorCode, + FeedsSuccess, + MoveToFolderError, + MoveToFolderErrorCode, + MoveToFolderSuccess, + MutationMoveToFolderArgs, + QueryFeedsArgs, +} from '../../generated/graphql' +import { feedRepository } from '../../repository/feed' +import { createPageSaveRequest } from '../../services/create_page_save_request' +import { updateLibraryItem } from '../../services/library_item' +import { analytics } from '../../utils/analytics' +import { + authorized, + libraryItemToArticleSavingRequest, +} from '../../utils/helpers' + +export const feedsResolver = authorized< + FeedsSuccess, + FeedsError, + QueryFeedsArgs +>(async (_, { input }, { log }) => { + try { + const startCursor = input.after || '' + const start = + startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 + const first = Math.min(input.first || 10, 100) // cap at 100 + + const { feeds, count } = await feedRepository.searchFeeds( + input.query || '', + first + 1, // fetch one extra to check if there is a next page + start, + input.sort?.by, + input.sort?.order || undefined + ) + + const hasNextPage = feeds.length > first + const endCursor = String(start + feeds.length - (hasNextPage ? 1 : 0)) + + if (hasNextPage) { + // remove an extra if exists + feeds.pop() + } + + const edges: FeedEdge[] = feeds.map((feed) => ({ + node: feed, + cursor: endCursor, + })) + + return { + __typename: 'FeedsSuccess', + edges, + pageInfo: { + hasPreviousPage: start > 0, + hasNextPage, + startCursor, + endCursor, + totalCount: count, + }, + } + } catch (error) { + log.error('Error fetching feeds', error) + + return { + errorCodes: [FeedsErrorCode.BadRequest], + } + } +}) + +export const moveToFolderResolver = authorized< + MoveToFolderSuccess, + MoveToFolderError, + MutationMoveToFolderArgs +>(async (_, { id, folder }, { authTrx, pubsub, uid }) => { + analytics.track({ + userId: uid, + event: 'move_to_folder', + properties: { + id, + folder, + }, + }) + + const item = await authTrx((tx) => + tx.getRepository(LibraryItem).findOne({ + where: { + id, + }, + relations: ['user'], + }) + ) + + if (!item) { + return { + errorCodes: [MoveToFolderErrorCode.Unauthorized], + } + } + + if (item.folder === folder) { + return { + errorCodes: [MoveToFolderErrorCode.AlreadyExists], + } + } + + const savedAt = new Date() + + // if the content is not fetched yet, create a page save request + if (!item.readableContent) { + const articleSavingRequest = await createPageSaveRequest({ + userId: uid, + url: item.originalUrl, + articleSavingRequestId: id, + priority: 'high', + publishedAt: item.publishedAt || undefined, + savedAt, + pubsub, + }) + + return { + __typename: 'MoveToFolderSuccess', + articleSavingRequest, + } + } + + const updatedItem = await updateLibraryItem( + item.id, + { + folder, + savedAt, + }, + uid, + pubsub + ) + + return { + __typename: 'MoveToFolderSuccess', + articleSavingRequest: libraryItemToArticleSavingRequest( + updatedItem.user, + updatedItem + ), + } +}) diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 365a32162..ff363e78f 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -29,6 +29,7 @@ import { generateUploadFilePathName, } from '../utils/uploads' import { optInFeatureResolver } from './features' +import { feedsResolver, moveToFolderResolver } from './following' import { uploadImportFileResolver } from './importers/uploadImportFileResolver' import { addPopularReadResolver, @@ -215,6 +216,7 @@ export const functionResolvers = { updateSubscription: updateSubscriptionResolver, updateFilter: updateFilterResolver, updateEmail: updateEmailResolver, + moveToFolder: moveToFolderResolver, }, Query: { me: getMeUserResolver, @@ -246,6 +248,7 @@ export const functionResolvers = { filters: filtersResolver, groups: groupsResolver, recentEmails: recentEmailsResolver, + feeds: feedsResolver, }, User: { async intercomHash( diff --git a/packages/api/src/resolvers/links/index.ts b/packages/api/src/resolvers/links/index.ts index 9e2dd3025..d16b9a155 100644 --- a/packages/api/src/resolvers/links/index.ts +++ b/packages/api/src/resolvers/links/index.ts @@ -1,4 +1,3 @@ -import { LibraryItemState } from '../../entity/library_item' import { env } from '../../env' import { ArchiveLinkError, @@ -9,6 +8,7 @@ import { import { updateLibraryItem } from '../../services/library_item' import { analytics } from '../../utils/analytics' import { authorized } from '../../utils/helpers' +import { InFilter } from '../../utils/search' // export const updateLinkShareInfoResolver = authorized< // UpdateLinkShareInfoSuccess, @@ -66,10 +66,8 @@ export const setLinkArchivedResolver = authorized< await updateLibraryItem( args.input.linkId, { - archivedAt: args.input.archived ? new Date() : null, - state: args.input.archived - ? LibraryItemState.Archived - : LibraryItemState.Succeeded, + savedAt: new Date(), + folder: args.input.archived ? InFilter.ARCHIVE : InFilter.INBOX, }, uid ) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 8d5b74a83..0680339ff 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -175,7 +175,7 @@ export const subscribeResolver = authorized< SubscribeSuccessPartial, SubscribeError, MutationSubscribeArgs ->(async (_, { input }, { uid, log }) => { +>(async (_, { input }, { authTrx, uid, log }) => { try { analytics.track({ userId: uid, @@ -213,6 +213,7 @@ export const subscribeResolver = authorized< scheduledDates: [new Date()], // fetch immediately fetchedDates: [updatedSubscription.lastFetchedAt || null], checksums: [updatedSubscription.lastFetchedChecksum || null], + addToLibraryFlags: [!!updatedSubscription.autoAddToLibrary], }) return { @@ -227,18 +228,20 @@ export const subscribeResolver = authorized< // limit number of rss subscriptions to 150 const results = (await getRepository(Subscription).query( - `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) - select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions + `insert into omnivore.subscriptions (name, url, description, type, user_id, icon, auto_add_to_library, is_private) + select $1, $2, $3, $4, $5, $6, $7, $8 from omnivore.subscriptions where user_id = $5 and type = 'RSS' and status = 'ACTIVE' - having count(*) < $7 + having count(*) < $9 returning *;`, [ feed.title, - input.url, + feed.feedUrl, feed.description || null, SubscriptionType.Rss, uid, feed.image?.url || null, + input.autoAddToLibrary ?? null, + input.isPrivate ?? null, MAX_RSS_SUBSCRIPTIONS, ] )) as Subscription[] @@ -259,6 +262,7 @@ export const subscribeResolver = authorized< scheduledDates: [new Date()], // fetch immediately fetchedDates: [null], checksums: [null], + addToLibraryFlags: [!!newSubscription.autoAddToLibrary], }) return { @@ -312,6 +316,8 @@ export const updateSubscriptionResolver = authorized< scheduledAt: input.scheduledAt ? new Date(input.scheduledAt) : undefined, + autoAddToLibrary: input.autoAddToLibrary ?? undefined, + isPrivate: input.isPrivate ?? undefined, }) return repo.findOneByOrFail({ diff --git a/packages/api/src/resolvers/update/index.ts b/packages/api/src/resolvers/update/index.ts index 255efd2b5..10eb68d32 100644 --- a/packages/api/src/resolvers/update/index.ts +++ b/packages/api/src/resolvers/update/index.ts @@ -1,4 +1,3 @@ -import { LibraryItemState } from '../../entity/library_item' import { MutationUpdatePageArgs, UpdatePageError, @@ -21,9 +20,6 @@ export const updatePageResolver = authorized< savedAt: input.savedAt ? new Date(input.savedAt) : undefined, publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, thumbnail: input.previewImage ?? undefined, - state: input.state - ? (input.state as unknown as LibraryItemState) - : undefined, }, uid ) diff --git a/packages/api/src/routers/svc/following.ts b/packages/api/src/routers/svc/following.ts new file mode 100644 index 000000000..44a191352 --- /dev/null +++ b/packages/api/src/routers/svc/following.ts @@ -0,0 +1,71 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import express from 'express' +import { saveFeedItemInFollowing } from '../../services/library_item' +import { logger } from '../../utils/logger' + +type SourceOfFollowing = 'feed' | 'newsletter' | 'user' + +export interface SaveFollowingItemRequest { + userIds: string[] + title: string + url: string + itemId: string + addedToFollowingBy: string + addedToFollowingFrom: SourceOfFollowing + author?: string + description?: string + links?: any + previewContent?: string + previewContentType?: string + publishedAt?: Date + savedAt: Date +} + +function isSaveFollowingItemRequest( + body: any +): body is SaveFollowingItemRequest { + return ( + 'userIds' in body && + 'addedToFollowingBy' in body && + 'addedToFollowingFrom' in body && + 'url' in body && + 'itemId' in body && + 'title' in body && + 'savedAt' in body + ) +} + +export function followingServiceRouter() { + const router = express.Router() + + router.post('/save', async (req, res) => { + logger.info('save following item request', req.body) + + if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) { + console.log('query does not include valid token') + return res.sendStatus(403) + } + + if (!isSaveFollowingItemRequest(req.body)) { + console.error('Invalid request body', req.body) + return res.status(400).send('INVALID_REQUEST_BODY') + } + + if (req.body.addedToFollowingFrom === 'feed') { + logger.info('saving feed item') + + const result = await saveFeedItemInFollowing(req.body) + if (result.identifiers.length === 0) { + logger.error('error saving feed item in following') + return res.status(500).send('ERROR_SAVING_FEED_ITEM') + } + + logger.info('feed item saved in following') + return res.sendStatus(200) + } + + res.sendStatus(200) + }) + + return router +} diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 3b6a21247..595a233ad 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -34,7 +34,8 @@ export function rssFeedRouter() { ARRAY_AGG(user_id) AS "userIds", ARRAY_AGG(last_fetched_at) AS "fetchedDates", ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", - ARRAY_AGG(last_fetched_checksum) AS checksums + ARRAY_AGG(last_fetched_checksum) AS checksums, + ARRAY_AGG(coalesce(auto_add_to_library, false)) AS "addToLibraryFlags" FROM omnivore.subscriptions WHERE diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 4e0692bcc..1ddb694b9 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1658,6 +1658,8 @@ const schema = gql` lastFetchedAt: Date createdAt: Date! updatedAt: Date + isPrivate: Boolean + autoAddToLibrary: Boolean } enum SubscriptionStatus { @@ -2554,6 +2556,8 @@ const schema = gql` input SubscribeInput { url: String! subscriptionType: SubscriptionType + isPrivate: Boolean + autoAddToLibrary: Boolean } input UpdateSubscriptionInput { @@ -2564,6 +2568,8 @@ const schema = gql` lastFetchedChecksum: String status: SubscriptionStatus scheduledAt: Date + isPrivate: Boolean + autoAddToLibrary: Boolean } union UpdateSubscriptionResult = @@ -2605,6 +2611,62 @@ const schema = gql` email: String! } + input FeedsInput { + after: String + first: Int + query: String @sanitize(maxLength: 255) + sort: SortParams + } + + union FeedsResult = FeedsSuccess | FeedsError + + type FeedsSuccess { + edges: [FeedEdge!]! + pageInfo: PageInfo! + } + + type FeedEdge { + cursor: String! + node: Feed! + } + + type FeedsError { + errorCodes: [FeedsErrorCode!]! + } + + enum FeedsErrorCode { + UNAUTHORIZED + BAD_REQUEST + } + + type Feed { + id: ID! + title: String! + url: String! + description: String + image: String + createdAt: Date! + updatedAt: Date! + publishedAt: Date + author: String + } + + union MoveToFolderResult = MoveToFolderSuccess | MoveToFolderError + + type MoveToFolderSuccess { + articleSavingRequest: ArticleSavingRequest! + } + + type MoveToFolderError { + errorCodes: [MoveToFolderErrorCode!]! + } + + enum MoveToFolderErrorCode { + UNAUTHORIZED + BAD_REQUEST + ALREADY_EXISTS + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2708,6 +2770,7 @@ const schema = gql` updateSubscription( input: UpdateSubscriptionInput! ): UpdateSubscriptionResult! + moveToFolder(id: ID!, folder: String!): MoveToFolderResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed @@ -2766,6 +2829,7 @@ const schema = gql` filters: FiltersResult! groups: GroupsResult! recentEmails: RecentEmailsResult! + feeds(input: FeedsInput!): FeedsResult! } ` diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 1209deb9e..888383e66 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -25,6 +25,7 @@ import { pageRouter } from './routers/page_router' import { contentServiceRouter } from './routers/svc/content' import { emailsServiceRouter } from './routers/svc/emails' import { emailAttachmentRouter } from './routers/svc/email_attachment' +import { followingServiceRouter } from './routers/svc/following' import { integrationsServiceRouter } from './routers/svc/integrations' import { linkServiceRouter } from './routers/svc/links' import { newsletterServiceRouter } from './routers/svc/newsletters' @@ -125,6 +126,7 @@ export const createApp = (): { app.use('/svc/pubsub/user', userServiceRouter()) // app.use('/svc/reminders', remindersServiceRouter()) app.use('/svc/email-attachment', emailAttachmentRouter()) + app.use('/svc/following', followingServiceRouter()) if (env.dev.isLocal) { app.use('/local/debug', localDebugRouter()) diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index bec2f2329..01db62eb2 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -130,7 +130,7 @@ export const createPageSaveRequest = async ({ pubsub ) } - // reset state to processing + // reset state to processing if not in following if (libraryItem.state !== LibraryItemState.Processing) { libraryItem = await updateLibraryItem( libraryItem.id, diff --git a/packages/api/src/services/create_user.ts b/packages/api/src/services/create_user.ts index 6fd8e2033..a04ff8164 100644 --- a/packages/api/src/services/create_user.ts +++ b/packages/api/src/services/create_user.ts @@ -159,7 +159,7 @@ const createDefaultFiltersForUser = name: 'Continue Reading', filter: 'in:inbox sort:read-desc is:reading', }, - { name: 'Non-Feed Items', filter: 'in:library' }, + { name: 'Non-Feed Items', filter: 'no:subscription' }, { name: 'Highlights', filter: 'has:highlights mode:highlights' }, { name: 'Unlabeled', filter: 'no:label' }, { name: 'Oldest First', filter: 'sort:saved-asc' }, diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 86a99f8ef..1ea4d6d87 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -1,55 +1,7 @@ import axios from 'axios' -import { LibraryItemState } from '../../entity/library_item' import { env } from '../../env' import { logger } from '../../utils/logger' -import { - IntegrationClient, - RetrievedResult, - RetrieveRequest, -} from './integration' - -interface PocketResponse { - status: number // 1 if success - complete: number // 1 if all items have been returned - list: { - [key: string]: PocketItem - } - since: number // unix timestamp in seconds - search_meta: { - search_type: string - } - error: string -} - -interface PocketItem { - item_id: string - resolved_id: string - given_url: string - resolved_url: string - given_title: string - resolved_title: string - favorite: string - status: string - excerpt: string - word_count: string - tags?: { - [key: string]: Tag - } - authors?: { - [key: string]: Author - } -} - -interface Tag { - item_id: string - tag: string -} - -interface Author { - item_id: string - author_id: string - name: string -} +import { IntegrationClient } from './integration' export class PocketClient implements IntegrationClient { name = 'POCKET' @@ -83,82 +35,4 @@ export class PocketClient implements IntegrationClient { return null } } - - retrievePocketData = async ( - accessToken: string, - since: number, // unix timestamp in seconds - count = 100, - offset = 0 - ): Promise => { - const url = `${this.apiUrl}/get` - try { - const response = await axios.post( - url, - { - consumer_key: env.pocket.consumerKey, - access_token: accessToken, - state: 'all', - detailType: 'complete', - since, - sort: 'oldest', - count, - offset, - }, - { - headers: this.headers, - timeout: 10000, // 10 seconds - } - ) - - return response.data - } catch (error) { - if (axios.isAxiosError(error)) { - logger.error(error.response) - } else { - logger.error(error) - } - - return null - } - } - - retrieve = async ({ - token, - since = 0, - count = 100, - offset = 0, - }: RetrieveRequest): Promise => { - const pocketData = await this.retrievePocketData( - token, - since / 1000, - count, - offset - ) - if (!pocketData) { - throw new Error('Error retrieving pocket data') - } - - const pocketItems = Object.values(pocketData.list) - const statusToState: Record = { - '0': LibraryItemState.Succeeded, - '1': LibraryItemState.Archived, - '2': LibraryItemState.Deleted, - } - const data = pocketItems.map((item) => ({ - url: item.given_url, - labels: item.tags - ? Object.values(item.tags).map((tag) => tag.tag) - : undefined, - state: statusToState[item.status], - })) - - if (pocketData.error) { - throw new Error(`Error retrieving pocket data: ${pocketData.error}`) - } - - return { - data, - since: pocketData.since * 1000, - } - } } diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 42f50f744..7764294d2 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -2,33 +2,6 @@ import axios from 'axios' import { logger } from '../../utils/logger' import { IntegrationClient } from './integration' -interface ReadwiseHighlight { - // The highlight text, (technically the only field required in a highlight object) - text: string - // The title of the page the highlight is on - title?: string - // The author of the page the highlight is on - author?: string - // The URL of the page image - image_url?: string - // The URL of the page - source_url?: string - // A meaningful unique identifier for your app - source_type?: string - // One of: books, articles, tweets or podcasts - category?: string - // Annotation note attached to the specific highlight - note?: string - // Highlight's location in the source text. Used to order the highlights - location?: number - // One of: page, order or time_offset - location_type?: string - // A datetime representing when the highlight was taken in the ISO 8601 format - highlighted_at?: string - // Unique url of the specific highlight - highlight_url?: string -} - export class ReadwiseClient implements IntegrationClient { name = 'READWISE' apiUrl = 'https://readwise.io/api/v2' diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 7fec735e0..6d66184a3 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -8,6 +8,8 @@ import { BulkActionType } from '../generated/graphql' import { createPubSubClient, EntityType } from '../pubsub' import { authTrx, getColumns } from '../repository' import { libraryItemRepository } from '../repository/library_item' +import { SaveFollowingItemRequest } from '../routers/svc/following' +import { SetClaimsRole } from '../utils/dictionary' import { wordsCount } from '../utils/helpers' import { DateFilter, @@ -29,7 +31,7 @@ export interface SearchArgs { size?: number sort?: Sort query?: string - inFilter?: InFilter + inFilter: InFilter readFilter?: ReadFilter typeFilter?: string labelFilters?: LabelFilter[] @@ -62,7 +64,6 @@ export interface SearchResultItem { title: string uploadFileId?: string | null url: string - archivedAt?: Date | null readingProgressTopPercent?: number readingProgressPercent: number readingProgressAnchorIndex: number @@ -104,33 +105,9 @@ const buildWhereClause = ( } if (args.inFilter !== InFilter.ALL) { - switch (args.inFilter) { - case InFilter.INBOX: - queryBuilder.andWhere('library_item.archived_at IS NULL') - break - case InFilter.ARCHIVE: - queryBuilder.andWhere('library_item.archived_at IS NOT NULL') - break - case InFilter.TRASH: - // return only deleted pages within 14 days - queryBuilder.andWhere( - "library_item.deleted_at >= now() - interval '14 days'" - ) - break - case InFilter.SUBSCRIPTION: - queryBuilder - .andWhere("NOT ('library' ILIKE ANY (library_item.label_names))") - .andWhere('library_item.archived_at IS NULL') - .andWhere('library_item.subscription IS NOT NULL') - break - case InFilter.LIBRARY: - queryBuilder - .andWhere( - "(library_item.subscription IS NULL OR 'library' ILIKE ANY (library_item.label_names))" - ) - .andWhere('library_item.archived_at IS NULL') - break - } + queryBuilder.andWhere('library_item.folder = :folder', { + folder: args.inFilter, + }) } if (args.readFilter !== ReadFilter.ALL) { @@ -162,6 +139,8 @@ const buildWhereClause = ( case HasFilter.LABELS: queryBuilder.andWhere("library_item.label_names <> '{}'") break + case HasFilter.SUBSCRIPTIONS: + queryBuilder.andWhere('library_item.subscription is NOT NULL') } }) } @@ -244,20 +223,18 @@ const buildWhereClause = ( } if (!args.includePending) { - queryBuilder.andWhere('library_item.state <> :state', { - state: LibraryItemState.Processing, - }) + queryBuilder.andWhere("library_item.state <> 'PROCESSING'") } if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) { - queryBuilder.andWhere('library_item.state <> :state', { - state: LibraryItemState.Deleted, - }) + queryBuilder.andWhere("library_item.folder <> 'trash'") } if (args.noFilters) { args.noFilters.forEach((filter) => { - queryBuilder.andWhere(`library_item.${filter.field} = '{}'`) + queryBuilder.andWhere( + `library_item.${filter.field} = '{}' OR library_item.${filter.field} IS NULL` + ) }) } @@ -318,7 +295,7 @@ export const searchLibraryItems = async ( const queryBuilder = tx .createQueryBuilder(LibraryItem, 'library_item') .select(selectColumns) - .where({ user: { id: userId } }) + .where('library_item.user_id = :userId', { userId }) // build the where clause buildWhereClause(queryBuilder, args) @@ -388,8 +365,6 @@ export const restoreLibraryItem = async ( { state: LibraryItemState.Succeeded, savedAt: new Date(), - archivedAt: null, - deletedAt: null, }, userId, pubsub @@ -405,22 +380,6 @@ export const updateLibraryItem = async ( const updatedLibraryItem = await authTrx( async (tx) => { const itemRepo = tx.withRepository(libraryItemRepository) - - // reset deletedAt and archivedAt - switch (libraryItem.state) { - case LibraryItemState.Archived: - libraryItem.archivedAt = new Date() - break - case LibraryItemState.Deleted: - libraryItem.deletedAt = new Date() - break - case LibraryItemState.Processing: - case LibraryItemState.Succeeded: - libraryItem.archivedAt = null - libraryItem.deletedAt = null - break - } - await itemRepo.update(id, libraryItem) return itemRepo.findOneByOrFail({ id }) @@ -560,6 +519,33 @@ export const createLibraryItem = async ( return newLibraryItem } +export const saveFeedItemInFollowing = (input: SaveFollowingItemRequest) => { + return authTrx( + async (tx) => { + const libraryItems: QueryDeepPartialEntity[] = + input.userIds.map((userId) => ({ + ...input, + user: { id: userId }, + originalUrl: input.url, + subscription: input.addedToFollowingBy, + folder: InFilter.FOLLOWING, + })) + + return tx + .getRepository(LibraryItem) + .createQueryBuilder() + .insert() + .values(libraryItems) + .orIgnore() // ignore if the item already exists + .returning('*') + .execute() + }, + undefined, + undefined, + SetClaimsRole.ADMIN + ) +} + export const findLibraryItemsByPrefix = async ( prefix: string, userId: string, @@ -613,14 +599,14 @@ export const updateLibraryItems = async ( switch (action) { case BulkActionType.Archive: values = { - archivedAt: new Date(), - state: LibraryItemState.Archived, + folder: InFilter.ARCHIVE, + savedAt: new Date(), } break case BulkActionType.Delete: values = { - deletedAt: new Date(), - state: LibraryItemState.Deleted, + savedAt: new Date(), + folder: InFilter.TRASH, } break case BulkActionType.AddLabels: diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index f428cb77a..bffe06fb8 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -1,7 +1,12 @@ import { LibraryItemState } from '../entity/library_item' import { User } from '../entity/user' import { homePageURL } from '../env' -import { SaveErrorCode, SaveFileInput, SaveResult } from '../generated/graphql' +import { + ArticleSavingRequestStatus, + SaveErrorCode, + SaveFileInput, + SaveResult, +} from '../generated/graphql' import { getStorageFileDetails } from '../utils/uploads' import { findOrCreateLabels, saveLabelsInLibraryItem } from './labels' import { updateLibraryItem } from './library_item' @@ -32,9 +37,11 @@ export const saveFile = async ( await updateLibraryItem( input.clientRequestId, { - state: input.state - ? (input.state as unknown as LibraryItemState) - : LibraryItemState.Succeeded, + state: LibraryItemState.Succeeded, + folder: + input.state === ArticleSavingRequestStatus.Archived + ? 'archive' + : 'inbox', }, user.id ) diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 6e6b7b954..5ee1b2708 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -261,9 +261,6 @@ export const parsedContentToLibraryItem = ({ uploadFileId: uploadFileId || undefined, readingProgressTopPercent: 0, readingProgressHighestReadAnchor: 0, - state: state - ? (state as unknown as LibraryItemState) - : LibraryItemState.Succeeded, createdAt: validatedDate(saveTime), savedAt: validatedDate(saveTime), siteName: parsedContent?.siteName, @@ -272,7 +269,7 @@ export const parsedContentToLibraryItem = ({ wordCount: wordsCount(parsedContent?.textContent || ''), contentReader: contentReaderForLibraryItem(itemType, uploadFileId), subscription: rssFeedUrl, - archivedAt: - state === ArticleSavingRequestStatus.Archived ? new Date() : undefined, + folder: state === ArticleSavingRequestStatus.Archived ? 'archive' : 'inbox', + state: LibraryItemState.Succeeded, } } diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 643772116..3c11e5028 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -615,6 +615,7 @@ export interface RssSubscriptionGroup { fetchedDates: (Date | null)[] scheduledDates: Date[] checksums: (string | null)[] + addToLibraryFlags: boolean[] } export const enqueueRssFeedFetch = async ( @@ -632,6 +633,7 @@ export const enqueueRssFeedFetch = async ( timestamp.getTime() ), // unix timestamp in milliseconds userIds: subscriptionGroup.userIds, + addToLibraryFlags: subscriptionGroup.addToLibraryFlags, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index b0ff7cfcf..80a883be7 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -30,6 +30,7 @@ import { validateUrl } from '../services/create_page_save_request' import { updateLibraryItem } from '../services/library_item' import { Merge } from '../util' import { logger } from './logger' +import { InFilter } from './search' interface InputObject { // eslint-disable-next-line @typescript-eslint/no-explicit-any [key: string]: any @@ -239,7 +240,7 @@ export const libraryItemToArticle = (item: LibraryItem): Article => ({ state: item.state as unknown as ArticleSavingRequestStatus, content: item.readableContent, hash: item.textContentHash || '', - isArchived: !!item.archivedAt, + isArchived: item.folder === InFilter.ARCHIVE, recommendations: item.recommendations?.map( recommandationDataToRecommendation ), @@ -258,7 +259,7 @@ export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({ url: item.originalUrl, state: item.state as unknown as ArticleSavingRequestStatus, content: item.readableContent, - isArchived: !!item.archivedAt, + isArchived: item.folder === InFilter.ARCHIVE, pageType: item.itemType as unknown as PageType, readingProgressPercent: item.readingProgressBottomPercent, contentReader: item.contentReader as unknown as ContentReader, diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index 39d9975d0..dde0e019d 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -3,6 +3,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-call */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { DateTime } from 'luxon' import { ISearchParserDictionary, parse, @@ -19,12 +20,11 @@ export enum ReadFilter { } export enum InFilter { - ALL, - INBOX, - ARCHIVE, - TRASH, - SUBSCRIPTION, - LIBRARY, + ALL = 'all', + INBOX = 'inbox', + ARCHIVE = 'archive', + TRASH = 'trash', + FOLLOWING = 'following', } export interface SearchFilter { @@ -57,6 +57,7 @@ export type LabelFilter = { export enum HasFilter { HIGHLIGHTS, LABELS, + SUBSCRIPTIONS, } export interface DateFilter { @@ -132,10 +133,8 @@ const parseInFilter = ( return InFilter.ARCHIVE case 'TRASH': return InFilter.TRASH - case 'SUBSCRIPTION': - return InFilter.SUBSCRIPTION - case 'LIBRARY': - return InFilter.LIBRARY + case 'FOLLOWING': + return InFilter.FOLLOWING } return query ? InFilter.ALL : InFilter.INBOX @@ -232,6 +231,8 @@ const parseHasFilter = (str?: string): HasFilter | undefined => { return HasFilter.HIGHLIGHTS case 'LABELS': return HasFilter.LABELS + case 'SUBSCRIPTIONS': + return HasFilter.SUBSCRIPTIONS } } @@ -243,21 +244,49 @@ const parseDateFilter = ( return undefined } + switch (field.toLowerCase()) { + case 'published': + field = 'published_at' + break + case 'saved': + field = 'saved_at' + break + case 'updated': + field = 'updated_at' + } + + // check for special date filters + switch (str.toLowerCase()) { + case 'today': + return { + field, + startDate: DateTime.local().startOf('day').toJSDate(), + } + case 'yesterday': { + const yesterday = DateTime.local().minus({ days: 1 }) + return { + field, + startDate: yesterday.startOf('day').toJSDate(), + endDate: yesterday.endOf('day').toJSDate(), + } + } + case 'this week': + return { + field, + startDate: DateTime.local().startOf('week').toJSDate(), + } + case 'this month': + return { + field, + startDate: DateTime.local().startOf('month').toJSDate(), + } + } + + // check for date ranges const [start, end] = str.split('..') const startDate = start && start !== '*' ? new Date(start) : undefined const endDate = end && end !== '*' ? new Date(end) : undefined - switch (field.toUpperCase()) { - case 'PUBLISHED': - field = 'published_at' - break - case 'SAVED': - field = 'saved_at' - break - case 'UPDATED': - field = 'updated_at' - } - return { field, startDate, @@ -357,13 +386,20 @@ const parseNoFilter = (str?: string): NoFilter | undefined => { return { field: 'highlight_annotations' } case 'label': return { field: 'label_names' } + case 'subscription': + return { field: 'subscription' } } return undefined } export const parseSearchQuery = (query: string | undefined): SearchFilter => { - const searchQuery = query ? query.replace(/\W\s":/g, '') : undefined + const searchQuery = query + ? query + .replace(/\W\s":/g, '') + .replace('in:subscription', 'has:subscriptions') // compatibility with old search + .replace('in:library', 'no:subscription') // compatibility with old search + : undefined const result: SearchFilter = { query: searchQuery, readFilter: ReadFilter.ALL, @@ -379,19 +415,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { } if (!searchQuery) { - return { - query: undefined, - inFilter: InFilter.INBOX, - readFilter: ReadFilter.ALL, - labelFilters: [], - hasFilters: [], - dateFilters: [], - termFilters: [], - matchFilters: [], - ids: [], - noFilters: [], - rangeFilters: [], - } + return result } const parsed = parse(searchQuery, { diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 0763a1a49..6fc8f5f88 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -410,8 +410,7 @@ describe('Article API', () => { title, user: { id: user.id }, originalUrl: url, - archivedAt: new Date(), - state: LibraryItemState.Archived, + folder: 'archive', }, user.id ) @@ -609,7 +608,7 @@ describe('Article API', () => { ).expect(200) const savedItem = await findLibraryItemByUrl(url, user.id) - expect(savedItem?.archivedAt).to.not.be.null + expect(savedItem?.folder).to.eql('archive') expect(savedItem?.labels?.map((l) => l.name)).to.eql(labels) }) }) @@ -692,7 +691,7 @@ describe('Article API', () => { 200 ) const item = await findLibraryItemById(itemId, user.id) - expect(item?.state).to.eql(LibraryItemState.Deleted) + expect(item?.folder).to.eql('trash') }) }) @@ -1031,8 +1030,7 @@ describe('Article API', () => { readableContent: '

test 1

', slug: 'test slug 1', originalUrl: `${url}/test1`, - archivedAt: new Date(), - state: LibraryItemState.Archived, + folder: 'archive', }, { user, @@ -1040,8 +1038,7 @@ describe('Article API', () => { readableContent: '

test 2

', slug: 'test slug 2', originalUrl: `${url}/test2`, - archivedAt: new Date(), - state: LibraryItemState.Archived, + folder: 'archive', }, { user, @@ -1153,12 +1150,12 @@ describe('Article API', () => { }) }) - context("when in:library label:test' is in the query", () => { + context("when in:inbox label:test' is in the query", () => { let items: LibraryItem[] = [] let label: Label before(async () => { - keyword = 'in:library label:test' + keyword = 'in:inbox label:test' // Create some test items label = await createLabel('test', '', user.id) items = await createLibraryItems( @@ -1170,29 +1167,19 @@ describe('Article API', () => { slug: 'test slug 1', originalUrl: `${url}/test1`, }, - { - user, - title: 'test title 2', - readableContent: '

test 2

', - slug: 'test slug 2', - originalUrl: `${url}/test2`, - subscription: 'test subscription', - }, { user, title: 'test title 3', readableContent: '

test 3

', slug: 'test slug 3', originalUrl: `${url}/test3`, - archivedAt: new Date(), - state: LibraryItemState.Archived, + folder: 'archive', }, ], user.id ) await saveLabelsInLibraryItem([label], items[0].id, user.id) await saveLabelsInLibraryItem([label], items[1].id, user.id) - await saveLabelsInLibraryItem([label], items[2].id, user.id) }) after(async () => { @@ -1276,7 +1263,7 @@ describe('Article API', () => { slug: 'test slug 1', originalUrl: `${url}/test1`, itemType: PageType.File, - archivedAt: new Date(), + folder: 'archive', }, { user, @@ -1284,7 +1271,7 @@ describe('Article API', () => { readableContent: '

test 2

', slug: 'test slug 2', originalUrl: `${url}/test2`, - archivedAt: new Date(), + folder: 'archive', readingProgressBottomPercent: 100, }, { @@ -1325,8 +1312,8 @@ describe('Article API', () => { readableContent: '

test 1

', slug: 'test slug 1', originalUrl: `${url}/test1`, - archivedAt: new Date(), subscription: 'feed', + folder: 'archive', }, { user, @@ -1342,7 +1329,7 @@ describe('Article API', () => { readableContent: '

test 3

', slug: 'test slug 3', originalUrl: `${url}/test3`, - archivedAt: new Date(), + folder: 'archive', }, ], user.id @@ -1375,7 +1362,7 @@ describe('Article API', () => { readableContent: '

test 1

', slug: 'test slug 1', originalUrl: `${url}/test1`, - deletedAt: new Date(), + folder: 'trash', }, { user, @@ -1383,8 +1370,8 @@ describe('Article API', () => { readableContent: '

test 2

', slug: 'test slug 2', originalUrl: `${url}/test2`, - deletedAt: new Date(), readingProgressBottomPercent: 100, + folder: 'trash', }, { user, @@ -1746,7 +1733,7 @@ describe('Article API', () => { for (let i = 0; i < 3; i++) { await updateLibraryItem( items[i].id, - { state: LibraryItemState.Deleted, deletedAt: new Date() }, + { folder: 'trash', savedAt: new Date() }, user.id ) deletedItems.push(items[i]) diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 02f7cd1b8..cfdf6b10e 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -332,7 +332,9 @@ describe('Subscriptions API', () => { const updatedSubscription = await getRepository(Subscription).findOneBy({ id: subscription.id, }) - expect(updatedSubscription?.status).to.eql(SubscriptionStatus.Unsubscribed) + expect(updatedSubscription?.status).to.eql( + SubscriptionStatus.Unsubscribed + ) // check if the email was sent expect(fake).to.have.been.calledOnceWith({ @@ -371,10 +373,15 @@ describe('Subscriptions API', () => { before(async () => { // fake rss parser - sinon.replace(Parser.prototype, 'parseURL', sinon.fake.resolves({ - title: 'RSS Feed', - description: 'RSS Feed Description', - })) + sinon.replace( + Parser.prototype, + 'parseURL', + sinon.fake.resolves({ + title: 'RSS Feed', + description: 'RSS Feed Description', + feedUrl: url, + }) + ) }) after(() => { @@ -439,11 +446,9 @@ describe('Subscriptions API', () => { }) it('creates a rss subscription', async () => { - const res = await graphqlRequest( - query, - authToken, - { input: { url, subscriptionType } }, - ).expect(200) + const res = await graphqlRequest(query, authToken, { + input: { url, subscriptionType }, + }).expect(200) expect(res.body.data.subscribe.subscriptions).to.have.lengthOf(1) expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') diff --git a/packages/api/test/routers/auth.test.ts b/packages/api/test/routers/auth.test.ts index 47d87737f..5cf40e3e3 100644 --- a/packages/api/test/routers/auth.test.ts +++ b/packages/api/test/routers/auth.test.ts @@ -15,6 +15,7 @@ import { generateVerificationToken, hashPassword, } from '../../src/utils/auth' +import { InFilter } from '../../src/utils/search' import * as util from '../../src/utils/sendEmail' import { createTestUser } from '../db' import { generateFakeUuid, request } from '../util' @@ -607,7 +608,10 @@ describe('auth router', () => { 'web' ).expect(200) const user = await userRepository.findOneByOrFail({ name }) - const { count } = await searchLibraryItems({}, user.id) + const { count } = await searchLibraryItems( + { inFilter: InFilter.ALL }, + user.id + ) expect(count).to.eql(3) }) @@ -628,7 +632,10 @@ describe('auth router', () => { 'ios' ).expect(200) const user = await userRepository.findOneByOrFail({ name }) - const { count } = await searchLibraryItems({}, user.id) + const { count } = await searchLibraryItems( + { inFilter: InFilter.ALL }, + user.id + ) expect(count).to.eql(4) }) diff --git a/packages/db/migrations/0146.do.following.sql b/packages/db/migrations/0146.do.following.sql new file mode 100755 index 000000000..61e59ccc7 --- /dev/null +++ b/packages/db/migrations/0146.do.following.sql @@ -0,0 +1,25 @@ +-- Type: DO +-- Name: following +-- Description: Create tables for following feature + +BEGIN; + +ALTER TABLE omnivore.subscriptions + ADD COLUMN is_private boolean, + ADD COLUMN auto_add_to_library boolean; + +ALTER TABLE omnivore.library_item + ADD COLUMN links jsonb, + ADD COLUMN preview_content text, + ADD COLUMN preview_content_type text, + ADD COLUMN folder text NOT NULL DEFAULT 'inbox'; + +UPDATE omnivore.library_item SET folder = 'archive' WHERE archived_at IS NOT NULL; +UPDATE omnivore.library_item SET folder = 'trash' WHERE deleted_at IS NOT NULL; + +CREATE POLICY library_item_admin_policy on omnivore.library_item + FOR ALL + TO omnivore_admin + USING (true); + +COMMIT; diff --git a/packages/db/migrations/0146.undo.following.sql b/packages/db/migrations/0146.undo.following.sql new file mode 100755 index 000000000..8d41f08e4 --- /dev/null +++ b/packages/db/migrations/0146.undo.following.sql @@ -0,0 +1,19 @@ +-- Type: UNDO +-- Name: following +-- Description: Create tables for following feature + +BEGIN; + +DROP policy library_item_admin_policy ON omnivore.library_item; + +ALTER TABLE omnivore.library_item + DROP COLUMN links, + DROP COLUMN preview_content, + DROP COLUMN preview_content_type, + DROP COLUMN folder; + +ALTER TABLE omnivore.subscriptions + DROP COLUMN is_private, + DROP COLUMN auto_add_to_library; + +COMMIT; diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 20583b3a3..1c06b413b 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -5,7 +5,11 @@ import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-d import * as jwt from 'jsonwebtoken' import Parser, { Item } from 'rss-parser' import { promisify } from 'util' -import { CONTENT_FETCH_URL, createCloudTask } from './task' +import { + CONTENT_FETCH_URL, + createCloudTask, + FOLLOWING_HANDLER_URL, +} from './task' interface RssFeedRequest { subscriptionIds: string[] @@ -14,6 +18,7 @@ interface RssFeedRequest { scheduledTimestamps: number[] // unix timestamp in milliseconds lastFetchedChecksums: string[] userIds: string[] + addToLibraryFlags: boolean[] } // link can be a string or an object @@ -26,7 +31,8 @@ function isRssFeedRequest(body: any): body is RssFeedRequest { 'lastFetchedTimestamps' in body && 'scheduledTimestamps' in body && 'userIds' in body && - 'lastFetchedChecksums' in body + 'lastFetchedChecksums' in body && + 'addToLibraryFlags' in body ) } @@ -120,6 +126,19 @@ const sendUpdateSubscriptionMutation = async ( } } +const createTask = async ( + userId: string, + feedUrl: string, + item: Item, + autoAddToLibrary: boolean +) => { + if (autoAddToLibrary) { + return createSavingItemTask(userId, feedUrl, item) + } + + return createFollowingTask(userId, feedUrl, item) +} + const createSavingItemTask = async ( userId: string, feedUrl: string, @@ -149,6 +168,38 @@ const createSavingItemTask = async ( } } +const createFollowingTask = async ( + userId: string, + feedUrl: string, + item: Item +) => { + const input = { + userIds: [userId], + url: item.link, + title: item.title, + author: item.creator, + description: item.summary, + addedToFollowingFrom: 'feed', + previewContent: item.content || item.contentSnippet, + addedToFollowingBy: feedUrl, + savedAt: item.isoDate, + publishedAt: item.isoDate, + previewContentType: 'text/html', // TODO: get content type from feed + } + + try { + console.log('Creating task', input.url) + // save page + const task = await createCloudTask(FOLLOWING_HANDLER_URL, input) + console.log('Created task', task) + + return !!task + } catch (error) { + console.error('Error while creating task', error) + return false + } +} + dotenv.config() Sentry.GCPFunction.init({ dsn: process.env.SENTRY_DSN, @@ -245,7 +296,21 @@ const processSubscription = async ( fetchResult: { content: string; checksum: string }, lastFetchedAt: number, scheduledAt: number, - lastFetchedChecksum: string + lastFetchedChecksum: string, + autoAddToLibrary: boolean, + feed: { + lastBuildDate: any + 'syn:updatePeriod': any + 'syn:updateFrequency': any + 'sy:updatePeriod': any + 'sy:updateFrequency': any + } & Parser.Output<{ + published: any + updated: any + created: any + link: any + links: any[] + }> ) => { let lastItemFetchedAt: Date | null = null let lastValidItem: Item | null = null @@ -258,8 +323,6 @@ const processSubscription = async ( // fetch feed let itemCount = 0 - const feed = await parser.parseString(fetchResult.content) - console.log('Fetched feed', feed.title, new Date()) const feedLastBuildDate = feed.lastBuildDate as string | undefined console.log('Feed last build date', feedLastBuildDate) @@ -317,7 +380,7 @@ const processSubscription = async ( continue } - const created = await createSavingItemTask(userId, feedUrl, item) + const created = await createTask(userId, feedUrl, item, autoAddToLibrary) if (!created) { console.error('Failed to create task for feed item', item.link) continue @@ -340,7 +403,12 @@ const processSubscription = async ( } // the feed has never been fetched, save at least the last valid item - const created = await createSavingItemTask(userId, feedUrl, lastValidItem) + const created = await createTask( + userId, + feedUrl, + lastValidItem, + autoAddToLibrary + ) if (!created) { console.error('Failed to create task for feed item', lastValidItem.link) throw new Error('Failed to create task for feed item') @@ -386,28 +454,29 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( scheduledTimestamps, userIds, lastFetchedChecksums, + addToLibraryFlags, } = req.body console.log('Processing feed', feedUrl) const fetchResult = await fetchAndChecksum(feedUrl) + const feed = await parser.parseString(fetchResult.content) + console.log('Fetched feed', feed.title, new Date()) - for (let i = 0; i < subscriptionIds.length; i++) { - const subscriptionId = subscriptionIds[i] - const lastFetchedAt = lastFetchedTimestamps[i] - const scheduledAt = scheduledTimestamps[i] - const userId = userIds[i] - const lastFetchedChecksum = lastFetchedChecksums[i] - - await processSubscription( - subscriptionId, - userId, - feedUrl, - fetchResult, - lastFetchedAt, - scheduledAt, - lastFetchedChecksum + await Promise.all( + subscriptionIds.map((_, i) => + processSubscription( + subscriptionIds[i], + userIds[i], + feedUrl, + fetchResult, + lastFetchedTimestamps[i], + scheduledTimestamps[i], + lastFetchedChecksums[i], + addToLibraryFlags[i], + feed + ) ) - } + ) res.send('ok') } catch (e) { diff --git a/packages/rss-handler/src/task.ts b/packages/rss-handler/src/task.ts index 8a89fbd43..9fe02fca7 100644 --- a/packages/rss-handler/src/task.ts +++ b/packages/rss-handler/src/task.ts @@ -4,6 +4,7 @@ import { CloudTasksClient, protos } from '@google-cloud/tasks' const cloudTask = new CloudTasksClient() export const CONTENT_FETCH_URL = process.env.CONTENT_FETCH_GCF_URL +export const FOLLOWING_HANDLER_URL = process.env.FOLLOWING_HANDLER_URL export const createCloudTask = async ( taskHandlerUrl: string | undefined, diff --git a/packages/web/components/templates/homeFeed/EmptyLibrary.tsx b/packages/web/components/templates/homeFeed/EmptyLibrary.tsx index 72da7891b..23dfe3d7b 100644 --- a/packages/web/components/templates/homeFeed/EmptyLibrary.tsx +++ b/packages/web/components/templates/homeFeed/EmptyLibrary.tsx @@ -169,7 +169,7 @@ export const EmptyLibrary = (props: EmptyLibraryProps) => { return 'inbox' case 'in:inbox sort:read-desc is:reading': return 'continue' - case 'in:library': + case 'no:subscription': return 'non-feed' case 'has:highlights mode:highlights': return 'highlight' @@ -181,7 +181,7 @@ export const EmptyLibrary = (props: EmptyLibraryProps) => { return 'archive' case 'label:RSS': return 'feed' - case 'in:subscription': + case 'has:subscriptions': return 'subscription' case 'label:Newsletter': return 'newsletter' diff --git a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx index 13a27dea4..d7bb99f9b 100644 --- a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx +++ b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx @@ -241,7 +241,7 @@ function Subscriptions( > {!collapsed ? ( <> - +