From b976b6098d8d09b9f8b61e4d4c0eeecea525dfeb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 29 Jul 2022 12:11:48 +0800 Subject: [PATCH 1/7] Add updatesSince API in GQL schema --- packages/api/src/generated/graphql.ts | 77 +++++++++++++++++++++++ packages/api/src/generated/schema.graphql | 29 +++++++++ packages/api/src/schema.ts | 29 +++++++++ 3 files changed, 135 insertions(+) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 2310ca181..c9fc8f810 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1264,6 +1264,7 @@ export type Query = { sharedArticle: SharedArticleResult; subscriptions: SubscriptionsResult; typeaheadSearch: TypeaheadSearchResult; + updatesSince: UpdatesSinceResult; user: UserResult; users: UsersResult; validateUsername: Scalars['Boolean']; @@ -1341,6 +1342,13 @@ export type QueryTypeaheadSearchArgs = { }; +export type QueryUpdatesSinceArgs = { + after?: InputMaybe; + first?: InputMaybe; + since: Scalars['Date']; +}; + + export type QueryUserArgs = { userId?: InputMaybe; username?: InputMaybe; @@ -1892,6 +1900,14 @@ export type SubscriptionsSuccess = { subscriptions: Array; }; +export type SyncUpdatedItemEdge = { + __typename?: 'SyncUpdatedItemEdge'; + cursor: Scalars['String']; + itemID: Scalars['ID']; + node?: Maybe; + updateReason: UpdateReason; +}; + export type TypeaheadSearchError = { __typename?: 'TypeaheadSearchError'; errorCodes: Array; @@ -2059,6 +2075,12 @@ export type UpdatePageSuccess = { updatedPage: Article; }; +export enum UpdateReason { + Created = 'CREATED', + Deleted = 'DELETED', + Updated = 'UPDATED' +} + export type UpdateReminderError = { __typename?: 'UpdateReminderError'; errorCodes: Array; @@ -2158,6 +2180,23 @@ export type UpdateUserSuccess = { user: User; }; +export type UpdatesSinceError = { + __typename?: 'UpdatesSinceError'; + errorCodes?: Maybe>; +}; + +export enum UpdatesSinceErrorCode { + Unauthorized = 'UNAUTHORIZED' +} + +export type UpdatesSinceResult = UpdatesSinceError | UpdatesSinceSuccess; + +export type UpdatesSinceSuccess = { + __typename?: 'UpdatesSinceSuccess'; + edges: Array; + pageInfo: PageInfo; +}; + export type UploadFileRequestError = { __typename?: 'UploadFileRequestError'; errorCodes: Array; @@ -2653,6 +2692,7 @@ export type ResolversTypes = { SubscriptionsErrorCode: SubscriptionsErrorCode; SubscriptionsResult: ResolversTypes['SubscriptionsError'] | ResolversTypes['SubscriptionsSuccess']; SubscriptionsSuccess: ResolverTypeWrapper; + SyncUpdatedItemEdge: ResolverTypeWrapper; TypeaheadSearchError: ResolverTypeWrapper; TypeaheadSearchErrorCode: TypeaheadSearchErrorCode; TypeaheadSearchItem: ResolverTypeWrapper; @@ -2687,6 +2727,7 @@ export type ResolversTypes = { UpdatePageInput: UpdatePageInput; UpdatePageResult: ResolversTypes['UpdatePageError'] | ResolversTypes['UpdatePageSuccess']; UpdatePageSuccess: ResolverTypeWrapper; + UpdateReason: UpdateReason; UpdateReminderError: ResolverTypeWrapper; UpdateReminderErrorCode: UpdateReminderErrorCode; UpdateReminderInput: UpdateReminderInput; @@ -2707,6 +2748,10 @@ export type ResolversTypes = { UpdateUserProfileSuccess: ResolverTypeWrapper; UpdateUserResult: ResolversTypes['UpdateUserError'] | ResolversTypes['UpdateUserSuccess']; UpdateUserSuccess: ResolverTypeWrapper; + UpdatesSinceError: ResolverTypeWrapper; + UpdatesSinceErrorCode: UpdatesSinceErrorCode; + UpdatesSinceResult: ResolversTypes['UpdatesSinceError'] | ResolversTypes['UpdatesSinceSuccess']; + UpdatesSinceSuccess: ResolverTypeWrapper; UploadFileRequestError: ResolverTypeWrapper; UploadFileRequestErrorCode: UploadFileRequestErrorCode; UploadFileRequestInput: UploadFileRequestInput; @@ -2950,6 +2995,7 @@ export type ResolversParentTypes = { SubscriptionsError: SubscriptionsError; SubscriptionsResult: ResolversParentTypes['SubscriptionsError'] | ResolversParentTypes['SubscriptionsSuccess']; SubscriptionsSuccess: SubscriptionsSuccess; + SyncUpdatedItemEdge: SyncUpdatedItemEdge; TypeaheadSearchError: TypeaheadSearchError; TypeaheadSearchItem: TypeaheadSearchItem; TypeaheadSearchResult: ResolversParentTypes['TypeaheadSearchError'] | ResolversParentTypes['TypeaheadSearchSuccess']; @@ -2993,6 +3039,9 @@ export type ResolversParentTypes = { UpdateUserProfileSuccess: UpdateUserProfileSuccess; UpdateUserResult: ResolversParentTypes['UpdateUserError'] | ResolversParentTypes['UpdateUserSuccess']; UpdateUserSuccess: UpdateUserSuccess; + UpdatesSinceError: UpdatesSinceError; + UpdatesSinceResult: ResolversParentTypes['UpdatesSinceError'] | ResolversParentTypes['UpdatesSinceSuccess']; + UpdatesSinceSuccess: UpdatesSinceSuccess; UploadFileRequestError: UploadFileRequestError; UploadFileRequestInput: UploadFileRequestInput; UploadFileRequestResult: ResolversParentTypes['UploadFileRequestError'] | ResolversParentTypes['UploadFileRequestSuccess']; @@ -3774,6 +3823,7 @@ export type QueryResolvers>; subscriptions?: Resolver>; typeaheadSearch?: Resolver>; + updatesSince?: Resolver>; user?: Resolver>; users?: Resolver; validateUsername?: Resolver>; @@ -4116,6 +4166,14 @@ export type SubscriptionsSuccessResolvers; }; +export type SyncUpdatedItemEdgeResolvers = { + cursor?: Resolver; + itemID?: Resolver; + node?: Resolver, ParentType, ContextType>; + updateReason?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type TypeaheadSearchErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -4279,6 +4337,21 @@ export type UpdateUserSuccessResolvers; }; +export type UpdatesSinceErrorResolvers = { + errorCodes?: Resolver>, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type UpdatesSinceResultResolvers = { + __resolveType: TypeResolveFn<'UpdatesSinceError' | 'UpdatesSinceSuccess', ParentType, ContextType>; +}; + +export type UpdatesSinceSuccessResolvers = { + edges?: Resolver, ParentType, ContextType>; + pageInfo?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type UploadFileRequestErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -4571,6 +4644,7 @@ export type Resolvers = { SubscriptionsError?: SubscriptionsErrorResolvers; SubscriptionsResult?: SubscriptionsResultResolvers; SubscriptionsSuccess?: SubscriptionsSuccessResolvers; + SyncUpdatedItemEdge?: SyncUpdatedItemEdgeResolvers; TypeaheadSearchError?: TypeaheadSearchErrorResolvers; TypeaheadSearchItem?: TypeaheadSearchItemResolvers; TypeaheadSearchResult?: TypeaheadSearchResultResolvers; @@ -4605,6 +4679,9 @@ export type Resolvers = { UpdateUserProfileSuccess?: UpdateUserProfileSuccessResolvers; UpdateUserResult?: UpdateUserResultResolvers; UpdateUserSuccess?: UpdateUserSuccessResolvers; + UpdatesSinceError?: UpdatesSinceErrorResolvers; + UpdatesSinceResult?: UpdatesSinceResultResolvers; + UpdatesSinceSuccess?: UpdatesSinceSuccessResolvers; UploadFileRequestError?: UploadFileRequestErrorResolvers; UploadFileRequestResult?: UploadFileRequestResultResolvers; UploadFileRequestSuccess?: UploadFileRequestSuccessResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index d3170ed12..133c0e135 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -918,6 +918,7 @@ type Query { sharedArticle(selectedHighlightId: String, slug: String!, username: String!): SharedArticleResult! subscriptions(sort: SortParams): SubscriptionsResult! typeaheadSearch(first: Int, query: String!): TypeaheadSearchResult! + updatesSince(after: String, first: Int, since: Date!): UpdatesSinceResult! user(userId: ID, username: String): UserResult! users: UsersResult! validateUsername(username: String!): Boolean! @@ -1419,6 +1420,13 @@ type SubscriptionsSuccess { subscriptions: [Subscription!]! } +type SyncUpdatedItemEdge { + cursor: String! + itemID: ID! + node: SearchItem + updateReason: UpdateReason! +} + type TypeaheadSearchError { errorCodes: [TypeaheadSearchErrorCode!]! } @@ -1571,6 +1579,12 @@ type UpdatePageSuccess { updatedPage: Article! } +enum UpdateReason { + CREATED + DELETED + UPDATED +} + type UpdateReminderError { errorCodes: [UpdateReminderErrorCode!]! } @@ -1662,6 +1676,21 @@ type UpdateUserSuccess { user: User! } +type UpdatesSinceError { + errorCodes: [UpdatesSinceErrorCode!] +} + +enum UpdatesSinceErrorCode { + UNAUTHORIZED +} + +union UpdatesSinceResult = UpdatesSinceError | UpdatesSinceSuccess + +type UpdatesSinceSuccess { + edges: [SyncUpdatedItemEdge!]! + pageInfo: PageInfo! +} + type UploadFileRequestError { errorCodes: [UploadFileRequestErrorCode!]! } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 336056564..7a2b6457d 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1761,6 +1761,34 @@ const schema = gql` siteName: String } + union UpdatesSinceResult = UpdatesSinceSuccess | UpdatesSinceError + + type UpdatesSinceSuccess { + edges: [SyncUpdatedItemEdge!]! + pageInfo: PageInfo! + } + + type SyncUpdatedItemEdge { + cursor: String! + updateReason: UpdateReason! + itemID: ID! + node: SearchItem # for created or updated items, null for deletions */ + } + + enum UpdateReason { + CREATED + UPDATED + DELETED + } + + type UpdatesSinceError { + errorCodes: [UpdatesSinceErrorCode!] + } + + enum UpdatesSinceErrorCode { + UNAUTHORIZED + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -1873,6 +1901,7 @@ const schema = gql` webhook(id: ID!): WebhookResult! apiKeys: ApiKeysResult! typeaheadSearch(query: String!, first: Int): TypeaheadSearchResult! + updatesSince(after: String, first: Int, since: Date!): UpdatesSinceResult! } ` From 07c6214e8831077c545a414a668ffa73780503ae Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 29 Jul 2022 14:08:35 +0800 Subject: [PATCH 2/7] Add DELETED state to the article --- packages/api/src/elastic/types.ts | 1 + packages/api/src/generated/graphql.ts | 1 + packages/api/src/generated/schema.graphql | 1 + packages/api/src/resolvers/article/index.ts | 1 - packages/api/src/schema.ts | 1 + 5 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 9381ca5cf..0e7fb7a2f 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -146,6 +146,7 @@ export enum ArticleSavingRequestStatus { Failed = 'FAILED', Processing = 'PROCESSING', Succeeded = 'SUCCEEDED', + Deleted = 'DELETED', } export interface Label { diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index c9fc8f810..8640e4346 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -183,6 +183,7 @@ export enum ArticleSavingRequestErrorCode { export type ArticleSavingRequestResult = ArticleSavingRequestError | ArticleSavingRequestSuccess; export enum ArticleSavingRequestStatus { + Deleted = 'DELETED', Failed = 'FAILED', Processing = 'PROCESSING', Succeeded = 'SUCCEEDED' diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 133c0e135..c70e400d9 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -148,6 +148,7 @@ enum ArticleSavingRequestErrorCode { union ArticleSavingRequestResult = ArticleSavingRequestError | ArticleSavingRequestSuccess enum ArticleSavingRequestStatus { + DELETED FAILED PROCESSING SUCCEEDED diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index e8cb98d40..28d0f6dd6 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -40,7 +40,6 @@ import { TypeaheadSearchErrorCode, TypeaheadSearchSuccess, } from '../../generated/graphql' -/* eslint-disable @typescript-eslint/no-explicit-any */ import { Merge } from '../../util' import { getStorageFileDetails, diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 7a2b6457d..cff2e32f5 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1000,6 +1000,7 @@ const schema = gql` PROCESSING SUCCEEDED FAILED + DELETED } type ArticleSavingRequest { From 83f988bb356339f076f0320dce2abea39633ad18 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 29 Jul 2022 15:53:13 +0800 Subject: [PATCH 3/7] Soft delete articles --- packages/api/src/elastic/labels.ts | 2 +- packages/api/src/elastic/pages.ts | 32 ++++++++++----------- packages/api/src/elastic/types.ts | 26 +++++++++++++++++ packages/api/src/resolvers/article/index.ts | 8 ++++-- packages/api/test/elastic/index.test.ts | 24 ++++++---------- packages/api/test/resolvers/article.test.ts | 19 +++++------- packages/api/test/resolvers/labels.test.ts | 4 +-- 7 files changed, 67 insertions(+), 48 deletions(-) diff --git a/packages/api/src/elastic/labels.ts b/packages/api/src/elastic/labels.ts index ce02655e2..3c1423f7c 100644 --- a/packages/api/src/elastic/labels.ts +++ b/packages/api/src/elastic/labels.ts @@ -18,7 +18,7 @@ export const addLabelInPage = async ( ctx._source.labels = [params.label]; ctx._source.updatedAt = params.updatedAt } else if (!ctx._source.labels.any(label -> label.name == params.label.name)) { - ctx._source.labels.add(params.label) ; + ctx._source.labels.add(params.label); ctx._source.updatedAt = params.updatedAt } else { ctx.op = 'none' }`, lang: 'painless', diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 3f0b1538f..4dbc40fbe 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -2,6 +2,7 @@ import { ArticleSavingRequestStatus, Page, PageContext, + PageSearchArgs, PageType, ParamSet, SearchBody, @@ -17,7 +18,6 @@ import { ReadFilter, SortBy, SortOrder, - SortParams, } from '../utils/search' import { client, INDEX_ALIAS } from './index' import { EntityType } from '../datalayer/pubsub' @@ -337,21 +337,7 @@ export const getPageById = async (id: string): Promise => { } export const searchPages = async ( - args: { - from?: number - size?: number - sort?: SortParams - query?: string - inFilter?: InFilter - readFilter?: ReadFilter - typeFilter?: PageType - labelFilters: LabelFilter[] - hasFilters: HasFilter[] - dateFilters: DateFilter[] - termFilters?: FieldFilter[] - matchFilters?: FieldFilter[] - includePending?: boolean | null - }, + args: PageSearchArgs, userId: string ): Promise<[Page[], number] | undefined> => { try { @@ -448,6 +434,14 @@ export const searchPages = async ( }) } + if (!args.includeDeleted) { + body.query.bool.must_not.push({ + term: { + state: ArticleSavingRequestStatus.Deleted, + }, + }) + } + console.log('searching pages in elastic', JSON.stringify(body)) const response = await client.search, SearchBody>({ @@ -533,6 +527,7 @@ export const deletePagesByParam = async ( const { body } = await client.deleteByQuery({ index: INDEX_ALIAS, body: params, + conflicts: 'proceed', }) if (body.deleted > 0) { @@ -566,6 +561,11 @@ export const searchAsYouType = async ( userId, }, }, + { + term: { + state: ArticleSavingRequestStatus.Succeeded, + }, + }, { multi_match: { query, diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 0e7fb7a2f..f4bf7b7c4 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -1,6 +1,15 @@ // Define the type of the body for the Search request import { PickTuple } from '../util' import { PubsubClient } from '../datalayer/pubsub' +import { + DateFilter, + FieldFilter, + HasFilter, + InFilter, + LabelFilter, + ReadFilter, + SortParams, +} from '../utils/search' export interface SearchBody { query: { @@ -246,3 +255,20 @@ export interface PageContext { refresh?: boolean uid: string } + +export interface PageSearchArgs { + from?: number + size?: number + sort?: SortParams + query?: string + inFilter?: InFilter + readFilter?: ReadFilter + typeFilter?: PageType + labelFilters: LabelFilter[] + hasFilters: HasFilter[] + dateFilters: DateFilter[] + termFilters?: FieldFilter[] + matchFilters?: FieldFilter[] + includePending?: boolean | null + includeDeleted?: boolean +} diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 28d0f6dd6..3ee5c1c29 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -82,7 +82,6 @@ import { } from '../../elastic/types' import { createPage, - deletePage, getPageById, getPageByParam, searchAsYouType, @@ -636,7 +635,12 @@ export const setBookmarkArticleResolver = authorized< return { errorCodes: [SetBookmarkArticleErrorCode.NotFound] } } - await deletePage(pageRemoved.id, { pubsub, uid }) + // delete the page + await updatePage( + pageRemoved.id, + { state: ArticleSavingRequestStatus.Deleted }, + { pubsub, uid } + ) const highlightsUnshared = await authTrx(async (tx) => { return models.highlight.unshareAllHighlights(articleID, uid, tx) diff --git a/packages/api/test/elastic/index.test.ts b/packages/api/test/elastic/index.test.ts index 7d7dc556d..ff415a41d 100644 --- a/packages/api/test/elastic/index.test.ts +++ b/packages/api/test/elastic/index.test.ts @@ -70,11 +70,7 @@ describe('elastic api', () => { ], state: ArticleSavingRequestStatus.Succeeded, } - const pageId = await createPage(page, ctx) - if (!pageId) { - expect.fail('Failed to create page') - } - page.id = pageId + page.id = (await createPage(page, ctx))! }) after(async () => { @@ -83,12 +79,10 @@ describe('elastic api', () => { }) describe('createPage', () => { - let newPageId: string | undefined + let newPageId: string after(async () => { - if (newPageId) { - await deletePage(newPageId, ctx) - } + await deletePage(newPageId, ctx) }) it('creates a page', async () => { @@ -108,9 +102,7 @@ describe('elastic api', () => { url: 'https://blog.omnivore.app/testUrl', state: ArticleSavingRequestStatus.Succeeded, } - - newPageId = await createPage(newPageData, ctx) - + newPageId = (await createPage(newPageData, ctx))! expect(newPageId).to.be.a('string') }) }) @@ -344,9 +336,11 @@ describe('elastic api', () => { }) describe('searchAsYouType', () => { + let pageId: string + before(async () => { // create a testing page - await createPage( + pageId = (await createPage( { content: '', createdAt: new Date(), @@ -363,12 +357,12 @@ describe('elastic api', () => { userId, }, ctx - ) + ))! }) after(async () => { // delete the testing page - await deletePagesByParam({ userId }, ctx) + await deletePage(pageId, ctx) }) it('searches pages', async () => { diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index a7b135406..bb8a8c58e 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -649,7 +649,7 @@ describe('Article API', () => { let query = '' let articleId = '' let bookmark = true - let pageId = '' + let pageId: string before(async () => { const page: Page = { @@ -667,16 +667,11 @@ describe('Article API', () => { readingProgressAnchorIndex: 0, state: ArticleSavingRequestStatus.Succeeded, } - const newPageId = await createPage(page, ctx) - if (newPageId) { - pageId = newPageId - } + pageId = (await createPage(page, ctx))! }) after(async () => { - if (pageId) { - await deletePage(pageId, ctx) - } + await deletePage(pageId, ctx) }) beforeEach(() => { @@ -684,7 +679,7 @@ describe('Article API', () => { }) context('when we set a bookmark on an article', () => { - before(async () => { + before(() => { articleId = pageId bookmark = true }) @@ -698,15 +693,15 @@ describe('Article API', () => { }) context('when we unset a bookmark on an article', () => { - before(async () => { + before(() => { articleId = pageId bookmark = false }) it('should delete an article', async () => { await graphqlRequest(query, authToken).expect(200) - const pageId = await getPageById(articleId) - expect(pageId).to.undefined + const page = await getPageById(articleId) + expect(page?.state).to.eql(ArticleSavingRequestStatus.Deleted) }) }) }) diff --git a/packages/api/test/resolvers/labels.test.ts b/packages/api/test/resolvers/labels.test.ts index 9b10b9a6f..afd3a6c85 100644 --- a/packages/api/test/resolvers/labels.test.ts +++ b/packages/api/test/resolvers/labels.test.ts @@ -11,7 +11,7 @@ import 'mocha' import { User } from '../../src/entity/user' import { Highlight, Page, PageContext } from '../../src/elastic/types' import { getRepository } from '../../src/entity/utils' -import { deletePagesByParam, getPageById } from '../../src/elastic/pages' +import { deletePage, getPageById } from '../../src/elastic/pages' import { addLabelInPage } from '../../src/elastic/labels' import { createPubSubClient } from '../../src/datalayer/pubsub' import { @@ -66,7 +66,7 @@ describe('Labels API', () => { after(async () => { // clean up - await deletePagesByParam({ userId: user.id }, ctx) + await deletePage(page.id, ctx) await deleteTestUser(username) }) From 7ce183d6f8341f4e19620e6704baf0c794d1e11c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 29 Jul 2022 18:10:05 +0800 Subject: [PATCH 4/7] Add soft delete of pages by updating page state to Deleted --- packages/api/src/elastic/pages.ts | 24 ++++++++++++------------ packages/api/src/elastic/types.ts | 6 +++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 4dbc40fbe..9abffd52e 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -224,12 +224,16 @@ export const updatePage = async ( if (body.result !== 'updated') return false + if (page.state === ArticleSavingRequestStatus.Deleted) { + await ctx.pubsub.entityDeleted(EntityType.PAGE, id, ctx.uid) + return true + } + await ctx.pubsub.entityUpdated>( EntityType.PAGE, { ...page, id }, ctx.uid ) - return true } catch (e) { if ( @@ -255,11 +259,7 @@ export const deletePage = async ( refresh: ctx.refresh, }) - if (body.deleted === 0) return false - - await ctx.pubsub.entityDeleted(EntityType.PAGE, id, ctx.uid) - - return true + return body.deleted !== 0 } catch (e) { if ( e instanceof ResponseError && @@ -359,10 +359,10 @@ export const searchPages = async ( const sortOrder = sort?.order || SortOrder.DESCENDING // default sort by saved_at const sortField = sort?.by || SortBy.SAVED - const includeLabels = labelFilters.filter( + const includeLabels = labelFilters?.filter( (filter) => filter.type === LabelFilterType.INCLUDE ) - const excludeLabels = labelFilters.filter( + const excludeLabels = labelFilters?.filter( (filter) => filter.type === LabelFilterType.EXCLUDE ) @@ -407,16 +407,16 @@ export const searchPages = async ( if (readFilter !== ReadFilter.ALL) { appendReadFilter(body, readFilter) } - if (hasFilters.length > 0) { + if (hasFilters && hasFilters.length > 0) { appendHasFilters(body, hasFilters) } - if (includeLabels.length > 0) { + if (includeLabels && includeLabels.length > 0) { appendIncludeLabelFilter(body, includeLabels) } - if (excludeLabels.length > 0) { + if (excludeLabels && excludeLabels.length > 0) { appendExcludeLabelFilter(body, excludeLabels) } - if (dateFilters.length > 0) { + if (dateFilters && dateFilters.length > 0) { appendDateFilters(body, dateFilters) } if (termFilters) { diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index f4bf7b7c4..a601082fb 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -264,9 +264,9 @@ export interface PageSearchArgs { inFilter?: InFilter readFilter?: ReadFilter typeFilter?: PageType - labelFilters: LabelFilter[] - hasFilters: HasFilter[] - dateFilters: DateFilter[] + labelFilters?: LabelFilter[] + hasFilters?: HasFilter[] + dateFilters?: DateFilter[] termFilters?: FieldFilter[] matchFilters?: FieldFilter[] includePending?: boolean | null From 336d06375aebaa2b8d7eb82afe03da75ed382a17 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 29 Jul 2022 18:11:18 +0800 Subject: [PATCH 5/7] Add updatesSince API resolver --- packages/api/src/generated/graphql.ts | 4 +- packages/api/src/generated/schema.graphql | 2 +- packages/api/src/resolvers/article/index.ts | 115 ++++++++++++++++-- .../api/src/resolvers/function_resolvers.ts | 3 + packages/api/src/schema.ts | 2 +- 5 files changed, 113 insertions(+), 13 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 8640e4346..20f1fd596 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2183,7 +2183,7 @@ export type UpdateUserSuccess = { export type UpdatesSinceError = { __typename?: 'UpdatesSinceError'; - errorCodes?: Maybe>; + errorCodes: Array; }; export enum UpdatesSinceErrorCode { @@ -4339,7 +4339,7 @@ export type UpdateUserSuccessResolvers = { - errorCodes?: Resolver>, ParentType, ContextType>; + errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index c70e400d9..260d5a601 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1678,7 +1678,7 @@ type UpdateUserSuccess { } type UpdatesSinceError { - errorCodes: [UpdatesSinceErrorCode!] + errorCodes: [UpdatesSinceErrorCode!]! } enum UpdatesSinceErrorCode { diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 3ee5c1c29..bd9e26ee5 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -23,6 +23,7 @@ import { QueryArticlesArgs, QuerySearchArgs, QueryTypeaheadSearchArgs, + QueryUpdatesSinceArgs, ResolverFn, SaveArticleReadingProgressError, SaveArticleReadingProgressErrorCode, @@ -39,6 +40,10 @@ import { TypeaheadSearchError, TypeaheadSearchErrorCode, TypeaheadSearchSuccess, + UpdateReason, + UpdatesSinceError, + UpdatesSinceErrorCode, + UpdatesSinceSuccess, } from '../../generated/graphql' import { Merge } from '../../util' import { @@ -68,7 +73,7 @@ import { createImageProxyUrl } from '../../utils/imageproxy' import normalizeUrl from 'normalize-url' import { WithDataSourcesContext } from '../types' -import { parseSearchQuery } from '../../utils/search' +import { parseSearchQuery, SortBy, SortOrder } from '../../utils/search' import { createPageSaveRequest } from '../../services/create_page_save_request' import { analytics } from '../../utils/analytics' import { env } from '../../env' @@ -618,7 +623,7 @@ export const setBookmarkArticleResolver = authorized< async ( _, { input: { articleID, bookmark } }, - { models, authTrx, claims: { uid }, log, pubsub } + { claims: { uid }, log, pubsub } ) => { const page = await getPageById(articleID) if (!page) { @@ -636,15 +641,14 @@ export const setBookmarkArticleResolver = authorized< } // delete the page - await updatePage( + const deleted = await updatePage( pageRemoved.id, { state: ArticleSavingRequestStatus.Deleted }, { pubsub, uid } ) - - const highlightsUnshared = await authTrx(async (tx) => { - return models.highlight.unshareAllHighlights(articleID, uid, tx) - }) + if (!deleted) { + return { errorCodes: [SetBookmarkArticleErrorCode.NotFound] } + } analytics.track({ userId: uid, @@ -660,7 +664,6 @@ export const setBookmarkArticleResolver = authorized< content: undefined, originalHtml: undefined, }), - highlightsUnshared: highlightsUnshared.length, labels: { source: 'resolver', resolver: 'setBookmarkArticleResolver', @@ -683,7 +686,13 @@ export const setBookmarkArticleResolver = authorized< userId: uid, slug: generateSlug(page.title), } - await updatePage(articleID, pageUpdated, { pubsub, uid }) + const updated = await updatePage(articleID, pageUpdated, { + pubsub, + uid, + }) + if (!updated) { + return { errorCodes: [SetBookmarkArticleErrorCode.NotFound] } + } log.info('Article bookmarked', { page: Object.assign({}, page, { @@ -927,3 +936,91 @@ export const typeaheadSearchResolver = authorized< return { items: await searchAsYouType(claims.uid, query, first || undefined) } }) + +export const updatesSinceResolver = authorized< + UpdatesSinceSuccess, + UpdatesSinceError, + QueryUpdatesSinceArgs +>(async (_obj, { since, first, after }, { claims: { uid } }) => { + if (!uid) { + return { errorCodes: [UpdatesSinceErrorCode.Unauthorized] } + } + + analytics.track({ + userId: uid, + event: 'updatesSince', + properties: { + env: env.server.apiEnv, + since, + first, + after, + }, + }) + + const startCursor = after || '' + const size = first || 10 + const startDate = new Date(since) + const [pages, totalCount] = (await searchPages( + { + from: Number(startCursor), + size: size + 1, // fetch one more item to get next cursor + includeDeleted: true, + dateFilters: [{ field: 'updatedAt', startDate }], + sort: { by: SortBy.UPDATED, order: SortOrder.ASCENDING }, + }, + uid + )) || [[], 0] + + const start = + startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 + const hasNextPage = pages.length > size + const endCursor = String(start + pages.length - (hasNextPage ? 1 : 0)) + + //TODO: refactor so that the lastCursor included + if (hasNextPage) { + // remove an extra if exists + pages.pop() + } + + const edges = pages.map((p) => { + const updateReason = getUpdateReason(p, startDate) + return { + node: + updateReason === UpdateReason.Deleted + ? null + : ({ + ...p, + image: p.image && createImageProxyUrl(p.image, 88, 88), + isArchived: !!p.archivedAt, + contentReader: + p.pageType === PageType.File + ? ContentReader.Pdf + : ContentReader.Web, + } as SearchItem), + cursor: endCursor, + itemID: p.id, + updateReason, + } + }) + + return { + edges, + pageInfo: { + hasPreviousPage: false, + startCursor, + hasNextPage, + endCursor, + totalCount, + }, + } +}) + +const getUpdateReason = (page: Page, since: Date) => { + if (page.state === ArticleSavingRequestStatus.Deleted) { + return UpdateReason.Deleted + } + if (page.createdAt >= since) { + return UpdateReason.Created + } + return UpdateReason.Updated +} diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 8ed87340f..3b05d3a02 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -82,6 +82,7 @@ import { updatePageResolver, updateReminderResolver, updateSharedCommentResolver, + updatesSinceResolver, updateUserProfileResolver, updateUserResolver, uploadFileRequestResolver, @@ -186,6 +187,7 @@ export const functionResolvers = { webhook: webhookResolver, apiKeys: apiKeysResolver, typeaheadSearch: typeaheadSearchResolver, + updatesSince: updatesSinceResolver, }, User: { async sharedArticles( @@ -586,4 +588,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('RevokeApiKey'), ...resultResolveTypeResolver('DeleteAccount'), ...resultResolveTypeResolver('TypeaheadSearch'), + ...resultResolveTypeResolver('UpdatesSince'), } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index cff2e32f5..111671b4d 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1783,7 +1783,7 @@ const schema = gql` } type UpdatesSinceError { - errorCodes: [UpdatesSinceErrorCode!] + errorCodes: [UpdatesSinceErrorCode!]! } enum UpdatesSinceErrorCode { From a9a88a21d24c4a6a6d34025c17c63414a3d01574 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 29 Jul 2022 18:11:52 +0800 Subject: [PATCH 6/7] Add integration tests for updatesSince API resolver --- packages/api/test/resolvers/article.test.ts | 102 +++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index bb8a8c58e..f90ae91a6 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -10,7 +10,7 @@ import { expect } from 'chai' import 'mocha' import { User } from '../../src/entity/user' import chaiString from 'chai-string' -import { UploadFileStatus } from '../../src/generated/graphql' +import { UpdateReason, UploadFileStatus } from '../../src/generated/graphql' import { ArticleSavingRequestStatus, Highlight, @@ -972,4 +972,104 @@ describe('Article API', () => { expect(res.body.data.typeaheadSearch.items[4].id).to.eq(pages[4].id) }) }) + + describe('UpdatesSince API', () => { + const updatesSinceQuery = (since: string) => ` + query { + updatesSince( + since: "${since}") { + ... on UpdatesSinceSuccess { + edges { + cursor + node { + id + createdAt + updatedAt + } + itemID + updateReason + } + pageInfo { + hasNextPage + hasPreviousPage + startCursor + endCursor + totalCount + } + } + ... on UpdatesSinceError { + errorCodes + } + } + } + ` + let since: string + let pageIds: string[] = [] + let deletedPageIds: string[] = [] + + before(async () => { + // Create some test pages + for (let i = 0; i < 5; i++) { + const page: Page = { + id: '', + hash: '', + userId: user.id, + pageType: PageType.Article, + title: 'test page', + content: '', + slug: '', + createdAt: new Date(), + updatedAt: new Date(), + readingProgressPercent: 0, + readingProgressAnchorIndex: 0, + url: '', + savedAt: new Date(), + state: ArticleSavingRequestStatus.Succeeded, + } + const pageId = (await createPage(page, ctx))! + pageIds.push(pageId) + } + + // set the since to be the date before deletion + since = new Date().toISOString() + + // Delete some pages + for (let i = 0; i < 3; i++) { + await updatePage( + pageIds[i], + { state: ArticleSavingRequestStatus.Deleted }, + ctx + ) + deletedPageIds.push(pageIds[i]) + } + }) + + after(async () => { + // Delete all pages + for (let i = 0; i < pageIds.length; i++) { + await deletePage(pageIds[i], ctx) + } + }) + + it('returns pages deleted after since', async () => { + const res = await graphqlRequest( + updatesSinceQuery(since), + authToken + ).expect(200) + + expect(res.body.data.updatesSince.edges.length).to.eql(3) + expect(res.body.data.updatesSince.edges[0].itemID).to.eq( + deletedPageIds[0] + ) + expect(res.body.data.updatesSince.edges[1].itemID).to.eq( + deletedPageIds[1] + ) + expect(res.body.data.updatesSince.edges[2].itemID).to.eq( + deletedPageIds[2] + ) + expect(res.body.data.updatesSince.edges[2].updateReason).to.eq( + UpdateReason.Deleted + ) + }) + }) }) From 446a9903e853c19321bab2dc2989fc8595b6315f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 29 Jul 2022 21:22:55 +0800 Subject: [PATCH 7/7] Fix tests --- packages/api/test/resolvers/article.test.ts | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index f90ae91a6..31eb49757 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -1004,8 +1004,8 @@ describe('Article API', () => { } ` let since: string - let pageIds: string[] = [] - let deletedPageIds: string[] = [] + let pages: Page[] = [] + let deletedPages: Page[] = [] before(async () => { // Create some test pages @@ -1026,28 +1026,28 @@ describe('Article API', () => { savedAt: new Date(), state: ArticleSavingRequestStatus.Succeeded, } - const pageId = (await createPage(page, ctx))! - pageIds.push(pageId) + page.id = (await createPage(page, ctx))! + pages.push(page) } - // set the since to be the date before deletion - since = new Date().toISOString() + // set the since to be the timestamp before deletion + since = pages[4].createdAt.toISOString() // Delete some pages for (let i = 0; i < 3; i++) { await updatePage( - pageIds[i], + pages[i].id, { state: ArticleSavingRequestStatus.Deleted }, ctx ) - deletedPageIds.push(pageIds[i]) + deletedPages.push(pages[i]) } }) after(async () => { // Delete all pages - for (let i = 0; i < pageIds.length; i++) { - await deletePage(pageIds[i], ctx) + for (let i = 0; i < pages.length; i++) { + await deletePage(pages[i].id, ctx) } }) @@ -1059,13 +1059,13 @@ describe('Article API', () => { expect(res.body.data.updatesSince.edges.length).to.eql(3) expect(res.body.data.updatesSince.edges[0].itemID).to.eq( - deletedPageIds[0] + deletedPages[0].id ) expect(res.body.data.updatesSince.edges[1].itemID).to.eq( - deletedPageIds[1] + deletedPages[1].id ) expect(res.body.data.updatesSince.edges[2].itemID).to.eq( - deletedPageIds[2] + deletedPages[2].id ) expect(res.body.data.updatesSince.edges[2].updateReason).to.eq( UpdateReason.Deleted