From 53b854b69f1c1d1cb5e6f2337ae86c54b5a916c1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 30 Jun 2023 15:32:09 +0800 Subject: [PATCH 1/5] fix: improve full-text search result * always sort by score first * boost title score by 3 times * tie breaker is 0.3 --- packages/api/src/elastic/pages.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 4f29984f3..538c272b9 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -28,14 +28,21 @@ import { } from './types' const appendQuery = (builder: ESBuilder, query: string): ESBuilder => { - const fields = ['title', 'content', 'author', 'description', 'siteName'] + const fields = [ + { field: 'title', boost: 3 }, + { field: 'content', boost: 1 }, + { field: 'author', boost: 1 }, + { field: 'description', boost: 1 }, + { field: 'siteName', boost: 1 }, + ] + // wildcard query if (query.includes('*')) { - // wildcard query fields.forEach((field) => { builder = builder.orQuery('wildcard', { - [field]: { + [field.field]: { value: query, case_insensitive: true, + boost: field.boost, }, }) }) @@ -45,9 +52,11 @@ const appendQuery = (builder: ESBuilder, query: string): ESBuilder => { return builder .orQuery('multi_match', { query, - fields, - operator: 'and', - type: 'cross_fields', + fields: fields.map( + (field) => `${field.field}${field.boost > 1 ? `^${field.boost}` : ''}` + ), + type: 'best_fields', + tie_breaker: 0.3, }) .queryMinimumShouldMatch(1) } @@ -523,6 +532,7 @@ export const searchPages = async ( // build the query const builder = buildSearchBody(userId, args) const body = builder + .sort('_score', 'desc') // sort by score first .sort(sortField, sortOrder) .from(from) .size(size) From 6be9afb619c2334badbe41612e0511fe19e96f69 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 30 Jun 2023 16:02:49 +0800 Subject: [PATCH 2/5] feat: allow searching by note: --- packages/api/src/elastic/pages.ts | 13 +++++++++++++ packages/api/src/utils/search.ts | 16 ++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 538c272b9..c53f20551 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -201,6 +201,19 @@ const appendMatchFilters = ( filters: FieldFilter[] ): ESBuilder => { filters.forEach((filter) => { + if (filter.nested) { + // nested query + builder = builder.query('nested', { + path: filter.field.split('.')[0], // get the nested field name + query: { + match: { + [filter.field]: filter.value, + }, + }, + }) + return + } + builder = builder.query('match', { [filter.field]: filter.value, }) diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index 7021e91df..737572bec 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -82,6 +82,7 @@ export interface SortParams { } export interface FieldFilter { + nested?: boolean field: string value: string } @@ -260,10 +261,19 @@ const parseFieldFilter = ( return undefined } + let nested = false + // normalize the term to lower case + const value = str.toLowerCase() + + if (field === 'note') { + field = 'highlights.annotation' + nested = true + } + return { + nested, field, - // normalize the term to lower case - value: str.toLowerCase(), + value, } } @@ -343,6 +353,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { 'no', 'mode', 'site', + 'note', ], tokenize: true, }) @@ -414,6 +425,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { case 'author': case 'title': case 'description': + case 'note': case 'content': { const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value) fieldFilter && result.matchFilters.push(fieldFilter) From 10e5059db71c58da3cfd07122c84c62b1737a296 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 30 Jun 2023 17:28:37 +0800 Subject: [PATCH 3/5] fix test --- packages/api/test/resolvers/article.test.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 8df253e27..8d8bd4138 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -911,6 +911,7 @@ describe('Article API', () => { const url = 'https://blog.omnivore.app/p/getting-started-with-omnivore' const pages: Page[] = [] const highlights: Highlight[] = [] + const searchedKeyword = 'aaabbbccc' let query = '' let keyword = '' @@ -924,13 +925,13 @@ describe('Article API', () => { userId: user.id, pageType: PageType.Article, title: 'test title', - content: '

test search api

', + content: `

test ${searchedKeyword}

`, slug: 'test slug', createdAt: new Date(), updatedAt: new Date(), readingProgressPercent: 0, readingProgressAnchorIndex: 0, - url: url, + url: `${url}/${i}`, savedAt: new Date(), state: ArticleSavingRequestStatus.Succeeded, siteName: 'Example', @@ -964,7 +965,7 @@ describe('Article API', () => { context('when type:highlights is not in the query', () => { before(() => { - keyword = 'search api' + keyword = searchedKeyword }) it('should return pages in descending order', async () => { @@ -990,7 +991,7 @@ describe('Article API', () => { context('when type:highlights is in the query', () => { before(() => { - keyword = "'search api' type:highlights" + keyword = `'${searchedKeyword}' type:highlights` }) it('should return highlights in descending order', async () => { @@ -1007,7 +1008,7 @@ describe('Article API', () => { context('when is:unread is in the query', () => { before(() => { - keyword = "'search api' is:unread" + keyword = `'${searchedKeyword}' is:unread` }) it('should return unread articles in descending order', async () => { @@ -1024,7 +1025,7 @@ describe('Article API', () => { context('when no:label is in the query', () => { before(async () => { - keyword = "'search api' no:label" + keyword = `'${searchedKeyword}' no:label` }) it('returns non-labeled items in descending order', async () => { @@ -1036,7 +1037,7 @@ describe('Article API', () => { context('when no:highlight is in the query', () => { before(async () => { - keyword = "'search api' no:highlight" + keyword = `'${searchedKeyword}' no:highlight` }) it('returns non-highlighted items in descending order', async () => { @@ -1048,7 +1049,7 @@ describe('Article API', () => { context('when site:${site_name} is in the query', () => { before(async () => { - keyword = "'search api' site:example" + keyword = `'${searchedKeyword}' site:example` }) it('returns items from the site', async () => { From ba35fa4447189ea63e97581fc145719eca3530b8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 30 Jun 2023 18:05:41 +0800 Subject: [PATCH 4/5] feat: show soft deleted pages within 14 days by searching in:trash --- packages/api/src/elastic/pages.ts | 13 ++++++++++++- packages/api/src/utils/search.ts | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index c53f20551..d4b3f9314 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -92,6 +92,17 @@ const appendInFilter = (builder: ESBuilder, filter: InFilter): ESBuilder => { return builder.query('exists', { field: 'archivedAt' }) case InFilter.INBOX: return builder.notQuery('exists', { field: 'archivedAt' }) + case InFilter.TRASH: + // return only deleted pages within 14 days + return builder + .query('term', { + state: ArticleSavingRequestStatus.Deleted, + }) + .andQuery('range', { + updatedAt: { + gte: 'now-14d', + }, + }) } return builder } @@ -515,7 +526,7 @@ const buildSearchBody = (userId: string, args: PageSearchArgs) => { state: ArticleSavingRequestStatus.Processing, }) } - if (!args.includeDeleted) { + if (!args.includeDeleted && inFilter !== InFilter.TRASH) { builder = builder.notQuery('term', { state: ArticleSavingRequestStatus.Deleted, }) diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index 737572bec..4caca6c8b 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -21,6 +21,7 @@ export enum InFilter { ALL, INBOX, ARCHIVE, + TRASH, } export interface SearchFilter { @@ -120,6 +121,8 @@ const parseInFilter = ( return InFilter.INBOX case 'ARCHIVE': return InFilter.ARCHIVE + case 'TRASH': + return InFilter.TRASH } return query ? InFilter.ALL : InFilter.INBOX } From 88dc514d27d4eb338cabb32c53cd40a290ad7dea Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 30 Jun 2023 18:39:43 +0800 Subject: [PATCH 5/5] feat: allow recovering a soft deleted item through updatePage api --- packages/api/src/generated/graphql.ts | 1 + packages/api/src/generated/schema.graphql | 1 + packages/api/src/resolvers/update/index.ts | 1 + packages/api/src/schema.ts | 1 + 4 files changed, 4 insertions(+) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 2c2af3081..58ced306c 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2910,6 +2910,7 @@ export type UpdatePageInput = { previewImage?: InputMaybe; publishedAt?: InputMaybe; savedAt?: InputMaybe; + state?: InputMaybe; title?: InputMaybe; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 7919e1273..23f8b463c 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2245,6 +2245,7 @@ input UpdatePageInput { previewImage: String publishedAt: Date savedAt: Date + state: ArticleSavingRequestStatus title: String } diff --git a/packages/api/src/resolvers/update/index.ts b/packages/api/src/resolvers/update/index.ts index 27c21f794..8462ff832 100644 --- a/packages/api/src/resolvers/update/index.ts +++ b/packages/api/src/resolvers/update/index.ts @@ -43,6 +43,7 @@ export const updatePageResolver = authorized< savedAt: input.savedAt ? new Date(input.savedAt) : undefined, publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, image: input.previewImage ?? undefined, + state: input.state ?? undefined, } const updateResult = await updatePage(input.pageId, pageData, { diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 9a7ec8071..44ee4bc49 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -581,6 +581,7 @@ const schema = gql` savedAt: Date publishedAt: Date previewImage: String @sanitize + state: ArticleSavingRequestStatus } type UpdatePageSuccess {