Merge pull request #2447 from omnivore-app/fix/full-text-search

fix/full text search

packages/api/src/elastic/pages.ts

@@ -28,14 +28,21 @@ import {
 } from './types'
 
 const appendQuery = (builder: ESBuilder, query: string): ESBuilder => {
-  const fields = ['title', 'content', 'author', 'description', 'siteName']
+  const fields = [
+    { field: 'title', boost: 3 },
+    { field: 'content', boost: 1 },
+    { field: 'author', boost: 1 },
+    { field: 'description', boost: 1 },
+    { field: 'siteName', boost: 1 },
+  ]
+  // wildcard query
   if (query.includes('*')) {
-    // wildcard query
     fields.forEach((field) => {
       builder = builder.orQuery('wildcard', {
-        [field]: {
+        [field.field]: {
           value: query,
           case_insensitive: true,
+          boost: field.boost,
         },
       })
     })
@@ -45,9 +52,11 @@ const appendQuery = (builder: ESBuilder, query: string): ESBuilder => {
   return builder
     .orQuery('multi_match', {
       query,
-      fields,
-      operator: 'and',
-      type: 'cross_fields',
+      fields: fields.map(
+        (field) => `${field.field}${field.boost > 1 ? `^${field.boost}` : ''}`
+      ),
+      type: 'best_fields',
+      tie_breaker: 0.3,
     })
     .queryMinimumShouldMatch(1)
 }
@@ -83,6 +92,17 @@ const appendInFilter = (builder: ESBuilder, filter: InFilter): ESBuilder => {
       return builder.query('exists', { field: 'archivedAt' })
     case InFilter.INBOX:
       return builder.notQuery('exists', { field: 'archivedAt' })
+    case InFilter.TRASH:
+      // return only deleted pages within 14 days
+      return builder
+        .query('term', {
+          state: ArticleSavingRequestStatus.Deleted,
+        })
+        .andQuery('range', {
+          updatedAt: {
+            gte: 'now-14d',
+          },
+        })
   }
   return builder
 }
@@ -192,6 +212,19 @@ const appendMatchFilters = (
   filters: FieldFilter[]
 ): ESBuilder => {
   filters.forEach((filter) => {
+    if (filter.nested) {
+      // nested query
+      builder = builder.query('nested', {
+        path: filter.field.split('.')[0], // get the nested field name
+        query: {
+          match: {
+            [filter.field]: filter.value,
+          },
+        },
+      })
+      return
+    }
+
     builder = builder.query('match', {
       [filter.field]: filter.value,
     })
@@ -493,7 +526,7 @@ const buildSearchBody = (userId: string, args: PageSearchArgs) => {
       state: ArticleSavingRequestStatus.Processing,
     })
   }
-  if (!args.includeDeleted) {
+  if (!args.includeDeleted && inFilter !== InFilter.TRASH) {
     builder = builder.notQuery('term', {
       state: ArticleSavingRequestStatus.Deleted,
     })
@@ -523,6 +556,7 @@ export const searchPages = async (
     // build the query
     const builder = buildSearchBody(userId, args)
     const body = builder
+      .sort('_score', 'desc') // sort by score first
       .sort(sortField, sortOrder)
       .from(from)
       .size(size)

packages/api/src/generated/graphql.ts

@@ -2910,6 +2910,7 @@ export type UpdatePageInput = {
   previewImage?: InputMaybe<Scalars['String']>;
   publishedAt?: InputMaybe<Scalars['Date']>;
   savedAt?: InputMaybe<Scalars['Date']>;
+  state?: InputMaybe<ArticleSavingRequestStatus>;
   title?: InputMaybe<Scalars['String']>;
 };
 

packages/api/src/generated/schema.graphql

@@ -2245,6 +2245,7 @@ input UpdatePageInput {
   previewImage: String
   publishedAt: Date
   savedAt: Date
+  state: ArticleSavingRequestStatus
   title: String
 }
 

packages/api/src/resolvers/update/index.ts

@@ -43,6 +43,7 @@ export const updatePageResolver = authorized<
     savedAt: input.savedAt ? new Date(input.savedAt) : undefined,
     publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined,
     image: input.previewImage ?? undefined,
+    state: input.state ?? undefined,
   }
 
   const updateResult = await updatePage(input.pageId, pageData, {

packages/api/src/schema.ts

@@ -581,6 +581,7 @@ const schema = gql`
     savedAt: Date
     publishedAt: Date
     previewImage: String @sanitize
+    state: ArticleSavingRequestStatus
   }
 
   type UpdatePageSuccess {

packages/api/src/utils/search.ts

@@ -21,6 +21,7 @@ export enum InFilter {
   ALL,
   INBOX,
   ARCHIVE,
+  TRASH,
 }
 
 export interface SearchFilter {
@@ -82,6 +83,7 @@ export interface SortParams {
 }
 
 export interface FieldFilter {
+  nested?: boolean
   field: string
   value: string
 }
@@ -119,6 +121,8 @@ const parseInFilter = (
       return InFilter.INBOX
     case 'ARCHIVE':
       return InFilter.ARCHIVE
+    case 'TRASH':
+      return InFilter.TRASH
   }
   return query ? InFilter.ALL : InFilter.INBOX
 }
@@ -260,10 +264,19 @@ const parseFieldFilter = (
     return undefined
   }
 
+  let nested = false
+  // normalize the term to lower case
+  const value = str.toLowerCase()
+
+  if (field === 'note') {
+    field = 'highlights.annotation'
+    nested = true
+  }
+
   return {
+    nested,
     field,
-    // normalize the term to lower case
-    value: str.toLowerCase(),
+    value,
   }
 }
 
@@ -343,6 +356,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
       'no',
       'mode',
       'site',
+      'note',
     ],
     tokenize: true,
   })
@@ -414,6 +428,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
         case 'author':
         case 'title':
         case 'description':
+        case 'note':
         case 'content': {
           const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value)
           fieldFilter && result.matchFilters.push(fieldFilter)

packages/api/test/resolvers/article.test.ts

@@ -911,6 +911,7 @@ describe('Article API', () => {
     const url = 'https://blog.omnivore.app/p/getting-started-with-omnivore'
     const pages: Page[] = []
     const highlights: Highlight[] = []
+    const searchedKeyword = 'aaabbbccc'
 
     let query = ''
     let keyword = ''
@@ -924,13 +925,13 @@ describe('Article API', () => {
           userId: user.id,
           pageType: PageType.Article,
           title: 'test title',
-          content: '<p>test search api</p>',
+          content: `<p>test ${searchedKeyword}</p>`,
           slug: 'test slug',
           createdAt: new Date(),
           updatedAt: new Date(),
           readingProgressPercent: 0,
           readingProgressAnchorIndex: 0,
-          url: url,
+          url: `${url}/${i}`,
           savedAt: new Date(),
           state: ArticleSavingRequestStatus.Succeeded,
           siteName: 'Example',
@@ -964,7 +965,7 @@ describe('Article API', () => {
 
     context('when type:highlights is not in the query', () => {
       before(() => {
-        keyword = 'search api'
+        keyword = searchedKeyword
       })
 
       it('should return pages in descending order', async () => {
@@ -990,7 +991,7 @@ describe('Article API', () => {
 
     context('when type:highlights is in the query', () => {
       before(() => {
-        keyword = "'search api' type:highlights"
+        keyword = `'${searchedKeyword}' type:highlights`
       })
 
       it('should return highlights in descending order', async () => {
@@ -1007,7 +1008,7 @@ describe('Article API', () => {
 
     context('when is:unread is in the query', () => {
       before(() => {
-        keyword = "'search api' is:unread"
+        keyword = `'${searchedKeyword}' is:unread`
       })
 
       it('should return unread articles in descending order', async () => {
@@ -1024,7 +1025,7 @@ describe('Article API', () => {
 
     context('when no:label is in the query', () => {
       before(async () => {
-        keyword = "'search api' no:label"
+        keyword = `'${searchedKeyword}' no:label`
       })
 
       it('returns non-labeled items in descending order', async () => {
@@ -1036,7 +1037,7 @@ describe('Article API', () => {
 
     context('when no:highlight is in the query', () => {
       before(async () => {
-        keyword = "'search api' no:highlight"
+        keyword = `'${searchedKeyword}' no:highlight`
       })
 
       it('returns non-highlighted items in descending order', async () => {
@@ -1048,7 +1049,7 @@ describe('Article API', () => {
 
     context('when site:${site_name} is in the query', () => {
       before(async () => {
-        keyword = "'search api' site:example"
+        keyword = `'${searchedKeyword}' site:example`
       })
 
       it('returns items from the site', async () => {