diff --git a/packages/api/package.json b/packages/api/package.json index 983d263ee..8697ac80f 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -54,6 +54,7 @@ "dompurify": "^2.0.17", "dot-case": "^3.0.4", "dotenv": "^8.2.0", + "elastic-ts": "^0.9.0", "express": "^4.17.1", "express-http-context": "^1.2.4", "express-rate-limit": "^6.3.0", diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 0c4ebb047..2ff88cfe3 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -1,4 +1,5 @@ import { ResponseError } from '@elastic/elasticsearch/lib/errors' +import { BuiltQuery, ESBuilder, esBuilder } from 'elastic-ts' import { EntityType } from '../datalayer/pubsub' import { BulkActionType } from '../generated/graphql' import { @@ -21,173 +22,160 @@ import { PageSearchArgs, PageType, ParamSet, - SearchBody, SearchResponse, } from './types' -const appendQuery = (body: SearchBody, query: string): void => { - body.query.bool.should.push({ - multi_match: { +const appendQuery = (builder: ESBuilder, query: string): ESBuilder => { + return builder + .orQuery('multi_match', { query, fields: ['title', 'content', 'author', 'description', 'siteName'], operator: 'and', type: 'cross_fields', - }, - }) - body.query.bool.minimum_should_match = 1 + }) + .queryMinimumShouldMatch(1) } -const appendTypeFilter = (body: SearchBody, filter: PageType): void => { - body.query.bool.must.push({ - term: { - pageType: filter, - }, - }) +const appendTypeFilter = (builder: ESBuilder, filter: PageType): ESBuilder => { + return builder.query('term', { pageType: filter }) } -const appendReadFilter = (body: SearchBody, filter: ReadFilter): void => { +const appendReadFilter = ( + builder: ESBuilder, + filter: ReadFilter +): ESBuilder => { switch (filter) { case ReadFilter.UNREAD: - body.query.bool.must.push({ - range: { - readingProgressPercent: { - lt: 98, - }, + return builder.query('range', { + readingProgressPercent: { + lt: 98, }, }) - break case ReadFilter.READ: - body.query.bool.must.push({ - range: { - readingProgressPercent: { - gte: 98, - }, + return builder.query('range', { + readingProgressPercent: { + gte: 98, }, }) } + return builder } -const appendInFilter = (body: SearchBody, filter: InFilter): void => { +const appendInFilter = (builder: ESBuilder, filter: InFilter): ESBuilder => { switch (filter) { case InFilter.ARCHIVE: - body.query.bool.must.push({ - exists: { - field: 'archivedAt', - }, - }) - break + return builder.query('exists', { field: 'archivedAt' }) case InFilter.INBOX: - body.query.bool.must_not.push({ - exists: { - field: 'archivedAt', - }, - }) + return builder.notQuery('exists', { field: 'archivedAt' }) } + return builder } -const appendHasFilters = (body: SearchBody, filters: HasFilter[]): void => { +const appendHasFilters = ( + builder: ESBuilder, + filters: HasFilter[] +): ESBuilder => { filters.forEach((filter) => { switch (filter) { case HasFilter.HIGHLIGHTS: - body.query.bool.must.push({ - nested: { - path: 'highlights', - query: { - exists: { - field: 'highlights', - }, + builder = builder.query('nested', { + path: 'highlights', + query: { + exists: { + field: 'highlights', }, }, }) break case HasFilter.SHARED_AT: - body.query.bool.must.push({ - exists: { - field: 'sharedAt', - }, - }) + builder = builder.query('exists', { field: 'sharedAt' }) break } }) + return builder } const appendExcludeLabelFilter = ( - body: SearchBody, + builder: ESBuilder, filters: LabelFilter[] -): void => { +): ESBuilder => { const labels = filters.map((filter) => filter.labels).flat() - body.query.bool.must_not.push({ - nested: { - path: 'labels', - query: { - terms: { - 'labels.name': labels, - }, + return builder.notQuery('nested', { + path: 'labels', + query: { + terms: { + 'labels.name': labels, }, }, }) } const appendIncludeLabelFilter = ( - body: SearchBody, + builder: ESBuilder, filters: LabelFilter[] -): void => { +): ESBuilder => { filters.forEach((filter) => { - body.query.bool.must.push({ - nested: { - path: 'labels', - query: { - terms: { - 'labels.name': filter.labels, - }, + builder = builder.query('nested', { + path: 'labels', + query: { + terms: { + 'labels.name': filter.labels, }, }, }) }) + return builder } -const appendDateFilters = (body: SearchBody, filters: DateFilter[]): void => { +const appendDateFilters = ( + builder: ESBuilder, + filters: DateFilter[] +): ESBuilder => { filters.forEach((filter) => { - body.query.bool.must.push({ - range: { - [filter.field]: { - gt: filter.startDate, - lt: filter.endDate, - }, + builder = builder.query('range', { + [filter.field]: { + gt: filter.startDate?.toISOString(), + lt: filter.endDate?.toISOString(), }, }) }) + return builder } -const appendTermFilters = (body: SearchBody, filters: FieldFilter[]): void => { +const appendTermFilters = ( + builder: ESBuilder, + filters: FieldFilter[] +): ESBuilder => { filters.forEach((filter) => { - body.query.bool.must.push({ - term: { - [filter.field]: filter.value, - }, + builder = builder.query('term', { + [filter.field]: filter.value, }) }) + return builder } -const appendMatchFilters = (body: SearchBody, filters: FieldFilter[]): void => { +const appendMatchFilters = ( + builder: ESBuilder, + filters: FieldFilter[] +): ESBuilder => { filters.forEach((filter) => { - body.query.bool.must.push({ - match: { - [filter.field]: filter.value, - }, + builder = builder.query('match', { + [filter.field]: filter.value, }) }) + return builder } -const appendIdsFilter = (body: SearchBody, ids: string[]): void => { - body.query.bool.must.push({ - terms: { - _id: ids, - }, +const appendIdsFilter = (builder: ESBuilder, ids: string[]): ESBuilder => { + return builder.query('terms', { + _id: ids, }) } -const appendRecommendedBy = (body: SearchBody, recommendedBy: string): void => { +const appendRecommendedBy = ( + builder: ESBuilder, + recommendedBy: string +): ESBuilder => { const query = recommendedBy === '*' ? { @@ -200,35 +188,48 @@ const appendRecommendedBy = (body: SearchBody, recommendedBy: string): void => { 'recommendations.name': recommendedBy, }, } - - body.query.bool.must.push({ - nested: { - path: 'recommendations', - query, - }, + return builder.query('nested', { + path: 'recommendations', + query, }) } -const appendNoFilters = (body: SearchBody, noFilters: NoFilter[]): void => { +const appendNoFilters = ( + builder: ESBuilder, + noFilters: NoFilter[] +): ESBuilder => { noFilters.forEach((filter) => { - body.query.bool.must_not.push({ - nested: { - path: filter.field, - query: { - exists: { - field: filter.field, - }, + builder = builder.notQuery('nested', { + path: filter.field, + query: { + exists: { + field: filter.field, }, }, }) }) + return builder } -const appendSiteNameFilter = (body: SearchBody, siteName: string): void => { - body.query.bool.must.push({ - match: { - siteName, - }, +const appendSiteNameFilter = ( + builder: ESBuilder, + siteName: string +): ESBuilder => { + return builder.query('bool', { + should: [ + { + match: { + siteName, + }, + }, + { + wildcard: { + // siteName is a domain name, so we need to wildcard the end + url: `*${siteName}*`, + }, + }, + ], + minimum_should_match: 1, }) } @@ -424,100 +425,76 @@ export const searchPages = async ( const excludeLabels = labelFilters?.filter( (filter) => filter.type === LabelFilterType.EXCLUDE ) - - const body: SearchBody = { - query: { - bool: { - must: [ - { - term: { - userId, - }, - }, - ], - should: [], - must_not: [], - }, - }, - sort: [ - { - [sortField]: { - order: sortOrder, - }, - }, - ], - from, - size, - _source: { + // start building the query + let builder = esBuilder() + .query('term', { userId }) + .sort(sortField, sortOrder) + .from(from) + .size(size) + .rawOption('_source', { excludes: includeContent ? [] : ['originalHtml', 'content'], - }, - } - + }) // append filters if (query) { - appendQuery(body, query) + builder = appendQuery(builder, query) } if (typeFilter) { - appendTypeFilter(body, typeFilter) + builder = appendTypeFilter(builder, typeFilter) } if (inFilter !== InFilter.ALL) { - appendInFilter(body, inFilter) + builder = appendInFilter(builder, inFilter) } if (readFilter !== ReadFilter.ALL) { - appendReadFilter(body, readFilter) + builder = appendReadFilter(builder, readFilter) } if (hasFilters && hasFilters.length > 0) { - appendHasFilters(body, hasFilters) + builder = appendHasFilters(builder, hasFilters) } if (includeLabels && includeLabels.length > 0) { - appendIncludeLabelFilter(body, includeLabels) + builder = appendIncludeLabelFilter(builder, includeLabels) } if (excludeLabels && excludeLabels.length > 0) { - appendExcludeLabelFilter(body, excludeLabels) + builder = appendExcludeLabelFilter(builder, excludeLabels) } if (dateFilters && dateFilters.length > 0) { - appendDateFilters(body, dateFilters) + builder = appendDateFilters(builder, dateFilters) } if (termFilters) { - appendTermFilters(body, termFilters) + builder = appendTermFilters(builder, termFilters) } if (matchFilters) { - appendMatchFilters(body, matchFilters) + builder = appendMatchFilters(builder, matchFilters) } if (ids && ids.length > 0) { - appendIdsFilter(body, ids) + builder = appendIdsFilter(builder, ids) } - if (args.recommendedBy) { - appendRecommendedBy(body, args.recommendedBy) + builder = appendRecommendedBy(builder, args.recommendedBy) } - if (!args.includePending) { - body.query.bool.must_not.push({ - term: { - state: ArticleSavingRequestStatus.Processing, - }, + builder = builder.notQuery('term', { + state: ArticleSavingRequestStatus.Processing, }) } - if (!args.includeDeleted) { - body.query.bool.must_not.push({ - term: { - state: ArticleSavingRequestStatus.Deleted, - }, + builder = builder.notQuery('term', { + state: ArticleSavingRequestStatus.Deleted, }) } - - noFilters && appendNoFilters(body, noFilters) - siteName && appendSiteNameFilter(body, siteName) + if (noFilters) { + builder = appendNoFilters(builder, noFilters) + } + if (siteName) { + builder = appendSiteNameFilter(builder, siteName) + } + // build the query + const body = builder.build() console.debug('searching pages in elastic', JSON.stringify(body)) - - const response = await client.search, SearchBody>({ + const response = await client.search, BuiltQuery>({ index: INDEX_ALIAS, body, }) - if (response.body.hits.total.value === 0) { return [[], 0] } diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 70ee52beb..52527ade6 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -12,124 +12,6 @@ import { SortParams, } from '../utils/search' -export interface SearchBody { - query: { - bool: { - must: ( - | { - term: { - [K: string]: string - } - } - | { exists: { field: string } } - | { - range: { - readingProgressPercent: { gte: number } | { lt: number } - } - } - | { - range: { - [K: string]: { gt: Date | undefined } | { lt: Date | undefined } - } - } - | { - nested: { - path: 'labels' - query: { - terms: { - 'labels.name': string[] - } - } - } - } - | { - nested: { - path: 'highlights' - query: { - exists: { - field: 'highlights' - } - } - } - } - | { - nested: { - path: 'recommendations' - query: { - exists?: { - field: string - } - term?: { - 'recommendations.name': string - } - } - } - } - | { - match: { - [K: string]: string - } - } - | { - terms: { - [K: string]: string[] - } - } - )[] - should: { - multi_match: { - query: string - fields: string[] - operator?: 'and' | 'or' - type?: - | 'best_fields' - | 'most_fields' - | 'cross_fields' - | 'phrase' - | 'phrase_prefix' - | 'bool_prefix' - analyzer?: string - } - }[] - minimum_should_match?: number - must_not: ( - | { term: { state: ArticleSavingRequestStatus } } - | { - exists: { - field: string - } - } - | { - nested: { - path: 'labels' - query: { - terms: { - 'labels.name': string[] - } - } - } - } - | { - nested: { - path: string - query: { - exists: { - field: string - } - } - } - } - )[] - } - } - sort: [Record] - from: number - size: number - _source: { - excludes: string[] - } -} - // Complete definition of the Search response export interface ShardsResponse { total: number @@ -164,7 +46,7 @@ export interface SearchResponse { _explanation?: Explanation fields?: never highlight?: never - inner_hits?: any + inner_hits?: unknown matched_queries?: string[] sort?: string[] }> diff --git a/yarn.lock b/yarn.lock index aacec739b..a961e1803 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5996,6 +5996,11 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ== +"@sindresorhus/is@^4.0.0": + version "4.6.0" + resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" + integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== + "@sinonjs/commons@^1", "@sinonjs/commons@^1.3.0", "@sinonjs/commons@^1.4.0", "@sinonjs/commons@^1.6.0", "@sinonjs/commons@^1.7.0", "@sinonjs/commons@^1.8.3": version "1.8.3" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.3.tgz#3802ddd21a50a949b6721ddd72da36e67e7f1b2d" @@ -13507,6 +13512,13 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= +elastic-ts@^0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/elastic-ts/-/elastic-ts-0.9.0.tgz#919b7646cd31d753235f3f9a336f7f26c5e3654d" + integrity sha512-w9Xj/67ygllZG1RS1uMnmWLMfvTEE8zUMyn9zYiEOIgt8RFLKu5X5qCT1N2eYo6IQNTCX5DUdlIw87ii36Udtg== + dependencies: + "@sindresorhus/is" "^4.0.0" + electron-to-chromium@^1.3.811: version "1.3.816" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.816.tgz#ab6488b126de92670a6459fe3e746050e0c6276f"