mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
Use elasticsearch body builder
This commit is contained in:
Hongbo Wu
parent
45eed5c40d
commit
adae20e7e1
4 changed files with 158 additions and 286 deletions
|
|
@ -54,6 +54,7 @@
|
|||
"dompurify": "^2.0.17",
|
||||
"dot-case": "^3.0.4",
|
||||
"dotenv": "^8.2.0",
|
||||
"elastic-ts": "^0.9.0",
|
||||
"express": "^4.17.1",
|
||||
"express-http-context": "^1.2.4",
|
||||
"express-rate-limit": "^6.3.0",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { ResponseError } from '@elastic/elasticsearch/lib/errors'
|
||||
import { BuiltQuery, ESBuilder, esBuilder } from 'elastic-ts'
|
||||
import { EntityType } from '../datalayer/pubsub'
|
||||
import { BulkActionType } from '../generated/graphql'
|
||||
import {
|
||||
|
|
@ -21,173 +22,160 @@ import {
|
|||
PageSearchArgs,
|
||||
PageType,
|
||||
ParamSet,
|
||||
SearchBody,
|
||||
SearchResponse,
|
||||
} from './types'
|
||||
|
||||
const appendQuery = (body: SearchBody, query: string): void => {
|
||||
body.query.bool.should.push({
|
||||
multi_match: {
|
||||
const appendQuery = (builder: ESBuilder, query: string): ESBuilder => {
|
||||
return builder
|
||||
.orQuery('multi_match', {
|
||||
query,
|
||||
fields: ['title', 'content', 'author', 'description', 'siteName'],
|
||||
operator: 'and',
|
||||
type: 'cross_fields',
|
||||
},
|
||||
})
|
||||
body.query.bool.minimum_should_match = 1
|
||||
})
|
||||
.queryMinimumShouldMatch(1)
|
||||
}
|
||||
|
||||
const appendTypeFilter = (body: SearchBody, filter: PageType): void => {
|
||||
body.query.bool.must.push({
|
||||
term: {
|
||||
pageType: filter,
|
||||
},
|
||||
})
|
||||
const appendTypeFilter = (builder: ESBuilder, filter: PageType): ESBuilder => {
|
||||
return builder.query('term', { pageType: filter })
|
||||
}
|
||||
|
||||
const appendReadFilter = (body: SearchBody, filter: ReadFilter): void => {
|
||||
const appendReadFilter = (
|
||||
builder: ESBuilder,
|
||||
filter: ReadFilter
|
||||
): ESBuilder => {
|
||||
switch (filter) {
|
||||
case ReadFilter.UNREAD:
|
||||
body.query.bool.must.push({
|
||||
range: {
|
||||
readingProgressPercent: {
|
||||
lt: 98,
|
||||
},
|
||||
return builder.query('range', {
|
||||
readingProgressPercent: {
|
||||
lt: 98,
|
||||
},
|
||||
})
|
||||
break
|
||||
case ReadFilter.READ:
|
||||
body.query.bool.must.push({
|
||||
range: {
|
||||
readingProgressPercent: {
|
||||
gte: 98,
|
||||
},
|
||||
return builder.query('range', {
|
||||
readingProgressPercent: {
|
||||
gte: 98,
|
||||
},
|
||||
})
|
||||
}
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendInFilter = (body: SearchBody, filter: InFilter): void => {
|
||||
const appendInFilter = (builder: ESBuilder, filter: InFilter): ESBuilder => {
|
||||
switch (filter) {
|
||||
case InFilter.ARCHIVE:
|
||||
body.query.bool.must.push({
|
||||
exists: {
|
||||
field: 'archivedAt',
|
||||
},
|
||||
})
|
||||
break
|
||||
return builder.query('exists', { field: 'archivedAt' })
|
||||
case InFilter.INBOX:
|
||||
body.query.bool.must_not.push({
|
||||
exists: {
|
||||
field: 'archivedAt',
|
||||
},
|
||||
})
|
||||
return builder.notQuery('exists', { field: 'archivedAt' })
|
||||
}
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendHasFilters = (body: SearchBody, filters: HasFilter[]): void => {
|
||||
const appendHasFilters = (
|
||||
builder: ESBuilder,
|
||||
filters: HasFilter[]
|
||||
): ESBuilder => {
|
||||
filters.forEach((filter) => {
|
||||
switch (filter) {
|
||||
case HasFilter.HIGHLIGHTS:
|
||||
body.query.bool.must.push({
|
||||
nested: {
|
||||
path: 'highlights',
|
||||
query: {
|
||||
exists: {
|
||||
field: 'highlights',
|
||||
},
|
||||
builder = builder.query('nested', {
|
||||
path: 'highlights',
|
||||
query: {
|
||||
exists: {
|
||||
field: 'highlights',
|
||||
},
|
||||
},
|
||||
})
|
||||
break
|
||||
case HasFilter.SHARED_AT:
|
||||
body.query.bool.must.push({
|
||||
exists: {
|
||||
field: 'sharedAt',
|
||||
},
|
||||
})
|
||||
builder = builder.query('exists', { field: 'sharedAt' })
|
||||
break
|
||||
}
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendExcludeLabelFilter = (
|
||||
body: SearchBody,
|
||||
builder: ESBuilder,
|
||||
filters: LabelFilter[]
|
||||
): void => {
|
||||
): ESBuilder => {
|
||||
const labels = filters.map((filter) => filter.labels).flat()
|
||||
body.query.bool.must_not.push({
|
||||
nested: {
|
||||
path: 'labels',
|
||||
query: {
|
||||
terms: {
|
||||
'labels.name': labels,
|
||||
},
|
||||
return builder.notQuery('nested', {
|
||||
path: 'labels',
|
||||
query: {
|
||||
terms: {
|
||||
'labels.name': labels,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const appendIncludeLabelFilter = (
|
||||
body: SearchBody,
|
||||
builder: ESBuilder,
|
||||
filters: LabelFilter[]
|
||||
): void => {
|
||||
): ESBuilder => {
|
||||
filters.forEach((filter) => {
|
||||
body.query.bool.must.push({
|
||||
nested: {
|
||||
path: 'labels',
|
||||
query: {
|
||||
terms: {
|
||||
'labels.name': filter.labels,
|
||||
},
|
||||
builder = builder.query('nested', {
|
||||
path: 'labels',
|
||||
query: {
|
||||
terms: {
|
||||
'labels.name': filter.labels,
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendDateFilters = (body: SearchBody, filters: DateFilter[]): void => {
|
||||
const appendDateFilters = (
|
||||
builder: ESBuilder,
|
||||
filters: DateFilter[]
|
||||
): ESBuilder => {
|
||||
filters.forEach((filter) => {
|
||||
body.query.bool.must.push({
|
||||
range: {
|
||||
[filter.field]: {
|
||||
gt: filter.startDate,
|
||||
lt: filter.endDate,
|
||||
},
|
||||
builder = builder.query('range', {
|
||||
[filter.field]: {
|
||||
gt: filter.startDate?.toISOString(),
|
||||
lt: filter.endDate?.toISOString(),
|
||||
},
|
||||
})
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendTermFilters = (body: SearchBody, filters: FieldFilter[]): void => {
|
||||
const appendTermFilters = (
|
||||
builder: ESBuilder,
|
||||
filters: FieldFilter[]
|
||||
): ESBuilder => {
|
||||
filters.forEach((filter) => {
|
||||
body.query.bool.must.push({
|
||||
term: {
|
||||
[filter.field]: filter.value,
|
||||
},
|
||||
builder = builder.query('term', {
|
||||
[filter.field]: filter.value,
|
||||
})
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendMatchFilters = (body: SearchBody, filters: FieldFilter[]): void => {
|
||||
const appendMatchFilters = (
|
||||
builder: ESBuilder,
|
||||
filters: FieldFilter[]
|
||||
): ESBuilder => {
|
||||
filters.forEach((filter) => {
|
||||
body.query.bool.must.push({
|
||||
match: {
|
||||
[filter.field]: filter.value,
|
||||
},
|
||||
builder = builder.query('match', {
|
||||
[filter.field]: filter.value,
|
||||
})
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendIdsFilter = (body: SearchBody, ids: string[]): void => {
|
||||
body.query.bool.must.push({
|
||||
terms: {
|
||||
_id: ids,
|
||||
},
|
||||
const appendIdsFilter = (builder: ESBuilder, ids: string[]): ESBuilder => {
|
||||
return builder.query('terms', {
|
||||
_id: ids,
|
||||
})
|
||||
}
|
||||
|
||||
const appendRecommendedBy = (body: SearchBody, recommendedBy: string): void => {
|
||||
const appendRecommendedBy = (
|
||||
builder: ESBuilder,
|
||||
recommendedBy: string
|
||||
): ESBuilder => {
|
||||
const query =
|
||||
recommendedBy === '*'
|
||||
? {
|
||||
|
|
@ -200,35 +188,48 @@ const appendRecommendedBy = (body: SearchBody, recommendedBy: string): void => {
|
|||
'recommendations.name': recommendedBy,
|
||||
},
|
||||
}
|
||||
|
||||
body.query.bool.must.push({
|
||||
nested: {
|
||||
path: 'recommendations',
|
||||
query,
|
||||
},
|
||||
return builder.query('nested', {
|
||||
path: 'recommendations',
|
||||
query,
|
||||
})
|
||||
}
|
||||
|
||||
const appendNoFilters = (body: SearchBody, noFilters: NoFilter[]): void => {
|
||||
const appendNoFilters = (
|
||||
builder: ESBuilder,
|
||||
noFilters: NoFilter[]
|
||||
): ESBuilder => {
|
||||
noFilters.forEach((filter) => {
|
||||
body.query.bool.must_not.push({
|
||||
nested: {
|
||||
path: filter.field,
|
||||
query: {
|
||||
exists: {
|
||||
field: filter.field,
|
||||
},
|
||||
builder = builder.notQuery('nested', {
|
||||
path: filter.field,
|
||||
query: {
|
||||
exists: {
|
||||
field: filter.field,
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendSiteNameFilter = (body: SearchBody, siteName: string): void => {
|
||||
body.query.bool.must.push({
|
||||
match: {
|
||||
siteName,
|
||||
},
|
||||
const appendSiteNameFilter = (
|
||||
builder: ESBuilder,
|
||||
siteName: string
|
||||
): ESBuilder => {
|
||||
return builder.query('bool', {
|
||||
should: [
|
||||
{
|
||||
match: {
|
||||
siteName,
|
||||
},
|
||||
},
|
||||
{
|
||||
wildcard: {
|
||||
// siteName is a domain name, so we need to wildcard the end
|
||||
url: `*${siteName}*`,
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -424,100 +425,76 @@ export const searchPages = async (
|
|||
const excludeLabels = labelFilters?.filter(
|
||||
(filter) => filter.type === LabelFilterType.EXCLUDE
|
||||
)
|
||||
|
||||
const body: SearchBody = {
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{
|
||||
term: {
|
||||
userId,
|
||||
},
|
||||
},
|
||||
],
|
||||
should: [],
|
||||
must_not: [],
|
||||
},
|
||||
},
|
||||
sort: [
|
||||
{
|
||||
[sortField]: {
|
||||
order: sortOrder,
|
||||
},
|
||||
},
|
||||
],
|
||||
from,
|
||||
size,
|
||||
_source: {
|
||||
// start building the query
|
||||
let builder = esBuilder()
|
||||
.query('term', { userId })
|
||||
.sort(sortField, sortOrder)
|
||||
.from(from)
|
||||
.size(size)
|
||||
.rawOption('_source', {
|
||||
excludes: includeContent ? [] : ['originalHtml', 'content'],
|
||||
},
|
||||
}
|
||||
|
||||
})
|
||||
// append filters
|
||||
if (query) {
|
||||
appendQuery(body, query)
|
||||
builder = appendQuery(builder, query)
|
||||
}
|
||||
if (typeFilter) {
|
||||
appendTypeFilter(body, typeFilter)
|
||||
builder = appendTypeFilter(builder, typeFilter)
|
||||
}
|
||||
if (inFilter !== InFilter.ALL) {
|
||||
appendInFilter(body, inFilter)
|
||||
builder = appendInFilter(builder, inFilter)
|
||||
}
|
||||
if (readFilter !== ReadFilter.ALL) {
|
||||
appendReadFilter(body, readFilter)
|
||||
builder = appendReadFilter(builder, readFilter)
|
||||
}
|
||||
if (hasFilters && hasFilters.length > 0) {
|
||||
appendHasFilters(body, hasFilters)
|
||||
builder = appendHasFilters(builder, hasFilters)
|
||||
}
|
||||
if (includeLabels && includeLabels.length > 0) {
|
||||
appendIncludeLabelFilter(body, includeLabels)
|
||||
builder = appendIncludeLabelFilter(builder, includeLabels)
|
||||
}
|
||||
if (excludeLabels && excludeLabels.length > 0) {
|
||||
appendExcludeLabelFilter(body, excludeLabels)
|
||||
builder = appendExcludeLabelFilter(builder, excludeLabels)
|
||||
}
|
||||
if (dateFilters && dateFilters.length > 0) {
|
||||
appendDateFilters(body, dateFilters)
|
||||
builder = appendDateFilters(builder, dateFilters)
|
||||
}
|
||||
if (termFilters) {
|
||||
appendTermFilters(body, termFilters)
|
||||
builder = appendTermFilters(builder, termFilters)
|
||||
}
|
||||
if (matchFilters) {
|
||||
appendMatchFilters(body, matchFilters)
|
||||
builder = appendMatchFilters(builder, matchFilters)
|
||||
}
|
||||
if (ids && ids.length > 0) {
|
||||
appendIdsFilter(body, ids)
|
||||
builder = appendIdsFilter(builder, ids)
|
||||
}
|
||||
|
||||
if (args.recommendedBy) {
|
||||
appendRecommendedBy(body, args.recommendedBy)
|
||||
builder = appendRecommendedBy(builder, args.recommendedBy)
|
||||
}
|
||||
|
||||
if (!args.includePending) {
|
||||
body.query.bool.must_not.push({
|
||||
term: {
|
||||
state: ArticleSavingRequestStatus.Processing,
|
||||
},
|
||||
builder = builder.notQuery('term', {
|
||||
state: ArticleSavingRequestStatus.Processing,
|
||||
})
|
||||
}
|
||||
|
||||
if (!args.includeDeleted) {
|
||||
body.query.bool.must_not.push({
|
||||
term: {
|
||||
state: ArticleSavingRequestStatus.Deleted,
|
||||
},
|
||||
builder = builder.notQuery('term', {
|
||||
state: ArticleSavingRequestStatus.Deleted,
|
||||
})
|
||||
}
|
||||
|
||||
noFilters && appendNoFilters(body, noFilters)
|
||||
siteName && appendSiteNameFilter(body, siteName)
|
||||
if (noFilters) {
|
||||
builder = appendNoFilters(builder, noFilters)
|
||||
}
|
||||
if (siteName) {
|
||||
builder = appendSiteNameFilter(builder, siteName)
|
||||
}
|
||||
// build the query
|
||||
const body = builder.build()
|
||||
|
||||
console.debug('searching pages in elastic', JSON.stringify(body))
|
||||
|
||||
const response = await client.search<SearchResponse<Page>, SearchBody>({
|
||||
const response = await client.search<SearchResponse<Page>, BuiltQuery>({
|
||||
index: INDEX_ALIAS,
|
||||
body,
|
||||
})
|
||||
|
||||
if (response.body.hits.total.value === 0) {
|
||||
return [[], 0]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,124 +12,6 @@ import {
|
|||
SortParams,
|
||||
} from '../utils/search'
|
||||
|
||||
export interface SearchBody {
|
||||
query: {
|
||||
bool: {
|
||||
must: (
|
||||
| {
|
||||
term: {
|
||||
[K: string]: string
|
||||
}
|
||||
}
|
||||
| { exists: { field: string } }
|
||||
| {
|
||||
range: {
|
||||
readingProgressPercent: { gte: number } | { lt: number }
|
||||
}
|
||||
}
|
||||
| {
|
||||
range: {
|
||||
[K: string]: { gt: Date | undefined } | { lt: Date | undefined }
|
||||
}
|
||||
}
|
||||
| {
|
||||
nested: {
|
||||
path: 'labels'
|
||||
query: {
|
||||
terms: {
|
||||
'labels.name': string[]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
| {
|
||||
nested: {
|
||||
path: 'highlights'
|
||||
query: {
|
||||
exists: {
|
||||
field: 'highlights'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
| {
|
||||
nested: {
|
||||
path: 'recommendations'
|
||||
query: {
|
||||
exists?: {
|
||||
field: string
|
||||
}
|
||||
term?: {
|
||||
'recommendations.name': string
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
| {
|
||||
match: {
|
||||
[K: string]: string
|
||||
}
|
||||
}
|
||||
| {
|
||||
terms: {
|
||||
[K: string]: string[]
|
||||
}
|
||||
}
|
||||
)[]
|
||||
should: {
|
||||
multi_match: {
|
||||
query: string
|
||||
fields: string[]
|
||||
operator?: 'and' | 'or'
|
||||
type?:
|
||||
| 'best_fields'
|
||||
| 'most_fields'
|
||||
| 'cross_fields'
|
||||
| 'phrase'
|
||||
| 'phrase_prefix'
|
||||
| 'bool_prefix'
|
||||
analyzer?: string
|
||||
}
|
||||
}[]
|
||||
minimum_should_match?: number
|
||||
must_not: (
|
||||
| { term: { state: ArticleSavingRequestStatus } }
|
||||
| {
|
||||
exists: {
|
||||
field: string
|
||||
}
|
||||
}
|
||||
| {
|
||||
nested: {
|
||||
path: 'labels'
|
||||
query: {
|
||||
terms: {
|
||||
'labels.name': string[]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
| {
|
||||
nested: {
|
||||
path: string
|
||||
query: {
|
||||
exists: {
|
||||
field: string
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)[]
|
||||
}
|
||||
}
|
||||
sort: [Record<string, { order: string }>]
|
||||
from: number
|
||||
size: number
|
||||
_source: {
|
||||
excludes: string[]
|
||||
}
|
||||
}
|
||||
|
||||
// Complete definition of the Search response
|
||||
export interface ShardsResponse {
|
||||
total: number
|
||||
|
|
@ -164,7 +46,7 @@ export interface SearchResponse<T> {
|
|||
_explanation?: Explanation
|
||||
fields?: never
|
||||
highlight?: never
|
||||
inner_hits?: any
|
||||
inner_hits?: unknown
|
||||
matched_queries?: string[]
|
||||
sort?: string[]
|
||||
}>
|
||||
|
|
|
|||
12
yarn.lock
12
yarn.lock
|
|
@ -5996,6 +5996,11 @@
|
|||
resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea"
|
||||
integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==
|
||||
|
||||
"@sindresorhus/is@^4.0.0":
|
||||
version "4.6.0"
|
||||
resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f"
|
||||
integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==
|
||||
|
||||
"@sinonjs/commons@^1", "@sinonjs/commons@^1.3.0", "@sinonjs/commons@^1.4.0", "@sinonjs/commons@^1.6.0", "@sinonjs/commons@^1.7.0", "@sinonjs/commons@^1.8.3":
|
||||
version "1.8.3"
|
||||
resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.3.tgz#3802ddd21a50a949b6721ddd72da36e67e7f1b2d"
|
||||
|
|
@ -13507,6 +13512,13 @@ ee-first@1.1.1:
|
|||
resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
|
||||
integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=
|
||||
|
||||
elastic-ts@^0.9.0:
|
||||
version "0.9.0"
|
||||
resolved "https://registry.yarnpkg.com/elastic-ts/-/elastic-ts-0.9.0.tgz#919b7646cd31d753235f3f9a336f7f26c5e3654d"
|
||||
integrity sha512-w9Xj/67ygllZG1RS1uMnmWLMfvTEE8zUMyn9zYiEOIgt8RFLKu5X5qCT1N2eYo6IQNTCX5DUdlIw87ii36Udtg==
|
||||
dependencies:
|
||||
"@sindresorhus/is" "^4.0.0"
|
||||
|
||||
electron-to-chromium@^1.3.811:
|
||||
version "1.3.816"
|
||||
resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.816.tgz#ab6488b126de92670a6459fe3e746050e0c6276f"
|
||||
|
|
|
|||
Loading…
Reference in a new issue