mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
Merge pull request #3189 from omnivore-app/feature/query-parser
feat: allow logical operator in search query
This commit is contained in:
Hongbo Wu
GitHub
commit
9b210d6ac0
8 changed files with 628 additions and 1020 deletions
|
|
@ -71,6 +71,7 @@
|
|||
"jsonwebtoken": "^8.5.1",
|
||||
"jwks-rsa": "^2.0.3",
|
||||
"linkedom": "^0.14.9",
|
||||
"liqe": "^3.8.0",
|
||||
"lodash": "^4.17.21",
|
||||
"luxon": "^3.2.1",
|
||||
"nanoid": "^3.1.25",
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ import {
|
|||
findLibraryItemByUrl,
|
||||
findLibraryItemsByPrefix,
|
||||
searchLibraryItems,
|
||||
sortParamsToSort,
|
||||
updateLibraryItem,
|
||||
updateLibraryItemReadingProgress,
|
||||
updateLibraryItems,
|
||||
|
|
@ -89,7 +90,6 @@ import {
|
|||
generateSlug,
|
||||
isParsingTimeout,
|
||||
libraryItemToArticle,
|
||||
libraryItemToArticleSavingRequest,
|
||||
libraryItemToSearchItem,
|
||||
titleForFilePath,
|
||||
userDataToUser,
|
||||
|
|
@ -101,11 +101,6 @@ import {
|
|||
ParsedContentPuppeteer,
|
||||
parsePreparedContent,
|
||||
} from '../../utils/parser'
|
||||
import {
|
||||
InFilter,
|
||||
parseSearchQuery,
|
||||
sortParamsToSort,
|
||||
} from '../../utils/search'
|
||||
import { getStorageFileDetails } from '../../utils/uploads'
|
||||
import { itemTypeForContentType } from '../upload_files'
|
||||
|
||||
|
|
@ -655,16 +650,15 @@ export const searchResolver = authorized<
|
|||
return { errorCodes: [SearchErrorCode.QueryTooLong] }
|
||||
}
|
||||
|
||||
const searchQuery = parseSearchQuery(params.query || undefined)
|
||||
|
||||
const { libraryItems, count } = await searchLibraryItems(
|
||||
{
|
||||
from: Number(startCursor),
|
||||
size: first + 1, // fetch one more item to get next cursor
|
||||
sort: searchQuery.sort,
|
||||
includePending: true,
|
||||
includeContent: !!params.includeContent,
|
||||
...searchQuery,
|
||||
includeDeleted: params.query?.includes('in:trash'),
|
||||
query: params.query,
|
||||
useFolders: params.query?.includes('use:folders'),
|
||||
},
|
||||
uid
|
||||
)
|
||||
|
|
@ -761,14 +755,18 @@ export const updatesSinceResolver = authorized<
|
|||
startDate = new Date(0)
|
||||
}
|
||||
|
||||
// create a search query
|
||||
const query = `updated:${startDate.toISOString()}${
|
||||
folder ? ' in:' + folder : ''
|
||||
}`
|
||||
|
||||
const { libraryItems, count } = await searchLibraryItems(
|
||||
{
|
||||
from: Number(startCursor),
|
||||
size: size + 1, // fetch one more item to get next cursor
|
||||
includeDeleted: true,
|
||||
dateFilters: [{ field: 'updatedAt', startDate }],
|
||||
sort,
|
||||
inFilter: (folder as InFilter) || InFilter.ALL,
|
||||
query,
|
||||
},
|
||||
uid
|
||||
)
|
||||
|
|
@ -826,9 +824,8 @@ export const bulkActionResolver = authorized<
|
|||
},
|
||||
})
|
||||
|
||||
// parse query
|
||||
const searchQuery = parseSearchQuery(query)
|
||||
if (searchQuery.ids.length > 100) {
|
||||
// the query size is limited to 255 characters
|
||||
if (!query || query.length > 255) {
|
||||
return { errorCodes: [BulkActionErrorCode.BadRequest] }
|
||||
}
|
||||
|
||||
|
|
@ -842,7 +839,7 @@ export const bulkActionResolver = authorized<
|
|||
labels = await findLabelsByIds(labelIds, uid)
|
||||
}
|
||||
|
||||
await updateLibraryItems(action, searchQuery, uid, labels, args)
|
||||
await updateLibraryItems(action, query, uid, labels, args)
|
||||
|
||||
return { success: true }
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -1,52 +1,50 @@
|
|||
import { Brackets, DeepPartial, SelectQueryBuilder } from 'typeorm'
|
||||
import { LiqeQuery } from 'liqe'
|
||||
import { DateTime } from 'luxon'
|
||||
import { DeepPartial, ObjectLiteral } from 'typeorm'
|
||||
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'
|
||||
import { EntityLabel } from '../entity/entity_label'
|
||||
import { Highlight } from '../entity/highlight'
|
||||
import { Label } from '../entity/label'
|
||||
import { LibraryItem, LibraryItemState } from '../entity/library_item'
|
||||
import { BulkActionType } from '../generated/graphql'
|
||||
import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql'
|
||||
import { createPubSubClient, EntityType } from '../pubsub'
|
||||
import { authTrx, getColumns } from '../repository'
|
||||
import { libraryItemRepository } from '../repository/library_item'
|
||||
import { SaveFollowingItemRequest } from '../routers/svc/following'
|
||||
import { generateSlug, wordsCount } from '../utils/helpers'
|
||||
import { createThumbnailUrl } from '../utils/imageproxy'
|
||||
import {
|
||||
DateFilter,
|
||||
FieldFilter,
|
||||
HasFilter,
|
||||
InFilter,
|
||||
LabelFilter,
|
||||
LabelFilterType,
|
||||
NoFilter,
|
||||
RangeFilter,
|
||||
ReadFilter,
|
||||
Sort,
|
||||
SortBy,
|
||||
SortOrder,
|
||||
} from '../utils/search'
|
||||
import { parseSearchQuery } from '../utils/search'
|
||||
|
||||
enum ReadFilter {
|
||||
ALL = 'all',
|
||||
READ = 'read',
|
||||
READING = 'reading',
|
||||
UNREAD = 'unread',
|
||||
}
|
||||
|
||||
enum InFilter {
|
||||
ALL = 'all',
|
||||
INBOX = 'inbox',
|
||||
ARCHIVE = 'archive',
|
||||
TRASH = 'trash',
|
||||
FOLLOWING = 'following',
|
||||
}
|
||||
|
||||
enum HasFilter {
|
||||
HIGHLIGHTS = 'highlights',
|
||||
LABELS = 'labels',
|
||||
SUBSCRIPTIONS = 'subscriptions',
|
||||
}
|
||||
|
||||
export interface SearchArgs {
|
||||
from?: number
|
||||
size?: number
|
||||
sort?: Sort
|
||||
query?: string
|
||||
inFilter: InFilter
|
||||
readFilter?: ReadFilter
|
||||
typeFilter?: string
|
||||
labelFilters?: LabelFilter[]
|
||||
hasFilters?: HasFilter[]
|
||||
dateFilters?: DateFilter[]
|
||||
termFilters?: FieldFilter[]
|
||||
matchFilters?: FieldFilter[]
|
||||
includePending?: boolean | null
|
||||
includeDeleted?: boolean
|
||||
ids?: string[]
|
||||
recommendedBy?: string
|
||||
includeContent?: boolean
|
||||
noFilters?: NoFilter[]
|
||||
rangeFilters?: RangeFilter[]
|
||||
useFolders?: boolean
|
||||
query?: string | null
|
||||
}
|
||||
|
||||
export interface SearchResultItem {
|
||||
|
|
@ -82,252 +80,547 @@ export interface SearchResultItem {
|
|||
content?: string
|
||||
}
|
||||
|
||||
const buildWhereClause = (
|
||||
queryBuilder: SelectQueryBuilder<LibraryItem>,
|
||||
args: SearchArgs
|
||||
export enum SortBy {
|
||||
SAVED = 'saved_at',
|
||||
UPDATED = 'updated_at',
|
||||
PUBLISHED = 'published_at',
|
||||
READ = 'read_at',
|
||||
WORDS_COUNT = 'word_count',
|
||||
}
|
||||
|
||||
export enum SortOrder {
|
||||
ASCENDING = 'ASC',
|
||||
DESCENDING = 'DESC',
|
||||
}
|
||||
|
||||
export interface Sort {
|
||||
by: SortBy
|
||||
order?: SortOrder
|
||||
}
|
||||
|
||||
interface Select {
|
||||
column: string
|
||||
alias: string
|
||||
}
|
||||
|
||||
export const sortParamsToSort = (
|
||||
sortParams: InputMaybe<SortParams> | undefined
|
||||
) => {
|
||||
if (args.query) {
|
||||
queryBuilder
|
||||
.addSelect(
|
||||
"ts_rank_cd(library_item.search_tsv, websearch_to_tsquery('english', :query))",
|
||||
'rank'
|
||||
)
|
||||
.andWhere(
|
||||
"websearch_to_tsquery('english', :query) @@ library_item.search_tsv"
|
||||
)
|
||||
.setParameter('query', args.query)
|
||||
.orderBy('rank', 'DESC')
|
||||
}
|
||||
const sort = { by: SortBy.UPDATED, order: SortOrder.DESCENDING }
|
||||
|
||||
if (args.typeFilter) {
|
||||
queryBuilder.andWhere('lower(library_item.item_type) = :typeFilter', {
|
||||
typeFilter: args.typeFilter.toLowerCase(),
|
||||
})
|
||||
}
|
||||
|
||||
if (args.inFilter !== InFilter.ALL) {
|
||||
switch (args.inFilter) {
|
||||
case InFilter.INBOX: {
|
||||
// if useFolders is true, we only return items in the inbox folder
|
||||
args.useFolders &&
|
||||
queryBuilder.andWhere("library_item.folder = 'inbox'")
|
||||
// for old clients, we return items that are not archived
|
||||
queryBuilder.andWhere('library_item.archived_at IS NULL')
|
||||
if (sortParams) {
|
||||
sortParams.order === 'ASCENDING' && (sort.order = SortOrder.ASCENDING)
|
||||
switch (sortParams.by) {
|
||||
case 'UPDATED_TIME':
|
||||
sort.by = SortBy.UPDATED
|
||||
break
|
||||
}
|
||||
case InFilter.ARCHIVE:
|
||||
queryBuilder.andWhere('library_item.archived_at IS NOT NULL')
|
||||
case 'PUBLISHED_AT':
|
||||
sort.by = SortBy.PUBLISHED
|
||||
break
|
||||
case InFilter.TRASH:
|
||||
// return only deleted pages within 14 days
|
||||
queryBuilder.andWhere(
|
||||
"library_item.deleted_at >= now() - interval '14 days'"
|
||||
)
|
||||
break
|
||||
default:
|
||||
queryBuilder.andWhere('library_item.folder = :folder', {
|
||||
folder: args.inFilter,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (args.readFilter !== ReadFilter.ALL) {
|
||||
switch (args.readFilter) {
|
||||
case ReadFilter.READ:
|
||||
queryBuilder.andWhere(
|
||||
'library_item.reading_progress_bottom_percent > 98'
|
||||
)
|
||||
break
|
||||
case ReadFilter.READING:
|
||||
queryBuilder.andWhere(
|
||||
'library_item.reading_progress_bottom_percent BETWEEN 2 AND 98'
|
||||
)
|
||||
break
|
||||
case ReadFilter.UNREAD:
|
||||
queryBuilder.andWhere(
|
||||
'library_item.reading_progress_bottom_percent < 2'
|
||||
)
|
||||
case 'SAVED_AT':
|
||||
sort.by = SortBy.SAVED
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (args.hasFilters && args.hasFilters.length > 0) {
|
||||
args.hasFilters.forEach((filter) => {
|
||||
switch (filter) {
|
||||
case HasFilter.HIGHLIGHTS:
|
||||
queryBuilder.andWhere("library_item.highlight_annotations <> '{}'")
|
||||
break
|
||||
case HasFilter.LABELS:
|
||||
queryBuilder.andWhere("library_item.label_names <> '{}'")
|
||||
break
|
||||
case HasFilter.SUBSCRIPTIONS:
|
||||
queryBuilder.andWhere('library_item.subscription is NOT NULL')
|
||||
}
|
||||
})
|
||||
return sort
|
||||
}
|
||||
|
||||
const getColumnName = (field: string) => {
|
||||
switch (field) {
|
||||
case 'language':
|
||||
return 'item_language'
|
||||
case 'subscription':
|
||||
case 'rss':
|
||||
return 'subscription'
|
||||
case 'site':
|
||||
return 'site_name'
|
||||
case 'wordsCount':
|
||||
return 'word_count'
|
||||
case 'readPosition':
|
||||
return 'reading_progress_bottom_percent'
|
||||
case 'saved':
|
||||
case 'read':
|
||||
case 'updated':
|
||||
case 'published':
|
||||
return `${field}_at`
|
||||
default:
|
||||
return field
|
||||
}
|
||||
}
|
||||
|
||||
export const buildQuery = (
|
||||
searchQuery: LiqeQuery,
|
||||
parameters: ObjectLiteral[] = [],
|
||||
selects: Select[] = [],
|
||||
orders: { by: string; order?: SortOrder }[] = [],
|
||||
useFolders = false
|
||||
) => {
|
||||
const escapeQueryWithParameters = (
|
||||
query: string,
|
||||
parameter: ObjectLiteral
|
||||
) => {
|
||||
parameters.push(parameter)
|
||||
return query
|
||||
}
|
||||
|
||||
if (args.labelFilters && args.labelFilters.length > 0) {
|
||||
const includeLabels = args.labelFilters?.filter(
|
||||
(filter) => filter.type === LabelFilterType.INCLUDE
|
||||
)
|
||||
const excludeLabels = args.labelFilters?.filter(
|
||||
(filter) => filter.type === LabelFilterType.EXCLUDE
|
||||
)
|
||||
const serializeTagExpression = (ast: LiqeQuery): string | null => {
|
||||
if (ast.type !== 'Tag') {
|
||||
throw new Error('Expected a tag expression.')
|
||||
}
|
||||
|
||||
if (includeLabels && includeLabels.length > 0) {
|
||||
includeLabels.forEach((includeLabel, i) => {
|
||||
const param = `includeLabels_${i}`
|
||||
const hasWildcard = includeLabel.labels.some((label) =>
|
||||
label.includes('*')
|
||||
)
|
||||
if (hasWildcard) {
|
||||
queryBuilder.andWhere(
|
||||
new Brackets((qb) => {
|
||||
includeLabel.labels.forEach((label, j) => {
|
||||
const param = `includeLabels_${i}_${j}`
|
||||
qb.orWhere(
|
||||
`array_to_string(array_cat(library_item.label_names, library_item.highlight_labels)::text[], ',') ILIKE :${param}`,
|
||||
const { field, expression } = ast
|
||||
|
||||
if (field.type === 'ImplicitField') {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
const value = expression.value?.toString()
|
||||
|
||||
if (value === undefined || value === '') {
|
||||
return null
|
||||
}
|
||||
|
||||
const param = 'implicit_field'
|
||||
const alias = 'rank'
|
||||
selects.push({
|
||||
column: `ts_rank_cd(library_item.search_tsv, websearch_to_tsquery('english', :${param}))`,
|
||||
alias,
|
||||
})
|
||||
|
||||
orders.push({
|
||||
by: alias,
|
||||
order: SortOrder.DESCENDING,
|
||||
})
|
||||
|
||||
return escapeQueryWithParameters(
|
||||
`websearch_to_tsquery('english', :${param}) @@ library_item.search_tsv`,
|
||||
{ [param]: value }
|
||||
)
|
||||
} else {
|
||||
switch (field.name) {
|
||||
case 'in': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
const folder = expression.value?.toString()
|
||||
if (!folder) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
switch (folder) {
|
||||
case InFilter.ALL:
|
||||
return null
|
||||
case InFilter.ARCHIVE:
|
||||
return 'library_item.archived_at IS NOT NULL'
|
||||
case InFilter.TRASH:
|
||||
// return only deleted pages within 14 days
|
||||
return "library_item.deleted_at >= now() - interval '14 days'"
|
||||
default: {
|
||||
let sql = 'library_item.archived_at IS NULL'
|
||||
if (useFolders) {
|
||||
const param = `folder_${parameters.length}`
|
||||
const folderSql = escapeQueryWithParameters(
|
||||
`library_item.folder = :${param}`,
|
||||
{ [param]: folder }
|
||||
)
|
||||
sql = `(${sql} AND ${folderSql})`
|
||||
}
|
||||
|
||||
return sql
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
case 'is': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
const value = expression.value?.toString()
|
||||
if (!value) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
switch (value) {
|
||||
case ReadFilter.READ:
|
||||
return 'library_item.reading_progress_bottom_percent > 98'
|
||||
case ReadFilter.READING:
|
||||
return 'library_item.reading_progress_bottom_percent BETWEEN 2 AND 98'
|
||||
case ReadFilter.UNREAD:
|
||||
return 'library_item.reading_progress_bottom_percent < 2'
|
||||
default:
|
||||
throw new Error(`Unexpected keyword: ${value}`)
|
||||
}
|
||||
}
|
||||
case 'type': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
const value = expression.value?.toString()
|
||||
if (!value) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
const param = `type_${parameters.length}`
|
||||
|
||||
return escapeQueryWithParameters(
|
||||
`LOWER(library_item.item_type) = :${param}`,
|
||||
{
|
||||
[param]: value.toLowerCase(),
|
||||
}
|
||||
)
|
||||
}
|
||||
case 'label': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
const value = expression.value?.toString()?.toLowerCase()
|
||||
if (!value) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
const labels = value.split(',')
|
||||
return (
|
||||
labels
|
||||
.map((label) => {
|
||||
const param = `label_${parameters.length}`
|
||||
|
||||
const hasWildcard = label.includes('*')
|
||||
if (hasWildcard) {
|
||||
return escapeQueryWithParameters(
|
||||
`exists (select 1 from unnest(array_cat(library_item.label_names, library_item.highlight_labels)::text[]) as label where label ILIKE :${param})`,
|
||||
{
|
||||
[param]: label.replace(/\*/g, '%'),
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
return escapeQueryWithParameters(
|
||||
`:${param} = ANY(lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[])`,
|
||||
{
|
||||
[param]: label.replace(/\*/g, '%'),
|
||||
[param]: label,
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
.join(' OR ')
|
||||
// wrap in brackets to avoid precedence issues
|
||||
.replace(/^(.*)$/, '($1)')
|
||||
)
|
||||
} else {
|
||||
queryBuilder.andWhere(
|
||||
`lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[] && ARRAY[:...${param}]::text[]`,
|
||||
}
|
||||
case 'sort': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
const value = expression.value?.toString()
|
||||
if (!value) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
const [sort, sortOrder] = value.split('-')
|
||||
const order =
|
||||
sortOrder?.toUpperCase() === 'ASC'
|
||||
? SortOrder.ASCENDING
|
||||
: SortOrder.DESCENDING
|
||||
|
||||
const column = getColumnName(sort)
|
||||
orders.push({ by: `library_item.${column}`, order })
|
||||
return null
|
||||
}
|
||||
case 'has': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
const value = expression.value?.toString()
|
||||
if (!value) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
switch (value) {
|
||||
case HasFilter.HIGHLIGHTS:
|
||||
return "library_item.highlight_annotations <> '{}'"
|
||||
case HasFilter.LABELS:
|
||||
return "library_item.label_names <> '{}'"
|
||||
case HasFilter.SUBSCRIPTIONS:
|
||||
return 'library_item.subscription is NOT NULL'
|
||||
default:
|
||||
throw new Error(`Unexpected keyword: ${value}`)
|
||||
}
|
||||
}
|
||||
case 'saved':
|
||||
case 'read':
|
||||
case 'updated':
|
||||
case 'published': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
const date = expression.value?.toString()
|
||||
if (!date) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
let startDate: Date | undefined
|
||||
let endDate: Date | undefined
|
||||
// check for special date filters
|
||||
switch (date.toLowerCase()) {
|
||||
case 'today':
|
||||
startDate = DateTime.local().startOf('day').toJSDate()
|
||||
break
|
||||
case 'yesterday': {
|
||||
const yesterday = DateTime.local().minus({ days: 1 })
|
||||
startDate = yesterday.startOf('day').toJSDate()
|
||||
endDate = yesterday.endOf('day').toJSDate()
|
||||
break
|
||||
}
|
||||
case 'this week':
|
||||
startDate = DateTime.local().startOf('week').toJSDate()
|
||||
break
|
||||
case 'this month':
|
||||
startDate = DateTime.local().startOf('month').toJSDate()
|
||||
break
|
||||
default: {
|
||||
// check for date ranges
|
||||
const [start, end] = date.split('..')
|
||||
startDate = start && start !== '*' ? new Date(start) : undefined
|
||||
endDate = end && end !== '*' ? new Date(end) : undefined
|
||||
}
|
||||
}
|
||||
|
||||
const startParam = `${field.name}_start_${parameters.length}`
|
||||
const endParam = `${field.name}_end_${parameters.length}`
|
||||
|
||||
return escapeQueryWithParameters(
|
||||
`library_item.${field.name}_at BETWEEN :${startParam} AND :${endParam}`,
|
||||
{
|
||||
[param]: includeLabel.labels,
|
||||
[startParam]: startDate ?? new Date(0),
|
||||
[endParam]: endDate ?? new Date(),
|
||||
}
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if (excludeLabels && excludeLabels.length > 0) {
|
||||
const labels = excludeLabels.flatMap((filter) => filter.labels)
|
||||
|
||||
const hasWildcard = labels.some((label) => label.includes('*'))
|
||||
|
||||
if (hasWildcard) {
|
||||
queryBuilder.andWhere(
|
||||
new Brackets((qb) => {
|
||||
labels.forEach((label, i) => {
|
||||
const param = `excludeLabels_${i}`
|
||||
qb.andWhere(
|
||||
`array_to_string(array_cat(library_item.label_names, library_item.highlight_labels)::text[], ',') NOT ILIKE :${param}`,
|
||||
{
|
||||
[param]: label.replace(/\*/g, '%'),
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
)
|
||||
} else {
|
||||
queryBuilder.andWhere(
|
||||
'NOT lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[] && ARRAY[:...excludeLabels]::text[]',
|
||||
{
|
||||
excludeLabels: labels,
|
||||
// term filters
|
||||
case 'subscription':
|
||||
case 'rss':
|
||||
case 'language': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
)
|
||||
|
||||
const value = expression.value?.toString()
|
||||
if (!value) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
const columnName = getColumnName(field.name)
|
||||
const param = `term_${field.name}_${parameters.length}`
|
||||
|
||||
return escapeQueryWithParameters(
|
||||
`library_item.${columnName} = :${param}`,
|
||||
{
|
||||
[param]: value,
|
||||
}
|
||||
)
|
||||
}
|
||||
// match filters
|
||||
case 'author':
|
||||
case 'title':
|
||||
case 'description':
|
||||
case 'note':
|
||||
case 'site': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
// normalize the term to lower case
|
||||
const value = expression.value?.toString()?.toLowerCase()
|
||||
if (!value) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
const columnName = getColumnName(field.name)
|
||||
const param = `match_${field.name}_${parameters.length}`
|
||||
const wildcardParam = `match_${field.name}_wildcard_${parameters.length}`
|
||||
|
||||
return escapeQueryWithParameters(
|
||||
`(websearch_to_tsquery('english', :${param}) @@ library_item.${columnName}_tsv OR library_item.${columnName} ILIKE :${wildcardParam})`,
|
||||
{
|
||||
[param]: value,
|
||||
[wildcardParam]: `%${value}%`,
|
||||
}
|
||||
)
|
||||
}
|
||||
case 'includes': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
const ids = expression.value?.toString()?.split(',')
|
||||
if (!ids || ids.length === 0) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
const param = `includes_${parameters.length}`
|
||||
|
||||
return escapeQueryWithParameters(`library_item.id = ANY(:${param})`, {
|
||||
[param]: ids,
|
||||
})
|
||||
}
|
||||
case 'recommendedBy': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
const value = expression.value?.toString()
|
||||
if (!value) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
const param = `recommendedBy_${parameters.length}`
|
||||
if (value === '*') {
|
||||
// select all if * is provided
|
||||
return "library_item.recommender_names <> '{}'"
|
||||
}
|
||||
|
||||
return escapeQueryWithParameters(
|
||||
`:${param} = ANY(lower(library_item.recommender_names::text)::text[])`,
|
||||
{
|
||||
[param]: value.toLowerCase(),
|
||||
}
|
||||
)
|
||||
}
|
||||
case 'no': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
const value = expression.value?.toString()
|
||||
if (!value) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
let column = ''
|
||||
switch (value) {
|
||||
case 'highlight':
|
||||
column = 'highlight_annotations'
|
||||
break
|
||||
case 'label':
|
||||
column = 'label_names'
|
||||
break
|
||||
case 'subscription':
|
||||
column = 'subscription'
|
||||
break
|
||||
default:
|
||||
throw new Error(`Unexpected keyword: ${value}`)
|
||||
}
|
||||
|
||||
return `(library_item.${column} = '{}' OR library_item.${column} IS NULL)`
|
||||
}
|
||||
case 'use':
|
||||
case 'mode':
|
||||
// mode is ignored and used only by the frontend
|
||||
return null
|
||||
case 'readPosition':
|
||||
case 'wordsCount': {
|
||||
if (expression.type !== 'LiteralExpression') {
|
||||
throw new Error('Expected a literal expression.')
|
||||
}
|
||||
|
||||
let value = expression.value?.toString()
|
||||
if (!value) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
const column = getColumnName(field.name)
|
||||
|
||||
const operatorRegex = /([<>]=?)/
|
||||
const operator = value.match(operatorRegex)?.[0]
|
||||
if (!operator) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
value = value.replace(operatorRegex, '')
|
||||
if (!value) {
|
||||
throw new Error('Expected a value.')
|
||||
}
|
||||
|
||||
const param = `range_${field.name}_${parameters.length}`
|
||||
|
||||
return escapeQueryWithParameters(
|
||||
`library_item.${column} ${operator} :${param}`,
|
||||
{
|
||||
[param]: parseInt(value, 10),
|
||||
}
|
||||
)
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unexpected keyword: ${field.name}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (args.dateFilters && args.dateFilters.length > 0) {
|
||||
args.dateFilters.forEach((filter) => {
|
||||
const startDate = `${filter.field}_start`
|
||||
const endDate = `${filter.field}_end`
|
||||
queryBuilder.andWhere(
|
||||
`library_item.${filter.field} between :${startDate} and :${endDate}`,
|
||||
{
|
||||
[startDate]: filter.startDate ?? new Date(0),
|
||||
[endDate]: filter.endDate ?? new Date(),
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
if (args.termFilters && args.termFilters.length > 0) {
|
||||
args.termFilters.forEach((filter) => {
|
||||
const param = `term_${filter.field}`
|
||||
queryBuilder.andWhere(`lower(library_item.${filter.field}) = :${param}`, {
|
||||
[param]: filter.value.toLowerCase(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
if (args.matchFilters && args.matchFilters.length > 0) {
|
||||
args.matchFilters.forEach((filter) => {
|
||||
const param = `match_${filter.field}`
|
||||
queryBuilder.andWhere(
|
||||
new Brackets((qb) => {
|
||||
qb.andWhere(
|
||||
`websearch_to_tsquery('english', :${param}) @@ library_item.${filter.field}_tsv`,
|
||||
{
|
||||
[param]: filter.value,
|
||||
}
|
||||
).orWhere(`${filter.field} ILIKE :value`, {
|
||||
value: `%${filter.value}%`,
|
||||
})
|
||||
})
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
if (args.ids && args.ids.length > 0) {
|
||||
queryBuilder.andWhere('library_item.id = ANY(:ids)', {
|
||||
ids: args.ids,
|
||||
})
|
||||
}
|
||||
|
||||
if (!args.includePending) {
|
||||
queryBuilder.andWhere("library_item.state <> 'PROCESSING'")
|
||||
}
|
||||
|
||||
if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) {
|
||||
queryBuilder.andWhere("library_item.state <> 'DELETED'")
|
||||
}
|
||||
|
||||
if (args.noFilters) {
|
||||
args.noFilters.forEach((filter) => {
|
||||
queryBuilder.andWhere(
|
||||
`(library_item.${filter.field} = '{}' OR library_item.${filter.field} IS NULL)`
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
if (args.recommendedBy) {
|
||||
if (args.recommendedBy === '*') {
|
||||
// select all if * is provided
|
||||
queryBuilder.andWhere(`library_item.recommender_names <> '{}'`)
|
||||
} else {
|
||||
// select only if the user is recommended by the provided user
|
||||
queryBuilder.andWhere(
|
||||
'lower(library_item.recommender_names::text)::text[] && ARRAY[:recommendedBy]::text[]',
|
||||
{
|
||||
recommendedBy: args.recommendedBy.toLowerCase(),
|
||||
}
|
||||
)
|
||||
const serialize = (ast: LiqeQuery): string | null => {
|
||||
if (ast.type === 'Tag') {
|
||||
return serializeTagExpression(ast)
|
||||
}
|
||||
|
||||
if (ast.type === 'LogicalExpression') {
|
||||
let operator = ''
|
||||
if (ast.operator.operator === 'AND') {
|
||||
operator = 'AND'
|
||||
} else if (ast.operator.operator === 'OR') {
|
||||
operator = 'OR'
|
||||
} else {
|
||||
throw new Error('Unexpected operator.')
|
||||
}
|
||||
|
||||
const left = serialize(ast.left)
|
||||
const right = serialize(ast.right)
|
||||
|
||||
if (!left && !right) {
|
||||
return null
|
||||
}
|
||||
|
||||
if (!left) {
|
||||
return right
|
||||
}
|
||||
|
||||
if (!right) {
|
||||
return left
|
||||
}
|
||||
|
||||
return `${left} ${operator} ${right}`
|
||||
}
|
||||
|
||||
if (ast.type === 'UnaryOperator') {
|
||||
const serialized = serialize(ast.operand)
|
||||
|
||||
if (!serialized) {
|
||||
return null
|
||||
}
|
||||
|
||||
return `NOT ${serialized}`
|
||||
}
|
||||
|
||||
if (ast.type === 'ParenthesizedExpression') {
|
||||
const serialized = serialize(ast.expression)
|
||||
|
||||
if (!serialized) {
|
||||
return null
|
||||
}
|
||||
|
||||
return `(${serialized})`
|
||||
}
|
||||
|
||||
throw new Error('Missing AST type.')
|
||||
}
|
||||
|
||||
if (args.includeContent) {
|
||||
queryBuilder.addSelect('library_item.readableContent')
|
||||
}
|
||||
|
||||
if (args.rangeFilters && args.rangeFilters.length > 0) {
|
||||
args.rangeFilters.forEach((filter, i) => {
|
||||
const param = `range_${filter.field}_${i}`
|
||||
queryBuilder.andWhere(
|
||||
`library_item.${filter.field} ${filter.operator} :${param}`,
|
||||
{
|
||||
[param]: filter.value,
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
return serialize(searchQuery)
|
||||
}
|
||||
|
||||
export const searchLibraryItems = async (
|
||||
|
|
@ -349,6 +642,11 @@ export const searchLibraryItems = async (
|
|||
column !== 'library_item.originalContent'
|
||||
)
|
||||
|
||||
let searchQuery: LiqeQuery | undefined
|
||||
if (args.query) {
|
||||
searchQuery = parseSearchQuery(args.query)
|
||||
}
|
||||
|
||||
// add pagination and sorting
|
||||
return authTrx(
|
||||
async (tx) => {
|
||||
|
|
@ -357,8 +655,39 @@ export const searchLibraryItems = async (
|
|||
.select(selectColumns)
|
||||
.where('library_item.user_id = :userId', { userId })
|
||||
|
||||
// build the where clause
|
||||
buildWhereClause(queryBuilder, args)
|
||||
if (searchQuery) {
|
||||
const parameters: ObjectLiteral[] = []
|
||||
const selects: Select[] = []
|
||||
const orders: Sort[] = []
|
||||
const whereClause = buildQuery(
|
||||
searchQuery,
|
||||
parameters,
|
||||
selects,
|
||||
orders,
|
||||
args.useFolders
|
||||
)
|
||||
whereClause &&
|
||||
queryBuilder
|
||||
.andWhere(whereClause)
|
||||
.setParameters(parameters.reduce((a, b) => ({ ...a, ...b }), {}))
|
||||
|
||||
selects.forEach((select) => {
|
||||
queryBuilder.addSelect(select.column, select.alias)
|
||||
})
|
||||
|
||||
// add order by
|
||||
orders.forEach((order) => {
|
||||
queryBuilder.addOrderBy(order.by, order.order, 'NULLS LAST')
|
||||
})
|
||||
}
|
||||
|
||||
if (!args.includePending) {
|
||||
queryBuilder.andWhere("library_item.state <> 'PROCESSING'")
|
||||
}
|
||||
|
||||
if (!args.includeDeleted) {
|
||||
queryBuilder.andWhere("library_item.state <> 'DELETED'")
|
||||
}
|
||||
|
||||
const libraryItems = await queryBuilder
|
||||
.addOrderBy(`library_item.${sortField}`, sortOrder, 'NULLS LAST')
|
||||
|
|
@ -671,7 +1000,7 @@ export const countByCreatedAt = async (
|
|||
|
||||
export const updateLibraryItems = async (
|
||||
action: BulkActionType,
|
||||
searchArgs: SearchArgs,
|
||||
query: string,
|
||||
userId: string,
|
||||
labels?: Label[],
|
||||
args?: unknown
|
||||
|
|
@ -725,13 +1054,20 @@ export const updateLibraryItems = async (
|
|||
throw new Error('Invalid bulk action')
|
||||
}
|
||||
|
||||
const searchQuery = parseSearchQuery(query)
|
||||
|
||||
await authTrx(async (tx) => {
|
||||
const queryBuilder = tx
|
||||
.createQueryBuilder(LibraryItem, 'library_item')
|
||||
.where('library_item.user_id = :userId', { userId })
|
||||
|
||||
// build the where clause
|
||||
buildWhereClause(queryBuilder, searchArgs)
|
||||
const parameters: ObjectLiteral[] = []
|
||||
const whereClause = buildQuery(searchQuery, parameters)
|
||||
if (whereClause) {
|
||||
queryBuilder
|
||||
.andWhere(whereClause)
|
||||
.setParameters(parameters.reduce((a, b) => ({ ...a, ...b }), {}))
|
||||
}
|
||||
|
||||
if (addLabels) {
|
||||
if (!labels) {
|
||||
|
|
|
|||
|
|
@ -1,582 +1,12 @@
|
|||
/* eslint-disable @typescript-eslint/restrict-template-expressions */
|
||||
/* eslint-disable @typescript-eslint/no-unsafe-return */
|
||||
/* eslint-disable @typescript-eslint/no-unsafe-call */
|
||||
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
|
||||
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
|
||||
import { DateTime } from 'luxon'
|
||||
import {
|
||||
ISearchParserDictionary,
|
||||
parse,
|
||||
SearchParserKeyWordOffset,
|
||||
SearchParserTextOffset,
|
||||
} from 'search-query-parser'
|
||||
import { InputMaybe, PageType, SortParams } from '../generated/graphql'
|
||||
import { LiqeQuery, parse } from 'liqe'
|
||||
|
||||
export enum ReadFilter {
|
||||
ALL,
|
||||
READ,
|
||||
READING,
|
||||
UNREAD,
|
||||
}
|
||||
|
||||
export enum InFilter {
|
||||
ALL = 'all',
|
||||
INBOX = 'inbox',
|
||||
ARCHIVE = 'archive',
|
||||
TRASH = 'trash',
|
||||
FOLLOWING = 'following',
|
||||
}
|
||||
|
||||
export interface SearchFilter {
|
||||
query: string | undefined
|
||||
inFilter: InFilter
|
||||
readFilter: ReadFilter
|
||||
typeFilter?: string
|
||||
labelFilters: LabelFilter[]
|
||||
sort?: Sort
|
||||
hasFilters: HasFilter[]
|
||||
dateFilters: DateFilter[]
|
||||
termFilters: FieldFilter[]
|
||||
matchFilters: FieldFilter[]
|
||||
ids: string[]
|
||||
recommendedBy?: string
|
||||
noFilters: NoFilter[]
|
||||
rangeFilters: RangeFilter[]
|
||||
useFolders?: boolean
|
||||
}
|
||||
|
||||
export enum LabelFilterType {
|
||||
INCLUDE,
|
||||
EXCLUDE,
|
||||
}
|
||||
|
||||
export type LabelFilter = {
|
||||
type: LabelFilterType
|
||||
labels: string[]
|
||||
}
|
||||
|
||||
export enum HasFilter {
|
||||
HIGHLIGHTS,
|
||||
LABELS,
|
||||
SUBSCRIPTIONS,
|
||||
}
|
||||
|
||||
export interface DateFilter {
|
||||
field: string
|
||||
startDate?: Date
|
||||
endDate?: Date
|
||||
}
|
||||
|
||||
export interface RangeFilter {
|
||||
field: string
|
||||
operator: string
|
||||
value: number
|
||||
}
|
||||
|
||||
export enum SortBy {
|
||||
SAVED = 'savedAt',
|
||||
UPDATED = 'updatedAt',
|
||||
PUBLISHED = 'publishedAt',
|
||||
READ = 'readAt',
|
||||
WORDS_COUNT = 'wordCount',
|
||||
}
|
||||
|
||||
export enum SortOrder {
|
||||
ASCENDING = 'ASC',
|
||||
DESCENDING = 'DESC',
|
||||
}
|
||||
|
||||
export interface Sort {
|
||||
by: SortBy
|
||||
order?: SortOrder
|
||||
}
|
||||
|
||||
export interface FieldFilter {
|
||||
nested?: boolean
|
||||
field: string
|
||||
value: string
|
||||
}
|
||||
|
||||
export interface NoFilter {
|
||||
field: string
|
||||
}
|
||||
|
||||
const parseStringValue = (str?: string): string | undefined => {
|
||||
if (str === undefined) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return str.toLowerCase()
|
||||
}
|
||||
|
||||
const parseIsFilter = (str: string | undefined): ReadFilter => {
|
||||
switch (str?.toUpperCase()) {
|
||||
case 'READ':
|
||||
return ReadFilter.READ
|
||||
case 'READING':
|
||||
return ReadFilter.READING
|
||||
case 'UNREAD':
|
||||
return ReadFilter.UNREAD
|
||||
}
|
||||
return ReadFilter.ALL
|
||||
}
|
||||
|
||||
const parseInFilter = (
|
||||
str: string | undefined,
|
||||
query: string | undefined
|
||||
): InFilter => {
|
||||
switch (str?.toUpperCase()) {
|
||||
case 'ALL':
|
||||
return InFilter.ALL
|
||||
case 'INBOX':
|
||||
return InFilter.INBOX
|
||||
case 'ARCHIVE':
|
||||
return InFilter.ARCHIVE
|
||||
case 'TRASH':
|
||||
return InFilter.TRASH
|
||||
case 'FOLLOWING':
|
||||
return InFilter.FOLLOWING
|
||||
}
|
||||
|
||||
return query ? InFilter.ALL : InFilter.INBOX
|
||||
}
|
||||
|
||||
const parseTypeFilter = (str: string | undefined): string | undefined => {
|
||||
if (str === undefined) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
switch (str.toLowerCase()) {
|
||||
case 'article':
|
||||
return PageType.Article
|
||||
case 'book':
|
||||
return PageType.Book
|
||||
case 'pdf':
|
||||
case 'file':
|
||||
return PageType.File
|
||||
case 'profile':
|
||||
return PageType.Profile
|
||||
case 'website':
|
||||
return PageType.Website
|
||||
case 'unknown':
|
||||
return PageType.Unknown
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
const parseLabelFilter = (
|
||||
str?: string,
|
||||
exclude?: ISearchParserDictionary
|
||||
): LabelFilter | undefined => {
|
||||
if (str === undefined) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const labels = str.split(',')
|
||||
|
||||
// check if the labels are on the exclusion list
|
||||
const excluded = exclude?.label && exclude.label.includes(...labels)
|
||||
|
||||
return {
|
||||
type: excluded ? LabelFilterType.EXCLUDE : LabelFilterType.INCLUDE,
|
||||
// use lower case for label names
|
||||
labels: labels.map((label) => label.toLowerCase()),
|
||||
}
|
||||
}
|
||||
|
||||
const parseSort = (str?: string): Sort | undefined => {
|
||||
if (str === undefined) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const [sort, order] = str.split('-')
|
||||
const sortOrder =
|
||||
order?.toUpperCase() === 'ASC' ? SortOrder.ASCENDING : SortOrder.DESCENDING
|
||||
|
||||
switch (sort.toUpperCase()) {
|
||||
case 'UPDATED':
|
||||
return {
|
||||
by: SortBy.UPDATED,
|
||||
order: sortOrder,
|
||||
}
|
||||
case 'SAVED':
|
||||
return {
|
||||
by: SortBy.SAVED,
|
||||
order: sortOrder,
|
||||
}
|
||||
case 'PUBLISHED':
|
||||
return {
|
||||
by: SortBy.PUBLISHED,
|
||||
order: sortOrder,
|
||||
}
|
||||
case 'READ':
|
||||
return {
|
||||
by: SortBy.READ,
|
||||
order: sortOrder,
|
||||
}
|
||||
case 'WORDSCOUNT':
|
||||
return {
|
||||
by: SortBy.WORDS_COUNT,
|
||||
order: sortOrder,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const parseHasFilter = (str?: string): HasFilter | undefined => {
|
||||
if (str === undefined) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
switch (str.toUpperCase()) {
|
||||
case 'HIGHLIGHTS':
|
||||
return HasFilter.HIGHLIGHTS
|
||||
case 'LABELS':
|
||||
return HasFilter.LABELS
|
||||
case 'SUBSCRIPTIONS':
|
||||
return HasFilter.SUBSCRIPTIONS
|
||||
}
|
||||
}
|
||||
|
||||
const parseDateFilter = (
|
||||
field: string,
|
||||
str?: string
|
||||
): DateFilter | undefined => {
|
||||
if (str === undefined) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
switch (field.toLowerCase()) {
|
||||
case 'published':
|
||||
field = 'published_at'
|
||||
break
|
||||
case 'saved':
|
||||
field = 'saved_at'
|
||||
break
|
||||
case 'updated':
|
||||
field = 'updated_at'
|
||||
}
|
||||
|
||||
// check for special date filters
|
||||
switch (str.toLowerCase()) {
|
||||
case 'today':
|
||||
return {
|
||||
field,
|
||||
startDate: DateTime.local().startOf('day').toJSDate(),
|
||||
}
|
||||
case 'yesterday': {
|
||||
const yesterday = DateTime.local().minus({ days: 1 })
|
||||
return {
|
||||
field,
|
||||
startDate: yesterday.startOf('day').toJSDate(),
|
||||
endDate: yesterday.endOf('day').toJSDate(),
|
||||
}
|
||||
}
|
||||
case 'this week':
|
||||
return {
|
||||
field,
|
||||
startDate: DateTime.local().startOf('week').toJSDate(),
|
||||
}
|
||||
case 'this month':
|
||||
return {
|
||||
field,
|
||||
startDate: DateTime.local().startOf('month').toJSDate(),
|
||||
}
|
||||
}
|
||||
|
||||
// check for date ranges
|
||||
const [start, end] = str.split('..')
|
||||
const startDate = start && start !== '*' ? new Date(start) : undefined
|
||||
const endDate = end && end !== '*' ? new Date(end) : undefined
|
||||
|
||||
return {
|
||||
field,
|
||||
startDate,
|
||||
endDate,
|
||||
}
|
||||
}
|
||||
|
||||
const parseRangeFilter = (
|
||||
field: string,
|
||||
str?: string
|
||||
): RangeFilter | undefined => {
|
||||
if (str === undefined) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
switch (field.toUpperCase()) {
|
||||
case 'WORDSCOUNT':
|
||||
field = 'word_count'
|
||||
break
|
||||
case 'READPOSITION':
|
||||
field = 'reading_progress_bottom_percent'
|
||||
break
|
||||
default:
|
||||
return undefined
|
||||
}
|
||||
|
||||
const operatorRegex = /([<>]=?)/
|
||||
const operator = str.match(operatorRegex)?.[0]
|
||||
if (!operator) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const value = str.replace(operatorRegex, '')
|
||||
if (!value) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return {
|
||||
field,
|
||||
operator,
|
||||
value: Number(value),
|
||||
}
|
||||
}
|
||||
|
||||
const parseFieldFilter = (
|
||||
field: string,
|
||||
str?: string
|
||||
): FieldFilter | undefined => {
|
||||
if (str === undefined) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
// normalize the term to lower case
|
||||
const value = str.toLowerCase()
|
||||
|
||||
switch (field.toUpperCase()) {
|
||||
case 'LANGUAGE':
|
||||
return {
|
||||
field: 'item_language',
|
||||
value,
|
||||
}
|
||||
case 'SUBSCRIPTION':
|
||||
case 'RSS':
|
||||
return {
|
||||
field: 'subscription',
|
||||
value,
|
||||
}
|
||||
case 'SITE':
|
||||
return {
|
||||
field: 'site_name',
|
||||
value,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
field,
|
||||
value,
|
||||
}
|
||||
}
|
||||
|
||||
const parseIds = (str?: string): string[] | undefined => {
|
||||
if (str === undefined) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return str.split(',')
|
||||
}
|
||||
|
||||
const parseNoFilter = (str?: string): NoFilter | undefined => {
|
||||
if (str === undefined) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const strLower = str.toLowerCase()
|
||||
switch (strLower) {
|
||||
case 'highlight':
|
||||
return { field: 'highlight_annotations' }
|
||||
case 'label':
|
||||
return { field: 'label_names' }
|
||||
case 'subscription':
|
||||
return { field: 'subscription' }
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
export const parseSearchQuery = (query: string | undefined): SearchFilter => {
|
||||
export const parseSearchQuery = (query: string): LiqeQuery => {
|
||||
const searchQuery = query
|
||||
? query
|
||||
.replace(/\W\s":/g, '')
|
||||
.replace('in:subscription', 'has:subscriptions') // compatibility with old search
|
||||
.replace('in:library', 'no:subscription') // compatibility with old search
|
||||
: undefined
|
||||
const result: SearchFilter = {
|
||||
query: searchQuery,
|
||||
readFilter: ReadFilter.ALL,
|
||||
inFilter: searchQuery ? InFilter.ALL : InFilter.INBOX,
|
||||
labelFilters: [],
|
||||
hasFilters: [],
|
||||
dateFilters: [],
|
||||
termFilters: [],
|
||||
matchFilters: [],
|
||||
ids: [],
|
||||
noFilters: [],
|
||||
rangeFilters: [],
|
||||
}
|
||||
.replace(/\W\s":/g, '')
|
||||
.replace('in:subscription', 'has:subscriptions') // compatibility with old search
|
||||
.replace('in:library', 'no:subscription') // compatibility with old search
|
||||
// wrap the value behind colon in quotes if it's not already
|
||||
.replace(/(\w+):([^"\s]+)/g, '$1:"$2"')
|
||||
|
||||
if (!searchQuery) {
|
||||
return result
|
||||
}
|
||||
|
||||
const parsed = parse(searchQuery, {
|
||||
keywords: [
|
||||
'in',
|
||||
'is',
|
||||
'type',
|
||||
'label',
|
||||
'sort',
|
||||
'has',
|
||||
'saved',
|
||||
'author',
|
||||
'published',
|
||||
'subscription',
|
||||
'language',
|
||||
'title',
|
||||
'description',
|
||||
'content',
|
||||
'updated',
|
||||
'includes',
|
||||
'recommendedBy',
|
||||
'no',
|
||||
'mode',
|
||||
'site',
|
||||
'note',
|
||||
'rss',
|
||||
'wordsCount',
|
||||
'readPosition',
|
||||
'use',
|
||||
],
|
||||
tokenize: true,
|
||||
})
|
||||
if (parsed.offsets) {
|
||||
const texts = parsed.offsets
|
||||
.filter((offset) => 'text' in offset)
|
||||
.map((offset) => offset as SearchParserTextOffset)
|
||||
|
||||
if (texts.length > 0) {
|
||||
result.query = texts
|
||||
.map((offset: SearchParserTextOffset) => {
|
||||
// TODO: the parser library doesn't let us accurately
|
||||
// pull out quoted text, so we are just assuming
|
||||
// anything with spaces is quoted.
|
||||
if (offset.text.indexOf(' ') > -1) {
|
||||
return `"${offset.text}"`
|
||||
}
|
||||
return offset.text
|
||||
})
|
||||
.join(' ')
|
||||
} else {
|
||||
result.query = undefined
|
||||
}
|
||||
|
||||
const keywords = parsed.offsets
|
||||
.filter((offset) => 'keyword' in offset)
|
||||
.map((offset) => offset as SearchParserKeyWordOffset)
|
||||
|
||||
for (const keyword of keywords) {
|
||||
switch (keyword.keyword) {
|
||||
case 'in':
|
||||
result.inFilter = parseInFilter(keyword.value, result.query)
|
||||
break
|
||||
case 'is':
|
||||
result.readFilter = parseIsFilter(keyword.value)
|
||||
break
|
||||
case 'type':
|
||||
result.typeFilter = parseTypeFilter(keyword.value)
|
||||
break
|
||||
case 'label': {
|
||||
const labelFilter = parseLabelFilter(keyword.value, parsed.exclude)
|
||||
labelFilter && result.labelFilters.push(labelFilter)
|
||||
break
|
||||
}
|
||||
case 'sort':
|
||||
result.sort = parseSort(keyword.value)
|
||||
break
|
||||
case 'has': {
|
||||
const hasFilter = parseHasFilter(keyword.value)
|
||||
hasFilter !== undefined && result.hasFilters.push(hasFilter)
|
||||
break
|
||||
}
|
||||
case 'saved':
|
||||
case 'read':
|
||||
case 'updated':
|
||||
case 'published': {
|
||||
const dateFilter = parseDateFilter(keyword.keyword, keyword.value)
|
||||
dateFilter && result.dateFilters.push(dateFilter)
|
||||
break
|
||||
}
|
||||
// term filters
|
||||
case 'subscription':
|
||||
case 'rss':
|
||||
case 'language': {
|
||||
const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value)
|
||||
fieldFilter && result.termFilters.push(fieldFilter)
|
||||
break
|
||||
}
|
||||
// match filters
|
||||
case 'author':
|
||||
case 'title':
|
||||
case 'description':
|
||||
case 'note':
|
||||
case 'site':
|
||||
case 'content': {
|
||||
const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value)
|
||||
fieldFilter && result.matchFilters.push(fieldFilter)
|
||||
break
|
||||
}
|
||||
case 'includes': {
|
||||
const ids = parseIds(keyword.value)
|
||||
ids && result.ids.push(...ids)
|
||||
break
|
||||
}
|
||||
case 'recommendedBy': {
|
||||
result.recommendedBy = parseStringValue(keyword.value)
|
||||
break
|
||||
}
|
||||
case 'no': {
|
||||
const noFilter = parseNoFilter(keyword.value)
|
||||
noFilter && result.noFilters.push(noFilter)
|
||||
break
|
||||
}
|
||||
case 'mode':
|
||||
// mode is ignored and used only by the frontend
|
||||
break
|
||||
case 'readPosition':
|
||||
case 'wordsCount': {
|
||||
const rangeFilter = parseRangeFilter(keyword.keyword, keyword.value)
|
||||
rangeFilter && result.rangeFilters.push(rangeFilter)
|
||||
break
|
||||
}
|
||||
case 'use':
|
||||
result.useFolders = keyword.value === 'folders'
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export const sortParamsToSort = (
|
||||
sortParams: InputMaybe<SortParams> | undefined
|
||||
) => {
|
||||
const sort = { by: SortBy.UPDATED, order: SortOrder.DESCENDING }
|
||||
|
||||
if (sortParams) {
|
||||
sortParams.order === 'ASCENDING' && (sort.order = SortOrder.ASCENDING)
|
||||
switch (sortParams.by) {
|
||||
case 'UPDATED_TIME':
|
||||
sort.by = SortBy.UPDATED
|
||||
break
|
||||
case 'PUBLISHED_AT':
|
||||
sort.by = SortBy.PUBLISHED
|
||||
break
|
||||
case 'SAVED_AT':
|
||||
sort.by = SortBy.SAVED
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return sort
|
||||
return parse(searchQuery)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -569,14 +569,14 @@ describe('Article API', () => {
|
|||
).expect(200)
|
||||
|
||||
// Save a link, then archive it
|
||||
let allLinks = await graphqlRequest(searchQuery(''), authToken).expect(
|
||||
let allLinks = await graphqlRequest(searchQuery('in:inbox'), authToken).expect(
|
||||
200
|
||||
)
|
||||
const justSavedId = allLinks.body.data.search.edges[0].node.id
|
||||
await archiveLink(authToken, justSavedId)
|
||||
|
||||
// test the negative case, ensuring the archive link wasn't returned
|
||||
allLinks = await graphqlRequest(searchQuery(''), authToken).expect(200)
|
||||
allLinks = await graphqlRequest(searchQuery('in:inbox'), authToken).expect(200)
|
||||
expect(allLinks.body.data.search.edges[0]?.node?.url).to.not.eq(url)
|
||||
|
||||
// Now save the link again, and ensure it is returned
|
||||
|
|
@ -585,7 +585,7 @@ describe('Article API', () => {
|
|||
authToken
|
||||
).expect(200)
|
||||
|
||||
allLinks = await graphqlRequest(searchQuery(''), authToken).expect(200)
|
||||
allLinks = await graphqlRequest(searchQuery('in:inbox'), authToken).expect(200)
|
||||
expect(allLinks.body.data.search.edges[0].node.id).to.eq(justSavedId)
|
||||
expect(allLinks.body.data.search.edges[0].node.url).to.eq(url)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ import {
|
|||
generateVerificationToken,
|
||||
hashPassword,
|
||||
} from '../../src/utils/auth'
|
||||
import { InFilter } from '../../src/utils/search'
|
||||
import * as util from '../../src/utils/sendEmail'
|
||||
import { createTestUser } from '../db'
|
||||
import { generateFakeUuid, request } from '../util'
|
||||
|
|
@ -609,7 +608,7 @@ describe('auth router', () => {
|
|||
).expect(200)
|
||||
const user = await userRepository.findOneByOrFail({ name })
|
||||
const { count } = await searchLibraryItems(
|
||||
{ inFilter: InFilter.ALL },
|
||||
{ query: 'in:all' },
|
||||
user.id
|
||||
)
|
||||
|
||||
|
|
@ -633,7 +632,7 @@ describe('auth router', () => {
|
|||
).expect(200)
|
||||
const user = await userRepository.findOneByOrFail({ name })
|
||||
const { count } = await searchLibraryItems(
|
||||
{ inFilter: InFilter.ALL },
|
||||
{ query: 'in:all' },
|
||||
user.id
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,168 +0,0 @@
|
|||
import 'mocha'
|
||||
import { expect } from 'chai'
|
||||
import { InFilter, parseSearchQuery, ReadFilter } from '../../src/utils/search'
|
||||
import { PageType } from '../../src/generated/graphql'
|
||||
|
||||
describe('undefined query', () => {
|
||||
it('returns an empty result with read state ALL and no typeFilter', () => {
|
||||
const result = parseSearchQuery(undefined)
|
||||
expect(result.query).to.be.undefined
|
||||
expect(result.readFilter).to.eq(ReadFilter.ALL)
|
||||
expect(result.typeFilter).to.be.undefined
|
||||
})
|
||||
})
|
||||
|
||||
describe('empty query', () => {
|
||||
it('returns an empty result with read state ALL and no typefilter', () => {
|
||||
const result = parseSearchQuery('')
|
||||
expect(result.query).to.be.undefined
|
||||
expect(result.readFilter).to.eq(ReadFilter.ALL)
|
||||
expect(result.typeFilter).to.be.undefined
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with READ read state', () => {
|
||||
it('returns a READ result', () => {
|
||||
const result = parseSearchQuery('is:read')
|
||||
expect(result.query).to.be.undefined
|
||||
expect(result.readFilter).to.eq(ReadFilter.READ)
|
||||
expect(result.typeFilter).to.be.undefined
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with UNREAD read state', () => {
|
||||
it('returns a UNREAD result', () => {
|
||||
const result = parseSearchQuery('is:unread')
|
||||
expect(result.query).to.be.undefined
|
||||
expect(result.readFilter).to.eq(ReadFilter.UNREAD)
|
||||
expect(result.typeFilter).to.be.undefined
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with multiple read states', () => {
|
||||
it('just uses the last one', () => {
|
||||
const result = parseSearchQuery('is:unread is:read')
|
||||
expect(result.query).to.be.undefined
|
||||
expect(result.readFilter).to.eq(ReadFilter.READ)
|
||||
expect(result.typeFilter).to.be.undefined
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with invalid read states', () => {
|
||||
it('returns ALL', () => {
|
||||
const result = parseSearchQuery('is:invalid')
|
||||
expect(result.query).to.be.undefined
|
||||
expect(result.readFilter).to.eq(ReadFilter.ALL)
|
||||
expect(result.typeFilter).to.be.undefined
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with read state before search query', () => {
|
||||
it('sets read state and query', () => {
|
||||
const result = parseSearchQuery('is:read "machine learning"')
|
||||
expect(result.query).to.eq(`"machine learning"`)
|
||||
expect(result.readFilter).to.eq(ReadFilter.READ)
|
||||
expect(result.typeFilter).to.be.undefined
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with read state after search query', () => {
|
||||
it('sets read state and query', () => {
|
||||
const result = parseSearchQuery('machine learning techniques is:read')
|
||||
expect(result.query).to.eq(`machine learning techniques`)
|
||||
expect(result.readFilter).to.eq(ReadFilter.READ)
|
||||
expect(result.typeFilter).to.be.undefined
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with quoted text', () => {
|
||||
it('sets the text as quoted and returns the default read state', () => {
|
||||
const result = parseSearchQuery('"machine learning" techniques"')
|
||||
expect(result.query).to.eq(`"machine learning" techniques`)
|
||||
expect(result.readFilter).to.eq(ReadFilter.ALL)
|
||||
expect(result.typeFilter).to.be.undefined
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with a file type', () => {
|
||||
it('sets the type to the supplied type', () => {
|
||||
const result = parseSearchQuery('"my string" type:file"')
|
||||
expect(result.query).to.eq(`"my string"`)
|
||||
expect(result.readFilter).to.eq(ReadFilter.ALL)
|
||||
expect(result.typeFilter).to.eq(PageType.File)
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with an article type', () => {
|
||||
it('sets the type to the supplied type', () => {
|
||||
const result = parseSearchQuery('"my string" type:article"')
|
||||
expect(result.query).to.eq(`"my string"`)
|
||||
expect(result.readFilter).to.eq(ReadFilter.ALL)
|
||||
expect(result.typeFilter).to.eq(PageType.Article)
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with pdf as its type', () => {
|
||||
it('sets the type to the supplied file type', () => {
|
||||
const result = parseSearchQuery('"my string" type:pdf"')
|
||||
expect(result.query).to.eq(`"my string"`)
|
||||
expect(result.readFilter).to.eq(ReadFilter.ALL)
|
||||
expect(result.typeFilter).to.eq(PageType.File)
|
||||
})
|
||||
})
|
||||
|
||||
describe('query without in param set', () => {
|
||||
it('returns ALL if there is a search term', () => {
|
||||
const result = parseSearchQuery('my search')
|
||||
expect(result.query).to.eq(`my search`)
|
||||
expect(result.inFilter).to.eq(InFilter.ALL)
|
||||
})
|
||||
it('returns INBOX if there is not a search term', () => {
|
||||
const result = parseSearchQuery('')
|
||||
expect(result.inFilter).to.eq(InFilter.INBOX)
|
||||
})
|
||||
it('returns INBOX if the search term is undefined', () => {
|
||||
const result = parseSearchQuery(undefined)
|
||||
expect(result.inFilter).to.eq(InFilter.INBOX)
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with in param set', () => {
|
||||
it('returns set value if there is a search term', () => {
|
||||
const result = parseSearchQuery('my search in:archive')
|
||||
expect(result.query).to.eq(`my search`)
|
||||
expect(result.inFilter).to.eq(InFilter.ARCHIVE)
|
||||
})
|
||||
it('returns set value if there is not a search term', () => {
|
||||
const result = parseSearchQuery('in:archive')
|
||||
expect(result.inFilter).to.eq(InFilter.ARCHIVE)
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with in param set to invalid value', () => {
|
||||
it('returns all if there is a query', () => {
|
||||
const result = parseSearchQuery('my search in:blahblah')
|
||||
expect(result.query).to.eq(`my search`)
|
||||
expect(result.inFilter).to.eq(InFilter.ALL)
|
||||
})
|
||||
it('returns set value if there is not a search term', () => {
|
||||
const result = parseSearchQuery('in:blahblah')
|
||||
expect(result.inFilter).to.eq(InFilter.INBOX)
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with author set', () => {
|
||||
it('adds author to the match filters', () => {
|
||||
const result = parseSearchQuery('author:"Omnivore Blog"')
|
||||
expect(result.matchFilters[0].field).to.equal('author')
|
||||
expect(result.matchFilters[0].value).to.equal('omnivore blog')
|
||||
})
|
||||
})
|
||||
|
||||
describe('query with site set', () => {
|
||||
it('adds site_name to the match filters', () => {
|
||||
const result = parseSearchQuery('site:omnivore.app')
|
||||
expect(result.matchFilters[0].field).to.equal('site_name')
|
||||
expect(result.matchFilters[0].value).to.equal('omnivore.app')
|
||||
})
|
||||
})
|
||||
13
yarn.lock
13
yarn.lock
|
|
@ -18804,6 +18804,14 @@ linkify-it@^4.0.1:
|
|||
dependencies:
|
||||
uc.micro "^1.0.1"
|
||||
|
||||
liqe@^3.8.0:
|
||||
version "3.8.0"
|
||||
resolved "https://registry.yarnpkg.com/liqe/-/liqe-3.8.0.tgz#e96c0733ecd3efb613d4404023433f15ae4f5849"
|
||||
integrity sha512-cZ1rDx4XzxONBTskSPBp7/KwJ9qbUdF8EPnY4VjKXwHF1Krz9lgnlMTh1G7kd+KtPYvUte1mhuZeQSnk7KiSBg==
|
||||
dependencies:
|
||||
nearley "^2.20.1"
|
||||
ts-error "^1.0.6"
|
||||
|
||||
listr-silent-renderer@^1.1.1:
|
||||
version "1.1.1"
|
||||
resolved "https://registry.yarnpkg.com/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz#924b5a3757153770bf1a8e3fbf74b8bbf3f9242e"
|
||||
|
|
@ -26919,6 +26927,11 @@ ts-dedent@^2.0.0, ts-dedent@^2.2.0:
|
|||
resolved "https://registry.yarnpkg.com/ts-dedent/-/ts-dedent-2.2.0.tgz#39e4bd297cd036292ae2394eb3412be63f563bb5"
|
||||
integrity sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==
|
||||
|
||||
ts-error@^1.0.6:
|
||||
version "1.0.6"
|
||||
resolved "https://registry.yarnpkg.com/ts-error/-/ts-error-1.0.6.tgz#277496f2a28de6c184cfce8dfd5cdd03a4e6b0fc"
|
||||
integrity sha512-tLJxacIQUM82IR7JO1UUkKlYuUTmoY9HBJAmNWFzheSlDS5SPMcNIepejHJa4BpPQLAcbRhRf3GDJzyj6rbKvA==
|
||||
|
||||
ts-loader@^9.3.0:
|
||||
version "9.3.0"
|
||||
resolved "https://registry.yarnpkg.com/ts-loader/-/ts-loader-9.3.0.tgz#980f4dbfb60e517179e15e10ed98e454b132159f"
|
||||
|
|
|
|||
Loading…
Reference in a new issue