From a3d3f4daf4023dd78f9ecb1389af6bc8bfc7ded4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 30 Nov 2023 16:21:47 +0800 Subject: [PATCH 01/10] feat: allow logical operator in search query --- packages/api/package.json | 1 + packages/api/src/resolvers/article/index.ts | 20 +- packages/api/src/services/library_item.ts | 426 +++++++++++++++++++- packages/api/src/utils/search.ts | 328 ++++++++------- yarn.lock | 13 + 5 files changed, 606 insertions(+), 182 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 3418c2962..ff7513d85 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -71,6 +71,7 @@ "jsonwebtoken": "^8.5.1", "jwks-rsa": "^2.0.3", "linkedom": "^0.14.9", + "liqe": "^3.8.0", "lodash": "^4.17.21", "luxon": "^3.2.1", "nanoid": "^3.1.25", diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 585d98a16..9b154f0c1 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -655,16 +655,15 @@ export const searchResolver = authorized< return { errorCodes: [SearchErrorCode.QueryTooLong] } } - const searchQuery = parseSearchQuery(params.query || undefined) + const searchQuery = params.query ? parseSearchQuery(params.query) : undefined const { libraryItems, count } = await searchLibraryItems( { from: Number(startCursor), size: first + 1, // fetch one more item to get next cursor - sort: searchQuery.sort, includePending: true, includeContent: !!params.includeContent, - ...searchQuery, + searchQuery, }, uid ) @@ -826,11 +825,16 @@ export const bulkActionResolver = authorized< }, }) - // parse query - const searchQuery = parseSearchQuery(query) - if (searchQuery.ids.length > 100) { - return { errorCodes: [BulkActionErrorCode.BadRequest] } - } + if (!query) { + return { errorCodes: [BulkActionErrorCode.BadRequest] } + } + + // parse query + const searchQuery = parseSearchQuery(query) + const ids = searchQuery.getValue?.('includes') as string[] + if (!ids || ids.length === 0 || ids.length > 100) { + return { errorCodes: [BulkActionErrorCode.BadRequest] } + } // get labels if needed let labels = undefined diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index e809cfbbf..d6f59f099 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1,4 +1,11 @@ -import { Brackets, DeepPartial, SelectQueryBuilder } from 'typeorm' +import { LiqeQuery } from 'liqe' +import { DateTime } from 'luxon' +import { + Brackets, + DeepPartial, + ObjectLiteral, + SelectQueryBuilder, +} from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' @@ -31,7 +38,7 @@ export interface SearchArgs { size?: number sort?: Sort query?: string - inFilter: InFilter + inFilter?: InFilter readFilter?: ReadFilter typeFilter?: string labelFilters?: LabelFilter[] @@ -47,6 +54,7 @@ export interface SearchArgs { noFilters?: NoFilter[] rangeFilters?: RangeFilter[] useFolders?: boolean + searchQuery?: LiqeQuery } export interface SearchResultItem { @@ -82,6 +90,405 @@ export interface SearchResultItem { content?: string } +const getColumnName = (field: string) => { + switch (field) { + case 'language': + return 'item_language' + case 'subscription': + case 'rss': + return 'subscription' + case 'site': + return 'site_name' + case 'wordsCount': + return 'word_count' + case 'readPosition': + return 'reading_progress_bottom_percent' + default: + return field + } +} + +export const buildQuery = ( + searchQuery: LiqeQuery, + parameters: ObjectLiteral[] +) => { + const escapeQueryWithParameters = ( + query: string, + parameter: ObjectLiteral + ) => { + parameters.push(parameter) + return query + } + + const serializeTagExpression = (ast: LiqeQuery): string => { + if (ast.type !== 'Tag') { + throw new Error('Expected a tag expression.') + } + + const { field, expression } = ast + + if (field.type === 'ImplicitField') { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const value = expression.value?.toString() + + if (!value) { + return '' + } + + const param = `implicit_${parameters.length}` + + return escapeQueryWithParameters( + `websearch_to_tsquery('english', :${param}) @@ library_item.search_tsv`, + { [param]: value } + ) + } else { + switch (field.name) { + case 'in': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const folder = expression.value?.toString() + if (!folder) { + return '' + } + + switch (folder) { + case InFilter.INBOX: + return 'library_item.archived_at IS NULL' + case InFilter.ARCHIVE: + return 'library_item.archived_at IS NOT NULL' + case InFilter.TRASH: + // return only deleted pages within 14 days + return "library_item.deleted_at >= now() - interval '14 days'" + default: { + const param = `folder_${parameters.length}` + return escapeQueryWithParameters( + `library_item.folder = :${param}`, + { [param]: folder } + ) + } + } + } + + case 'is': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const value = expression.value?.toString() + if (!value) { + return '' + } + + switch (value) { + case ReadFilter.READ: + return 'library_item.reading_progress_bottom_percent > 98' + case ReadFilter.READING: + return 'library_item.reading_progress_bottom_percent BETWEEN 2 AND 98' + case ReadFilter.UNREAD: + return 'library_item.reading_progress_bottom_percent < 2' + default: + throw new Error(`Unexpected keyword: ${value}`) + } + } + case 'type': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const value = expression.value?.toString() + if (!value) { + return '' + } + + const param = `type_${parameters.length}` + + return escapeQueryWithParameters( + `LOWER(library_item.item_type) = :${param}`, + { + [param]: value.toLowerCase(), + } + ) + } + case 'label': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const label = expression.value?.toString()?.toLowerCase() + if (!label) { + return '' + } + + const param = `label_${parameters.length}` + + const hasWildcard = label.includes('*') + if (hasWildcard) { + return escapeQueryWithParameters( + `exists (select 1 from unnest(array_cat(library_item.label_names, library_item.highlight_labels)::text[]) as label where label ILIKE :${param})`, + { + [param]: label.replace(/\*/g, '%'), + } + ) + } + + return escapeQueryWithParameters( + `:${param} = ANY(lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[])`, + { + [param]: label, + } + ) + } + // case 'sort': + // result.sort = parseSort(keyword.value) + // break + case 'has': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const value = expression.value?.toString() + if (!value) { + return '' + } + + switch (value) { + case HasFilter.HIGHLIGHTS: + return "library_item.highlight_annotations <> '{}'" + case HasFilter.LABELS: + return "library_item.label_names <> '{}'" + case HasFilter.SUBSCRIPTIONS: + return 'library_item.subscription is NOT NULL' + default: + throw new Error(`Unexpected keyword: ${value}`) + } + } + case 'saved': + case 'read': + case 'updated': + case 'published': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const date = expression.value?.toString() + if (!date) { + return '' + } + + let startDate: Date | undefined + let endDate: Date | undefined + // check for special date filters + switch (date.toLowerCase()) { + case 'today': + startDate = DateTime.local().startOf('day').toJSDate() + break + case 'yesterday': { + const yesterday = DateTime.local().minus({ days: 1 }) + startDate = yesterday.startOf('day').toJSDate() + endDate = yesterday.endOf('day').toJSDate() + break + } + case 'this week': + startDate = DateTime.local().startOf('week').toJSDate() + break + case 'this month': + startDate = DateTime.local().startOf('month').toJSDate() + break + default: { + // check for date ranges + const [start, end] = date.split('..') + startDate = start && start !== '*' ? new Date(start) : undefined + endDate = end && end !== '*' ? new Date(end) : undefined + } + } + + const startParam = `${field.name}_start_${parameters.length}` + const endParam = `${field.name}_end_${parameters.length}` + + return escapeQueryWithParameters( + `library_item.${field.name}_at BETWEEN :${startParam} AND :${endParam}`, + { + [startParam]: startDate ?? new Date(0), + [endParam]: endDate ?? new Date(), + } + ) + } + // term filters + case 'subscription': + case 'rss': + case 'language': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const value = expression.value?.toString() + if (!value) { + return '' + } + + const columnName = getColumnName(field.name) + const param = `term_${field.name}_${parameters.length}` + + return escapeQueryWithParameters( + `library_item.${columnName} = :${param}`, + { + [param]: value, + } + ) + } + // match filters + case 'author': + case 'title': + case 'description': + case 'note': + case 'site': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + // normalize the term to lower case + const value = expression.value?.toString()?.toLowerCase() + if (!value) { + return '' + } + + const columnName = getColumnName(field.name) + const param = `match_${field.name}_${parameters.length}` + const wildcardParam = `match_${field.name}_wildcard_${parameters.length}` + + return escapeQueryWithParameters( + `(websearch_to_tsquery('english', :${param}) @@ library_item.${columnName}_tsv OR library_item.${columnName} ILIKE :${wildcardParam})`, + { + [param]: value, + [wildcardParam]: `%${value}%`, + } + ) + } + case 'includes': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const ids = expression.value?.toString()?.split(',') + if (!ids || ids.length === 0) { + return '' + } + + const param = `includes_${parameters.length}` + + return escapeQueryWithParameters(`library_item.id = ANY(:${param})`, { + [param]: ids, + }) + } + // case 'recommendedBy': { + // result.recommendedBy = parseStringValue(keyword.value) + // break + // } + case 'no': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const value = expression.value?.toString() + if (!value) { + return '' + } + + let column = '' + switch (value) { + case 'highlight': + column = 'highlight_annotations' + break + case 'label': + column = 'label_names' + break + case 'subscription': + column = 'subscription' + break + default: + throw new Error(`Unexpected keyword: ${value}`) + } + + return `(library_item.${column} = '{}' OR library_item.${column} IS NULL)` + } + case 'mode': + // mode is ignored and used only by the frontend + return '' + case 'readPosition': + case 'wordsCount': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + let value = expression.value?.toString() + if (!value) { + return '' + } + + const column = getColumnName(field.name) + + const operatorRegex = /([<>]=?)/ + const operator = value.match(operatorRegex)?.[0] + if (!operator) { + return '' + } + + value = value.replace(operatorRegex, '') + if (!value) { + return '' + } + + const param = `range_${field.name}_${parameters.length}` + + return escapeQueryWithParameters( + `library_item.${column} ${operator} :${param}`, + { + [param]: parseInt(value, 10), + } + ) + } + default: + throw new Error(`Unexpected keyword: ${field.name}`) + } + } + } + + const serialize = (ast: LiqeQuery): string => { + if (ast.type === 'Tag') { + return serializeTagExpression(ast) + } + + if (ast.type === 'LogicalExpression') { + let operator = '' + if (ast.operator.operator === 'AND') { + operator = 'AND' + } else if (ast.operator.operator === 'OR') { + operator = 'OR' + } else { + throw new Error('Unexpected operator.') + } + + return `${serialize(ast.left)} ${operator} ${serialize(ast.right)}` + } + + if (ast.type === 'UnaryOperator') { + return `NOT ${serialize(ast.operand)}` + } + if (ast.type === 'ParenthesizedExpression') { + return `(${serialize(ast.expression)})` + } + + throw new Error('Missing AST type.') + } + + return serialize(searchQuery) +} + const buildWhereClause = ( queryBuilder: SelectQueryBuilder, args: SearchArgs @@ -357,8 +764,15 @@ export const searchLibraryItems = async ( .select(selectColumns) .where('library_item.user_id = :userId', { userId }) - // build the where clause - buildWhereClause(queryBuilder, args) + if (args.searchQuery) { + const parameters: ObjectLiteral[] = [] + const whereClause = buildQuery(args.searchQuery, parameters) + whereClause && + queryBuilder.andWhere( + whereClause, + parameters.reduce((a, b) => ({ ...a, ...b }), {}) + ) + } const libraryItems = await queryBuilder .addOrderBy(`library_item.${sortField}`, sortOrder, 'NULLS LAST') @@ -671,7 +1085,7 @@ export const countByCreatedAt = async ( export const updateLibraryItems = async ( action: BulkActionType, - searchArgs: SearchArgs, + searchQuery: LiqeQuery, userId: string, labels?: Label[], args?: unknown @@ -731,7 +1145,7 @@ export const updateLibraryItems = async ( .where('library_item.user_id = :userId', { userId }) // build the where clause - buildWhereClause(queryBuilder, searchArgs) + // buildWhereClause(queryBuilder, searchQuery) if (addLabels) { if (!labels) { diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index bfd0fe1e0..d4098ffc3 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -3,20 +3,16 @@ /* eslint-disable @typescript-eslint/no-unsafe-call */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { LiqeQuery, parse } from 'liqe' import { DateTime } from 'luxon' -import { - ISearchParserDictionary, - parse, - SearchParserKeyWordOffset, - SearchParserTextOffset, -} from 'search-query-parser' +import { ISearchParserDictionary } from 'search-query-parser' import { InputMaybe, PageType, SortParams } from '../generated/graphql' export enum ReadFilter { - ALL, - READ, - READING, - UNREAD, + ALL = 'all', + READ = 'read', + READING = 'reading', + UNREAD = 'unread', } export enum InFilter { @@ -56,9 +52,9 @@ export type LabelFilter = { } export enum HasFilter { - HIGHLIGHTS, - LABELS, - SUBSCRIPTIONS, + HIGHLIGHTS = 'highlights', + LABELS = 'labels', + SUBSCRIPTIONS = 'subscriptions', } export interface DateFilter { @@ -394,168 +390,164 @@ const parseNoFilter = (str?: string): NoFilter | undefined => { return undefined } -export const parseSearchQuery = (query: string | undefined): SearchFilter => { +export const parseSearchQuery = (query: string): LiqeQuery => { const searchQuery = query - ? query - .replace(/\W\s":/g, '') - .replace('in:subscription', 'has:subscriptions') // compatibility with old search - .replace('in:library', 'no:subscription') // compatibility with old search - : undefined - const result: SearchFilter = { - query: searchQuery, - readFilter: ReadFilter.ALL, - inFilter: searchQuery ? InFilter.ALL : InFilter.INBOX, - labelFilters: [], - hasFilters: [], - dateFilters: [], - termFilters: [], - matchFilters: [], - ids: [], - noFilters: [], - rangeFilters: [], - } + .replace(/\W\s":/g, '') + .replace('in:subscription', 'has:subscriptions') // compatibility with old search + .replace('in:library', 'no:subscription') // compatibility with old search + // const result: SearchFilter = { + // query: searchQuery, + // readFilter: ReadFilter.ALL, + // inFilter: searchQuery ? InFilter.ALL : InFilter.INBOX, + // labelFilters: [], + // hasFilters: [], + // dateFilters: [], + // termFilters: [], + // matchFilters: [], + // ids: [], + // noFilters: [], + // rangeFilters: [], + // } - if (!searchQuery) { - return result - } + // if (!searchQuery) { + // return result + // } - const parsed = parse(searchQuery, { - keywords: [ - 'in', - 'is', - 'type', - 'label', - 'sort', - 'has', - 'saved', - 'author', - 'published', - 'subscription', - 'language', - 'title', - 'description', - 'content', - 'updated', - 'includes', - 'recommendedBy', - 'no', - 'mode', - 'site', - 'note', - 'rss', - 'wordsCount', - 'readPosition', - 'use', - ], - tokenize: true, - }) - if (parsed.offsets) { - const texts = parsed.offsets - .filter((offset) => 'text' in offset) - .map((offset) => offset as SearchParserTextOffset) + return parse(searchQuery) - if (texts.length > 0) { - result.query = texts - .map((offset: SearchParserTextOffset) => { - // TODO: the parser library doesn't let us accurately - // pull out quoted text, so we are just assuming - // anything with spaces is quoted. - if (offset.text.indexOf(' ') > -1) { - return `"${offset.text}"` - } - return offset.text - }) - .join(' ') - } else { - result.query = undefined - } + // const parsed = parse(searchQuery, { + // keywords: [ + // 'in', + // 'is', + // 'type', + // 'label', + // 'sort', + // 'has', + // 'saved', + // 'author', + // 'published', + // 'subscription', + // 'language', + // 'title', + // 'description', + // 'content', + // 'updated', + // 'includes', + // 'recommendedBy', + // 'no', + // 'mode', + // 'site', + // 'note', + // 'rss', + // 'wordsCount', + // 'readPosition', + // ], + // tokenize: true, + // }) + // if (parsed.offsets) { + // const texts = parsed.offsets + // .filter((offset) => 'text' in offset) + // .map((offset) => offset as SearchParserTextOffset) - const keywords = parsed.offsets - .filter((offset) => 'keyword' in offset) - .map((offset) => offset as SearchParserKeyWordOffset) + // if (texts.length > 0) { + // result.query = texts + // .map((offset: SearchParserTextOffset) => { + // // TODO: the parser library doesn't let us accurately + // // pull out quoted text, so we are just assuming + // // anything with spaces is quoted. + // if (offset.text.indexOf(' ') > -1) { + // return `"${offset.text}"` + // } + // return offset.text + // }) + // .join(' ') + // } else { + // result.query = undefined + // } - for (const keyword of keywords) { - switch (keyword.keyword) { - case 'in': - result.inFilter = parseInFilter(keyword.value, result.query) - break - case 'is': - result.readFilter = parseIsFilter(keyword.value) - break - case 'type': - result.typeFilter = parseTypeFilter(keyword.value) - break - case 'label': { - const labelFilter = parseLabelFilter(keyword.value, parsed.exclude) - labelFilter && result.labelFilters.push(labelFilter) - break - } - case 'sort': - result.sort = parseSort(keyword.value) - break - case 'has': { - const hasFilter = parseHasFilter(keyword.value) - hasFilter !== undefined && result.hasFilters.push(hasFilter) - break - } - case 'saved': - case 'read': - case 'updated': - case 'published': { - const dateFilter = parseDateFilter(keyword.keyword, keyword.value) - dateFilter && result.dateFilters.push(dateFilter) - break - } - // term filters - case 'subscription': - case 'rss': - case 'language': { - const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value) - fieldFilter && result.termFilters.push(fieldFilter) - break - } - // match filters - case 'author': - case 'title': - case 'description': - case 'note': - case 'site': - case 'content': { - const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value) - fieldFilter && result.matchFilters.push(fieldFilter) - break - } - case 'includes': { - const ids = parseIds(keyword.value) - ids && result.ids.push(...ids) - break - } - case 'recommendedBy': { - result.recommendedBy = parseStringValue(keyword.value) - break - } - case 'no': { - const noFilter = parseNoFilter(keyword.value) - noFilter && result.noFilters.push(noFilter) - break - } - case 'mode': - // mode is ignored and used only by the frontend - break - case 'readPosition': - case 'wordsCount': { - const rangeFilter = parseRangeFilter(keyword.keyword, keyword.value) - rangeFilter && result.rangeFilters.push(rangeFilter) - break - } - case 'use': - result.useFolders = keyword.value === 'folders' - break - } - } - } + // const keywords = parsed.offsets + // .filter((offset) => 'keyword' in offset) + // .map((offset) => offset as SearchParserKeyWordOffset) - return result + // for (const keyword of keywords) { + // switch (keyword.keyword) { + // case 'in': + // result.inFilter = parseInFilter(keyword.value, result.query) + // break + // case 'is': + // result.readFilter = parseIsFilter(keyword.value) + // break + // case 'type': + // result.typeFilter = parseTypeFilter(keyword.value) + // break + // case 'label': { + // const labelFilter = parseLabelFilter(keyword.value, parsed.exclude) + // labelFilter && result.labelFilters.push(labelFilter) + // break + // } + // case 'sort': + // result.sort = parseSort(keyword.value) + // break + // case 'has': { + // const hasFilter = parseHasFilter(keyword.value) + // hasFilter !== undefined && result.hasFilters.push(hasFilter) + // break + // } + // case 'saved': + // case 'read': + // case 'updated': + // case 'published': { + // const dateFilter = parseDateFilter(keyword.keyword, keyword.value) + // dateFilter && result.dateFilters.push(dateFilter) + // break + // } + // // term filters + // case 'subscription': + // case 'rss': + // case 'language': { + // const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value) + // fieldFilter && result.termFilters.push(fieldFilter) + // break + // } + // // match filters + // case 'author': + // case 'title': + // case 'description': + // case 'note': + // case 'site': + // case 'content': { + // const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value) + // fieldFilter && result.matchFilters.push(fieldFilter) + // break + // } + // case 'includes': { + // const ids = parseIds(keyword.value) + // ids && result.ids.push(...ids) + // break + // } + // case 'recommendedBy': { + // result.recommendedBy = parseStringValue(keyword.value) + // break + // } + // case 'no': { + // const noFilter = parseNoFilter(keyword.value) + // noFilter && result.noFilters.push(noFilter) + // break + // } + // case 'mode': + // // mode is ignored and used only by the frontend + // break + // case 'readPosition': + // case 'wordsCount': { + // const rangeFilter = parseRangeFilter(keyword.keyword, keyword.value) + // rangeFilter && result.rangeFilters.push(rangeFilter) + // break + // } + // } + // } + // } + + // return result } export const sortParamsToSort = ( diff --git a/yarn.lock b/yarn.lock index 10eb47ae5..f5c3158be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18804,6 +18804,14 @@ linkify-it@^4.0.1: dependencies: uc.micro "^1.0.1" +liqe@^3.8.0: + version "3.8.0" + resolved "https://registry.yarnpkg.com/liqe/-/liqe-3.8.0.tgz#e96c0733ecd3efb613d4404023433f15ae4f5849" + integrity sha512-cZ1rDx4XzxONBTskSPBp7/KwJ9qbUdF8EPnY4VjKXwHF1Krz9lgnlMTh1G7kd+KtPYvUte1mhuZeQSnk7KiSBg== + dependencies: + nearley "^2.20.1" + ts-error "^1.0.6" + listr-silent-renderer@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz#924b5a3757153770bf1a8e3fbf74b8bbf3f9242e" @@ -26919,6 +26927,11 @@ ts-dedent@^2.0.0, ts-dedent@^2.2.0: resolved "https://registry.yarnpkg.com/ts-dedent/-/ts-dedent-2.2.0.tgz#39e4bd297cd036292ae2394eb3412be63f563bb5" integrity sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ== +ts-error@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/ts-error/-/ts-error-1.0.6.tgz#277496f2a28de6c184cfce8dfd5cdd03a4e6b0fc" + integrity sha512-tLJxacIQUM82IR7JO1UUkKlYuUTmoY9HBJAmNWFzheSlDS5SPMcNIepejHJa4BpPQLAcbRhRf3GDJzyj6rbKvA== + ts-loader@^9.3.0: version "9.3.0" resolved "https://registry.yarnpkg.com/ts-loader/-/ts-loader-9.3.0.tgz#980f4dbfb60e517179e15e10ed98e454b132159f" From 88a61d962f01640053e5180b456d19d464ccef64 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 30 Nov 2023 18:13:49 +0800 Subject: [PATCH 02/10] parse sort parameter --- packages/api/src/resolvers/article/index.ts | 7 +- packages/api/src/services/library_item.ts | 199 +++++++++++++++----- packages/api/src/utils/search.ts | 82 +------- 3 files changed, 160 insertions(+), 128 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 9b154f0c1..e45de8910 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -70,6 +70,7 @@ import { findLibraryItemByUrl, findLibraryItemsByPrefix, searchLibraryItems, + sortParamsToSort, updateLibraryItem, updateLibraryItemReadingProgress, updateLibraryItems, @@ -101,11 +102,7 @@ import { ParsedContentPuppeteer, parsePreparedContent, } from '../../utils/parser' -import { - InFilter, - parseSearchQuery, - sortParamsToSort, -} from '../../utils/search' +import { InFilter, parseSearchQuery } from '../../utils/search' import { getStorageFileDetails } from '../../utils/uploads' import { itemTypeForContentType } from '../upload_files' diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index d6f59f099..ae151ba71 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -11,7 +11,7 @@ import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' import { LibraryItem, LibraryItemState } from '../entity/library_item' -import { BulkActionType } from '../generated/graphql' +import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql' import { createPubSubClient, EntityType } from '../pubsub' import { authTrx, getColumns } from '../repository' import { libraryItemRepository } from '../repository/library_item' @@ -28,9 +28,6 @@ import { NoFilter, RangeFilter, ReadFilter, - Sort, - SortBy, - SortOrder, } from '../utils/search' export interface SearchArgs { @@ -90,6 +87,52 @@ export interface SearchResultItem { content?: string } +export enum SortBy { + SAVED = 'saved_at', + UPDATED = 'updated_at', + PUBLISHED = 'published_at', + READ = 'read_at', + WORDS_COUNT = 'word_count', +} + +export enum SortOrder { + ASCENDING = 'ASC', + DESCENDING = 'DESC', +} + +export interface Sort { + by: SortBy + order?: SortOrder +} + +interface Select { + column: string + alias: string +} + +export const sortParamsToSort = ( + sortParams: InputMaybe | undefined +) => { + const sort = { by: SortBy.UPDATED, order: SortOrder.DESCENDING } + + if (sortParams) { + sortParams.order === 'ASCENDING' && (sort.order = SortOrder.ASCENDING) + switch (sortParams.by) { + case 'UPDATED_TIME': + sort.by = SortBy.UPDATED + break + case 'PUBLISHED_AT': + sort.by = SortBy.PUBLISHED + break + case 'SAVED_AT': + sort.by = SortBy.SAVED + break + } + } + + return sort +} + const getColumnName = (field: string) => { switch (field) { case 'language': @@ -103,6 +146,11 @@ const getColumnName = (field: string) => { return 'word_count' case 'readPosition': return 'reading_progress_bottom_percent' + case 'saved': + case 'read': + case 'updated': + case 'published': + return `${field}_at` default: return field } @@ -110,7 +158,8 @@ const getColumnName = (field: string) => { export const buildQuery = ( searchQuery: LiqeQuery, - parameters: ObjectLiteral[] + parameters: ObjectLiteral[], + orders: Sort[] ) => { const escapeQueryWithParameters = ( query: string, @@ -120,7 +169,7 @@ export const buildQuery = ( return query } - const serializeTagExpression = (ast: LiqeQuery): string => { + const serializeTagExpression = (ast: LiqeQuery): string | null => { if (ast.type !== 'Tag') { throw new Error('Expected a tag expression.') } @@ -134,8 +183,8 @@ export const buildQuery = ( const value = expression.value?.toString() - if (!value) { - return '' + if (value === undefined || value === '') { + return null } const param = `implicit_${parameters.length}` @@ -153,7 +202,7 @@ export const buildQuery = ( const folder = expression.value?.toString() if (!folder) { - return '' + throw new Error('Expected a value.') } switch (folder) { @@ -181,7 +230,7 @@ export const buildQuery = ( const value = expression.value?.toString() if (!value) { - return '' + throw new Error('Expected a value.') } switch (value) { @@ -202,7 +251,7 @@ export const buildQuery = ( const value = expression.value?.toString() if (!value) { - return '' + throw new Error('Expected a value.') } const param = `type_${parameters.length}` @@ -221,7 +270,7 @@ export const buildQuery = ( const label = expression.value?.toString()?.toLowerCase() if (!label) { - return '' + throw new Error('Expected a value.') } const param = `label_${parameters.length}` @@ -243,9 +292,26 @@ export const buildQuery = ( } ) } - // case 'sort': - // result.sort = parseSort(keyword.value) - // break + case 'sort': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const value = expression.value?.toString() + if (!value) { + throw new Error('Expected a value.') + } + + const [sort, sortOrder] = value.split('-') + const order = + sortOrder?.toUpperCase() === 'ASC' + ? SortOrder.ASCENDING + : SortOrder.DESCENDING + + const column = getColumnName(sort) + orders.push({ by: column as SortBy, order }) + return null + } case 'has': { if (expression.type !== 'LiteralExpression') { throw new Error('Expected a literal expression.') @@ -253,7 +319,7 @@ export const buildQuery = ( const value = expression.value?.toString() if (!value) { - return '' + throw new Error('Expected a value.') } switch (value) { @@ -277,7 +343,7 @@ export const buildQuery = ( const date = expression.value?.toString() if (!date) { - return '' + throw new Error('Expected a value.') } let startDate: Date | undefined @@ -328,7 +394,7 @@ export const buildQuery = ( const value = expression.value?.toString() if (!value) { - return '' + throw new Error('Expected a value.') } const columnName = getColumnName(field.name) @@ -354,7 +420,7 @@ export const buildQuery = ( // normalize the term to lower case const value = expression.value?.toString()?.toLowerCase() if (!value) { - return '' + throw new Error('Expected a value.') } const columnName = getColumnName(field.name) @@ -376,7 +442,7 @@ export const buildQuery = ( const ids = expression.value?.toString()?.split(',') if (!ids || ids.length === 0) { - return '' + throw new Error('Expected a value.') } const param = `includes_${parameters.length}` @@ -385,10 +451,25 @@ export const buildQuery = ( [param]: ids, }) } - // case 'recommendedBy': { - // result.recommendedBy = parseStringValue(keyword.value) - // break - // } + case 'recommendedBy': { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const value = expression.value?.toString() + if (!value) { + throw new Error('Expected a value.') + } + + const param = `recommendedBy_${parameters.length}` + + return escapeQueryWithParameters( + `:recommendedBy = ANY(lower(library_item.recommender_names::text)::text[])`, + { + [param]: value.toLowerCase(), + } + ) + } case 'no': { if (expression.type !== 'LiteralExpression') { throw new Error('Expected a literal expression.') @@ -396,7 +477,7 @@ export const buildQuery = ( const value = expression.value?.toString() if (!value) { - return '' + throw new Error('Expected a value.') } let column = '' @@ -418,7 +499,7 @@ export const buildQuery = ( } case 'mode': // mode is ignored and used only by the frontend - return '' + return null case 'readPosition': case 'wordsCount': { if (expression.type !== 'LiteralExpression') { @@ -427,7 +508,7 @@ export const buildQuery = ( let value = expression.value?.toString() if (!value) { - return '' + throw new Error('Expected a value.') } const column = getColumnName(field.name) @@ -435,12 +516,12 @@ export const buildQuery = ( const operatorRegex = /([<>]=?)/ const operator = value.match(operatorRegex)?.[0] if (!operator) { - return '' + throw new Error('Expected a value.') } value = value.replace(operatorRegex, '') if (!value) { - return '' + throw new Error('Expected a value.') } const param = `range_${field.name}_${parameters.length}` @@ -458,7 +539,7 @@ export const buildQuery = ( } } - const serialize = (ast: LiqeQuery): string => { + const serialize = (ast: LiqeQuery): string | null => { if (ast.type === 'Tag') { return serializeTagExpression(ast) } @@ -473,14 +554,42 @@ export const buildQuery = ( throw new Error('Unexpected operator.') } - return `${serialize(ast.left)} ${operator} ${serialize(ast.right)}` + const left = serialize(ast.left) + const right = serialize(ast.right) + + if (!left && !right) { + return null + } + + if (!left) { + return right + } + + if (!right) { + return left + } + + return `(${left} ${operator} ${right})` } if (ast.type === 'UnaryOperator') { - return `NOT ${serialize(ast.operand)}` + const serialized = serialize(ast.operand) + + if (!serialized) { + return null + } + + return `NOT ${serialized}` } + if (ast.type === 'ParenthesizedExpression') { - return `(${serialize(ast.expression)})` + const serialized = serialize(ast.expression) + + if (!serialized) { + return null + } + + return `(${serialized})` } throw new Error('Missing AST type.') @@ -763,22 +872,28 @@ export const searchLibraryItems = async ( .createQueryBuilder(LibraryItem, 'library_item') .select(selectColumns) .where('library_item.user_id = :userId', { userId }) + .orderBy(`library_item.${sortField}`, sortOrder, 'NULLS LAST') if (args.searchQuery) { const parameters: ObjectLiteral[] = [] - const whereClause = buildQuery(args.searchQuery, parameters) + const orders: Sort[] = [] + const whereClause = buildQuery(args.searchQuery, parameters, orders) whereClause && - queryBuilder.andWhere( - whereClause, - parameters.reduce((a, b) => ({ ...a, ...b }), {}) + queryBuilder + .andWhere(whereClause) + .setParameters(parameters.reduce((a, b) => ({ ...a, ...b }), {})) + + // add order by + orders.forEach((order) => { + queryBuilder.addOrderBy( + `library_item.${order.by}`, + order.order, + 'NULLS LAST' ) + }) } - const libraryItems = await queryBuilder - .addOrderBy(`library_item.${sortField}`, sortOrder, 'NULLS LAST') - .skip(from) - .take(size) - .getMany() + const libraryItems = await queryBuilder.skip(from).take(size).getMany() const count = await queryBuilder.getCount() diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index d4098ffc3..f6bd870ac 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -6,7 +6,7 @@ import { LiqeQuery, parse } from 'liqe' import { DateTime } from 'luxon' import { ISearchParserDictionary } from 'search-query-parser' -import { InputMaybe, PageType, SortParams } from '../generated/graphql' +import { PageType } from '../generated/graphql' export enum ReadFilter { ALL = 'all', @@ -29,7 +29,6 @@ export interface SearchFilter { readFilter: ReadFilter typeFilter?: string labelFilters: LabelFilter[] - sort?: Sort hasFilters: HasFilter[] dateFilters: DateFilter[] termFilters: FieldFilter[] @@ -69,24 +68,6 @@ export interface RangeFilter { value: number } -export enum SortBy { - SAVED = 'savedAt', - UPDATED = 'updatedAt', - PUBLISHED = 'publishedAt', - READ = 'readAt', - WORDS_COUNT = 'wordCount', -} - -export enum SortOrder { - ASCENDING = 'ASC', - DESCENDING = 'DESC', -} - -export interface Sort { - by: SortBy - order?: SortOrder -} - export interface FieldFilter { nested?: boolean field: string @@ -180,44 +161,6 @@ const parseLabelFilter = ( } } -const parseSort = (str?: string): Sort | undefined => { - if (str === undefined) { - return undefined - } - - const [sort, order] = str.split('-') - const sortOrder = - order?.toUpperCase() === 'ASC' ? SortOrder.ASCENDING : SortOrder.DESCENDING - - switch (sort.toUpperCase()) { - case 'UPDATED': - return { - by: SortBy.UPDATED, - order: sortOrder, - } - case 'SAVED': - return { - by: SortBy.SAVED, - order: sortOrder, - } - case 'PUBLISHED': - return { - by: SortBy.PUBLISHED, - order: sortOrder, - } - case 'READ': - return { - by: SortBy.READ, - order: sortOrder, - } - case 'WORDSCOUNT': - return { - by: SortBy.WORDS_COUNT, - order: sortOrder, - } - } -} - const parseHasFilter = (str?: string): HasFilter | undefined => { if (str === undefined) { return undefined @@ -549,26 +492,3 @@ export const parseSearchQuery = (query: string): LiqeQuery => { // return result } - -export const sortParamsToSort = ( - sortParams: InputMaybe | undefined -) => { - const sort = { by: SortBy.UPDATED, order: SortOrder.DESCENDING } - - if (sortParams) { - sortParams.order === 'ASCENDING' && (sort.order = SortOrder.ASCENDING) - switch (sortParams.by) { - case 'UPDATED_TIME': - sort.by = SortBy.UPDATED - break - case 'PUBLISHED_AT': - sort.by = SortBy.PUBLISHED - break - case 'SAVED_AT': - sort.by = SortBy.SAVED - break - } - } - - return sort -} From 073bd8b43a209adceb9805322122ca786d8ed9b4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 30 Nov 2023 19:12:22 +0800 Subject: [PATCH 03/10] fix in:trash --- packages/api/src/resolvers/article/index.ts | 9 +++- packages/api/src/services/library_item.ts | 47 ++++++++++++++++----- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index e45de8910..9340d6d23 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -660,6 +660,7 @@ export const searchResolver = authorized< size: first + 1, // fetch one more item to get next cursor includePending: true, includeContent: !!params.includeContent, + includeDeleted: params.query?.includes('in:trash'), searchQuery, }, uid @@ -757,14 +758,18 @@ export const updatesSinceResolver = authorized< startDate = new Date(0) } + folder = folder || InFilter.ALL + const searchQuery = parseSearchQuery( + `in:${folder} updated:${startDate.toISOString()}` + ) + const { libraryItems, count } = await searchLibraryItems( { from: Number(startCursor), size: size + 1, // fetch one more item to get next cursor includeDeleted: true, - dateFilters: [{ field: 'updatedAt', startDate }], sort, - inFilter: (folder as InFilter) || InFilter.ALL, + searchQuery, }, uid ) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index ae151ba71..58d5254be 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -158,8 +158,9 @@ const getColumnName = (field: string) => { export const buildQuery = ( searchQuery: LiqeQuery, - parameters: ObjectLiteral[], - orders: Sort[] + parameters: ObjectLiteral[] = [], + selects: Select[] = [], + orders: { by: string; order?: SortOrder }[] = [] ) => { const escapeQueryWithParameters = ( query: string, @@ -187,7 +188,17 @@ export const buildQuery = ( return null } - const param = `implicit_${parameters.length}` + const param = 'implicit_field' + const alias = 'rank' + selects.push({ + column: `ts_rank_cd(library_item.search_tsv, websearch_to_tsquery('english', :${param}))`, + alias, + }) + + orders.push({ + by: alias, + order: SortOrder.DESCENDING, + }) return escapeQueryWithParameters( `websearch_to_tsquery('english', :${param}) @@ library_item.search_tsv`, @@ -206,6 +217,8 @@ export const buildQuery = ( } switch (folder) { + case InFilter.ALL: + return null case InFilter.INBOX: return 'library_item.archived_at IS NULL' case InFilter.ARCHIVE: @@ -309,7 +322,7 @@ export const buildQuery = ( : SortOrder.DESCENDING const column = getColumnName(sort) - orders.push({ by: column as SortBy, order }) + orders.push({ by: `library_item.${column}`, order }) return null } case 'has': { @@ -876,23 +889,37 @@ export const searchLibraryItems = async ( if (args.searchQuery) { const parameters: ObjectLiteral[] = [] + const selects: Select[] = [] const orders: Sort[] = [] - const whereClause = buildQuery(args.searchQuery, parameters, orders) + const whereClause = buildQuery( + args.searchQuery, + parameters, + selects, + orders + ) whereClause && queryBuilder .andWhere(whereClause) .setParameters(parameters.reduce((a, b) => ({ ...a, ...b }), {})) + selects.forEach((select) => { + queryBuilder.addSelect(select.column, select.alias) + }) + // add order by orders.forEach((order) => { - queryBuilder.addOrderBy( - `library_item.${order.by}`, - order.order, - 'NULLS LAST' - ) + queryBuilder.addOrderBy(order.by, order.order, 'NULLS LAST') }) } + if (!args.includePending) { + queryBuilder.andWhere("library_item.state <> 'PROCESSING'") + } + + if (!args.includeDeleted) { + queryBuilder.andWhere("library_item.state <> 'DELETED'") + } + const libraryItems = await queryBuilder.skip(from).take(size).getMany() const count = await queryBuilder.getCount() From 3c67d1911cd6497a54c1aaa1c2e2ce0b3fb9cd96 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Dec 2023 12:31:26 +0800 Subject: [PATCH 04/10] fix tests --- packages/api/src/resolvers/article/index.ts | 17 +- packages/api/src/services/library_item.ts | 67 +++++--- packages/api/src/utils/search.ts | 19 +-- packages/api/test/resolvers/article.test.ts | 6 +- packages/api/test/utils/search.test.ts | 168 -------------------- 5 files changed, 55 insertions(+), 222 deletions(-) delete mode 100644 packages/api/test/utils/search.test.ts diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 9340d6d23..d3b598623 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -90,7 +90,6 @@ import { generateSlug, isParsingTimeout, libraryItemToArticle, - libraryItemToArticleSavingRequest, libraryItemToSearchItem, titleForFilePath, userDataToUser, @@ -758,6 +757,7 @@ export const updatesSinceResolver = authorized< startDate = new Date(0) } + // create a search query folder = folder || InFilter.ALL const searchQuery = parseSearchQuery( `in:${folder} updated:${startDate.toISOString()}` @@ -827,16 +827,10 @@ export const bulkActionResolver = authorized< }, }) - if (!query) { - return { errorCodes: [BulkActionErrorCode.BadRequest] } - } - - // parse query - const searchQuery = parseSearchQuery(query) - const ids = searchQuery.getValue?.('includes') as string[] - if (!ids || ids.length === 0 || ids.length > 100) { - return { errorCodes: [BulkActionErrorCode.BadRequest] } - } + // the query size is limited to 255 characters + if (!query || query.length > 255) { + return { errorCodes: [BulkActionErrorCode.BadRequest] } + } // get labels if needed let labels = undefined @@ -848,6 +842,7 @@ export const bulkActionResolver = authorized< labels = await findLabelsByIds(labelIds, uid) } + const searchQuery = parseSearchQuery(query) await updateLibraryItems(action, searchQuery, uid, labels, args) return { success: true } diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 58d5254be..fdd646380 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -281,28 +281,37 @@ export const buildQuery = ( throw new Error('Expected a literal expression.') } - const label = expression.value?.toString()?.toLowerCase() - if (!label) { + const value = expression.value?.toString()?.toLowerCase() + if (!value) { throw new Error('Expected a value.') } - const param = `label_${parameters.length}` + const labels = value.split(',') + return ( + labels + .map((label) => { + const param = `label_${parameters.length}` - const hasWildcard = label.includes('*') - if (hasWildcard) { - return escapeQueryWithParameters( - `exists (select 1 from unnest(array_cat(library_item.label_names, library_item.highlight_labels)::text[]) as label where label ILIKE :${param})`, - { - [param]: label.replace(/\*/g, '%'), - } - ) - } + const hasWildcard = label.includes('*') + if (hasWildcard) { + return escapeQueryWithParameters( + `exists (select 1 from unnest(array_cat(library_item.label_names, library_item.highlight_labels)::text[]) as label where label ILIKE :${param})`, + { + [param]: label.replace(/\*/g, '%'), + } + ) + } - return escapeQueryWithParameters( - `:${param} = ANY(lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[])`, - { - [param]: label, - } + return escapeQueryWithParameters( + `:${param} = ANY(lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[])`, + { + [param]: label, + } + ) + }) + .join(' OR ') + // wrap in brackets to avoid precedence issues + .replace(/^(.*)$/, '($1)') ) } case 'sort': { @@ -475,9 +484,13 @@ export const buildQuery = ( } const param = `recommendedBy_${parameters.length}` + if (value === '*') { + // select all if * is provided + return "library_item.recommender_names <> '{}'" + } return escapeQueryWithParameters( - `:recommendedBy = ANY(lower(library_item.recommender_names::text)::text[])`, + `:${param} = ANY(lower(library_item.recommender_names::text)::text[])`, { [param]: value.toLowerCase(), } @@ -582,7 +595,7 @@ export const buildQuery = ( return left } - return `(${left} ${operator} ${right})` + return `${left} ${operator} ${right}` } if (ast.type === 'UnaryOperator') { @@ -885,7 +898,6 @@ export const searchLibraryItems = async ( .createQueryBuilder(LibraryItem, 'library_item') .select(selectColumns) .where('library_item.user_id = :userId', { userId }) - .orderBy(`library_item.${sortField}`, sortOrder, 'NULLS LAST') if (args.searchQuery) { const parameters: ObjectLiteral[] = [] @@ -920,7 +932,11 @@ export const searchLibraryItems = async ( queryBuilder.andWhere("library_item.state <> 'DELETED'") } - const libraryItems = await queryBuilder.skip(from).take(size).getMany() + const libraryItems = await queryBuilder + .addOrderBy(`library_item.${sortField}`, sortOrder, 'NULLS LAST') + .skip(from) + .take(size) + .getMany() const count = await queryBuilder.getCount() @@ -1286,8 +1302,13 @@ export const updateLibraryItems = async ( .createQueryBuilder(LibraryItem, 'library_item') .where('library_item.user_id = :userId', { userId }) - // build the where clause - // buildWhereClause(queryBuilder, searchQuery) + const parameters: ObjectLiteral[] = [] + const whereClause = buildQuery(searchQuery, parameters) + if (whereClause) { + queryBuilder + .andWhere(whereClause) + .setParameters(parameters.reduce((a, b) => ({ ...a, ...b }), {})) + } if (addLabels) { if (!labels) { diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index f6bd870ac..8c1d82669 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -338,23 +338,8 @@ export const parseSearchQuery = (query: string): LiqeQuery => { .replace(/\W\s":/g, '') .replace('in:subscription', 'has:subscriptions') // compatibility with old search .replace('in:library', 'no:subscription') // compatibility with old search - // const result: SearchFilter = { - // query: searchQuery, - // readFilter: ReadFilter.ALL, - // inFilter: searchQuery ? InFilter.ALL : InFilter.INBOX, - // labelFilters: [], - // hasFilters: [], - // dateFilters: [], - // termFilters: [], - // matchFilters: [], - // ids: [], - // noFilters: [], - // rangeFilters: [], - // } - - // if (!searchQuery) { - // return result - // } + // wrap the value behind colon in quotes if it's not already + .replace(/(\w+):([^"\s]+)/g, '$1:"$2"') return parse(searchQuery) diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index dda47a53f..17b66eff2 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -569,14 +569,14 @@ describe('Article API', () => { ).expect(200) // Save a link, then archive it - let allLinks = await graphqlRequest(searchQuery(''), authToken).expect( + let allLinks = await graphqlRequest(searchQuery('in:inbox'), authToken).expect( 200 ) const justSavedId = allLinks.body.data.search.edges[0].node.id await archiveLink(authToken, justSavedId) // test the negative case, ensuring the archive link wasn't returned - allLinks = await graphqlRequest(searchQuery(''), authToken).expect(200) + allLinks = await graphqlRequest(searchQuery('in:inbox'), authToken).expect(200) expect(allLinks.body.data.search.edges[0]?.node?.url).to.not.eq(url) // Now save the link again, and ensure it is returned @@ -585,7 +585,7 @@ describe('Article API', () => { authToken ).expect(200) - allLinks = await graphqlRequest(searchQuery(''), authToken).expect(200) + allLinks = await graphqlRequest(searchQuery('in:inbox'), authToken).expect(200) expect(allLinks.body.data.search.edges[0].node.id).to.eq(justSavedId) expect(allLinks.body.data.search.edges[0].node.url).to.eq(url) }) diff --git a/packages/api/test/utils/search.test.ts b/packages/api/test/utils/search.test.ts deleted file mode 100644 index 5518e84b0..000000000 --- a/packages/api/test/utils/search.test.ts +++ /dev/null @@ -1,168 +0,0 @@ -import 'mocha' -import { expect } from 'chai' -import { InFilter, parseSearchQuery, ReadFilter } from '../../src/utils/search' -import { PageType } from '../../src/generated/graphql' - -describe('undefined query', () => { - it('returns an empty result with read state ALL and no typeFilter', () => { - const result = parseSearchQuery(undefined) - expect(result.query).to.be.undefined - expect(result.readFilter).to.eq(ReadFilter.ALL) - expect(result.typeFilter).to.be.undefined - }) -}) - -describe('empty query', () => { - it('returns an empty result with read state ALL and no typefilter', () => { - const result = parseSearchQuery('') - expect(result.query).to.be.undefined - expect(result.readFilter).to.eq(ReadFilter.ALL) - expect(result.typeFilter).to.be.undefined - }) -}) - -describe('query with READ read state', () => { - it('returns a READ result', () => { - const result = parseSearchQuery('is:read') - expect(result.query).to.be.undefined - expect(result.readFilter).to.eq(ReadFilter.READ) - expect(result.typeFilter).to.be.undefined - }) -}) - -describe('query with UNREAD read state', () => { - it('returns a UNREAD result', () => { - const result = parseSearchQuery('is:unread') - expect(result.query).to.be.undefined - expect(result.readFilter).to.eq(ReadFilter.UNREAD) - expect(result.typeFilter).to.be.undefined - }) -}) - -describe('query with multiple read states', () => { - it('just uses the last one', () => { - const result = parseSearchQuery('is:unread is:read') - expect(result.query).to.be.undefined - expect(result.readFilter).to.eq(ReadFilter.READ) - expect(result.typeFilter).to.be.undefined - }) -}) - -describe('query with invalid read states', () => { - it('returns ALL', () => { - const result = parseSearchQuery('is:invalid') - expect(result.query).to.be.undefined - expect(result.readFilter).to.eq(ReadFilter.ALL) - expect(result.typeFilter).to.be.undefined - }) -}) - -describe('query with read state before search query', () => { - it('sets read state and query', () => { - const result = parseSearchQuery('is:read "machine learning"') - expect(result.query).to.eq(`"machine learning"`) - expect(result.readFilter).to.eq(ReadFilter.READ) - expect(result.typeFilter).to.be.undefined - }) -}) - -describe('query with read state after search query', () => { - it('sets read state and query', () => { - const result = parseSearchQuery('machine learning techniques is:read') - expect(result.query).to.eq(`machine learning techniques`) - expect(result.readFilter).to.eq(ReadFilter.READ) - expect(result.typeFilter).to.be.undefined - }) -}) - -describe('query with quoted text', () => { - it('sets the text as quoted and returns the default read state', () => { - const result = parseSearchQuery('"machine learning" techniques"') - expect(result.query).to.eq(`"machine learning" techniques`) - expect(result.readFilter).to.eq(ReadFilter.ALL) - expect(result.typeFilter).to.be.undefined - }) -}) - -describe('query with a file type', () => { - it('sets the type to the supplied type', () => { - const result = parseSearchQuery('"my string" type:file"') - expect(result.query).to.eq(`"my string"`) - expect(result.readFilter).to.eq(ReadFilter.ALL) - expect(result.typeFilter).to.eq(PageType.File) - }) -}) - -describe('query with an article type', () => { - it('sets the type to the supplied type', () => { - const result = parseSearchQuery('"my string" type:article"') - expect(result.query).to.eq(`"my string"`) - expect(result.readFilter).to.eq(ReadFilter.ALL) - expect(result.typeFilter).to.eq(PageType.Article) - }) -}) - -describe('query with pdf as its type', () => { - it('sets the type to the supplied file type', () => { - const result = parseSearchQuery('"my string" type:pdf"') - expect(result.query).to.eq(`"my string"`) - expect(result.readFilter).to.eq(ReadFilter.ALL) - expect(result.typeFilter).to.eq(PageType.File) - }) -}) - -describe('query without in param set', () => { - it('returns ALL if there is a search term', () => { - const result = parseSearchQuery('my search') - expect(result.query).to.eq(`my search`) - expect(result.inFilter).to.eq(InFilter.ALL) - }) - it('returns INBOX if there is not a search term', () => { - const result = parseSearchQuery('') - expect(result.inFilter).to.eq(InFilter.INBOX) - }) - it('returns INBOX if the search term is undefined', () => { - const result = parseSearchQuery(undefined) - expect(result.inFilter).to.eq(InFilter.INBOX) - }) -}) - -describe('query with in param set', () => { - it('returns set value if there is a search term', () => { - const result = parseSearchQuery('my search in:archive') - expect(result.query).to.eq(`my search`) - expect(result.inFilter).to.eq(InFilter.ARCHIVE) - }) - it('returns set value if there is not a search term', () => { - const result = parseSearchQuery('in:archive') - expect(result.inFilter).to.eq(InFilter.ARCHIVE) - }) -}) - -describe('query with in param set to invalid value', () => { - it('returns all if there is a query', () => { - const result = parseSearchQuery('my search in:blahblah') - expect(result.query).to.eq(`my search`) - expect(result.inFilter).to.eq(InFilter.ALL) - }) - it('returns set value if there is not a search term', () => { - const result = parseSearchQuery('in:blahblah') - expect(result.inFilter).to.eq(InFilter.INBOX) - }) -}) - -describe('query with author set', () => { - it('adds author to the match filters', () => { - const result = parseSearchQuery('author:"Omnivore Blog"') - expect(result.matchFilters[0].field).to.equal('author') - expect(result.matchFilters[0].value).to.equal('omnivore blog') - }) -}) - -describe('query with site set', () => { - it('adds site_name to the match filters', () => { - const result = parseSearchQuery('site:omnivore.app') - expect(result.matchFilters[0].field).to.equal('site_name') - expect(result.matchFilters[0].value).to.equal('omnivore.app') - }) -}) From e6322ffaf821b419cb84a04fc54fe1483a3cde95 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Dec 2023 13:04:07 +0800 Subject: [PATCH 05/10] remove unused code --- packages/api/src/resolvers/article/index.ts | 18 +- packages/api/src/services/library_item.ts | 321 ++------------ packages/api/src/utils/search.ts | 467 -------------------- 3 files changed, 41 insertions(+), 765 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index d3b598623..45c033419 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -101,7 +101,7 @@ import { ParsedContentPuppeteer, parsePreparedContent, } from '../../utils/parser' -import { InFilter, parseSearchQuery } from '../../utils/search' +import { parseSearchQuery } from '../../utils/search' import { getStorageFileDetails } from '../../utils/uploads' import { itemTypeForContentType } from '../upload_files' @@ -651,8 +651,6 @@ export const searchResolver = authorized< return { errorCodes: [SearchErrorCode.QueryTooLong] } } - const searchQuery = params.query ? parseSearchQuery(params.query) : undefined - const { libraryItems, count } = await searchLibraryItems( { from: Number(startCursor), @@ -660,7 +658,7 @@ export const searchResolver = authorized< includePending: true, includeContent: !!params.includeContent, includeDeleted: params.query?.includes('in:trash'), - searchQuery, + query: params.query, }, uid ) @@ -758,10 +756,9 @@ export const updatesSinceResolver = authorized< } // create a search query - folder = folder || InFilter.ALL - const searchQuery = parseSearchQuery( - `in:${folder} updated:${startDate.toISOString()}` - ) + const query = `updated:${startDate.toISOString()}${ + folder ? ' in:' + folder : '' + }` const { libraryItems, count } = await searchLibraryItems( { @@ -769,7 +766,7 @@ export const updatesSinceResolver = authorized< size: size + 1, // fetch one more item to get next cursor includeDeleted: true, sort, - searchQuery, + query, }, uid ) @@ -842,8 +839,7 @@ export const bulkActionResolver = authorized< labels = await findLabelsByIds(labelIds, uid) } - const searchQuery = parseSearchQuery(query) - await updateLibraryItems(action, searchQuery, uid, labels, args) + await updateLibraryItems(action, query, uid, labels, args) return { success: true } } catch (error) { diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index fdd646380..823a62852 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1,11 +1,6 @@ import { LiqeQuery } from 'liqe' import { DateTime } from 'luxon' -import { - Brackets, - DeepPartial, - ObjectLiteral, - SelectQueryBuilder, -} from 'typeorm' +import { DeepPartial, ObjectLiteral } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' @@ -18,40 +13,38 @@ import { libraryItemRepository } from '../repository/library_item' import { SaveFollowingItemRequest } from '../routers/svc/following' import { generateSlug, wordsCount } from '../utils/helpers' import { createThumbnailUrl } from '../utils/imageproxy' -import { - DateFilter, - FieldFilter, - HasFilter, - InFilter, - LabelFilter, - LabelFilterType, - NoFilter, - RangeFilter, - ReadFilter, -} from '../utils/search' +import { parseSearchQuery } from '../utils/search' + +enum ReadFilter { + ALL = 'all', + READ = 'read', + READING = 'reading', + UNREAD = 'unread', +} + +enum InFilter { + ALL = 'all', + INBOX = 'inbox', + ARCHIVE = 'archive', + TRASH = 'trash', + FOLLOWING = 'following', +} + +enum HasFilter { + HIGHLIGHTS = 'highlights', + LABELS = 'labels', + SUBSCRIPTIONS = 'subscriptions', +} export interface SearchArgs { from?: number size?: number sort?: Sort - query?: string - inFilter?: InFilter - readFilter?: ReadFilter - typeFilter?: string - labelFilters?: LabelFilter[] - hasFilters?: HasFilter[] - dateFilters?: DateFilter[] - termFilters?: FieldFilter[] - matchFilters?: FieldFilter[] includePending?: boolean | null includeDeleted?: boolean - ids?: string[] - recommendedBy?: string includeContent?: boolean - noFilters?: NoFilter[] - rangeFilters?: RangeFilter[] useFolders?: boolean - searchQuery?: LiqeQuery + query?: string | null } export interface SearchResultItem { @@ -624,254 +617,6 @@ export const buildQuery = ( return serialize(searchQuery) } -const buildWhereClause = ( - queryBuilder: SelectQueryBuilder, - args: SearchArgs -) => { - if (args.query) { - queryBuilder - .addSelect( - "ts_rank_cd(library_item.search_tsv, websearch_to_tsquery('english', :query))", - 'rank' - ) - .andWhere( - "websearch_to_tsquery('english', :query) @@ library_item.search_tsv" - ) - .setParameter('query', args.query) - .orderBy('rank', 'DESC') - } - - if (args.typeFilter) { - queryBuilder.andWhere('lower(library_item.item_type) = :typeFilter', { - typeFilter: args.typeFilter.toLowerCase(), - }) - } - - if (args.inFilter !== InFilter.ALL) { - switch (args.inFilter) { - case InFilter.INBOX: { - // if useFolders is true, we only return items in the inbox folder - args.useFolders && - queryBuilder.andWhere("library_item.folder = 'inbox'") - // for old clients, we return items that are not archived - queryBuilder.andWhere('library_item.archived_at IS NULL') - break - } - case InFilter.ARCHIVE: - queryBuilder.andWhere('library_item.archived_at IS NOT NULL') - break - case InFilter.TRASH: - // return only deleted pages within 14 days - queryBuilder.andWhere( - "library_item.deleted_at >= now() - interval '14 days'" - ) - break - default: - queryBuilder.andWhere('library_item.folder = :folder', { - folder: args.inFilter, - }) - } - } - - if (args.readFilter !== ReadFilter.ALL) { - switch (args.readFilter) { - case ReadFilter.READ: - queryBuilder.andWhere( - 'library_item.reading_progress_bottom_percent > 98' - ) - break - case ReadFilter.READING: - queryBuilder.andWhere( - 'library_item.reading_progress_bottom_percent BETWEEN 2 AND 98' - ) - break - case ReadFilter.UNREAD: - queryBuilder.andWhere( - 'library_item.reading_progress_bottom_percent < 2' - ) - break - } - } - - if (args.hasFilters && args.hasFilters.length > 0) { - args.hasFilters.forEach((filter) => { - switch (filter) { - case HasFilter.HIGHLIGHTS: - queryBuilder.andWhere("library_item.highlight_annotations <> '{}'") - break - case HasFilter.LABELS: - queryBuilder.andWhere("library_item.label_names <> '{}'") - break - case HasFilter.SUBSCRIPTIONS: - queryBuilder.andWhere('library_item.subscription is NOT NULL') - } - }) - } - - if (args.labelFilters && args.labelFilters.length > 0) { - const includeLabels = args.labelFilters?.filter( - (filter) => filter.type === LabelFilterType.INCLUDE - ) - const excludeLabels = args.labelFilters?.filter( - (filter) => filter.type === LabelFilterType.EXCLUDE - ) - - if (includeLabels && includeLabels.length > 0) { - includeLabels.forEach((includeLabel, i) => { - const param = `includeLabels_${i}` - const hasWildcard = includeLabel.labels.some((label) => - label.includes('*') - ) - if (hasWildcard) { - queryBuilder.andWhere( - new Brackets((qb) => { - includeLabel.labels.forEach((label, j) => { - const param = `includeLabels_${i}_${j}` - qb.orWhere( - `array_to_string(array_cat(library_item.label_names, library_item.highlight_labels)::text[], ',') ILIKE :${param}`, - { - [param]: label.replace(/\*/g, '%'), - } - ) - }) - }) - ) - } else { - queryBuilder.andWhere( - `lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[] && ARRAY[:...${param}]::text[]`, - { - [param]: includeLabel.labels, - } - ) - } - }) - } - - if (excludeLabels && excludeLabels.length > 0) { - const labels = excludeLabels.flatMap((filter) => filter.labels) - - const hasWildcard = labels.some((label) => label.includes('*')) - - if (hasWildcard) { - queryBuilder.andWhere( - new Brackets((qb) => { - labels.forEach((label, i) => { - const param = `excludeLabels_${i}` - qb.andWhere( - `array_to_string(array_cat(library_item.label_names, library_item.highlight_labels)::text[], ',') NOT ILIKE :${param}`, - { - [param]: label.replace(/\*/g, '%'), - } - ) - }) - }) - ) - } else { - queryBuilder.andWhere( - 'NOT lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[] && ARRAY[:...excludeLabels]::text[]', - { - excludeLabels: labels, - } - ) - } - } - } - - if (args.dateFilters && args.dateFilters.length > 0) { - args.dateFilters.forEach((filter) => { - const startDate = `${filter.field}_start` - const endDate = `${filter.field}_end` - queryBuilder.andWhere( - `library_item.${filter.field} between :${startDate} and :${endDate}`, - { - [startDate]: filter.startDate ?? new Date(0), - [endDate]: filter.endDate ?? new Date(), - } - ) - }) - } - - if (args.termFilters && args.termFilters.length > 0) { - args.termFilters.forEach((filter) => { - const param = `term_${filter.field}` - queryBuilder.andWhere(`lower(library_item.${filter.field}) = :${param}`, { - [param]: filter.value.toLowerCase(), - }) - }) - } - - if (args.matchFilters && args.matchFilters.length > 0) { - args.matchFilters.forEach((filter) => { - const param = `match_${filter.field}` - queryBuilder.andWhere( - new Brackets((qb) => { - qb.andWhere( - `websearch_to_tsquery('english', :${param}) @@ library_item.${filter.field}_tsv`, - { - [param]: filter.value, - } - ).orWhere(`${filter.field} ILIKE :value`, { - value: `%${filter.value}%`, - }) - }) - ) - }) - } - - if (args.ids && args.ids.length > 0) { - queryBuilder.andWhere('library_item.id = ANY(:ids)', { - ids: args.ids, - }) - } - - if (!args.includePending) { - queryBuilder.andWhere("library_item.state <> 'PROCESSING'") - } - - if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) { - queryBuilder.andWhere("library_item.state <> 'DELETED'") - } - - if (args.noFilters) { - args.noFilters.forEach((filter) => { - queryBuilder.andWhere( - `(library_item.${filter.field} = '{}' OR library_item.${filter.field} IS NULL)` - ) - }) - } - - if (args.recommendedBy) { - if (args.recommendedBy === '*') { - // select all if * is provided - queryBuilder.andWhere(`library_item.recommender_names <> '{}'`) - } else { - // select only if the user is recommended by the provided user - queryBuilder.andWhere( - 'lower(library_item.recommender_names::text)::text[] && ARRAY[:recommendedBy]::text[]', - { - recommendedBy: args.recommendedBy.toLowerCase(), - } - ) - } - } - - if (args.includeContent) { - queryBuilder.addSelect('library_item.readableContent') - } - - if (args.rangeFilters && args.rangeFilters.length > 0) { - args.rangeFilters.forEach((filter, i) => { - const param = `range_${filter.field}_${i}` - queryBuilder.andWhere( - `library_item.${filter.field} ${filter.operator} :${param}`, - { - [param]: filter.value, - } - ) - }) - } -} - export const searchLibraryItems = async ( args: SearchArgs, userId: string @@ -891,6 +636,11 @@ export const searchLibraryItems = async ( column !== 'library_item.originalContent' ) + let searchQuery: LiqeQuery | undefined + if (args.query) { + searchQuery = parseSearchQuery(args.query) + } + // add pagination and sorting return authTrx( async (tx) => { @@ -899,16 +649,11 @@ export const searchLibraryItems = async ( .select(selectColumns) .where('library_item.user_id = :userId', { userId }) - if (args.searchQuery) { + if (searchQuery) { const parameters: ObjectLiteral[] = [] const selects: Select[] = [] const orders: Sort[] = [] - const whereClause = buildQuery( - args.searchQuery, - parameters, - selects, - orders - ) + const whereClause = buildQuery(searchQuery, parameters, selects, orders) whereClause && queryBuilder .andWhere(whereClause) @@ -1243,7 +988,7 @@ export const countByCreatedAt = async ( export const updateLibraryItems = async ( action: BulkActionType, - searchQuery: LiqeQuery, + query: string, userId: string, labels?: Label[], args?: unknown @@ -1297,6 +1042,8 @@ export const updateLibraryItems = async ( throw new Error('Invalid bulk action') } + const searchQuery = parseSearchQuery(query) + await authTrx(async (tx) => { const queryBuilder = tx .createQueryBuilder(LibraryItem, 'library_item') diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index 8c1d82669..33329e13a 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -1,337 +1,4 @@ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { LiqeQuery, parse } from 'liqe' -import { DateTime } from 'luxon' -import { ISearchParserDictionary } from 'search-query-parser' -import { PageType } from '../generated/graphql' - -export enum ReadFilter { - ALL = 'all', - READ = 'read', - READING = 'reading', - UNREAD = 'unread', -} - -export enum InFilter { - ALL = 'all', - INBOX = 'inbox', - ARCHIVE = 'archive', - TRASH = 'trash', - FOLLOWING = 'following', -} - -export interface SearchFilter { - query: string | undefined - inFilter: InFilter - readFilter: ReadFilter - typeFilter?: string - labelFilters: LabelFilter[] - hasFilters: HasFilter[] - dateFilters: DateFilter[] - termFilters: FieldFilter[] - matchFilters: FieldFilter[] - ids: string[] - recommendedBy?: string - noFilters: NoFilter[] - rangeFilters: RangeFilter[] - useFolders?: boolean -} - -export enum LabelFilterType { - INCLUDE, - EXCLUDE, -} - -export type LabelFilter = { - type: LabelFilterType - labels: string[] -} - -export enum HasFilter { - HIGHLIGHTS = 'highlights', - LABELS = 'labels', - SUBSCRIPTIONS = 'subscriptions', -} - -export interface DateFilter { - field: string - startDate?: Date - endDate?: Date -} - -export interface RangeFilter { - field: string - operator: string - value: number -} - -export interface FieldFilter { - nested?: boolean - field: string - value: string -} - -export interface NoFilter { - field: string -} - -const parseStringValue = (str?: string): string | undefined => { - if (str === undefined) { - return undefined - } - - return str.toLowerCase() -} - -const parseIsFilter = (str: string | undefined): ReadFilter => { - switch (str?.toUpperCase()) { - case 'READ': - return ReadFilter.READ - case 'READING': - return ReadFilter.READING - case 'UNREAD': - return ReadFilter.UNREAD - } - return ReadFilter.ALL -} - -const parseInFilter = ( - str: string | undefined, - query: string | undefined -): InFilter => { - switch (str?.toUpperCase()) { - case 'ALL': - return InFilter.ALL - case 'INBOX': - return InFilter.INBOX - case 'ARCHIVE': - return InFilter.ARCHIVE - case 'TRASH': - return InFilter.TRASH - case 'FOLLOWING': - return InFilter.FOLLOWING - } - - return query ? InFilter.ALL : InFilter.INBOX -} - -const parseTypeFilter = (str: string | undefined): string | undefined => { - if (str === undefined) { - return undefined - } - - switch (str.toLowerCase()) { - case 'article': - return PageType.Article - case 'book': - return PageType.Book - case 'pdf': - case 'file': - return PageType.File - case 'profile': - return PageType.Profile - case 'website': - return PageType.Website - case 'unknown': - return PageType.Unknown - } - return undefined -} - -const parseLabelFilter = ( - str?: string, - exclude?: ISearchParserDictionary -): LabelFilter | undefined => { - if (str === undefined) { - return undefined - } - - const labels = str.split(',') - - // check if the labels are on the exclusion list - const excluded = exclude?.label && exclude.label.includes(...labels) - - return { - type: excluded ? LabelFilterType.EXCLUDE : LabelFilterType.INCLUDE, - // use lower case for label names - labels: labels.map((label) => label.toLowerCase()), - } -} - -const parseHasFilter = (str?: string): HasFilter | undefined => { - if (str === undefined) { - return undefined - } - - switch (str.toUpperCase()) { - case 'HIGHLIGHTS': - return HasFilter.HIGHLIGHTS - case 'LABELS': - return HasFilter.LABELS - case 'SUBSCRIPTIONS': - return HasFilter.SUBSCRIPTIONS - } -} - -const parseDateFilter = ( - field: string, - str?: string -): DateFilter | undefined => { - if (str === undefined) { - return undefined - } - - switch (field.toLowerCase()) { - case 'published': - field = 'published_at' - break - case 'saved': - field = 'saved_at' - break - case 'updated': - field = 'updated_at' - } - - // check for special date filters - switch (str.toLowerCase()) { - case 'today': - return { - field, - startDate: DateTime.local().startOf('day').toJSDate(), - } - case 'yesterday': { - const yesterday = DateTime.local().minus({ days: 1 }) - return { - field, - startDate: yesterday.startOf('day').toJSDate(), - endDate: yesterday.endOf('day').toJSDate(), - } - } - case 'this week': - return { - field, - startDate: DateTime.local().startOf('week').toJSDate(), - } - case 'this month': - return { - field, - startDate: DateTime.local().startOf('month').toJSDate(), - } - } - - // check for date ranges - const [start, end] = str.split('..') - const startDate = start && start !== '*' ? new Date(start) : undefined - const endDate = end && end !== '*' ? new Date(end) : undefined - - return { - field, - startDate, - endDate, - } -} - -const parseRangeFilter = ( - field: string, - str?: string -): RangeFilter | undefined => { - if (str === undefined) { - return undefined - } - - switch (field.toUpperCase()) { - case 'WORDSCOUNT': - field = 'word_count' - break - case 'READPOSITION': - field = 'reading_progress_bottom_percent' - break - default: - return undefined - } - - const operatorRegex = /([<>]=?)/ - const operator = str.match(operatorRegex)?.[0] - if (!operator) { - return undefined - } - - const value = str.replace(operatorRegex, '') - if (!value) { - return undefined - } - - return { - field, - operator, - value: Number(value), - } -} - -const parseFieldFilter = ( - field: string, - str?: string -): FieldFilter | undefined => { - if (str === undefined) { - return undefined - } - - // normalize the term to lower case - const value = str.toLowerCase() - - switch (field.toUpperCase()) { - case 'LANGUAGE': - return { - field: 'item_language', - value, - } - case 'SUBSCRIPTION': - case 'RSS': - return { - field: 'subscription', - value, - } - case 'SITE': - return { - field: 'site_name', - value, - } - } - - return { - field, - value, - } -} - -const parseIds = (str?: string): string[] | undefined => { - if (str === undefined) { - return undefined - } - - return str.split(',') -} - -const parseNoFilter = (str?: string): NoFilter | undefined => { - if (str === undefined) { - return undefined - } - - const strLower = str.toLowerCase() - switch (strLower) { - case 'highlight': - return { field: 'highlight_annotations' } - case 'label': - return { field: 'label_names' } - case 'subscription': - return { field: 'subscription' } - } - - return undefined -} export const parseSearchQuery = (query: string): LiqeQuery => { const searchQuery = query @@ -342,138 +9,4 @@ export const parseSearchQuery = (query: string): LiqeQuery => { .replace(/(\w+):([^"\s]+)/g, '$1:"$2"') return parse(searchQuery) - - // const parsed = parse(searchQuery, { - // keywords: [ - // 'in', - // 'is', - // 'type', - // 'label', - // 'sort', - // 'has', - // 'saved', - // 'author', - // 'published', - // 'subscription', - // 'language', - // 'title', - // 'description', - // 'content', - // 'updated', - // 'includes', - // 'recommendedBy', - // 'no', - // 'mode', - // 'site', - // 'note', - // 'rss', - // 'wordsCount', - // 'readPosition', - // ], - // tokenize: true, - // }) - // if (parsed.offsets) { - // const texts = parsed.offsets - // .filter((offset) => 'text' in offset) - // .map((offset) => offset as SearchParserTextOffset) - - // if (texts.length > 0) { - // result.query = texts - // .map((offset: SearchParserTextOffset) => { - // // TODO: the parser library doesn't let us accurately - // // pull out quoted text, so we are just assuming - // // anything with spaces is quoted. - // if (offset.text.indexOf(' ') > -1) { - // return `"${offset.text}"` - // } - // return offset.text - // }) - // .join(' ') - // } else { - // result.query = undefined - // } - - // const keywords = parsed.offsets - // .filter((offset) => 'keyword' in offset) - // .map((offset) => offset as SearchParserKeyWordOffset) - - // for (const keyword of keywords) { - // switch (keyword.keyword) { - // case 'in': - // result.inFilter = parseInFilter(keyword.value, result.query) - // break - // case 'is': - // result.readFilter = parseIsFilter(keyword.value) - // break - // case 'type': - // result.typeFilter = parseTypeFilter(keyword.value) - // break - // case 'label': { - // const labelFilter = parseLabelFilter(keyword.value, parsed.exclude) - // labelFilter && result.labelFilters.push(labelFilter) - // break - // } - // case 'sort': - // result.sort = parseSort(keyword.value) - // break - // case 'has': { - // const hasFilter = parseHasFilter(keyword.value) - // hasFilter !== undefined && result.hasFilters.push(hasFilter) - // break - // } - // case 'saved': - // case 'read': - // case 'updated': - // case 'published': { - // const dateFilter = parseDateFilter(keyword.keyword, keyword.value) - // dateFilter && result.dateFilters.push(dateFilter) - // break - // } - // // term filters - // case 'subscription': - // case 'rss': - // case 'language': { - // const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value) - // fieldFilter && result.termFilters.push(fieldFilter) - // break - // } - // // match filters - // case 'author': - // case 'title': - // case 'description': - // case 'note': - // case 'site': - // case 'content': { - // const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value) - // fieldFilter && result.matchFilters.push(fieldFilter) - // break - // } - // case 'includes': { - // const ids = parseIds(keyword.value) - // ids && result.ids.push(...ids) - // break - // } - // case 'recommendedBy': { - // result.recommendedBy = parseStringValue(keyword.value) - // break - // } - // case 'no': { - // const noFilter = parseNoFilter(keyword.value) - // noFilter && result.noFilters.push(noFilter) - // break - // } - // case 'mode': - // // mode is ignored and used only by the frontend - // break - // case 'readPosition': - // case 'wordsCount': { - // const rangeFilter = parseRangeFilter(keyword.keyword, keyword.value) - // rangeFilter && result.rangeFilters.push(rangeFilter) - // break - // } - // } - // } - // } - - // return result } From 7c5f90983edbd1d287360c9ff88906459f722dde Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Dec 2023 13:05:31 +0800 Subject: [PATCH 06/10] clean up --- packages/api/src/resolvers/article/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 45c033419..05b74a696 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -101,7 +101,6 @@ import { ParsedContentPuppeteer, parsePreparedContent, } from '../../utils/parser' -import { parseSearchQuery } from '../../utils/search' import { getStorageFileDetails } from '../../utils/uploads' import { itemTypeForContentType } from '../upload_files' From 312bf93d9b98b4b593e5a20d12fed90e4e48d86a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Dec 2023 17:07:40 +0800 Subject: [PATCH 07/10] fix tests --- packages/api/test/routers/auth.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/api/test/routers/auth.test.ts b/packages/api/test/routers/auth.test.ts index 5cf40e3e3..2bb3d5831 100644 --- a/packages/api/test/routers/auth.test.ts +++ b/packages/api/test/routers/auth.test.ts @@ -15,7 +15,6 @@ import { generateVerificationToken, hashPassword, } from '../../src/utils/auth' -import { InFilter } from '../../src/utils/search' import * as util from '../../src/utils/sendEmail' import { createTestUser } from '../db' import { generateFakeUuid, request } from '../util' @@ -609,7 +608,7 @@ describe('auth router', () => { ).expect(200) const user = await userRepository.findOneByOrFail({ name }) const { count } = await searchLibraryItems( - { inFilter: InFilter.ALL }, + { query: 'in:all' }, user.id ) @@ -633,7 +632,7 @@ describe('auth router', () => { ).expect(200) const user = await userRepository.findOneByOrFail({ name }) const { count } = await searchLibraryItems( - { inFilter: InFilter.ALL }, + { query: 'in:all' }, user.id ) From 29e2bf45e3321c4437f033c6cac1e512c47abae1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Dec 2023 20:40:02 +0800 Subject: [PATCH 08/10] allow use:folders in search string for new clients to filter inbox items --- packages/api/src/resolvers/article/index.ts | 1 + packages/api/src/services/library_item.ts | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 05b74a696..22413fe27 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -658,6 +658,7 @@ export const searchResolver = authorized< includeContent: !!params.includeContent, includeDeleted: params.query?.includes('in:trash'), query: params.query, + useFolders: params.query?.includes('use:folders'), }, uid ) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 823a62852..ee8bab352 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -153,7 +153,8 @@ export const buildQuery = ( searchQuery: LiqeQuery, parameters: ObjectLiteral[] = [], selects: Select[] = [], - orders: { by: string; order?: SortOrder }[] = [] + orders: { by: string; order?: SortOrder }[] = [], + useFolders = false ) => { const escapeQueryWithParameters = ( query: string, @@ -220,9 +221,13 @@ export const buildQuery = ( // return only deleted pages within 14 days return "library_item.deleted_at >= now() - interval '14 days'" default: { + if (!useFolders) { + throw new Error(`Unexpected keyword: ${folder}`) + } + const param = `folder_${parameters.length}` return escapeQueryWithParameters( - `library_item.folder = :${param}`, + `(library_item.folder = :${param} AND library_item.archived_at IS NULL)`, { [param]: folder } ) } @@ -653,7 +658,13 @@ export const searchLibraryItems = async ( const parameters: ObjectLiteral[] = [] const selects: Select[] = [] const orders: Sort[] = [] - const whereClause = buildQuery(searchQuery, parameters, selects, orders) + const whereClause = buildQuery( + searchQuery, + parameters, + selects, + orders, + args.useFolders + ) whereClause && queryBuilder .andWhere(whereClause) From 4dacceac01716251c311e6a67fd8b3f6def06839 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Dec 2023 20:43:23 +0800 Subject: [PATCH 09/10] allow use keyword --- packages/api/src/services/library_item.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index ee8bab352..1c54b80e0 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -521,6 +521,7 @@ export const buildQuery = ( return `(library_item.${column} = '{}' OR library_item.${column} IS NULL)` } + case 'use': case 'mode': // mode is ignored and used only by the frontend return null From 8579b79eb36cf5022277804e6f64ff59ee459573 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Dec 2023 20:58:08 +0800 Subject: [PATCH 10/10] fix tests --- packages/api/src/services/library_item.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 1c54b80e0..54442d635 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -213,23 +213,23 @@ export const buildQuery = ( switch (folder) { case InFilter.ALL: return null - case InFilter.INBOX: - return 'library_item.archived_at IS NULL' case InFilter.ARCHIVE: return 'library_item.archived_at IS NOT NULL' case InFilter.TRASH: // return only deleted pages within 14 days return "library_item.deleted_at >= now() - interval '14 days'" default: { - if (!useFolders) { - throw new Error(`Unexpected keyword: ${folder}`) + let sql = 'library_item.archived_at IS NULL' + if (useFolders) { + const param = `folder_${parameters.length}` + const folderSql = escapeQueryWithParameters( + `library_item.folder = :${param}`, + { [param]: folder } + ) + sql = `(${sql} AND ${folderSql})` } - const param = `folder_${parameters.length}` - return escapeQueryWithParameters( - `(library_item.folder = :${param} AND library_item.archived_at IS NULL)`, - { [param]: folder } - ) + return sql } } }