From 55cc5f57690074732e748507a0a67fbdbb07b89a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 12 Dec 2023 16:14:51 +0800 Subject: [PATCH 1/8] treat unknown fields as text search --- packages/api/src/services/library_item.ts | 74 +++++++++++++++-------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 4f74db41c..986971c12 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1,4 +1,4 @@ -import { LiqeQuery } from '@omnivore/liqe' +import { ExpressionToken, LiqeQuery } from '@omnivore/liqe' import { DateTime } from 'luxon' import { DeepPartial, ObjectLiteral } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' @@ -167,6 +167,34 @@ export const buildQuery = ( return query } + const serializeImplicitField = ( + expression: ExpressionToken + ): string | null => { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression.') + } + + const value = expression.value?.toString() + + if (value === undefined || value === '') { + return null + } + + const param = `implicit_field_${parameters.length}` + const alias = `rank_${parameters.length}` + selects.push({ + column: `ts_rank_cd(library_item.search_tsv, websearch_to_tsquery('english', :${param}))`, + alias, + }) + + orders.push({ by: alias, order: SortOrder.DESCENDING }) + + return escapeQueryWithParameters( + `websearch_to_tsquery('english', :${param}) @@ library_item.search_tsv`, + { [param]: value } + ) + } + const serializeTagExpression = (ast: LiqeQuery): string | null => { if (ast.type !== 'Tag') { throw new Error('Expected a tag expression.') @@ -175,29 +203,7 @@ export const buildQuery = ( const { field, expression } = ast if (field.type === 'ImplicitField') { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const value = expression.value?.toString() - - if (value === undefined || value === '') { - return null - } - - const param = `implicit_field_${parameters.length}` - const alias = `rank_${parameters.length}` - selects.push({ - column: `ts_rank_cd(library_item.search_tsv, websearch_to_tsquery('english', :${param}))`, - alias, - }) - - orders.push({ by: alias, order: SortOrder.DESCENDING }) - - return escapeQueryWithParameters( - `websearch_to_tsquery('english', :${param}) @@ library_item.search_tsv`, - { [param]: value } - ) + return serializeImplicitField(expression) } else { switch (field.name) { case 'in': { @@ -576,8 +582,24 @@ export const buildQuery = ( } ) } - default: - throw new Error(`Unexpected keyword: ${field.name}`) + default: { + if (expression.type !== 'LiteralExpression') { + // ignore unknown fields without values + return null + } + + const fieldValue = expression.value?.toString() + if (!fieldValue) { + // ignore empty values + return null + } + + // treat all other unknown fields as implicit fields + return serializeImplicitField({ + ...expression, + value: `${field.name}:${fieldValue}`, + }) + } } } } From 7622d8a139d2d6899b8ad763a262cb0fbc4bc27f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 12 Dec 2023 16:17:31 +0800 Subject: [PATCH 2/8] ignore field case --- packages/api/src/services/library_item.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 986971c12..74b2ea153 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -205,7 +205,7 @@ export const buildQuery = ( if (field.type === 'ImplicitField') { return serializeImplicitField(expression) } else { - switch (field.name) { + switch (field.name.toLowerCase()) { case 'in': { if (expression.type !== 'LiteralExpression') { throw new Error('Expected a literal expression.') From 609c46cdc35406faf733f7b74d3a9a8b2571b1ca Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 12 Dec 2023 16:35:50 +0800 Subject: [PATCH 3/8] ignore empty field value --- packages/api/src/services/library_item.ts | 171 ++++------------------ 1 file changed, 27 insertions(+), 144 deletions(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 74b2ea153..70c01c454 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -130,7 +130,7 @@ export const sortParamsToSort = ( } const getColumnName = (field: string) => { - switch (field) { + switch (field.toLowerCase()) { case 'language': return 'item_language' case 'subscription': @@ -205,18 +205,20 @@ export const buildQuery = ( if (field.type === 'ImplicitField') { return serializeImplicitField(expression) } else { + if (expression.type !== 'LiteralExpression') { + // ignore empty values + return null + } + + const value = expression.value?.toString() + if (!value) { + // ignore empty values + return null + } + switch (field.name.toLowerCase()) { case 'in': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const folder = expression.value?.toString() - if (!folder) { - throw new Error('Expected a value.') - } - - switch (folder) { + switch (value.toLowerCase()) { case InFilter.ALL: return null case InFilter.ARCHIVE: @@ -230,7 +232,7 @@ export const buildQuery = ( const param = `folder_${parameters.length}` const folderSql = escapeQueryWithParameters( `library_item.folder = :${param}`, - { [param]: folder } + { [param]: value } ) sql = `(${sql} AND ${folderSql})` } @@ -241,16 +243,7 @@ export const buildQuery = ( } case 'is': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const value = expression.value?.toString() - if (!value) { - throw new Error('Expected a value.') - } - - switch (value) { + switch (value.toLowerCase()) { case ReadFilter.READ: return 'library_item.reading_progress_bottom_percent > 98' case ReadFilter.READING: @@ -262,15 +255,6 @@ export const buildQuery = ( } } case 'type': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const value = expression.value?.toString() - if (!value) { - throw new Error('Expected a value.') - } - const param = `type_${parameters.length}` return escapeQueryWithParameters( @@ -281,16 +265,7 @@ export const buildQuery = ( ) } case 'label': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const value = expression.value?.toString()?.toLowerCase() - if (!value) { - throw new Error('Expected a value.') - } - - const labels = value.split(',') + const labels = value.toLowerCase().split(',') return ( labels .map((label) => { @@ -319,15 +294,6 @@ export const buildQuery = ( ) } case 'sort': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const value = expression.value?.toString() - if (!value) { - throw new Error('Expected a value.') - } - const [sort, sortOrder] = value.split('-') if (sort.toLowerCase() === 'score') { // score is not a column and is handled separately @@ -335,7 +301,7 @@ export const buildQuery = ( } const order = - sortOrder?.toUpperCase() === 'ASC' + sortOrder?.toLowerCase() === 'asc' ? SortOrder.ASCENDING : SortOrder.DESCENDING @@ -344,16 +310,7 @@ export const buildQuery = ( return null } case 'has': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const value = expression.value?.toString() - if (!value) { - throw new Error('Expected a value.') - } - - switch (value) { + switch (value.toLowerCase()) { case HasFilter.HIGHLIGHTS: return "library_item.highlight_annotations <> '{}'" case HasFilter.LABELS: @@ -368,19 +325,10 @@ export const buildQuery = ( case 'read': case 'updated': case 'published': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const date = expression.value?.toString() - if (!date) { - throw new Error('Expected a value.') - } - let startDate: Date | undefined let endDate: Date | undefined // check for special date filters - switch (date.toLowerCase()) { + switch (value.toLowerCase()) { case 'today': startDate = DateTime.local().startOf('day').toJSDate() break @@ -398,7 +346,7 @@ export const buildQuery = ( break default: { // check for date ranges - const [start, end] = date.split('..') + const [start, end] = value.split('..') // validate date if (start && start !== '*') { startDate = new Date(start) @@ -431,15 +379,6 @@ export const buildQuery = ( case 'subscription': case 'rss': case 'language': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const value = expression.value?.toString() - if (!value) { - throw new Error('Expected a value.') - } - const columnName = getColumnName(field.name) const param = `term_${field.name}_${parameters.length}` @@ -456,16 +395,6 @@ export const buildQuery = ( case 'description': case 'note': case 'site': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - // normalize the term to lower case - const value = expression.value?.toString()?.toLowerCase() - if (!value) { - throw new Error('Expected a value.') - } - const columnName = getColumnName(field.name) const param = `match_${field.name}_${parameters.length}` const wildcardParam = `match_${field.name}_wildcard_${parameters.length}` @@ -479,11 +408,7 @@ export const buildQuery = ( ) } case 'includes': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const ids = expression.value?.toString()?.split(',') + const ids = value.split(',') if (!ids || ids.length === 0) { throw new Error('Expected a value.') } @@ -495,15 +420,6 @@ export const buildQuery = ( }) } case 'recommendedBy': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const value = expression.value?.toString() - if (!value) { - throw new Error('Expected a value.') - } - const param = `recommendedBy_${parameters.length}` if (value === '*') { // select all if * is provided @@ -518,17 +434,8 @@ export const buildQuery = ( ) } case 'no': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - const value = expression.value?.toString() - if (!value) { - throw new Error('Expected a value.') - } - let column = '' - switch (value) { + switch (value.toLowerCase()) { case 'highlight': column = 'highlight_annotations' break @@ -551,15 +458,6 @@ export const buildQuery = ( return null case 'readPosition': case 'wordsCount': { - if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') - } - - let value = expression.value?.toString() - if (!value) { - throw new Error('Expected a value.') - } - const column = getColumnName(field.name) const operatorRegex = /([<>]=?)/ @@ -568,38 +466,23 @@ export const buildQuery = ( throw new Error('Expected a value.') } - value = value.replace(operatorRegex, '') - if (!value) { - throw new Error('Expected a value.') - } + const newValue = value.replace(operatorRegex, '') const param = `range_${field.name}_${parameters.length}` return escapeQueryWithParameters( `library_item.${column} ${operator} :${param}`, { - [param]: parseInt(value, 10), + [param]: parseInt(newValue, 10), } ) } - default: { - if (expression.type !== 'LiteralExpression') { - // ignore unknown fields without values - return null - } - - const fieldValue = expression.value?.toString() - if (!fieldValue) { - // ignore empty values - return null - } - - // treat all other unknown fields as implicit fields + default: + // treat unknown fields as implicit fields return serializeImplicitField({ ...expression, - value: `${field.name}:${fieldValue}`, + value: `${field.name}:${value}`, }) - } } } } From a7f9214cdad0ead4d720ed7e7ead26a1a1fe7ed9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 12 Dec 2023 17:56:10 +0800 Subject: [PATCH 4/8] ignore empty text field --- packages/api/src/services/library_item.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 70c01c454..fae25fb11 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -540,7 +540,7 @@ export const buildQuery = ( return `(${serialized})` } - throw new Error('Missing AST type.') + return null } return serialize(searchQuery) From 8dfa5f42269551a00565d09b1c84f000f2078440 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 12 Dec 2023 18:02:54 +0800 Subject: [PATCH 5/8] treat numbers as text field too --- packages/liqe/src/grammar.ne | 2 +- packages/liqe/src/grammar.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/liqe/src/grammar.ne b/packages/liqe/src/grammar.ne index 9bf25daaa..f79d5c71e 100644 --- a/packages/liqe/src/grammar.ne +++ b/packages/liqe/src/grammar.ne @@ -292,4 +292,4 @@ regex_flags -> [gmiyusd]:+ {% d => d[0].join('') %} unquoted_value -> - [a-zA-Z_*?@#$\u0080-\uFFFF] [a-zA-Z\.\-_*?@#$\u0080-\uFFFF]:* {% d => d[0] + d[1].join('') %} + [a-zA-Z_*?@#$\u0080-\uFFFF0-9] [a-zA-Z\.\-_*?@#$\u0080-\uFFFF0-9]:* {% d => d[0] + d[1].join('') %} diff --git a/packages/liqe/src/grammar.ts b/packages/liqe/src/grammar.ts index e46834888..2ad071d92 100644 --- a/packages/liqe/src/grammar.ts +++ b/packages/liqe/src/grammar.ts @@ -299,8 +299,8 @@ const grammar: Grammar = { {"name": "regex_flags$ebnf$1", "symbols": ["regex_flags$ebnf$1", /[gmiyusd]/], "postprocess": (d) => d[0].concat([d[1]])}, {"name": "regex_flags", "symbols": ["regex_flags$ebnf$1"], "postprocess": d => d[0].join('')}, {"name": "unquoted_value$ebnf$1", "symbols": []}, - {"name": "unquoted_value$ebnf$1", "symbols": ["unquoted_value$ebnf$1", /[a-zA-Z\.\-_*?@#$\u0080-\uFFFF]/], "postprocess": (d) => d[0].concat([d[1]])}, - {"name": "unquoted_value", "symbols": [/[a-zA-Z_*?@#$\u0080-\uFFFF]/, "unquoted_value$ebnf$1"], "postprocess": d => d[0] + d[1].join('')} + {"name": "unquoted_value$ebnf$1", "symbols": ["unquoted_value$ebnf$1", /[a-zA-Z\.\-_*?@#$\u0080-\uFFFF0-9]/], "postprocess": (d) => d[0].concat([d[1]])}, + {"name": "unquoted_value", "symbols": [/[a-zA-Z_*?@#$\u0080-\uFFFF0-9]/, "unquoted_value$ebnf$1"], "postprocess": d => d[0] + d[1].join('')} ], ParserStart: "main", }; From 07868c729d081098e9548dbc8a23a5fdd27c5504 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 12 Dec 2023 18:13:38 +0800 Subject: [PATCH 6/8] update comments --- packages/api/src/services/library_item.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index fae25fb11..0781d2927 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -171,7 +171,7 @@ export const buildQuery = ( expression: ExpressionToken ): string | null => { if (expression.type !== 'LiteralExpression') { - throw new Error('Expected a literal expression.') + throw new Error('Expected a literal expression') } const value = expression.value?.toString() @@ -197,7 +197,7 @@ export const buildQuery = ( const serializeTagExpression = (ast: LiqeQuery): string | null => { if (ast.type !== 'Tag') { - throw new Error('Expected a tag expression.') + throw new Error('Expected a tag expression') } const { field, expression } = ast @@ -351,14 +351,14 @@ export const buildQuery = ( if (start && start !== '*') { startDate = new Date(start) if (isNaN(startDate.getTime())) { - throw new Error('Invalid start date.') + throw new Error('Invalid start date') } } if (end && end !== '*') { endDate = new Date(end) if (isNaN(endDate.getTime())) { - throw new Error('Invalid end date.') + throw new Error('Invalid end date') } } } @@ -410,7 +410,7 @@ export const buildQuery = ( case 'includes': { const ids = value.split(',') if (!ids || ids.length === 0) { - throw new Error('Expected a value.') + throw new Error('Expected ids') } const param = `includes_${parameters.length}` @@ -463,7 +463,7 @@ export const buildQuery = ( const operatorRegex = /([<>]=?)/ const operator = value.match(operatorRegex)?.[0] if (!operator) { - throw new Error('Expected a value.') + throw new Error('Expected operator') } const newValue = value.replace(operatorRegex, '') @@ -499,7 +499,7 @@ export const buildQuery = ( } else if (ast.operator.operator === 'OR') { operator = 'OR' } else { - throw new Error('Unexpected operator.') + throw new Error('Unexpected operator') } const left = serialize(ast.left) From 5bac173d640f373714b12b229352472d60b3f6b8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 12 Dec 2023 19:04:53 +0800 Subject: [PATCH 7/8] skip converting to numeric value in liqe --- packages/liqe/src/grammar.ne | 82 ++++++++++++++-------------- packages/liqe/src/grammar.ts | 44 --------------- packages/liqe/test/liqe/filter.ts | 26 ++++----- packages/liqe/test/liqe/highlight.ts | 6 +- packages/liqe/test/liqe/parse.ts | 18 +++--- packages/liqe/test/liqe/serialize.ts | 18 +++--- 6 files changed, 75 insertions(+), 119 deletions(-) diff --git a/packages/liqe/src/grammar.ne b/packages/liqe/src/grammar.ne index f79d5c71e..ec1a7e4fe 100644 --- a/packages/liqe/src/grammar.ne +++ b/packages/liqe/src/grammar.ne @@ -8,14 +8,14 @@ __ -> whitespace_character:+ {% (data) => data[0].length %} whitespace_character -> [ \t\n\v\f] {% id %} -# Numbers -decimal -> "-":? [0-9]:+ ("." [0-9]:+):? {% - (data) => parseFloat( - (data[0] || "") + - data[1].join("") + - (data[2] ? "."+data[2][1].join("") : "") - ) -%} +# # Numbers +# decimal -> "-":? [0-9]:+ ("." [0-9]:+):? {% +# (data) => parseFloat( +# (data[0] || "") + +# data[1].join("") + +# (data[2] ? "."+data[2][1].join("") : "") +# ) +# %} # Double-quoted string dqstring -> "\"" dstrchar:* "\"" {% (data) => data[1].join('') %} @@ -198,9 +198,9 @@ field -> | dqstring {% (data, start) => ({type: 'LiteralExpression', name: data[0], quoted: true, quotes: 'double', location: {start, end: start + data[0].length + 2}}) %} expression -> - decimal {% (data, start) => ({type: 'Tag', expression: {location: {start, end: start + data.join('').length}, type: 'LiteralExpression', quoted: false, value: Number(data.join(''))}}) %} - | regex {% (data, start) => ({type: 'Tag', expression: {location: {start, end: start + data.join('').length}, type: 'RegexExpression', value: data.join('')}}) %} - | range {% (data) => data[0] %} +# decimal {% (data, start) => ({type: 'Tag', expression: {location: {start, end: start + data.join('').length}, type: 'LiteralExpression', quoted: false, value: Number(data.join(''))}}) %} + regex {% (data, start) => ({type: 'Tag', expression: {location: {start, end: start + data.join('').length}, type: 'RegexExpression', value: data.join('')}}) %} +# | range {% (data) => data[0] %} | unquoted_value {% (data, start, reject) => { const value = data.join(''); @@ -236,36 +236,36 @@ expression -> | sqstring {% (data, start) => ({type: 'Tag', expression: {location: {start, end: start + data.join('').length + 2}, type: 'LiteralExpression', quoted: true, quotes: 'single', value: data.join('')}}) %} | dqstring {% (data, start) => ({type: 'Tag', expression: {location: {start, end: start + data.join('').length + 2}, type: 'LiteralExpression', quoted: true, quotes: 'double', value: data.join('')}}) %} -range -> - range_open decimal " TO " decimal range_close {% (data, start) => { - return { - location: { - start, - }, - type: 'Tag', - expression: { - location: { - start: data[0].location.start, - end: data[4].location.start + 1, - }, - type: 'RangeExpression', - range: { - min: data[1], - minInclusive: data[0].inclusive, - maxInclusive: data[4].inclusive, - max: data[3], - } - } - } - } %} - -range_open -> - "[" {% (data, start) => ({location: {start}, inclusive: true}) %} - | "{" {% (data, start) => ({location: {start}, inclusive: false}) %} - -range_close -> - "]" {% (data, start) => ({location: {start}, inclusive: true}) %} - | "}" {% (data, start) => ({location: {start}, inclusive: false}) %} +# range -> +# range_open decimal " TO " decimal range_close {% (data, start) => { +# return { +# location: { +# start, +# }, +# type: 'Tag', +# expression: { +# location: { +# start: data[0].location.start, +# end: data[4].location.start + 1, +# }, +# type: 'RangeExpression', +# range: { +# min: data[1], +# minInclusive: data[0].inclusive, +# maxInclusive: data[4].inclusive, +# max: data[3], +# } +# } +# } +# } %} +# +# range_open -> +# "[" {% (data, start) => ({location: {start}, inclusive: true}) %} +# | "{" {% (data, start) => ({location: {start}, inclusive: false}) %} +# +# range_close -> +# "]" {% (data, start) => ({location: {start}, inclusive: true}) %} +# | "}" {% (data, start) => ({location: {start}, inclusive: false}) %} comparison_operator -> ( diff --git a/packages/liqe/src/grammar.ts b/packages/liqe/src/grammar.ts index 2ad071d92..e8140b7a2 100644 --- a/packages/liqe/src/grammar.ts +++ b/packages/liqe/src/grammar.ts @@ -42,22 +42,6 @@ const grammar: Grammar = { {"name": "__$ebnf$1", "symbols": ["__$ebnf$1", "whitespace_character"], "postprocess": (d) => d[0].concat([d[1]])}, {"name": "__", "symbols": ["__$ebnf$1"], "postprocess": (data) => data[0].length}, {"name": "whitespace_character", "symbols": [/[ \t\n\v\f]/], "postprocess": id}, - {"name": "decimal$ebnf$1", "symbols": [{"literal":"-"}], "postprocess": id}, - {"name": "decimal$ebnf$1", "symbols": [], "postprocess": () => null}, - {"name": "decimal$ebnf$2", "symbols": [/[0-9]/]}, - {"name": "decimal$ebnf$2", "symbols": ["decimal$ebnf$2", /[0-9]/], "postprocess": (d) => d[0].concat([d[1]])}, - {"name": "decimal$ebnf$3$subexpression$1$ebnf$1", "symbols": [/[0-9]/]}, - {"name": "decimal$ebnf$3$subexpression$1$ebnf$1", "symbols": ["decimal$ebnf$3$subexpression$1$ebnf$1", /[0-9]/], "postprocess": (d) => d[0].concat([d[1]])}, - {"name": "decimal$ebnf$3$subexpression$1", "symbols": [{"literal":"."}, "decimal$ebnf$3$subexpression$1$ebnf$1"]}, - {"name": "decimal$ebnf$3", "symbols": ["decimal$ebnf$3$subexpression$1"], "postprocess": id}, - {"name": "decimal$ebnf$3", "symbols": [], "postprocess": () => null}, - {"name": "decimal", "symbols": ["decimal$ebnf$1", "decimal$ebnf$2", "decimal$ebnf$3"], "postprocess": - (data) => parseFloat( - (data[0] || "") + - data[1].join("") + - (data[2] ? "."+data[2][1].join("") : "") - ) - }, {"name": "dqstring$ebnf$1", "symbols": []}, {"name": "dqstring$ebnf$1", "symbols": ["dqstring$ebnf$1", "dstrchar"], "postprocess": (d) => d[0].concat([d[1]])}, {"name": "dqstring", "symbols": [{"literal":"\""}, "dqstring$ebnf$1", {"literal":"\""}], "postprocess": (data) => data[1].join('')}, @@ -213,9 +197,7 @@ const grammar: Grammar = { {"name": "field", "symbols": [/[_a-zA-Z$]/, "field$ebnf$1"], "postprocess": (data, start) => ({type: 'LiteralExpression', name: data[0] + data[1].join(''), quoted: false, location: {start, end: start + (data[0] + data[1].join('')).length}})}, {"name": "field", "symbols": ["sqstring"], "postprocess": (data, start) => ({type: 'LiteralExpression', name: data[0], quoted: true, quotes: 'single', location: {start, end: start + data[0].length + 2}})}, {"name": "field", "symbols": ["dqstring"], "postprocess": (data, start) => ({type: 'LiteralExpression', name: data[0], quoted: true, quotes: 'double', location: {start, end: start + data[0].length + 2}})}, - {"name": "expression", "symbols": ["decimal"], "postprocess": (data, start) => ({type: 'Tag', expression: {location: {start, end: start + data.join('').length}, type: 'LiteralExpression', quoted: false, value: Number(data.join(''))}})}, {"name": "expression", "symbols": ["regex"], "postprocess": (data, start) => ({type: 'Tag', expression: {location: {start, end: start + data.join('').length}, type: 'RegexExpression', value: data.join('')}})}, - {"name": "expression", "symbols": ["range"], "postprocess": (data) => data[0]}, {"name": "expression", "symbols": ["unquoted_value"], "postprocess": (data, start, reject) => { const value = data.join(''); @@ -250,32 +232,6 @@ const grammar: Grammar = { } }, {"name": "expression", "symbols": ["sqstring"], "postprocess": (data, start) => ({type: 'Tag', expression: {location: {start, end: start + data.join('').length + 2}, type: 'LiteralExpression', quoted: true, quotes: 'single', value: data.join('')}})}, {"name": "expression", "symbols": ["dqstring"], "postprocess": (data, start) => ({type: 'Tag', expression: {location: {start, end: start + data.join('').length + 2}, type: 'LiteralExpression', quoted: true, quotes: 'double', value: data.join('')}})}, - {"name": "range$string$1", "symbols": [{"literal":" "}, {"literal":"T"}, {"literal":"O"}, {"literal":" "}], "postprocess": (d) => d.join('')}, - {"name": "range", "symbols": ["range_open", "decimal", "range$string$1", "decimal", "range_close"], "postprocess": (data, start) => { - return { - location: { - start, - }, - type: 'Tag', - expression: { - location: { - start: data[0].location.start, - end: data[4].location.start + 1, - }, - type: 'RangeExpression', - range: { - min: data[1], - minInclusive: data[0].inclusive, - maxInclusive: data[4].inclusive, - max: data[3], - } - } - } - } }, - {"name": "range_open", "symbols": [{"literal":"["}], "postprocess": (data, start) => ({location: {start}, inclusive: true})}, - {"name": "range_open", "symbols": [{"literal":"{"}], "postprocess": (data, start) => ({location: {start}, inclusive: false})}, - {"name": "range_close", "symbols": [{"literal":"]"}], "postprocess": (data, start) => ({location: {start}, inclusive: true})}, - {"name": "range_close", "symbols": [{"literal":"}"}], "postprocess": (data, start) => ({location: {start}, inclusive: false})}, {"name": "comparison_operator$subexpression$1", "symbols": [{"literal":":"}]}, {"name": "comparison_operator$subexpression$1$string$1", "symbols": [{"literal":":"}, {"literal":"="}], "postprocess": (d) => d.join('')}, {"name": "comparison_operator$subexpression$1", "symbols": ["comparison_operator$subexpression$1$string$1"]}, diff --git a/packages/liqe/test/liqe/filter.ts b/packages/liqe/test/liqe/filter.ts index 9ed377a60..4b597038a 100644 --- a/packages/liqe/test/liqe/filter.ts +++ b/packages/liqe/test/liqe/filter.ts @@ -93,12 +93,12 @@ test('name:/(david)|(john)/', testQuery, ['david', 'john']); test('name:/(David)|(John)/', testQuery, []); test('name:/(David)|(John)/i', testQuery, ['david', 'john']); -test('height:[200 TO 300]', testQuery, ['robert', 'noah']); -test('height:[220 TO 300]', testQuery, ['robert', 'noah']); -test('height:{220 TO 300]', testQuery, ['noah']); -test('height:[200 TO 225]', testQuery, ['robert', 'noah']); -test('height:[200 TO 225}', testQuery, ['robert']); -test('height:{220 TO 225}', testQuery, []); +test.skip('height:[200 TO 300]', testQuery, ['robert', 'noah']); +test.skip('height:[220 TO 300]', testQuery, ['robert', 'noah']); +test.skip('height:{220 TO 300]', testQuery, ['noah']); +test.skip('height:[200 TO 225]', testQuery, ['robert', 'noah']); +test.skip('height:[200 TO 225}', testQuery, ['robert']); +test.skip('height:{220 TO 225}', testQuery, []); test('NOT David', testQuery, ['john', 'mike', 'robert', 'noah', 'foo bar', 'fox']); test('-David', testQuery, ['john', 'mike', 'robert', 'noah', 'foo bar', 'fox']); @@ -115,12 +115,12 @@ test('name:David OR name:John', testQuery, ['david', 'john']); test('name:"david" OR name:"john"', testQuery, ['david', 'john']); test('name:"David" OR name:"John"', testQuery, []); -test('height:=175', testQuery, ['john', 'mike']); -test('height:>200', testQuery, ['robert', 'noah']); -test('height:>220', testQuery, ['noah']); -test('height:>=220', testQuery, ['robert', 'noah']); +test.skip('height:=175', testQuery, ['john', 'mike']); +test.skip('height:>200', testQuery, ['robert', 'noah']); +test.skip('height:>220', testQuery, ['noah']); +test.skip('height:>=220', testQuery, ['robert', 'noah']); -test('height:=175 AND NOT name:mike', testQuery, ['john']); +test.skip('height:=175 AND NOT name:mike', testQuery, ['john']); test('"member"', testQuery, ['robert']); @@ -138,9 +138,9 @@ test('subscribed:true', testQuery, ['noah']); test('email:/[^.:@\\s](?:[^:@\\s]*[^.:@\\s])?@[^.@\\s]+(?:\\.[^.@\\s]+)*/', testQuery, ['noah']); test('phoneNumber:"404-050-2611"', testQuery, ['noah']); -test('phoneNumber:404', testQuery, ['noah']); +test.skip('phoneNumber:404', testQuery, ['noah']); -test('balance:364', testQuery, ['noah']); +test.skip('balance:364', testQuery, ['noah']); test('(David)', testQuery, ['david']); test('(name:david OR name:john)', testQuery, ['david', 'john']); diff --git a/packages/liqe/test/liqe/highlight.ts b/packages/liqe/test/liqe/highlight.ts index 4ee4dc6ea..d3c989839 100644 --- a/packages/liqe/test/liqe/highlight.ts +++ b/packages/liqe/test/liqe/highlight.ts @@ -83,7 +83,7 @@ test( ], ); -test( +test.skip( 'matches or', testQuery, 'name:foo OR name:bar OR height:=180', @@ -181,7 +181,7 @@ test.skip( ], ); -test( +test.skip( 'matches number', testQuery, 'height:=180', @@ -195,7 +195,7 @@ test( ], ); -test( +test.skip( 'matches range', testQuery, 'height:[100 TO 200]', diff --git a/packages/liqe/test/liqe/parse.ts b/packages/liqe/test/liqe/parse.ts index d445e05a7..a8bdaaf4a 100644 --- a/packages/liqe/test/liqe/parse.ts +++ b/packages/liqe/test/liqe/parse.ts @@ -529,7 +529,7 @@ test.skip('foo: bar', testQuery, { type: 'Tag', }); -test('foo:123', testQuery, { +test.skip('foo:123', testQuery, { expression: { location: { end: 7, @@ -564,7 +564,7 @@ test('foo:123', testQuery, { type: 'Tag', }); -test('foo:=123', testQuery, { +test.skip('foo:=123', testQuery, { expression: { location: { end: 8, @@ -636,7 +636,7 @@ test.skip('foo:= 123', testQuery, { type: 'Tag', }); -test('foo:=-123', testQuery, { +test.skip('foo:=-123', testQuery, { expression: { location: { end: 9, @@ -671,7 +671,7 @@ test('foo:=-123', testQuery, { type: 'Tag', }); -test('foo:=123.4', testQuery, { +test.skip('foo:=123.4', testQuery, { expression: { location: { end: 10, @@ -706,7 +706,7 @@ test('foo:=123.4', testQuery, { type: 'Tag', }); -test('foo:>=123', testQuery, { +test.skip('foo:>=123', testQuery, { expression: { location: { end: 9, @@ -2415,7 +2415,7 @@ test('(foo:bar OR baz:qux) OR quuz:corge', testQuery, { type: 'LogicalExpression', }); -test('[1 TO 2]', testQuery, { +test.skip('[1 TO 2]', testQuery, { expression: { location: { end: 8, @@ -2438,7 +2438,7 @@ test('[1 TO 2]', testQuery, { type: 'Tag', }); -test('{1 TO 2]', testQuery, { +test.skip('{1 TO 2]', testQuery, { expression: { location: { end: 8, @@ -2461,7 +2461,7 @@ test('{1 TO 2]', testQuery, { type: 'Tag', }); -test('[1 TO 2}', testQuery, { +test.skip('[1 TO 2}', testQuery, { expression: { location: { end: 8, @@ -2484,7 +2484,7 @@ test('[1 TO 2}', testQuery, { type: 'Tag', }); -test('{1 TO 2}', testQuery, { +test.skip('{1 TO 2}', testQuery, { expression: { location: { end: 8, diff --git a/packages/liqe/test/liqe/serialize.ts b/packages/liqe/test/liqe/serialize.ts index 2d03d6f0d..4f8e85fa4 100644 --- a/packages/liqe/test/liqe/serialize.ts +++ b/packages/liqe/test/liqe/serialize.ts @@ -52,19 +52,19 @@ test('foo:bar', testQuery); // https://github.com/gajus/liqe/issues/19 test.skip('foo: bar', testQuery); -test('foo:123', testQuery); +test.skip('foo:123', testQuery); -test('foo:=123', testQuery); +test.skip('foo:=123', testQuery); // https://github.com/gajus/liqe/issues/18 // https://github.com/gajus/liqe/issues/19 test.skip('foo:= 123', testQuery); -test('foo:=-123', testQuery); +test.skip('foo:=-123', testQuery); -test('foo:=123.4', testQuery); +test.skip('foo:=123.4', testQuery); -test('foo:>=123', testQuery); +test.skip('foo:>=123', testQuery); test('foo:true', testQuery); @@ -124,10 +124,10 @@ test('(foo:bar OR (baz:qux OR quuz:corge))', testQuery); test('((foo:bar OR baz:qux) OR quuz:corge)', testQuery); -test('[1 TO 2]', testQuery); +test.skip('[1 TO 2]', testQuery); -test('{1 TO 2]', testQuery); +test.skip('{1 TO 2]', testQuery); -test('[1 TO 2}', testQuery); +test.skip('[1 TO 2}', testQuery); -test('{1 TO 2}', testQuery); +test.skip('{1 TO 2}', testQuery); From 2155be470409caf1a1923f92b25641df23f9b24c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 12 Dec 2023 19:42:09 +0800 Subject: [PATCH 8/8] fix tests --- packages/api/src/services/library_item.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 0781d2927..997870c4e 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -130,7 +130,8 @@ export const sortParamsToSort = ( } const getColumnName = (field: string) => { - switch (field.toLowerCase()) { + const lowerCaseField = field.toLowerCase() + switch (lowerCaseField) { case 'language': return 'item_language' case 'subscription': @@ -138,17 +139,17 @@ const getColumnName = (field: string) => { return 'subscription' case 'site': return 'site_name' - case 'wordsCount': + case 'wordscount': return 'word_count' - case 'readPosition': + case 'readposition': return 'reading_progress_bottom_percent' case 'saved': case 'read': case 'updated': case 'published': - return `${field}_at` + return `${lowerCaseField}_at` default: - return field + return lowerCaseField } } @@ -419,7 +420,7 @@ export const buildQuery = ( [param]: ids, }) } - case 'recommendedBy': { + case 'recommendedby': { const param = `recommendedBy_${parameters.length}` if (value === '*') { // select all if * is provided @@ -456,8 +457,8 @@ export const buildQuery = ( case 'event': // mode is ignored and used only by the frontend return null - case 'readPosition': - case 'wordsCount': { + case 'readposition': + case 'wordscount': { const column = getColumnName(field.name) const operatorRegex = /([<>]=?)/