diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index d3b598623..45c033419 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -101,7 +101,7 @@ import { ParsedContentPuppeteer, parsePreparedContent, } from '../../utils/parser' -import { InFilter, parseSearchQuery } from '../../utils/search' +import { parseSearchQuery } from '../../utils/search' import { getStorageFileDetails } from '../../utils/uploads' import { itemTypeForContentType } from '../upload_files' @@ -651,8 +651,6 @@ export const searchResolver = authorized< return { errorCodes: [SearchErrorCode.QueryTooLong] } } - const searchQuery = params.query ? parseSearchQuery(params.query) : undefined - const { libraryItems, count } = await searchLibraryItems( { from: Number(startCursor), @@ -660,7 +658,7 @@ export const searchResolver = authorized< includePending: true, includeContent: !!params.includeContent, includeDeleted: params.query?.includes('in:trash'), - searchQuery, + query: params.query, }, uid ) @@ -758,10 +756,9 @@ export const updatesSinceResolver = authorized< } // create a search query - folder = folder || InFilter.ALL - const searchQuery = parseSearchQuery( - `in:${folder} updated:${startDate.toISOString()}` - ) + const query = `updated:${startDate.toISOString()}${ + folder ? ' in:' + folder : '' + }` const { libraryItems, count } = await searchLibraryItems( { @@ -769,7 +766,7 @@ export const updatesSinceResolver = authorized< size: size + 1, // fetch one more item to get next cursor includeDeleted: true, sort, - searchQuery, + query, }, uid ) @@ -842,8 +839,7 @@ export const bulkActionResolver = authorized< labels = await findLabelsByIds(labelIds, uid) } - const searchQuery = parseSearchQuery(query) - await updateLibraryItems(action, searchQuery, uid, labels, args) + await updateLibraryItems(action, query, uid, labels, args) return { success: true } } catch (error) { diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index fdd646380..823a62852 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1,11 +1,6 @@ import { LiqeQuery } from 'liqe' import { DateTime } from 'luxon' -import { - Brackets, - DeepPartial, - ObjectLiteral, - SelectQueryBuilder, -} from 'typeorm' +import { DeepPartial, ObjectLiteral } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' @@ -18,40 +13,38 @@ import { libraryItemRepository } from '../repository/library_item' import { SaveFollowingItemRequest } from '../routers/svc/following' import { generateSlug, wordsCount } from '../utils/helpers' import { createThumbnailUrl } from '../utils/imageproxy' -import { - DateFilter, - FieldFilter, - HasFilter, - InFilter, - LabelFilter, - LabelFilterType, - NoFilter, - RangeFilter, - ReadFilter, -} from '../utils/search' +import { parseSearchQuery } from '../utils/search' + +enum ReadFilter { + ALL = 'all', + READ = 'read', + READING = 'reading', + UNREAD = 'unread', +} + +enum InFilter { + ALL = 'all', + INBOX = 'inbox', + ARCHIVE = 'archive', + TRASH = 'trash', + FOLLOWING = 'following', +} + +enum HasFilter { + HIGHLIGHTS = 'highlights', + LABELS = 'labels', + SUBSCRIPTIONS = 'subscriptions', +} export interface SearchArgs { from?: number size?: number sort?: Sort - query?: string - inFilter?: InFilter - readFilter?: ReadFilter - typeFilter?: string - labelFilters?: LabelFilter[] - hasFilters?: HasFilter[] - dateFilters?: DateFilter[] - termFilters?: FieldFilter[] - matchFilters?: FieldFilter[] includePending?: boolean | null includeDeleted?: boolean - ids?: string[] - recommendedBy?: string includeContent?: boolean - noFilters?: NoFilter[] - rangeFilters?: RangeFilter[] useFolders?: boolean - searchQuery?: LiqeQuery + query?: string | null } export interface SearchResultItem { @@ -624,254 +617,6 @@ export const buildQuery = ( return serialize(searchQuery) } -const buildWhereClause = ( - queryBuilder: SelectQueryBuilder, - args: SearchArgs -) => { - if (args.query) { - queryBuilder - .addSelect( - "ts_rank_cd(library_item.search_tsv, websearch_to_tsquery('english', :query))", - 'rank' - ) - .andWhere( - "websearch_to_tsquery('english', :query) @@ library_item.search_tsv" - ) - .setParameter('query', args.query) - .orderBy('rank', 'DESC') - } - - if (args.typeFilter) { - queryBuilder.andWhere('lower(library_item.item_type) = :typeFilter', { - typeFilter: args.typeFilter.toLowerCase(), - }) - } - - if (args.inFilter !== InFilter.ALL) { - switch (args.inFilter) { - case InFilter.INBOX: { - // if useFolders is true, we only return items in the inbox folder - args.useFolders && - queryBuilder.andWhere("library_item.folder = 'inbox'") - // for old clients, we return items that are not archived - queryBuilder.andWhere('library_item.archived_at IS NULL') - break - } - case InFilter.ARCHIVE: - queryBuilder.andWhere('library_item.archived_at IS NOT NULL') - break - case InFilter.TRASH: - // return only deleted pages within 14 days - queryBuilder.andWhere( - "library_item.deleted_at >= now() - interval '14 days'" - ) - break - default: - queryBuilder.andWhere('library_item.folder = :folder', { - folder: args.inFilter, - }) - } - } - - if (args.readFilter !== ReadFilter.ALL) { - switch (args.readFilter) { - case ReadFilter.READ: - queryBuilder.andWhere( - 'library_item.reading_progress_bottom_percent > 98' - ) - break - case ReadFilter.READING: - queryBuilder.andWhere( - 'library_item.reading_progress_bottom_percent BETWEEN 2 AND 98' - ) - break - case ReadFilter.UNREAD: - queryBuilder.andWhere( - 'library_item.reading_progress_bottom_percent < 2' - ) - break - } - } - - if (args.hasFilters && args.hasFilters.length > 0) { - args.hasFilters.forEach((filter) => { - switch (filter) { - case HasFilter.HIGHLIGHTS: - queryBuilder.andWhere("library_item.highlight_annotations <> '{}'") - break - case HasFilter.LABELS: - queryBuilder.andWhere("library_item.label_names <> '{}'") - break - case HasFilter.SUBSCRIPTIONS: - queryBuilder.andWhere('library_item.subscription is NOT NULL') - } - }) - } - - if (args.labelFilters && args.labelFilters.length > 0) { - const includeLabels = args.labelFilters?.filter( - (filter) => filter.type === LabelFilterType.INCLUDE - ) - const excludeLabels = args.labelFilters?.filter( - (filter) => filter.type === LabelFilterType.EXCLUDE - ) - - if (includeLabels && includeLabels.length > 0) { - includeLabels.forEach((includeLabel, i) => { - const param = `includeLabels_${i}` - const hasWildcard = includeLabel.labels.some((label) => - label.includes('*') - ) - if (hasWildcard) { - queryBuilder.andWhere( - new Brackets((qb) => { - includeLabel.labels.forEach((label, j) => { - const param = `includeLabels_${i}_${j}` - qb.orWhere( - `array_to_string(array_cat(library_item.label_names, library_item.highlight_labels)::text[], ',') ILIKE :${param}`, - { - [param]: label.replace(/\*/g, '%'), - } - ) - }) - }) - ) - } else { - queryBuilder.andWhere( - `lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[] && ARRAY[:...${param}]::text[]`, - { - [param]: includeLabel.labels, - } - ) - } - }) - } - - if (excludeLabels && excludeLabels.length > 0) { - const labels = excludeLabels.flatMap((filter) => filter.labels) - - const hasWildcard = labels.some((label) => label.includes('*')) - - if (hasWildcard) { - queryBuilder.andWhere( - new Brackets((qb) => { - labels.forEach((label, i) => { - const param = `excludeLabels_${i}` - qb.andWhere( - `array_to_string(array_cat(library_item.label_names, library_item.highlight_labels)::text[], ',') NOT ILIKE :${param}`, - { - [param]: label.replace(/\*/g, '%'), - } - ) - }) - }) - ) - } else { - queryBuilder.andWhere( - 'NOT lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[] && ARRAY[:...excludeLabels]::text[]', - { - excludeLabels: labels, - } - ) - } - } - } - - if (args.dateFilters && args.dateFilters.length > 0) { - args.dateFilters.forEach((filter) => { - const startDate = `${filter.field}_start` - const endDate = `${filter.field}_end` - queryBuilder.andWhere( - `library_item.${filter.field} between :${startDate} and :${endDate}`, - { - [startDate]: filter.startDate ?? new Date(0), - [endDate]: filter.endDate ?? new Date(), - } - ) - }) - } - - if (args.termFilters && args.termFilters.length > 0) { - args.termFilters.forEach((filter) => { - const param = `term_${filter.field}` - queryBuilder.andWhere(`lower(library_item.${filter.field}) = :${param}`, { - [param]: filter.value.toLowerCase(), - }) - }) - } - - if (args.matchFilters && args.matchFilters.length > 0) { - args.matchFilters.forEach((filter) => { - const param = `match_${filter.field}` - queryBuilder.andWhere( - new Brackets((qb) => { - qb.andWhere( - `websearch_to_tsquery('english', :${param}) @@ library_item.${filter.field}_tsv`, - { - [param]: filter.value, - } - ).orWhere(`${filter.field} ILIKE :value`, { - value: `%${filter.value}%`, - }) - }) - ) - }) - } - - if (args.ids && args.ids.length > 0) { - queryBuilder.andWhere('library_item.id = ANY(:ids)', { - ids: args.ids, - }) - } - - if (!args.includePending) { - queryBuilder.andWhere("library_item.state <> 'PROCESSING'") - } - - if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) { - queryBuilder.andWhere("library_item.state <> 'DELETED'") - } - - if (args.noFilters) { - args.noFilters.forEach((filter) => { - queryBuilder.andWhere( - `(library_item.${filter.field} = '{}' OR library_item.${filter.field} IS NULL)` - ) - }) - } - - if (args.recommendedBy) { - if (args.recommendedBy === '*') { - // select all if * is provided - queryBuilder.andWhere(`library_item.recommender_names <> '{}'`) - } else { - // select only if the user is recommended by the provided user - queryBuilder.andWhere( - 'lower(library_item.recommender_names::text)::text[] && ARRAY[:recommendedBy]::text[]', - { - recommendedBy: args.recommendedBy.toLowerCase(), - } - ) - } - } - - if (args.includeContent) { - queryBuilder.addSelect('library_item.readableContent') - } - - if (args.rangeFilters && args.rangeFilters.length > 0) { - args.rangeFilters.forEach((filter, i) => { - const param = `range_${filter.field}_${i}` - queryBuilder.andWhere( - `library_item.${filter.field} ${filter.operator} :${param}`, - { - [param]: filter.value, - } - ) - }) - } -} - export const searchLibraryItems = async ( args: SearchArgs, userId: string @@ -891,6 +636,11 @@ export const searchLibraryItems = async ( column !== 'library_item.originalContent' ) + let searchQuery: LiqeQuery | undefined + if (args.query) { + searchQuery = parseSearchQuery(args.query) + } + // add pagination and sorting return authTrx( async (tx) => { @@ -899,16 +649,11 @@ export const searchLibraryItems = async ( .select(selectColumns) .where('library_item.user_id = :userId', { userId }) - if (args.searchQuery) { + if (searchQuery) { const parameters: ObjectLiteral[] = [] const selects: Select[] = [] const orders: Sort[] = [] - const whereClause = buildQuery( - args.searchQuery, - parameters, - selects, - orders - ) + const whereClause = buildQuery(searchQuery, parameters, selects, orders) whereClause && queryBuilder .andWhere(whereClause) @@ -1243,7 +988,7 @@ export const countByCreatedAt = async ( export const updateLibraryItems = async ( action: BulkActionType, - searchQuery: LiqeQuery, + query: string, userId: string, labels?: Label[], args?: unknown @@ -1297,6 +1042,8 @@ export const updateLibraryItems = async ( throw new Error('Invalid bulk action') } + const searchQuery = parseSearchQuery(query) + await authTrx(async (tx) => { const queryBuilder = tx .createQueryBuilder(LibraryItem, 'library_item') diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index 8c1d82669..33329e13a 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -1,337 +1,4 @@ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { LiqeQuery, parse } from 'liqe' -import { DateTime } from 'luxon' -import { ISearchParserDictionary } from 'search-query-parser' -import { PageType } from '../generated/graphql' - -export enum ReadFilter { - ALL = 'all', - READ = 'read', - READING = 'reading', - UNREAD = 'unread', -} - -export enum InFilter { - ALL = 'all', - INBOX = 'inbox', - ARCHIVE = 'archive', - TRASH = 'trash', - FOLLOWING = 'following', -} - -export interface SearchFilter { - query: string | undefined - inFilter: InFilter - readFilter: ReadFilter - typeFilter?: string - labelFilters: LabelFilter[] - hasFilters: HasFilter[] - dateFilters: DateFilter[] - termFilters: FieldFilter[] - matchFilters: FieldFilter[] - ids: string[] - recommendedBy?: string - noFilters: NoFilter[] - rangeFilters: RangeFilter[] - useFolders?: boolean -} - -export enum LabelFilterType { - INCLUDE, - EXCLUDE, -} - -export type LabelFilter = { - type: LabelFilterType - labels: string[] -} - -export enum HasFilter { - HIGHLIGHTS = 'highlights', - LABELS = 'labels', - SUBSCRIPTIONS = 'subscriptions', -} - -export interface DateFilter { - field: string - startDate?: Date - endDate?: Date -} - -export interface RangeFilter { - field: string - operator: string - value: number -} - -export interface FieldFilter { - nested?: boolean - field: string - value: string -} - -export interface NoFilter { - field: string -} - -const parseStringValue = (str?: string): string | undefined => { - if (str === undefined) { - return undefined - } - - return str.toLowerCase() -} - -const parseIsFilter = (str: string | undefined): ReadFilter => { - switch (str?.toUpperCase()) { - case 'READ': - return ReadFilter.READ - case 'READING': - return ReadFilter.READING - case 'UNREAD': - return ReadFilter.UNREAD - } - return ReadFilter.ALL -} - -const parseInFilter = ( - str: string | undefined, - query: string | undefined -): InFilter => { - switch (str?.toUpperCase()) { - case 'ALL': - return InFilter.ALL - case 'INBOX': - return InFilter.INBOX - case 'ARCHIVE': - return InFilter.ARCHIVE - case 'TRASH': - return InFilter.TRASH - case 'FOLLOWING': - return InFilter.FOLLOWING - } - - return query ? InFilter.ALL : InFilter.INBOX -} - -const parseTypeFilter = (str: string | undefined): string | undefined => { - if (str === undefined) { - return undefined - } - - switch (str.toLowerCase()) { - case 'article': - return PageType.Article - case 'book': - return PageType.Book - case 'pdf': - case 'file': - return PageType.File - case 'profile': - return PageType.Profile - case 'website': - return PageType.Website - case 'unknown': - return PageType.Unknown - } - return undefined -} - -const parseLabelFilter = ( - str?: string, - exclude?: ISearchParserDictionary -): LabelFilter | undefined => { - if (str === undefined) { - return undefined - } - - const labels = str.split(',') - - // check if the labels are on the exclusion list - const excluded = exclude?.label && exclude.label.includes(...labels) - - return { - type: excluded ? LabelFilterType.EXCLUDE : LabelFilterType.INCLUDE, - // use lower case for label names - labels: labels.map((label) => label.toLowerCase()), - } -} - -const parseHasFilter = (str?: string): HasFilter | undefined => { - if (str === undefined) { - return undefined - } - - switch (str.toUpperCase()) { - case 'HIGHLIGHTS': - return HasFilter.HIGHLIGHTS - case 'LABELS': - return HasFilter.LABELS - case 'SUBSCRIPTIONS': - return HasFilter.SUBSCRIPTIONS - } -} - -const parseDateFilter = ( - field: string, - str?: string -): DateFilter | undefined => { - if (str === undefined) { - return undefined - } - - switch (field.toLowerCase()) { - case 'published': - field = 'published_at' - break - case 'saved': - field = 'saved_at' - break - case 'updated': - field = 'updated_at' - } - - // check for special date filters - switch (str.toLowerCase()) { - case 'today': - return { - field, - startDate: DateTime.local().startOf('day').toJSDate(), - } - case 'yesterday': { - const yesterday = DateTime.local().minus({ days: 1 }) - return { - field, - startDate: yesterday.startOf('day').toJSDate(), - endDate: yesterday.endOf('day').toJSDate(), - } - } - case 'this week': - return { - field, - startDate: DateTime.local().startOf('week').toJSDate(), - } - case 'this month': - return { - field, - startDate: DateTime.local().startOf('month').toJSDate(), - } - } - - // check for date ranges - const [start, end] = str.split('..') - const startDate = start && start !== '*' ? new Date(start) : undefined - const endDate = end && end !== '*' ? new Date(end) : undefined - - return { - field, - startDate, - endDate, - } -} - -const parseRangeFilter = ( - field: string, - str?: string -): RangeFilter | undefined => { - if (str === undefined) { - return undefined - } - - switch (field.toUpperCase()) { - case 'WORDSCOUNT': - field = 'word_count' - break - case 'READPOSITION': - field = 'reading_progress_bottom_percent' - break - default: - return undefined - } - - const operatorRegex = /([<>]=?)/ - const operator = str.match(operatorRegex)?.[0] - if (!operator) { - return undefined - } - - const value = str.replace(operatorRegex, '') - if (!value) { - return undefined - } - - return { - field, - operator, - value: Number(value), - } -} - -const parseFieldFilter = ( - field: string, - str?: string -): FieldFilter | undefined => { - if (str === undefined) { - return undefined - } - - // normalize the term to lower case - const value = str.toLowerCase() - - switch (field.toUpperCase()) { - case 'LANGUAGE': - return { - field: 'item_language', - value, - } - case 'SUBSCRIPTION': - case 'RSS': - return { - field: 'subscription', - value, - } - case 'SITE': - return { - field: 'site_name', - value, - } - } - - return { - field, - value, - } -} - -const parseIds = (str?: string): string[] | undefined => { - if (str === undefined) { - return undefined - } - - return str.split(',') -} - -const parseNoFilter = (str?: string): NoFilter | undefined => { - if (str === undefined) { - return undefined - } - - const strLower = str.toLowerCase() - switch (strLower) { - case 'highlight': - return { field: 'highlight_annotations' } - case 'label': - return { field: 'label_names' } - case 'subscription': - return { field: 'subscription' } - } - - return undefined -} export const parseSearchQuery = (query: string): LiqeQuery => { const searchQuery = query @@ -342,138 +9,4 @@ export const parseSearchQuery = (query: string): LiqeQuery => { .replace(/(\w+):([^"\s]+)/g, '$1:"$2"') return parse(searchQuery) - - // const parsed = parse(searchQuery, { - // keywords: [ - // 'in', - // 'is', - // 'type', - // 'label', - // 'sort', - // 'has', - // 'saved', - // 'author', - // 'published', - // 'subscription', - // 'language', - // 'title', - // 'description', - // 'content', - // 'updated', - // 'includes', - // 'recommendedBy', - // 'no', - // 'mode', - // 'site', - // 'note', - // 'rss', - // 'wordsCount', - // 'readPosition', - // ], - // tokenize: true, - // }) - // if (parsed.offsets) { - // const texts = parsed.offsets - // .filter((offset) => 'text' in offset) - // .map((offset) => offset as SearchParserTextOffset) - - // if (texts.length > 0) { - // result.query = texts - // .map((offset: SearchParserTextOffset) => { - // // TODO: the parser library doesn't let us accurately - // // pull out quoted text, so we are just assuming - // // anything with spaces is quoted. - // if (offset.text.indexOf(' ') > -1) { - // return `"${offset.text}"` - // } - // return offset.text - // }) - // .join(' ') - // } else { - // result.query = undefined - // } - - // const keywords = parsed.offsets - // .filter((offset) => 'keyword' in offset) - // .map((offset) => offset as SearchParserKeyWordOffset) - - // for (const keyword of keywords) { - // switch (keyword.keyword) { - // case 'in': - // result.inFilter = parseInFilter(keyword.value, result.query) - // break - // case 'is': - // result.readFilter = parseIsFilter(keyword.value) - // break - // case 'type': - // result.typeFilter = parseTypeFilter(keyword.value) - // break - // case 'label': { - // const labelFilter = parseLabelFilter(keyword.value, parsed.exclude) - // labelFilter && result.labelFilters.push(labelFilter) - // break - // } - // case 'sort': - // result.sort = parseSort(keyword.value) - // break - // case 'has': { - // const hasFilter = parseHasFilter(keyword.value) - // hasFilter !== undefined && result.hasFilters.push(hasFilter) - // break - // } - // case 'saved': - // case 'read': - // case 'updated': - // case 'published': { - // const dateFilter = parseDateFilter(keyword.keyword, keyword.value) - // dateFilter && result.dateFilters.push(dateFilter) - // break - // } - // // term filters - // case 'subscription': - // case 'rss': - // case 'language': { - // const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value) - // fieldFilter && result.termFilters.push(fieldFilter) - // break - // } - // // match filters - // case 'author': - // case 'title': - // case 'description': - // case 'note': - // case 'site': - // case 'content': { - // const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value) - // fieldFilter && result.matchFilters.push(fieldFilter) - // break - // } - // case 'includes': { - // const ids = parseIds(keyword.value) - // ids && result.ids.push(...ids) - // break - // } - // case 'recommendedBy': { - // result.recommendedBy = parseStringValue(keyword.value) - // break - // } - // case 'no': { - // const noFilter = parseNoFilter(keyword.value) - // noFilter && result.noFilters.push(noFilter) - // break - // } - // case 'mode': - // // mode is ignored and used only by the frontend - // break - // case 'readPosition': - // case 'wordsCount': { - // const rangeFilter = parseRangeFilter(keyword.keyword, keyword.value) - // rangeFilter && result.rangeFilters.push(rangeFilter) - // break - // } - // } - // } - // } - - // return result }