remove unused code

This commit is contained in:
Hongbo Wu 2023-12-05 13:04:07 +08:00
parent 3c67d1911c
commit e6322ffaf8
3 changed files with 41 additions and 765 deletions

View file

@ -101,7 +101,7 @@ import {
ParsedContentPuppeteer,
parsePreparedContent,
} from '../../utils/parser'
import { InFilter, parseSearchQuery } from '../../utils/search'
import { parseSearchQuery } from '../../utils/search'
import { getStorageFileDetails } from '../../utils/uploads'
import { itemTypeForContentType } from '../upload_files'
@ -651,8 +651,6 @@ export const searchResolver = authorized<
return { errorCodes: [SearchErrorCode.QueryTooLong] }
}
const searchQuery = params.query ? parseSearchQuery(params.query) : undefined
const { libraryItems, count } = await searchLibraryItems(
{
from: Number(startCursor),
@ -660,7 +658,7 @@ export const searchResolver = authorized<
includePending: true,
includeContent: !!params.includeContent,
includeDeleted: params.query?.includes('in:trash'),
searchQuery,
query: params.query,
},
uid
)
@ -758,10 +756,9 @@ export const updatesSinceResolver = authorized<
}
// create a search query
folder = folder || InFilter.ALL
const searchQuery = parseSearchQuery(
`in:${folder} updated:${startDate.toISOString()}`
)
const query = `updated:${startDate.toISOString()}${
folder ? ' in:' + folder : ''
}`
const { libraryItems, count } = await searchLibraryItems(
{
@ -769,7 +766,7 @@ export const updatesSinceResolver = authorized<
size: size + 1, // fetch one more item to get next cursor
includeDeleted: true,
sort,
searchQuery,
query,
},
uid
)
@ -842,8 +839,7 @@ export const bulkActionResolver = authorized<
labels = await findLabelsByIds(labelIds, uid)
}
const searchQuery = parseSearchQuery(query)
await updateLibraryItems(action, searchQuery, uid, labels, args)
await updateLibraryItems(action, query, uid, labels, args)
return { success: true }
} catch (error) {

View file

@ -1,11 +1,6 @@
import { LiqeQuery } from 'liqe'
import { DateTime } from 'luxon'
import {
Brackets,
DeepPartial,
ObjectLiteral,
SelectQueryBuilder,
} from 'typeorm'
import { DeepPartial, ObjectLiteral } from 'typeorm'
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'
import { EntityLabel } from '../entity/entity_label'
import { Highlight } from '../entity/highlight'
@ -18,40 +13,38 @@ import { libraryItemRepository } from '../repository/library_item'
import { SaveFollowingItemRequest } from '../routers/svc/following'
import { generateSlug, wordsCount } from '../utils/helpers'
import { createThumbnailUrl } from '../utils/imageproxy'
import {
DateFilter,
FieldFilter,
HasFilter,
InFilter,
LabelFilter,
LabelFilterType,
NoFilter,
RangeFilter,
ReadFilter,
} from '../utils/search'
import { parseSearchQuery } from '../utils/search'
enum ReadFilter {
ALL = 'all',
READ = 'read',
READING = 'reading',
UNREAD = 'unread',
}
enum InFilter {
ALL = 'all',
INBOX = 'inbox',
ARCHIVE = 'archive',
TRASH = 'trash',
FOLLOWING = 'following',
}
enum HasFilter {
HIGHLIGHTS = 'highlights',
LABELS = 'labels',
SUBSCRIPTIONS = 'subscriptions',
}
export interface SearchArgs {
from?: number
size?: number
sort?: Sort
query?: string
inFilter?: InFilter
readFilter?: ReadFilter
typeFilter?: string
labelFilters?: LabelFilter[]
hasFilters?: HasFilter[]
dateFilters?: DateFilter[]
termFilters?: FieldFilter[]
matchFilters?: FieldFilter[]
includePending?: boolean | null
includeDeleted?: boolean
ids?: string[]
recommendedBy?: string
includeContent?: boolean
noFilters?: NoFilter[]
rangeFilters?: RangeFilter[]
useFolders?: boolean
searchQuery?: LiqeQuery
query?: string | null
}
export interface SearchResultItem {
@ -624,254 +617,6 @@ export const buildQuery = (
return serialize(searchQuery)
}
const buildWhereClause = (
queryBuilder: SelectQueryBuilder<LibraryItem>,
args: SearchArgs
) => {
if (args.query) {
queryBuilder
.addSelect(
"ts_rank_cd(library_item.search_tsv, websearch_to_tsquery('english', :query))",
'rank'
)
.andWhere(
"websearch_to_tsquery('english', :query) @@ library_item.search_tsv"
)
.setParameter('query', args.query)
.orderBy('rank', 'DESC')
}
if (args.typeFilter) {
queryBuilder.andWhere('lower(library_item.item_type) = :typeFilter', {
typeFilter: args.typeFilter.toLowerCase(),
})
}
if (args.inFilter !== InFilter.ALL) {
switch (args.inFilter) {
case InFilter.INBOX: {
// if useFolders is true, we only return items in the inbox folder
args.useFolders &&
queryBuilder.andWhere("library_item.folder = 'inbox'")
// for old clients, we return items that are not archived
queryBuilder.andWhere('library_item.archived_at IS NULL')
break
}
case InFilter.ARCHIVE:
queryBuilder.andWhere('library_item.archived_at IS NOT NULL')
break
case InFilter.TRASH:
// return only deleted pages within 14 days
queryBuilder.andWhere(
"library_item.deleted_at >= now() - interval '14 days'"
)
break
default:
queryBuilder.andWhere('library_item.folder = :folder', {
folder: args.inFilter,
})
}
}
if (args.readFilter !== ReadFilter.ALL) {
switch (args.readFilter) {
case ReadFilter.READ:
queryBuilder.andWhere(
'library_item.reading_progress_bottom_percent > 98'
)
break
case ReadFilter.READING:
queryBuilder.andWhere(
'library_item.reading_progress_bottom_percent BETWEEN 2 AND 98'
)
break
case ReadFilter.UNREAD:
queryBuilder.andWhere(
'library_item.reading_progress_bottom_percent < 2'
)
break
}
}
if (args.hasFilters && args.hasFilters.length > 0) {
args.hasFilters.forEach((filter) => {
switch (filter) {
case HasFilter.HIGHLIGHTS:
queryBuilder.andWhere("library_item.highlight_annotations <> '{}'")
break
case HasFilter.LABELS:
queryBuilder.andWhere("library_item.label_names <> '{}'")
break
case HasFilter.SUBSCRIPTIONS:
queryBuilder.andWhere('library_item.subscription is NOT NULL')
}
})
}
if (args.labelFilters && args.labelFilters.length > 0) {
const includeLabels = args.labelFilters?.filter(
(filter) => filter.type === LabelFilterType.INCLUDE
)
const excludeLabels = args.labelFilters?.filter(
(filter) => filter.type === LabelFilterType.EXCLUDE
)
if (includeLabels && includeLabels.length > 0) {
includeLabels.forEach((includeLabel, i) => {
const param = `includeLabels_${i}`
const hasWildcard = includeLabel.labels.some((label) =>
label.includes('*')
)
if (hasWildcard) {
queryBuilder.andWhere(
new Brackets((qb) => {
includeLabel.labels.forEach((label, j) => {
const param = `includeLabels_${i}_${j}`
qb.orWhere(
`array_to_string(array_cat(library_item.label_names, library_item.highlight_labels)::text[], ',') ILIKE :${param}`,
{
[param]: label.replace(/\*/g, '%'),
}
)
})
})
)
} else {
queryBuilder.andWhere(
`lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[] && ARRAY[:...${param}]::text[]`,
{
[param]: includeLabel.labels,
}
)
}
})
}
if (excludeLabels && excludeLabels.length > 0) {
const labels = excludeLabels.flatMap((filter) => filter.labels)
const hasWildcard = labels.some((label) => label.includes('*'))
if (hasWildcard) {
queryBuilder.andWhere(
new Brackets((qb) => {
labels.forEach((label, i) => {
const param = `excludeLabels_${i}`
qb.andWhere(
`array_to_string(array_cat(library_item.label_names, library_item.highlight_labels)::text[], ',') NOT ILIKE :${param}`,
{
[param]: label.replace(/\*/g, '%'),
}
)
})
})
)
} else {
queryBuilder.andWhere(
'NOT lower(array_cat(library_item.label_names, library_item.highlight_labels)::text)::text[] && ARRAY[:...excludeLabels]::text[]',
{
excludeLabels: labels,
}
)
}
}
}
if (args.dateFilters && args.dateFilters.length > 0) {
args.dateFilters.forEach((filter) => {
const startDate = `${filter.field}_start`
const endDate = `${filter.field}_end`
queryBuilder.andWhere(
`library_item.${filter.field} between :${startDate} and :${endDate}`,
{
[startDate]: filter.startDate ?? new Date(0),
[endDate]: filter.endDate ?? new Date(),
}
)
})
}
if (args.termFilters && args.termFilters.length > 0) {
args.termFilters.forEach((filter) => {
const param = `term_${filter.field}`
queryBuilder.andWhere(`lower(library_item.${filter.field}) = :${param}`, {
[param]: filter.value.toLowerCase(),
})
})
}
if (args.matchFilters && args.matchFilters.length > 0) {
args.matchFilters.forEach((filter) => {
const param = `match_${filter.field}`
queryBuilder.andWhere(
new Brackets((qb) => {
qb.andWhere(
`websearch_to_tsquery('english', :${param}) @@ library_item.${filter.field}_tsv`,
{
[param]: filter.value,
}
).orWhere(`${filter.field} ILIKE :value`, {
value: `%${filter.value}%`,
})
})
)
})
}
if (args.ids && args.ids.length > 0) {
queryBuilder.andWhere('library_item.id = ANY(:ids)', {
ids: args.ids,
})
}
if (!args.includePending) {
queryBuilder.andWhere("library_item.state <> 'PROCESSING'")
}
if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) {
queryBuilder.andWhere("library_item.state <> 'DELETED'")
}
if (args.noFilters) {
args.noFilters.forEach((filter) => {
queryBuilder.andWhere(
`(library_item.${filter.field} = '{}' OR library_item.${filter.field} IS NULL)`
)
})
}
if (args.recommendedBy) {
if (args.recommendedBy === '*') {
// select all if * is provided
queryBuilder.andWhere(`library_item.recommender_names <> '{}'`)
} else {
// select only if the user is recommended by the provided user
queryBuilder.andWhere(
'lower(library_item.recommender_names::text)::text[] && ARRAY[:recommendedBy]::text[]',
{
recommendedBy: args.recommendedBy.toLowerCase(),
}
)
}
}
if (args.includeContent) {
queryBuilder.addSelect('library_item.readableContent')
}
if (args.rangeFilters && args.rangeFilters.length > 0) {
args.rangeFilters.forEach((filter, i) => {
const param = `range_${filter.field}_${i}`
queryBuilder.andWhere(
`library_item.${filter.field} ${filter.operator} :${param}`,
{
[param]: filter.value,
}
)
})
}
}
export const searchLibraryItems = async (
args: SearchArgs,
userId: string
@ -891,6 +636,11 @@ export const searchLibraryItems = async (
column !== 'library_item.originalContent'
)
let searchQuery: LiqeQuery | undefined
if (args.query) {
searchQuery = parseSearchQuery(args.query)
}
// add pagination and sorting
return authTrx(
async (tx) => {
@ -899,16 +649,11 @@ export const searchLibraryItems = async (
.select(selectColumns)
.where('library_item.user_id = :userId', { userId })
if (args.searchQuery) {
if (searchQuery) {
const parameters: ObjectLiteral[] = []
const selects: Select[] = []
const orders: Sort[] = []
const whereClause = buildQuery(
args.searchQuery,
parameters,
selects,
orders
)
const whereClause = buildQuery(searchQuery, parameters, selects, orders)
whereClause &&
queryBuilder
.andWhere(whereClause)
@ -1243,7 +988,7 @@ export const countByCreatedAt = async (
export const updateLibraryItems = async (
action: BulkActionType,
searchQuery: LiqeQuery,
query: string,
userId: string,
labels?: Label[],
args?: unknown
@ -1297,6 +1042,8 @@ export const updateLibraryItems = async (
throw new Error('Invalid bulk action')
}
const searchQuery = parseSearchQuery(query)
await authTrx(async (tx) => {
const queryBuilder = tx
.createQueryBuilder(LibraryItem, 'library_item')

View file

@ -1,337 +1,4 @@
/* eslint-disable @typescript-eslint/restrict-template-expressions */
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { LiqeQuery, parse } from 'liqe'
import { DateTime } from 'luxon'
import { ISearchParserDictionary } from 'search-query-parser'
import { PageType } from '../generated/graphql'
export enum ReadFilter {
ALL = 'all',
READ = 'read',
READING = 'reading',
UNREAD = 'unread',
}
export enum InFilter {
ALL = 'all',
INBOX = 'inbox',
ARCHIVE = 'archive',
TRASH = 'trash',
FOLLOWING = 'following',
}
export interface SearchFilter {
query: string | undefined
inFilter: InFilter
readFilter: ReadFilter
typeFilter?: string
labelFilters: LabelFilter[]
hasFilters: HasFilter[]
dateFilters: DateFilter[]
termFilters: FieldFilter[]
matchFilters: FieldFilter[]
ids: string[]
recommendedBy?: string
noFilters: NoFilter[]
rangeFilters: RangeFilter[]
useFolders?: boolean
}
export enum LabelFilterType {
INCLUDE,
EXCLUDE,
}
export type LabelFilter = {
type: LabelFilterType
labels: string[]
}
export enum HasFilter {
HIGHLIGHTS = 'highlights',
LABELS = 'labels',
SUBSCRIPTIONS = 'subscriptions',
}
export interface DateFilter {
field: string
startDate?: Date
endDate?: Date
}
export interface RangeFilter {
field: string
operator: string
value: number
}
export interface FieldFilter {
nested?: boolean
field: string
value: string
}
export interface NoFilter {
field: string
}
const parseStringValue = (str?: string): string | undefined => {
if (str === undefined) {
return undefined
}
return str.toLowerCase()
}
const parseIsFilter = (str: string | undefined): ReadFilter => {
switch (str?.toUpperCase()) {
case 'READ':
return ReadFilter.READ
case 'READING':
return ReadFilter.READING
case 'UNREAD':
return ReadFilter.UNREAD
}
return ReadFilter.ALL
}
const parseInFilter = (
str: string | undefined,
query: string | undefined
): InFilter => {
switch (str?.toUpperCase()) {
case 'ALL':
return InFilter.ALL
case 'INBOX':
return InFilter.INBOX
case 'ARCHIVE':
return InFilter.ARCHIVE
case 'TRASH':
return InFilter.TRASH
case 'FOLLOWING':
return InFilter.FOLLOWING
}
return query ? InFilter.ALL : InFilter.INBOX
}
const parseTypeFilter = (str: string | undefined): string | undefined => {
if (str === undefined) {
return undefined
}
switch (str.toLowerCase()) {
case 'article':
return PageType.Article
case 'book':
return PageType.Book
case 'pdf':
case 'file':
return PageType.File
case 'profile':
return PageType.Profile
case 'website':
return PageType.Website
case 'unknown':
return PageType.Unknown
}
return undefined
}
const parseLabelFilter = (
str?: string,
exclude?: ISearchParserDictionary
): LabelFilter | undefined => {
if (str === undefined) {
return undefined
}
const labels = str.split(',')
// check if the labels are on the exclusion list
const excluded = exclude?.label && exclude.label.includes(...labels)
return {
type: excluded ? LabelFilterType.EXCLUDE : LabelFilterType.INCLUDE,
// use lower case for label names
labels: labels.map((label) => label.toLowerCase()),
}
}
const parseHasFilter = (str?: string): HasFilter | undefined => {
if (str === undefined) {
return undefined
}
switch (str.toUpperCase()) {
case 'HIGHLIGHTS':
return HasFilter.HIGHLIGHTS
case 'LABELS':
return HasFilter.LABELS
case 'SUBSCRIPTIONS':
return HasFilter.SUBSCRIPTIONS
}
}
const parseDateFilter = (
field: string,
str?: string
): DateFilter | undefined => {
if (str === undefined) {
return undefined
}
switch (field.toLowerCase()) {
case 'published':
field = 'published_at'
break
case 'saved':
field = 'saved_at'
break
case 'updated':
field = 'updated_at'
}
// check for special date filters
switch (str.toLowerCase()) {
case 'today':
return {
field,
startDate: DateTime.local().startOf('day').toJSDate(),
}
case 'yesterday': {
const yesterday = DateTime.local().minus({ days: 1 })
return {
field,
startDate: yesterday.startOf('day').toJSDate(),
endDate: yesterday.endOf('day').toJSDate(),
}
}
case 'this week':
return {
field,
startDate: DateTime.local().startOf('week').toJSDate(),
}
case 'this month':
return {
field,
startDate: DateTime.local().startOf('month').toJSDate(),
}
}
// check for date ranges
const [start, end] = str.split('..')
const startDate = start && start !== '*' ? new Date(start) : undefined
const endDate = end && end !== '*' ? new Date(end) : undefined
return {
field,
startDate,
endDate,
}
}
const parseRangeFilter = (
field: string,
str?: string
): RangeFilter | undefined => {
if (str === undefined) {
return undefined
}
switch (field.toUpperCase()) {
case 'WORDSCOUNT':
field = 'word_count'
break
case 'READPOSITION':
field = 'reading_progress_bottom_percent'
break
default:
return undefined
}
const operatorRegex = /([<>]=?)/
const operator = str.match(operatorRegex)?.[0]
if (!operator) {
return undefined
}
const value = str.replace(operatorRegex, '')
if (!value) {
return undefined
}
return {
field,
operator,
value: Number(value),
}
}
const parseFieldFilter = (
field: string,
str?: string
): FieldFilter | undefined => {
if (str === undefined) {
return undefined
}
// normalize the term to lower case
const value = str.toLowerCase()
switch (field.toUpperCase()) {
case 'LANGUAGE':
return {
field: 'item_language',
value,
}
case 'SUBSCRIPTION':
case 'RSS':
return {
field: 'subscription',
value,
}
case 'SITE':
return {
field: 'site_name',
value,
}
}
return {
field,
value,
}
}
const parseIds = (str?: string): string[] | undefined => {
if (str === undefined) {
return undefined
}
return str.split(',')
}
const parseNoFilter = (str?: string): NoFilter | undefined => {
if (str === undefined) {
return undefined
}
const strLower = str.toLowerCase()
switch (strLower) {
case 'highlight':
return { field: 'highlight_annotations' }
case 'label':
return { field: 'label_names' }
case 'subscription':
return { field: 'subscription' }
}
return undefined
}
export const parseSearchQuery = (query: string): LiqeQuery => {
const searchQuery = query
@ -342,138 +9,4 @@ export const parseSearchQuery = (query: string): LiqeQuery => {
.replace(/(\w+):([^"\s]+)/g, '$1:"$2"')
return parse(searchQuery)
// const parsed = parse(searchQuery, {
// keywords: [
// 'in',
// 'is',
// 'type',
// 'label',
// 'sort',
// 'has',
// 'saved',
// 'author',
// 'published',
// 'subscription',
// 'language',
// 'title',
// 'description',
// 'content',
// 'updated',
// 'includes',
// 'recommendedBy',
// 'no',
// 'mode',
// 'site',
// 'note',
// 'rss',
// 'wordsCount',
// 'readPosition',
// ],
// tokenize: true,
// })
// if (parsed.offsets) {
// const texts = parsed.offsets
// .filter((offset) => 'text' in offset)
// .map((offset) => offset as SearchParserTextOffset)
// if (texts.length > 0) {
// result.query = texts
// .map((offset: SearchParserTextOffset) => {
// // TODO: the parser library doesn't let us accurately
// // pull out quoted text, so we are just assuming
// // anything with spaces is quoted.
// if (offset.text.indexOf(' ') > -1) {
// return `"${offset.text}"`
// }
// return offset.text
// })
// .join(' ')
// } else {
// result.query = undefined
// }
// const keywords = parsed.offsets
// .filter((offset) => 'keyword' in offset)
// .map((offset) => offset as SearchParserKeyWordOffset)
// for (const keyword of keywords) {
// switch (keyword.keyword) {
// case 'in':
// result.inFilter = parseInFilter(keyword.value, result.query)
// break
// case 'is':
// result.readFilter = parseIsFilter(keyword.value)
// break
// case 'type':
// result.typeFilter = parseTypeFilter(keyword.value)
// break
// case 'label': {
// const labelFilter = parseLabelFilter(keyword.value, parsed.exclude)
// labelFilter && result.labelFilters.push(labelFilter)
// break
// }
// case 'sort':
// result.sort = parseSort(keyword.value)
// break
// case 'has': {
// const hasFilter = parseHasFilter(keyword.value)
// hasFilter !== undefined && result.hasFilters.push(hasFilter)
// break
// }
// case 'saved':
// case 'read':
// case 'updated':
// case 'published': {
// const dateFilter = parseDateFilter(keyword.keyword, keyword.value)
// dateFilter && result.dateFilters.push(dateFilter)
// break
// }
// // term filters
// case 'subscription':
// case 'rss':
// case 'language': {
// const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value)
// fieldFilter && result.termFilters.push(fieldFilter)
// break
// }
// // match filters
// case 'author':
// case 'title':
// case 'description':
// case 'note':
// case 'site':
// case 'content': {
// const fieldFilter = parseFieldFilter(keyword.keyword, keyword.value)
// fieldFilter && result.matchFilters.push(fieldFilter)
// break
// }
// case 'includes': {
// const ids = parseIds(keyword.value)
// ids && result.ids.push(...ids)
// break
// }
// case 'recommendedBy': {
// result.recommendedBy = parseStringValue(keyword.value)
// break
// }
// case 'no': {
// const noFilter = parseNoFilter(keyword.value)
// noFilter && result.noFilters.push(noFilter)
// break
// }
// case 'mode':
// // mode is ignored and used only by the frontend
// break
// case 'readPosition':
// case 'wordsCount': {
// const rangeFilter = parseRangeFilter(keyword.keyword, keyword.value)
// rangeFilter && result.rangeFilters.push(rangeFilter)
// break
// }
// }
// }
// }
// return result
}