diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 22413fe27..a1e7c33cb 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -745,8 +745,6 @@ export const updatesSinceResolver = authorized< UpdatesSinceError, QueryUpdatesSinceArgs >(async (_obj, { since, first, after, sort: sortParams, folder }, { uid }) => { - const sort = sortParamsToSort(sortParams) - const startCursor = after || '' const size = Math.min(first || 10, 100) // limit to 100 items let startDate = new Date(since) @@ -754,18 +752,18 @@ export const updatesSinceResolver = authorized< // for android app compatibility startDate = new Date(0) } + const sort = sortParamsToSort(sortParams) // create a search query const query = `updated:${startDate.toISOString()}${ folder ? ' in:' + folder : '' - }` + } sort:${sort.by}-${sort.order}` const { libraryItems, count } = await searchLibraryItems( { from: Number(startCursor), size: size + 1, // fetch one more item to get next cursor includeDeleted: true, - sort, query, }, uid diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 98e0d9c79..77464dbec 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -1,6 +1,5 @@ import axios from 'axios' import { parseHTML } from 'linkedom' -import Parser from 'rss-parser' import { Brackets } from 'typeorm' import { Subscription } from '../../entity/subscription' import { env } from '../../env' @@ -47,18 +46,6 @@ import { parseFeed, parseOpml } from '../../utils/parser' type PartialSubscription = Omit -const parser = new Parser({ - timeout: 30000, // 30 seconds - maxRedirects: 5, - headers: { - // some rss feeds require user agent - 'User-Agent': - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - Accept: - 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4', - }, -}) - export type SubscriptionsSuccessPartial = Merge< SubscriptionsSuccess, { subscriptions: PartialSubscription[] } diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 54442d635..4264fbc1a 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -39,7 +39,6 @@ enum HasFilter { export interface SearchArgs { from?: number size?: number - sort?: Sort includePending?: boolean | null includeDeleted?: boolean includeContent?: boolean @@ -94,13 +93,13 @@ export enum SortOrder { } export interface Sort { - by: SortBy + by: string order?: SortOrder } interface Select { column: string - alias: string + alias?: string } export const sortParamsToSort = ( @@ -153,7 +152,7 @@ export const buildQuery = ( searchQuery: LiqeQuery, parameters: ObjectLiteral[] = [], selects: Select[] = [], - orders: { by: string; order?: SortOrder }[] = [], + orders: Sort[] = [], useFolders = false ) => { const escapeQueryWithParameters = ( @@ -189,10 +188,8 @@ export const buildQuery = ( alias, }) - orders.push({ - by: alias, - order: SortOrder.DESCENDING, - }) + // always sort by rank first + orders.unshift({ by: alias, order: SortOrder.DESCENDING }) return escapeQueryWithParameters( `websearch_to_tsquery('english', :${param}) @@ library_item.search_tsv`, @@ -388,8 +385,20 @@ export const buildQuery = ( default: { // check for date ranges const [start, end] = date.split('..') - startDate = start && start !== '*' ? new Date(start) : undefined - endDate = end && end !== '*' ? new Date(end) : undefined + // validate date + if (start && start !== '*') { + startDate = new Date(start) + if (isNaN(startDate.getTime())) { + throw new Error('Invalid start date.') + } + } + + if (end && end !== '*') { + endDate = new Date(end) + if (isNaN(endDate.getTime())) { + throw new Error('Invalid end date.') + } + } } } @@ -627,24 +636,32 @@ export const searchLibraryItems = async ( args: SearchArgs, userId: string ): Promise<{ libraryItems: LibraryItem[]; count: number }> => { - const { from = 0, size = 10, sort } = args + const { from = 0, size = 10 } = args - // default order is descending - const sortOrder = sort?.order || SortOrder.DESCENDING - // default sort by saved_at - const sortField = sort?.by || SortBy.SAVED - - const selectColumns = getColumns(libraryItemRepository) - .map((column) => `library_item.${column}`) + // select all columns except content + const selects: Select[] = getColumns(libraryItemRepository) + .map((column) => ({ column: `library_item.${column}` })) .filter( - (column) => - column !== 'library_item.readableContent' && - column !== 'library_item.originalContent' + (select) => + select.column !== 'library_item.readableContent' && + select.column !== 'library_item.originalContent' ) - let searchQuery: LiqeQuery | undefined + const parameters: ObjectLiteral[] = [] + const orders: Sort[] = [] + let query: string | null = null + if (args.query) { - searchQuery = parseSearchQuery(args.query) + const searchQuery = parseSearchQuery(args.query) + + // build query and save parameters + query = buildQuery( + searchQuery, + parameters, + selects, + orders, + args.useFolders + ) } // add pagination and sorting @@ -652,34 +669,12 @@ export const searchLibraryItems = async ( async (tx) => { const queryBuilder = tx .createQueryBuilder(LibraryItem, 'library_item') - .select(selectColumns) .where('library_item.user_id = :userId', { userId }) - if (searchQuery) { - const parameters: ObjectLiteral[] = [] - const selects: Select[] = [] - const orders: Sort[] = [] - const whereClause = buildQuery( - searchQuery, - parameters, - selects, - orders, - args.useFolders - ) - whereClause && - queryBuilder - .andWhere(whereClause) - .setParameters(parameters.reduce((a, b) => ({ ...a, ...b }), {})) - - selects.forEach((select) => { - queryBuilder.addSelect(select.column, select.alias) - }) - - // add order by - orders.forEach((order) => { - queryBuilder.addOrderBy(order.by, order.order, 'NULLS LAST') - }) - } + // add select + selects.forEach((select) => { + queryBuilder.addSelect(select.column, select.alias) + }) if (!args.includePending) { queryBuilder.andWhere("library_item.state <> 'PROCESSING'") @@ -689,14 +684,30 @@ export const searchLibraryItems = async ( queryBuilder.andWhere("library_item.state <> 'DELETED'") } - const libraryItems = await queryBuilder - .addOrderBy(`library_item.${sortField}`, sortOrder, 'NULLS LAST') - .skip(from) - .take(size) - .getMany() + if (query) { + // add where clause from query + queryBuilder + .andWhere(query) + .setParameters(parameters.reduce((a, b) => ({ ...a, ...b }), {})) + } const count = await queryBuilder.getCount() + // default order by saved at descending + if (!orders.find((order) => order.by === 'library_item.saved_at')) { + orders.push({ + by: 'library_item.saved_at', + order: SortOrder.DESCENDING, + }) + } + + // add order by + orders.forEach((order) => { + queryBuilder.addOrderBy(order.by, order.order, 'NULLS LAST') + }) + + const libraryItems = await queryBuilder.skip(from).take(size).getMany() + return { libraryItems, count } }, undefined, diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index 33329e13a..167b80acf 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -6,7 +6,7 @@ export const parseSearchQuery = (query: string): LiqeQuery => { .replace('in:subscription', 'has:subscriptions') // compatibility with old search .replace('in:library', 'no:subscription') // compatibility with old search // wrap the value behind colon in quotes if it's not already - .replace(/(\w+):([^"\s]+)/g, '$1:"$2"') + .replace(/(\w+):("([^"]+)"|([^")\s]+))/g, '$1:"$3$4"') return parse(searchQuery) } diff --git a/packages/integration-handler/src/index.ts b/packages/integration-handler/src/index.ts index 7fb805682..f9d657297 100644 --- a/packages/integration-handler/src/index.ts +++ b/packages/integration-handler/src/index.ts @@ -97,16 +97,22 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction( const client = getIntegrationClient(integrationName) // get paginated items from the backend - const first = '50' + const first = 50 let hasMore = true let after = '0' while (hasMore) { - console.log('searching for items...') + const updatedSince = new Date(syncAt) + console.log('searching for items...', { + userId: claims.uid, + first, + after, + updatedSince, + }) const response = await search( REST_BACKEND_ENDPOINT, systemToken, client.highlightOnly, - new Date(syncAt), + updatedSince, first, after ) @@ -125,7 +131,11 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction( break } - console.log('exporting items...') + console.log('exporting items...', { + userId: claims.uid, + total: items.length, + hasMore, + }) const synced = await client.export(claims.token, items) if (!synced) { console.error('failed to export item', { @@ -134,7 +144,11 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction( return res.status(400).send('Failed to sync') } - console.log('updating integration...') + console.log('updating integration...', { + userId: claims.uid, + integrationId, + syncedAt: items[items.length - 1].updatedAt, + }) // update integration syncedAt if successful const updated = await updateIntegration( REST_BACKEND_ENDPOINT, @@ -152,9 +166,6 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction( }) return res.status(400).send('Failed to update integration') } - - // avoid rate limiting - await wait(500) } console.log('done') diff --git a/packages/integration-handler/src/item.ts b/packages/integration-handler/src/item.ts index cb535904c..df9da7725 100644 --- a/packages/integration-handler/src/item.ts +++ b/packages/integration-handler/src/item.ts @@ -44,7 +44,7 @@ export const search = async ( token: string, highlightOnly: boolean, updatedSince: Date, - first = '50', + first = 50, after = '0' ): Promise => { const query = `updated:${updatedSince.toISOString()} ${ @@ -52,8 +52,8 @@ export const search = async ( } sort:updated-asc` const requestData = JSON.stringify({ - query: `query Search($query: String) { - search(query: $query) { + query: `query Search($query: String, $first: Int, $after: String) { + search(query: $query, first: $first, after: $after) { ... on SearchSuccess { edges { node {