Merge pull request #3211 from omnivore-app/fix/search

fix/search

packages/api/src/resolvers/article/index.ts

@@ -745,8 +745,6 @@ export const updatesSinceResolver = authorized<
   UpdatesSinceError,
   QueryUpdatesSinceArgs
 >(async (_obj, { since, first, after, sort: sortParams, folder }, { uid }) => {
-  const sort = sortParamsToSort(sortParams)
-
   const startCursor = after || ''
   const size = Math.min(first || 10, 100) // limit to 100 items
   let startDate = new Date(since)
@@ -754,18 +752,18 @@ export const updatesSinceResolver = authorized<
     // for android app compatibility
     startDate = new Date(0)
   }
+  const sort = sortParamsToSort(sortParams)
 
   // create a search query
   const query = `updated:${startDate.toISOString()}${
     folder ? ' in:' + folder : ''
-  }`
+  } sort:${sort.by}-${sort.order}`
 
   const { libraryItems, count } = await searchLibraryItems(
     {
       from: Number(startCursor),
       size: size + 1, // fetch one more item to get next cursor
       includeDeleted: true,
-      sort,
       query,
     },
     uid

packages/api/src/resolvers/subscriptions/index.ts

@@ -1,6 +1,5 @@
 import axios from 'axios'
 import { parseHTML } from 'linkedom'
-import Parser from 'rss-parser'
 import { Brackets } from 'typeorm'
 import { Subscription } from '../../entity/subscription'
 import { env } from '../../env'
@@ -47,18 +46,6 @@ import { parseFeed, parseOpml } from '../../utils/parser'
 
 type PartialSubscription = Omit<Subscription, 'newsletterEmail'>
 
-const parser = new Parser({
-  timeout: 30000, // 30 seconds
-  maxRedirects: 5,
-  headers: {
-    // some rss feeds require user agent
-    'User-Agent':
-      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
-    Accept:
-      'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4',
-  },
-})
-
 export type SubscriptionsSuccessPartial = Merge<
   SubscriptionsSuccess,
   { subscriptions: PartialSubscription[] }

packages/api/src/services/library_item.ts

@@ -39,7 +39,6 @@ enum HasFilter {
 export interface SearchArgs {
   from?: number
   size?: number
-  sort?: Sort
   includePending?: boolean | null
   includeDeleted?: boolean
   includeContent?: boolean
@@ -94,13 +93,13 @@ export enum SortOrder {
 }
 
 export interface Sort {
-  by: SortBy
+  by: string
   order?: SortOrder
 }
 
 interface Select {
   column: string
-  alias: string
+  alias?: string
 }
 
 export const sortParamsToSort = (
@@ -153,7 +152,7 @@ export const buildQuery = (
   searchQuery: LiqeQuery,
   parameters: ObjectLiteral[] = [],
   selects: Select[] = [],
-  orders: { by: string; order?: SortOrder }[] = [],
+  orders: Sort[] = [],
   useFolders = false
 ) => {
   const escapeQueryWithParameters = (
@@ -189,10 +188,8 @@ export const buildQuery = (
         alias,
       })
 
-      orders.push({
-        by: alias,
-        order: SortOrder.DESCENDING,
-      })
+      // always sort by rank first
+      orders.unshift({ by: alias, order: SortOrder.DESCENDING })
 
       return escapeQueryWithParameters(
         `websearch_to_tsquery('english', :${param}) @@ library_item.search_tsv`,
@@ -388,8 +385,20 @@ export const buildQuery = (
             default: {
               // check for date ranges
               const [start, end] = date.split('..')
-              startDate = start && start !== '*' ? new Date(start) : undefined
-              endDate = end && end !== '*' ? new Date(end) : undefined
+              // validate date
+              if (start && start !== '*') {
+                startDate = new Date(start)
+                if (isNaN(startDate.getTime())) {
+                  throw new Error('Invalid start date.')
+                }
+              }
+
+              if (end && end !== '*') {
+                endDate = new Date(end)
+                if (isNaN(endDate.getTime())) {
+                  throw new Error('Invalid end date.')
+                }
+              }
             }
           }
 
@@ -627,24 +636,32 @@ export const searchLibraryItems = async (
   args: SearchArgs,
   userId: string
 ): Promise<{ libraryItems: LibraryItem[]; count: number }> => {
-  const { from = 0, size = 10, sort } = args
+  const { from = 0, size = 10 } = args
 
-  // default order is descending
-  const sortOrder = sort?.order || SortOrder.DESCENDING
-  // default sort by saved_at
-  const sortField = sort?.by || SortBy.SAVED
-
-  const selectColumns = getColumns(libraryItemRepository)
-    .map((column) => `library_item.${column}`)
+  // select all columns except content
+  const selects: Select[] = getColumns(libraryItemRepository)
+    .map((column) => ({ column: `library_item.${column}` }))
     .filter(
-      (column) =>
-        column !== 'library_item.readableContent' &&
-        column !== 'library_item.originalContent'
+      (select) =>
+        select.column !== 'library_item.readableContent' &&
+        select.column !== 'library_item.originalContent'
     )
 
-  let searchQuery: LiqeQuery | undefined
+  const parameters: ObjectLiteral[] = []
+  const orders: Sort[] = []
+  let query: string | null = null
+
   if (args.query) {
-    searchQuery = parseSearchQuery(args.query)
+    const searchQuery = parseSearchQuery(args.query)
+
+    // build query and save parameters
+    query = buildQuery(
+      searchQuery,
+      parameters,
+      selects,
+      orders,
+      args.useFolders
+    )
   }
 
   // add pagination and sorting
@@ -652,34 +669,12 @@ export const searchLibraryItems = async (
     async (tx) => {
       const queryBuilder = tx
         .createQueryBuilder(LibraryItem, 'library_item')
-        .select(selectColumns)
         .where('library_item.user_id = :userId', { userId })
 
-      if (searchQuery) {
-        const parameters: ObjectLiteral[] = []
-        const selects: Select[] = []
-        const orders: Sort[] = []
-        const whereClause = buildQuery(
-          searchQuery,
-          parameters,
-          selects,
-          orders,
-          args.useFolders
-        )
-        whereClause &&
-          queryBuilder
-            .andWhere(whereClause)
-            .setParameters(parameters.reduce((a, b) => ({ ...a, ...b }), {}))
-
-        selects.forEach((select) => {
-          queryBuilder.addSelect(select.column, select.alias)
-        })
-
-        // add order by
-        orders.forEach((order) => {
-          queryBuilder.addOrderBy(order.by, order.order, 'NULLS LAST')
-        })
-      }
+      // add select
+      selects.forEach((select) => {
+        queryBuilder.addSelect(select.column, select.alias)
+      })
 
       if (!args.includePending) {
         queryBuilder.andWhere("library_item.state <> 'PROCESSING'")
@@ -689,14 +684,30 @@ export const searchLibraryItems = async (
         queryBuilder.andWhere("library_item.state <> 'DELETED'")
       }
 
-      const libraryItems = await queryBuilder
-        .addOrderBy(`library_item.${sortField}`, sortOrder, 'NULLS LAST')
-        .skip(from)
-        .take(size)
-        .getMany()
+      if (query) {
+        // add where clause from query
+        queryBuilder
+          .andWhere(query)
+          .setParameters(parameters.reduce((a, b) => ({ ...a, ...b }), {}))
+      }
 
       const count = await queryBuilder.getCount()
 
+      // default order by saved at descending
+      if (!orders.find((order) => order.by === 'library_item.saved_at')) {
+        orders.push({
+          by: 'library_item.saved_at',
+          order: SortOrder.DESCENDING,
+        })
+      }
+
+      // add order by
+      orders.forEach((order) => {
+        queryBuilder.addOrderBy(order.by, order.order, 'NULLS LAST')
+      })
+
+      const libraryItems = await queryBuilder.skip(from).take(size).getMany()
+
       return { libraryItems, count }
     },
     undefined,

packages/api/src/utils/search.ts

@@ -6,7 +6,7 @@ export const parseSearchQuery = (query: string): LiqeQuery => {
     .replace('in:subscription', 'has:subscriptions') // compatibility with old search
     .replace('in:library', 'no:subscription') // compatibility with old search
     // wrap the value behind colon in quotes if it's not already
-    .replace(/(\w+):([^"\s]+)/g, '$1:"$2"')
+    .replace(/(\w+):("([^"]+)"|([^")\s]+))/g, '$1:"$3$4"')
 
   return parse(searchQuery)
 }

packages/integration-handler/src/index.ts

@@ -97,16 +97,22 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction(
       const client = getIntegrationClient(integrationName)
 
       // get paginated items from the backend
-      const first = '50'
+      const first = 50
       let hasMore = true
       let after = '0'
       while (hasMore) {
-        console.log('searching for items...')
+        const updatedSince = new Date(syncAt)
+        console.log('searching for items...', {
+          userId: claims.uid,
+          first,
+          after,
+          updatedSince,
+        })
         const response = await search(
           REST_BACKEND_ENDPOINT,
           systemToken,
           client.highlightOnly,
-          new Date(syncAt),
+          updatedSince,
           first,
           after
         )
@@ -125,7 +131,11 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction(
           break
         }
 
-        console.log('exporting items...')
+        console.log('exporting items...', {
+          userId: claims.uid,
+          total: items.length,
+          hasMore,
+        })
         const synced = await client.export(claims.token, items)
         if (!synced) {
           console.error('failed to export item', {
@@ -134,7 +144,11 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction(
           return res.status(400).send('Failed to sync')
         }
 
-        console.log('updating integration...')
+        console.log('updating integration...', {
+          userId: claims.uid,
+          integrationId,
+          syncedAt: items[items.length - 1].updatedAt,
+        })
         // update integration syncedAt if successful
         const updated = await updateIntegration(
           REST_BACKEND_ENDPOINT,
@@ -152,9 +166,6 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction(
           })
           return res.status(400).send('Failed to update integration')
         }
-
-        // avoid rate limiting
-        await wait(500)
       }
 
       console.log('done')

packages/integration-handler/src/item.ts

@@ -44,7 +44,7 @@ export const search = async (
   token: string,
   highlightOnly: boolean,
   updatedSince: Date,
-  first = '50',
+  first = 50,
   after = '0'
 ): Promise<SearchResponse | null> => {
   const query = `updated:${updatedSince.toISOString()} ${
@@ -52,8 +52,8 @@ export const search = async (
   } sort:updated-asc`
 
   const requestData = JSON.stringify({
-    query: `query Search($query: String) {
-              search(query: $query) {
+    query: `query Search($query: String, $first: Int, $after: String) {
+              search(query: $query, first: $first, after: $after) {
                 ... on SearchSuccess {
                   edges {
                     node {