Merge pull request #2329 from omnivore-app/feature/bulk-api

update bulk api to allow mark as read and add labels

packages/api/src/elastic/pages.ts

@@ -18,6 +18,7 @@ import {
 import { client, INDEX_ALIAS } from './index'
 import {
   ArticleSavingRequestStatus,
+  Label,
   Page,
   PageContext,
   PageSearchArgs,
@@ -424,103 +425,111 @@ export const getPageById = async (id: string): Promise<Page | undefined> => {
   }
 }
 
+const buildSearchBody = (userId: string, args: PageSearchArgs) => {
+  const {
+    query,
+    readFilter = ReadFilter.ALL,
+    typeFilter,
+    labelFilters,
+    inFilter = InFilter.ALL,
+    hasFilters,
+    dateFilters,
+    termFilters,
+    matchFilters,
+    ids,
+    noFilters,
+    siteName,
+  } = args
+
+  const includeLabels = labelFilters?.filter(
+    (filter) => filter.type === LabelFilterType.INCLUDE
+  )
+  const excludeLabels = labelFilters?.filter(
+    (filter) => filter.type === LabelFilterType.EXCLUDE
+  )
+
+  // start building the query
+  let builder = esBuilder().query('term', { userId })
+
+  // append filters
+  if (query) {
+    builder = appendQuery(builder, query)
+  }
+  if (typeFilter) {
+    builder = appendTypeFilter(builder, typeFilter)
+  }
+  if (inFilter !== InFilter.ALL) {
+    builder = appendInFilter(builder, inFilter)
+  }
+  if (readFilter !== ReadFilter.ALL) {
+    builder = appendReadFilter(builder, readFilter)
+  }
+  if (hasFilters && hasFilters.length > 0) {
+    builder = appendHasFilters(builder, hasFilters)
+  }
+  if (includeLabels && includeLabels.length > 0) {
+    builder = appendIncludeLabelFilter(builder, includeLabels)
+  }
+  if (excludeLabels && excludeLabels.length > 0) {
+    builder = appendExcludeLabelFilter(builder, excludeLabels)
+  }
+  if (dateFilters && dateFilters.length > 0) {
+    builder = appendDateFilters(builder, dateFilters)
+  }
+  if (termFilters) {
+    builder = appendTermFilters(builder, termFilters)
+  }
+  if (matchFilters) {
+    builder = appendMatchFilters(builder, matchFilters)
+  }
+  if (ids && ids.length > 0) {
+    builder = appendIdsFilter(builder, ids)
+  }
+  if (args.recommendedBy) {
+    builder = appendRecommendedBy(builder, args.recommendedBy)
+  }
+  if (!args.includePending) {
+    builder = builder.notQuery('term', {
+      state: ArticleSavingRequestStatus.Processing,
+    })
+  }
+  if (!args.includeDeleted) {
+    builder = builder.notQuery('term', {
+      state: ArticleSavingRequestStatus.Deleted,
+    })
+  }
+  if (noFilters) {
+    builder = appendNoFilters(builder, noFilters)
+  }
+  if (siteName) {
+    builder = appendSiteNameFilter(builder, siteName)
+  }
+
+  return builder
+}
+
 export const searchPages = async (
   args: PageSearchArgs,
   userId: string
 ): Promise<[Page[], number] | undefined> => {
   try {
-    const {
-      from = 0,
-      size = 10,
-      sort,
-      query,
-      readFilter = ReadFilter.ALL,
-      typeFilter,
-      labelFilters,
-      inFilter = InFilter.ALL,
-      hasFilters,
-      dateFilters,
-      termFilters,
-      matchFilters,
-      ids,
-      includeContent,
-      noFilters,
-      siteName,
-    } = args
+    const { from = 0, size = 10, sort, includeContent } = args
+
     // default order is descending
     const sortOrder = sort?.order || SortOrder.DESCENDING
     // default sort by saved_at
     const sortField = sort?.by || SortBy.SAVED
-    const includeLabels = labelFilters?.filter(
-      (filter) => filter.type === LabelFilterType.INCLUDE
-    )
-    const excludeLabels = labelFilters?.filter(
-      (filter) => filter.type === LabelFilterType.EXCLUDE
-    )
-    // start building the query
-    let builder = esBuilder()
-      .query('term', { userId })
+
+    // build the query
+    const builder = buildSearchBody(userId, args)
+    const body = builder
       .sort(sortField, sortOrder)
       .from(from)
       .size(size)
       .rawOption('_source', {
         excludes: includeContent ? [] : ['originalHtml', 'content'],
       })
-    // append filters
-    if (query) {
-      builder = appendQuery(builder, query)
-    }
-    if (typeFilter) {
-      builder = appendTypeFilter(builder, typeFilter)
-    }
-    if (inFilter !== InFilter.ALL) {
-      builder = appendInFilter(builder, inFilter)
-    }
-    if (readFilter !== ReadFilter.ALL) {
-      builder = appendReadFilter(builder, readFilter)
-    }
-    if (hasFilters && hasFilters.length > 0) {
-      builder = appendHasFilters(builder, hasFilters)
-    }
-    if (includeLabels && includeLabels.length > 0) {
-      builder = appendIncludeLabelFilter(builder, includeLabels)
-    }
-    if (excludeLabels && excludeLabels.length > 0) {
-      builder = appendExcludeLabelFilter(builder, excludeLabels)
-    }
-    if (dateFilters && dateFilters.length > 0) {
-      builder = appendDateFilters(builder, dateFilters)
-    }
-    if (termFilters) {
-      builder = appendTermFilters(builder, termFilters)
-    }
-    if (matchFilters) {
-      builder = appendMatchFilters(builder, matchFilters)
-    }
-    if (ids && ids.length > 0) {
-      builder = appendIdsFilter(builder, ids)
-    }
-    if (args.recommendedBy) {
-      builder = appendRecommendedBy(builder, args.recommendedBy)
-    }
-    if (!args.includePending) {
-      builder = builder.notQuery('term', {
-        state: ArticleSavingRequestStatus.Processing,
-      })
-    }
-    if (!args.includeDeleted) {
-      builder = builder.notQuery('term', {
-        state: ArticleSavingRequestStatus.Deleted,
-      })
-    }
-    if (noFilters) {
-      builder = appendNoFilters(builder, noFilters)
-    }
-    if (siteName) {
-      builder = appendSiteNameFilter(builder, siteName)
-    }
-    // build the query
-    const body = builder.build()
+      .build()
 
     console.debug('searching pages in elastic', JSON.stringify(body))
     const response = await client.search<SearchResponse<Page>, BuiltQuery>({
@@ -688,61 +697,89 @@ export const searchAsYouType = async (
 
 export const updatePagesAsync = async (
   userId: string,
-  action: BulkActionType
+  action: BulkActionType,
+  args: PageSearchArgs,
+  labels?: Label[]
 ): Promise<string | null> => {
-  // default action is archive
-  let must_not = [
-    {
-      exists: {
-        field: 'archivedAt',
-      },
-    },
-  ]
-  let params: Record<string, unknown> = { archivedAt: new Date() }
-  if (action === BulkActionType.Delete) {
-    must_not = []
-    params = { state: ArticleSavingRequestStatus.Deleted }
+  // build the script
+  let script = {
+    source: '',
+    params: {},
   }
-  // get update field
-  const field = Object.keys(params)[0]
+  switch (action) {
+    case BulkActionType.Archive:
+      script = {
+        source: `ctx._source.archivedAt = params.archivedAt;`,
+        params: {
+          archivedAt: new Date(),
+        },
+      }
+      break
+    case BulkActionType.Delete:
+      script = {
+        source: `ctx._source.state = params.state;`,
+        params: {
+          state: ArticleSavingRequestStatus.Deleted,
+        },
+      }
+      break
+    case BulkActionType.AddLabels:
+      script = {
+        source: `if (ctx._source.labels == null) {
+          ctx._source.labels = params.labels
+        } else {
+          for (label in params.labels) {
+            if (!ctx._source.labels.any(l -> l.name == label.name)) {
+              ctx._source.labels.add(label)
+            }
+          }
+        }`,
+        params: {
+          labels,
+        },
+      }
+      break
+    case BulkActionType.MarkAsRead:
+      script = {
+        source: `ctx._source.readAt = params.readAt;
+        ctx._source.readingProgressPercent = params.readingProgressPercent;`,
+        params: {
+          readAt: new Date(),
+          readingProgressPercent: 100,
+        },
+      }
+      break
+    default:
+      throw new Error('Invalid bulk action')
+  }
+
+  // add updatedAt to the script
+  const updatedScript = {
+    source: `${script.source} ctx._source.updatedAt = params.updatedAt`,
+    lang: 'painless',
+    params: {
+      ...script.params,
+      updatedAt: new Date(),
+    },
+  }
+
+  // build the query
+  const searchBody = buildSearchBody(userId, args)
+    .rawOption('script', updatedScript)
+    .build()
+
+  console.debug('updating pages in elastic', JSON.stringify(searchBody))
 
   try {
     const { body } = await client.updateByQuery({
       index: INDEX_ALIAS,
       conflicts: 'proceed',
       wait_for_completion: false,
-      body: {
-        query: {
-          bool: {
-            filter: [
-              {
-                term: {
-                  userId,
-                },
-              },
-              {
-                terms: {
-                  state: [
-                    ArticleSavingRequestStatus.Succeeded,
-                    ArticleSavingRequestStatus.Failed,
-                    ArticleSavingRequestStatus.Processing,
-                  ],
-                },
-              },
-            ],
-            must_not,
-          },
-        },
-        script: {
-          source: `ctx._source.${field} = params.${field}`,
-          lang: 'painless',
-          params,
-        },
-      },
+      body: searchBody,
     })
 
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    if (body.failures?.length > 0) {
+    if (body.failures && body.failures.length > 0) {
       console.log('failed to update pages in elastic', body.failures)
       return null
     }

packages/api/src/generated/graphql.ts

@@ -228,6 +228,7 @@ export type BulkActionError = {
 };
 
 export enum BulkActionErrorCode {
+  BadRequest = 'BAD_REQUEST',
   Unauthorized = 'UNAUTHORIZED'
 }
 
@@ -239,8 +240,10 @@ export type BulkActionSuccess = {
 };
 
 export enum BulkActionType {
+  AddLabels = 'ADD_LABELS',
   Archive = 'ARCHIVE',
-  Delete = 'DELETE'
+  Delete = 'DELETE',
+  MarkAsRead = 'MARK_AS_READ'
 }
 
 export enum ContentReader {
@@ -1299,7 +1302,8 @@ export type MutationAddPopularReadArgs = {
 
 export type MutationBulkActionArgs = {
   action: BulkActionType;
-  query?: InputMaybe<Scalars['String']>;
+  labelIds?: InputMaybe<Array<Scalars['ID']>>;
+  query: Scalars['String'];
 };
 
 
@@ -5012,7 +5016,7 @@ export type MoveLabelSuccessResolvers<ContextType = ResolverContext, ParentType
 
 export type MutationResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['Mutation'] = ResolversParentTypes['Mutation']> = {
   addPopularRead?: Resolver<ResolversTypes['AddPopularReadResult'], ParentType, ContextType, RequireFields<MutationAddPopularReadArgs, 'name'>>;
-  bulkAction?: Resolver<ResolversTypes['BulkActionResult'], ParentType, ContextType, RequireFields<MutationBulkActionArgs, 'action'>>;
+  bulkAction?: Resolver<ResolversTypes['BulkActionResult'], ParentType, ContextType, RequireFields<MutationBulkActionArgs, 'action' | 'query'>>;
   createArticle?: Resolver<ResolversTypes['CreateArticleResult'], ParentType, ContextType, RequireFields<MutationCreateArticleArgs, 'input'>>;
   createArticleSavingRequest?: Resolver<ResolversTypes['CreateArticleSavingRequestResult'], ParentType, ContextType, RequireFields<MutationCreateArticleSavingRequestArgs, 'input'>>;
   createGroup?: Resolver<ResolversTypes['CreateGroupResult'], ParentType, ContextType, RequireFields<MutationCreateGroupArgs, 'input'>>;

packages/api/src/generated/schema.graphql

@@ -188,6 +188,7 @@ type BulkActionError {
 }
 
 enum BulkActionErrorCode {
+  BAD_REQUEST
   UNAUTHORIZED
 }
 
@@ -198,8 +199,10 @@ type BulkActionSuccess {
 }
 
 enum BulkActionType {
+  ADD_LABELS
   ARCHIVE
   DELETE
+  MARK_AS_READ
 }
 
 enum ContentReader {
@@ -1090,7 +1093,7 @@ type MoveLabelSuccess {
 
 type Mutation {
   addPopularRead(name: String!): AddPopularReadResult!
-  bulkAction(action: BulkActionType!, query: String): BulkActionResult!
+  bulkAction(action: BulkActionType!, labelIds: [ID!], query: String!): BulkActionResult!
   createArticle(input: CreateArticleInput!): CreateArticleResult!
   createArticleSavingRequest(input: CreateArticleSavingRequestInput!): CreateArticleSavingRequestResult!
   createGroup(input: CreateGroupInput!): CreateGroupResult!

packages/api/src/resolvers/article/index.ts

@@ -31,6 +31,7 @@ import {
   BulkActionError,
   BulkActionErrorCode,
   BulkActionSuccess,
+  BulkActionType,
   ContentReader,
   CreateArticleError,
   CreateArticleErrorCode,
@@ -73,7 +74,7 @@ import {
   UpdatesSinceSuccess,
 } from '../../generated/graphql'
 import { createPageSaveRequest } from '../../services/create_page_save_request'
-import { createLabels } from '../../services/labels'
+import { createLabels, getLabelsByIds } from '../../services/labels'
 import { parsedContentToPage } from '../../services/save_page'
 import { traceAs } from '../../tracing'
 import { Merge } from '../../util'
@@ -1097,26 +1098,43 @@ export const bulkActionResolver = authorized<
   BulkActionSuccess,
   BulkActionError,
   MutationBulkActionArgs
->(async (_parent, { action }, { claims: { uid }, log }) => {
+>(async (_parent, { query, action, labelIds }, { claims: { uid }, log }) => {
   log.info('bulkActionResolver')
 
-  if (!uid) {
-    log.error('bulkActionResolver', { error: 'Unauthorized' })
-    return { errorCodes: [BulkActionErrorCode.Unauthorized] }
-  }
-
   analytics.track({
     userId: uid,
     event: 'BulkAction',
     properties: {
       env: env.server.apiEnv,
+      action,
     },
   })
 
-  // TODO: get search filters from query
+  if (!uid) {
+    log.log('bulkActionResolver', { error: 'Unauthorized' })
+    return { errorCodes: [BulkActionErrorCode.Unauthorized] }
+  }
+
+  if (!query) {
+    log.log('bulkActionResolver', { error: 'no query' })
+    return { errorCodes: [BulkActionErrorCode.BadRequest] }
+  }
+
+  // get labels if needed
+  let labels = undefined
+  if (action === BulkActionType.AddLabels) {
+    if (!labelIds || labelIds.length === 0) {
+      return { errorCodes: [BulkActionErrorCode.BadRequest] }
+    }
+
+    labels = await getLabelsByIds(uid, labelIds)
+  }
+
+  // parse query
+  const searchQuery = parseSearchQuery(query)
 
   // start a task to update pages
-  const taskId = await updatePagesAsync(uid, action)
+  const taskId = await updatePagesAsync(uid, action, searchQuery, labels)
 
   return { success: !!taskId }
 })

packages/api/src/schema.ts

@@ -2416,6 +2416,8 @@ const schema = gql`
   enum BulkActionType {
     DELETE
     ARCHIVE
+    MARK_AS_READ
+    ADD_LABELS
   }
 
   union BulkActionResult = BulkActionSuccess | BulkActionError
@@ -2430,6 +2432,7 @@ const schema = gql`
 
   enum BulkActionErrorCode {
     UNAUTHORIZED
+    BAD_REQUEST
   }
 
   union ImportFromIntegrationResult =
@@ -2538,7 +2541,11 @@ const schema = gql`
       contentType: String!
     ): UploadImportFileResult!
     markEmailAsItem(recentEmailId: ID!): MarkEmailAsItemResult!
-    bulkAction(query: String, action: BulkActionType!): BulkActionResult!
+    bulkAction(
+      query: String!
+      action: BulkActionType!
+      labelIds: [ID!]
+    ): BulkActionResult!
     importFromIntegration(integrationId: ID!): ImportFromIntegrationResult!
   }
 

packages/api/test/resolvers/article.test.ts

@@ -1214,9 +1214,9 @@ describe('Article API', () => {
   })
 
   describe('BulkAction API', () => {
-    const bulkActionQuery = (action: BulkActionType) => `
+    const bulkActionQuery = (action: BulkActionType, query = 'in:all') => `
       mutation {
-        bulkAction (action: ${action}) {
+        bulkAction (action: ${action}, query: "${query}") {
           ... on BulkActionSuccess {
             success
           }