Merge pull request #1449 from omnivore-app/more-filters-in-rules

Use search API to filter page in rule engine

packages/api/src/elastic/pages.ts

@@ -179,6 +179,14 @@ const appendMatchFilters = (body: SearchBody, filters: FieldFilter[]): void => {
   })
 }
 
+const appendIdsFilter = (body: SearchBody, ids: string[]): void => {
+  body.query.bool.must.push({
+    terms: {
+      _id: ids,
+    },
+  })
+}
+
 export const createPage = async (
   page: Page,
   ctx: PageContext
@@ -359,6 +367,7 @@ export const searchPages = async (
       dateFilters,
       termFilters,
       matchFilters,
+      ids,
     } = args
     // default order is descending
     const sortOrder = sort?.order || SortOrder.DESCENDING
@@ -430,6 +439,9 @@ export const searchPages = async (
     if (matchFilters) {
       appendMatchFilters(body, matchFilters)
     }
+    if (ids && ids.length > 0) {
+      appendIdsFilter(body, ids)
+    }
 
     if (!args.includePending) {
       body.query.bool.must_not.push({

packages/api/src/elastic/types.ts

@@ -56,6 +56,11 @@ export interface SearchBody {
               [K: string]: string
             }
           }
+        | {
+            terms: {
+              [K: string]: string[]
+            }
+          }
       )[]
       should: {
         multi_match: {
@@ -278,4 +283,5 @@ export interface PageSearchArgs {
   matchFilters?: FieldFilter[]
   includePending?: boolean | null
   includeDeleted?: boolean
+  ids?: string[]
 }

packages/api/src/utils/search.ts

@@ -34,6 +34,7 @@ export interface SearchFilter {
   dateFilters: DateFilter[]
   termFilters: FieldFilter[]
   matchFilters: FieldFilter[]
+  ids: string[]
 }
 
 export enum LabelFilterType {
@@ -245,6 +246,14 @@ const parseFieldFilter = (
   }
 }
 
+const parseIds = (field: string, str?: string): string[] | undefined => {
+  if (str === undefined) {
+    return undefined
+  }
+
+  return str.split(',')
+}
+
 export const parseSearchQuery = (query: string | undefined): SearchFilter => {
   const searchQuery = query ? query.replace(/\W\s":/g, '') : undefined
   const result: SearchFilter = {
@@ -256,6 +265,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
     dateFilters: [],
     termFilters: [],
     matchFilters: [],
+    ids: [],
   }
 
   if (!searchQuery) {
@@ -268,6 +278,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
       dateFilters: [],
       termFilters: [],
       matchFilters: [],
+      ids: [],
     }
   }
 
@@ -288,6 +299,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
       'description',
       'content',
       'updated',
+      'includes',
     ],
     tokenize: true,
   })
@@ -364,6 +376,11 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
           fieldFilter && result.matchFilters.push(fieldFilter)
           break
         }
+        case 'includes': {
+          const ids = parseIds(keyword.keyword, keyword.value)
+          ids && result.ids.push(...ids)
+          break
+        }
       }
     }
   }

packages/rule-handler/src/filter.ts

@@ -0,0 +1,81 @@
+import axios from 'axios'
+
+interface SearchResponse {
+  data: {
+    search: {
+      edges: Edge[]
+    }
+  }
+}
+
+interface Edge {
+  node: Node
+}
+
+interface Node {
+  id: string
+}
+
+export const search = async (
+  userId: string,
+  apiEndpoint: string,
+  auth: string,
+  query: string
+): Promise<Node[]> => {
+  const requestData = JSON.stringify({
+    query: `query Search($query: String) {
+              search(query: $query) {
+                ... on SearchSuccess {
+                  edges {
+                    node {
+                      id
+                    }
+                  }
+                }
+                ... on SearchError {
+                  errorCodes
+                }
+              }
+            }`,
+    variables: {
+      query,
+    },
+  })
+
+  try {
+    const response = await axios.post<SearchResponse>(
+      `${apiEndpoint}/graphql`,
+      requestData,
+      {
+        headers: {
+          Cookie: `auth=${auth};`,
+          'Content-Type': 'application/json',
+        },
+      }
+    )
+
+    const edges = response.data.data.search.edges
+    if (edges.length == 0) {
+      return []
+    }
+
+    return edges.map((edge: Edge) => edge.node)
+  } catch (e) {
+    console.error(e)
+
+    return []
+  }
+}
+
+export const isMatched = async (
+  userId: string,
+  apiEndpoint: string,
+  auth: string,
+  filter: string,
+  pageId: string
+): Promise<boolean> => {
+  filter += ` includes:${pageId}`
+  const nodes = await search(userId, apiEndpoint, auth, filter)
+
+  return nodes.length > 0
+}

packages/rule-handler/src/index.ts

@@ -25,6 +25,8 @@ export interface PubSubData {
   subscription: string
   image: string
   content: string
+  readingProgressPercent: number
+  pageType: string
 }
 
 enum EntityType {
@@ -123,6 +125,7 @@ export const ruleHandler = Sentry.GCPFunction.wrapHttpFunction(
         jwtSecret
       )
       if (triggeredActions.length === 0) {
+        console.log('No actions triggered')
         res.status(200).send('No Actions')
         return
       }

packages/rule-handler/src/label.ts

@@ -28,7 +28,7 @@ export const addLabels = async (
   })
 
   try {
-    await axios.post(`${apiEndpoint}/graphql`, data, {
+    return axios.post(`${apiEndpoint}/graphql`, data, {
       headers: {
         Cookie: `auth=${auth};`,
         'Content-Type': 'application/json',

packages/rule-handler/src/notification.ts

@@ -1,6 +1,6 @@
 import axios from 'axios'
 
-interface NotificationData {
+interface RequestData {
   body: string
   title?: string
   data?: Record<string, string>
@@ -11,19 +11,21 @@ interface NotificationData {
 export const sendNotification = async (
   apiEndpoint: string,
   auth: string,
-  message: string,
+  body: string,
   title?: string,
-  image?: string
+  image?: string,
+  data?: Record<string, string>
 ) => {
-  const data: NotificationData = {
-    body: message,
-    title: title || message,
+  const requestData: RequestData = {
+    body,
+    title,
     image,
     notificationType: 'rule',
+    data,
   }
 
   try {
-    await axios.post(`${apiEndpoint}/notification/send`, data, {
+    return axios.post(`${apiEndpoint}/notification/send`, requestData, {
       headers: {
         Cookie: `auth=${auth};`,
         'Content-Type': 'application/json',

packages/rule-handler/src/page.ts

@@ -27,7 +27,7 @@ export const archivePage = async (
   })
 
   try {
-    await axios.post(`${apiEndpoint}/graphql`, data, {
+    return axios.post(`${apiEndpoint}/graphql`, data, {
       headers: {
         Cookie: `auth=${auth};`,
         'Content-Type': 'application/json',
@@ -66,7 +66,7 @@ export const markPageAsRead = async (
   })
 
   try {
-    await axios.post(`${apiEndpoint}/graphql`, data, {
+    return axios.post(`${apiEndpoint}/graphql`, data, {
       headers: {
         Cookie: `auth=${auth};`,
         'Content-Type': 'application/json',

packages/rule-handler/src/rule.ts

@@ -1,12 +1,9 @@
 import { sendNotification } from './notification'
 import { getAuthToken, PubSubData } from './index'
-import axios from 'axios'
-import { parse, SearchParserKeyWordOffset } from 'search-query-parser'
+import axios, { AxiosResponse } from 'axios'
 import { addLabels } from './label'
 import { archivePage, markPageAsRead } from './page'
-import { SearchFilter } from './search_filter'
-import { SubscriptionFilter } from './search_filter/subscription_filter'
-import { ContentFilter } from './search_filter/content_filter'
+import { isMatched } from './filter'
 
 export enum RuleActionType {
   AddLabel = 'ADD_LABEL',
@@ -32,49 +29,6 @@ export interface Rule {
   updatedAt: Date
 }
 
-const parseSearchFilter = (filter: string): SearchFilter[] => {
-  const searchFilter = filter ? filter.replace(/\W\s":/g, '') : undefined
-  const result: SearchFilter[] = []
-
-  if (!searchFilter || searchFilter === '*') {
-    return result
-  }
-
-  const parsed = parse(searchFilter, {
-    keywords: ['subscription', 'content'],
-    tokenize: true,
-  })
-  if (parsed.offsets) {
-    const keywords = parsed.offsets
-      .filter((offset) => 'keyword' in offset)
-      .map((offset) => offset as SearchParserKeyWordOffset)
-
-    for (const keyword of keywords) {
-      switch (keyword.keyword) {
-        case 'subscription':
-          keyword.value && result.push(new SubscriptionFilter(keyword.value))
-          break
-        case 'content':
-          keyword.value && result.push(new ContentFilter(keyword.value))
-          break
-      }
-    }
-  }
-
-  return result
-}
-
-const isValidData = (filter: string, data: PubSubData): boolean => {
-  const searchFilters = parseSearchFilter(filter)
-
-  if (searchFilters.length === 0) {
-    console.debug('no search filters found')
-    return true
-  }
-
-  return searchFilters.every((searchFilter) => searchFilter.isValid(data))
-}
-
 export const getEnabledRules = async (
   userId: string,
   apiEndpoint: string,
@@ -121,49 +75,44 @@ export const triggerActions = async (
   apiEndpoint: string,
   jwtSecret: string
 ) => {
-  const triggeredActions: RuleAction[] = []
   const authToken = await getAuthToken(userId, jwtSecret)
+  const actionPromises: Promise<AxiosResponse<any, any> | undefined>[] = []
 
   for (const rule of rules) {
-    if (!isValidData(rule.filter, data)) {
+    if (
+      !(await isMatched(userId, apiEndpoint, authToken, rule.filter, data.id))
+    ) {
       continue
     }
 
-    for (const action of rule.actions) {
+    rule.actions.forEach((action) => {
       switch (action.type) {
         case RuleActionType.AddLabel:
-          if (!data.id || action.params.length === 0) {
-            console.log('invalid data for add label action')
-            continue
-          }
-          await addLabels(apiEndpoint, authToken, data.id, action.params)
-          triggeredActions.push(action)
+          data.id &&
+            actionPromises.push(
+              addLabels(apiEndpoint, authToken, data.id, action.params)
+            )
           break
         case RuleActionType.Archive:
-          if (!data.id) {
-            console.log('invalid data for archive action')
-            continue
-          }
-          await archivePage(apiEndpoint, authToken, data.id)
-          triggeredActions.push(action)
+          data.id &&
+            actionPromises.push(archivePage(apiEndpoint, authToken, data.id))
           break
         case RuleActionType.MarkAsRead:
-          if (!data.id) {
-            console.log('invalid data for mark as read action')
-            continue
-          }
-          await markPageAsRead(apiEndpoint, authToken, data.id)
-          triggeredActions.push(action)
+          data.id &&
+            actionPromises.push(markPageAsRead(apiEndpoint, authToken, data.id))
           break
         case RuleActionType.SendNotification:
-          for (const message of action.params) {
-            await sendNotification(apiEndpoint, authToken, message)
-          }
-          triggeredActions.push(action)
+          actionPromises.push(
+            sendNotification(
+              apiEndpoint,
+              authToken,
+              'New page added to your feed'
+            )
+          )
           break
       }
-    }
+    })
   }
 
-  return triggeredActions
+  return Promise.all(actionPromises)
 }

packages/rule-handler/src/search_filter/content_filter.ts

@@ -1,15 +0,0 @@
-import { SearchFilter } from './index'
-import { PubSubData } from '../index'
-
-export class ContentFilter extends SearchFilter {
-  public isValid(data: PubSubData): boolean {
-    console.debug('ContentFilter.isValid', this.filter, data.content)
-
-    if (!data.content) {
-      return false
-    }
-
-    // TODO: implement content filter with semantic search
-    return this.filter === '*' || data.content.includes(this.filter)
-  }
-}

packages/rule-handler/src/search_filter/index.ts

@@ -1,9 +0,0 @@
-import { PubSubData } from '../index'
-
-export abstract class SearchFilter {
-  constructor(protected filter: string) {
-    this.filter = filter
-  }
-
-  public abstract isValid(data: PubSubData): boolean
-}

packages/rule-handler/src/search_filter/subscription_filter.ts

@@ -1,18 +0,0 @@
-import { SearchFilter } from './index'
-import { PubSubData } from '../index'
-
-export class SubscriptionFilter extends SearchFilter {
-  public isValid(data: PubSubData): boolean {
-    console.debug('SubscriptionFilter.isValid', this.filter, data.subscription)
-
-    if (!data.subscription) {
-      return false
-    }
-
-    // compare subscription name case insensitive
-    return (
-      this.filter === '*' ||
-      data.subscription.toLowerCase() === this.filter.toLowerCase()
-    )
-  }
-}