From 07cee98ff9a36558e78ac34c349a2ff8b553eb84 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 21 Mar 2024 17:24:24 +0800
Subject: [PATCH 01/35] allow label and highlight events to trigger rules

---
 packages/api/src/entity/rule.ts         |  3 +++
 packages/api/src/jobs/trigger_rule.ts   | 12 +++++++++++-
 packages/api/src/pubsub.ts              | 14 ++++++--------
 packages/api/src/services/highlights.ts | 13 +++++++------
 packages/api/src/services/labels.ts     | 10 +++++-----
 5 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/packages/api/src/entity/rule.ts b/packages/api/src/entity/rule.ts
index 51c63b972..18538971c 100644
--- a/packages/api/src/entity/rule.ts
+++ b/packages/api/src/entity/rule.ts
@@ -15,11 +15,14 @@ export enum RuleActionType {
   Delete = 'DELETE',
   MarkAsRead = 'MARK_AS_READ',
   SendNotification = 'SEND_NOTIFICATION',
+  Webhook = 'WEBHOOK',
 }
 
 export enum RuleEventType {
   PageCreated = 'PAGE_CREATED',
   PageUpdated = 'PAGE_UPDATED',
+  LabelCreated = 'PAGE_CREATED',
+  HighlightCreated = 'HIGHLIGHT_CREATED',
 }
 
 export interface RuleAction {
diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts
index 596760295..3a72dac11 100644
--- a/packages/api/src/jobs/trigger_rule.ts
+++ b/packages/api/src/jobs/trigger_rule.ts
@@ -86,7 +86,9 @@ const sendNotification = async (obj: RuleActionObj) => {
   return sendPushNotifications(obj.userId, message, 'rule', data)
 }
 
-const getRuleAction = (actionType: RuleActionType): RuleActionFunc => {
+const getRuleAction = (
+  actionType: RuleActionType
+): RuleActionFunc | undefined => {
   switch (actionType) {
     case RuleActionType.AddLabel:
       return addLabels
@@ -98,6 +100,9 @@ const getRuleAction = (actionType: RuleActionType): RuleActionFunc => {
       return markPageAsRead
     case RuleActionType.SendNotification:
       return sendNotification
+    default:
+      logger.error('Unknown rule action type', actionType)
+      return undefined
   }
 }
 
@@ -150,6 +155,11 @@ const triggerActions = async (
 
     for (const action of rule.actions) {
       const actionFunc = getRuleAction(action.type)
+      if (!actionFunc) {
+        logger.error('No action function found for action', action.type)
+        continue
+      }
+
       const actionObj: RuleActionObj = {
         libraryItemId,
         userId,
diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts
index 4401cfad9..d8c6bba4d 100644
--- a/packages/api/src/pubsub.ts
+++ b/packages/api/src/pubsub.ts
@@ -53,14 +53,12 @@ export const createPubSubClient = (): PubsubClient => {
       libraryItemId: string
     ): Promise<void> => {
       // queue trigger rule job
-      if (type === EntityType.PAGE) {
-        await enqueueTriggerRuleJob({
-          userId,
-          ruleEventType: RuleEventType.PageCreated,
-          libraryItemId,
-          data,
-        })
-      }
+      await enqueueTriggerRuleJob({
+        userId,
+        ruleEventType: `${type.toUpperCase()}_CREATED` as RuleEventType,
+        libraryItemId,
+        data,
+      })
       // queue export item job
       await enqueueExportItem({
         userId,
diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index babf4d4a4..91249626b 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -9,6 +9,7 @@ import { createPubSubClient, EntityType } from '../pubsub'
 import { authTrx } from '../repository'
 import { highlightRepository } from '../repository/highlight'
 import { enqueueUpdateHighlight } from '../utils/createTask'
+import { UpdateItemEvent } from './library_item'
 
 type HighlightEvent = { id: string; pageId: string }
 type CreateHighlightEvent = DeepPartial<Highlight> & HighlightEvent
@@ -56,9 +57,9 @@ export const createHighlight = async (
     userId
   )
 
-  await pubsub.entityCreated<CreateHighlightEvent>(
+  await pubsub.entityCreated<UpdateItemEvent>(
     EntityType.HIGHLIGHT,
-    { ...newHighlight, pageId: libraryItemId },
+    { id: libraryItemId, highlights: [newHighlight] },
     userId,
     libraryItemId
   )
@@ -104,9 +105,9 @@ export const mergeHighlights = async (
     })
   })
 
-  await pubsub.entityCreated<CreateHighlightEvent>(
+  await pubsub.entityCreated<UpdateItemEvent>(
     EntityType.HIGHLIGHT,
-    { ...newHighlight, pageId: libraryItemId },
+    { id: libraryItemId, highlights: [newHighlight] },
     userId,
     libraryItemId
   )
@@ -139,9 +140,9 @@ export const updateHighlight = async (
   })
 
   const libraryItemId = updatedHighlight.libraryItem.id
-  await pubsub.entityUpdated<UpdateHighlightEvent>(
+  await pubsub.entityUpdated<UpdateItemEvent>(
     EntityType.HIGHLIGHT,
-    { ...highlight, id: highlightId, pageId: libraryItemId },
+    { id: libraryItemId, highlights: [highlight] },
     userId,
     libraryItemId
   )
diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts
index f4bae13b7..ca0e02d64 100644
--- a/packages/api/src/services/labels.ts
+++ b/packages/api/src/services/labels.ts
@@ -8,7 +8,7 @@ import { CreateLabelInput, labelRepository } from '../repository/label'
 import { bulkEnqueueUpdateLabels } from '../utils/createTask'
 import { logger } from '../utils/logger'
 import { findHighlightById } from './highlights'
-import { findLibraryItemIdsByLabelId } from './library_item'
+import { findLibraryItemIdsByLabelId, UpdateItemEvent } from './library_item'
 
 type AddLabelsToLibraryItemEvent = {
   pageId: string
@@ -144,9 +144,9 @@ export const saveLabelsInLibraryItem = async (
 
   if (source === 'user') {
     // create pubsub event
-    await pubsub.entityCreated<AddLabelsToLibraryItemEvent>(
+    await pubsub.entityCreated<UpdateItemEvent>(
       EntityType.LABEL,
-      { pageId: libraryItemId, labels, source },
+      { id: libraryItemId, labels },
       userId,
       libraryItemId
     )
@@ -215,9 +215,9 @@ export const saveLabelsInHighlight = async (
 
   const libraryItemId = highlight.libraryItemId
   // create pubsub event
-  await pubsub.entityCreated<AddLabelsToHighlightEvent>(
+  await pubsub.entityCreated<UpdateItemEvent>(
     EntityType.LABEL,
-    { highlightId, labels },
+    { id: libraryItemId, highlights: [{ id: highlightId, labels }] },
     userId,
     libraryItemId
   )

From c551d9fe6bdd9dad026b370ebfcff017ac2747b0 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 21 Mar 2024 18:13:42 +0800
Subject: [PATCH 02/35] allow send data to webhook with rules

---
 packages/api/src/entity/rule.ts           |  2 +-
 packages/api/src/generated/graphql.ts     |  5 ++-
 packages/api/src/generated/schema.graphql |  3 ++
 packages/api/src/jobs/trigger_rule.ts     | 33 +++++++++++++++-
 packages/api/src/pubsub.ts                | 31 +++++++--------
 packages/api/src/schema.ts                |  3 ++
 packages/api/src/services/highlights.ts   |  6 +--
 packages/api/src/services/labels.ts       |  4 +-
 packages/api/src/services/library_item.ts | 46 +++++++++--------------
 9 files changed, 80 insertions(+), 53 deletions(-)

diff --git a/packages/api/src/entity/rule.ts b/packages/api/src/entity/rule.ts
index 18538971c..9ef6d797e 100644
--- a/packages/api/src/entity/rule.ts
+++ b/packages/api/src/entity/rule.ts
@@ -21,7 +21,7 @@ export enum RuleActionType {
 export enum RuleEventType {
   PageCreated = 'PAGE_CREATED',
   PageUpdated = 'PAGE_UPDATED',
-  LabelCreated = 'PAGE_CREATED',
+  LabelCreated = 'LABEL_CREATED',
   HighlightCreated = 'HIGHLIGHT_CREATED',
 }
 
diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index ae5bcdbd8..3e3119db6 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -2502,10 +2502,13 @@ export enum RuleActionType {
   Archive = 'ARCHIVE',
   Delete = 'DELETE',
   MarkAsRead = 'MARK_AS_READ',
-  SendNotification = 'SEND_NOTIFICATION'
+  SendNotification = 'SEND_NOTIFICATION',
+  Webhook = 'WEBHOOK'
 }
 
 export enum RuleEventType {
+  HighlightCreated = 'HIGHLIGHT_CREATED',
+  LabelCreated = 'LABEL_CREATED',
   PageCreated = 'PAGE_CREATED',
   PageUpdated = 'PAGE_UPDATED'
 }
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 94695e350..1782328b8 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1879,9 +1879,12 @@ enum RuleActionType {
   DELETE
   MARK_AS_READ
   SEND_NOTIFICATION
+  WEBHOOK
 }
 
 enum RuleEventType {
+  HIGHLIGHT_CREATED
+  LABEL_CREATED
   PAGE_CREATED
   PAGE_UPDATED
 }
diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts
index 3a72dac11..132218ebb 100644
--- a/packages/api/src/jobs/trigger_rule.ts
+++ b/packages/api/src/jobs/trigger_rule.ts
@@ -1,4 +1,5 @@
 import { LiqeQuery } from '@omnivore/liqe'
+import axios, { Method } from 'axios'
 import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source'
 import { LibraryItem, LibraryItemState } from '../entity/library_item'
 import { Rule, RuleAction, RuleActionType, RuleEventType } from '../entity/rule'
@@ -28,6 +29,7 @@ interface RuleActionObj {
   userId: string
   action: RuleAction
   data: ItemEvent | LibraryItem
+  ruleEventType: RuleEventType
 }
 type RuleActionFunc = (obj: RuleActionObj) => Promise<unknown>
 
@@ -86,6 +88,29 @@ const sendNotification = async (obj: RuleActionObj) => {
   return sendPushNotifications(obj.userId, message, 'rule', data)
 }
 
+const sendToWebhook = async (obj: RuleActionObj) => {
+  const [url, method, contentType] = obj.action.params
+  const [type, action] = obj.ruleEventType.split('_')
+
+  const body = {
+    action,
+    userId: obj.userId,
+    [type]: obj.data,
+  }
+
+  logger.info('triggering webhook', { url, method })
+
+  return axios.request({
+    url,
+    method: method as Method,
+    headers: {
+      'Content-Type': contentType,
+    },
+    data: body,
+    timeout: 5000, // 5s
+  })
+}
+
 const getRuleAction = (
   actionType: RuleActionType
 ): RuleActionFunc | undefined => {
@@ -100,6 +125,8 @@ const getRuleAction = (
       return markPageAsRead
     case RuleActionType.SendNotification:
       return sendNotification
+    case RuleActionType.Webhook:
+      return sendToWebhook
     default:
       logger.error('Unknown rule action type', actionType)
       return undefined
@@ -110,7 +137,8 @@ const triggerActions = async (
   libraryItemId: string,
   userId: string,
   rules: Rule[],
-  data: ItemEvent
+  data: ItemEvent,
+  ruleEventType: RuleEventType
 ) => {
   const actionPromises: Promise<unknown>[] = []
 
@@ -165,6 +193,7 @@ const triggerActions = async (
         userId,
         action,
         data: results[0],
+        ruleEventType,
       }
 
       actionPromises.push(actionFunc(actionObj))
@@ -188,7 +217,7 @@ export const triggerRule = async (jobData: TriggerRuleJobData) => {
     return false
   }
 
-  await triggerActions(libraryItemId, userId, rules, data)
+  await triggerActions(libraryItemId, userId, rules, data, ruleEventType)
 
   return true
 }
diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts
index d8c6bba4d..d340f520b 100644
--- a/packages/api/src/pubsub.ts
+++ b/packages/api/src/pubsub.ts
@@ -12,6 +12,8 @@ import {
 import { buildLogger } from './utils/logger'
 import { isYouTubeVideoURL } from './utils/youtube'
 
+export type BaseEntityEvent = { id: string; userId: string }
+
 const logger = buildLogger('pubsub')
 
 const client = new PubSub()
@@ -46,7 +48,7 @@ export const createPubSubClient = (): PubsubClient => {
         Buffer.from(JSON.stringify({ userId, email, name, username }))
       )
     },
-    entityCreated: async <T extends Record<string, any>>(
+    entityCreated: async <T extends BaseEntityEvent>(
       type: EntityType,
       data: T,
       userId: string,
@@ -54,10 +56,10 @@ export const createPubSubClient = (): PubsubClient => {
     ): Promise<void> => {
       // queue trigger rule job
       await enqueueTriggerRuleJob({
-        userId,
         ruleEventType: `${type.toUpperCase()}_CREATED` as RuleEventType,
-        libraryItemId,
         data,
+        userId,
+        libraryItemId,
       })
       // queue export item job
       await enqueueExportItem({
@@ -92,21 +94,20 @@ export const createPubSubClient = (): PubsubClient => {
         }
       }
     },
-    entityUpdated: async <T extends Record<string, any>>(
+    entityUpdated: async <T extends BaseEntityEvent>(
       type: EntityType,
       data: T,
       userId: string,
       libraryItemId: string
     ): Promise<void> => {
       // queue trigger rule job
-      if (type === EntityType.PAGE) {
-        await enqueueTriggerRuleJob({
-          userId,
-          ruleEventType: RuleEventType.PageUpdated,
-          libraryItemId,
-          data,
-        })
-      }
+      await enqueueTriggerRuleJob({
+        userId,
+        ruleEventType: RuleEventType.PageUpdated,
+        libraryItemId,
+        data,
+      })
+
       // queue export item job
       await enqueueExportItem({
         userId,
@@ -145,7 +146,7 @@ export const createPubSubClient = (): PubsubClient => {
 }
 
 export enum EntityType {
-  PAGE = 'page',
+  ITEM = 'page',
   HIGHLIGHT = 'highlight',
   LABEL = 'label',
   RSS_FEED = 'feed',
@@ -158,13 +159,13 @@ export interface PubsubClient {
     name: string,
     username: string
   ) => Promise<void>
-  entityCreated: <T extends Record<string, any>>(
+  entityCreated: <T extends BaseEntityEvent>(
     type: EntityType,
     data: T,
     userId: string,
     libraryItemId: string
   ) => Promise<void>
-  entityUpdated: <T extends Record<string, any>>(
+  entityUpdated: <T extends BaseEntityEvent>(
     type: EntityType,
     data: T,
     userId: string,
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index e106514e4..d2279407e 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -2167,6 +2167,7 @@ const schema = gql`
     DELETE
     MARK_AS_READ
     SEND_NOTIFICATION
+    WEBHOOK
   }
 
   type RulesError {
@@ -2181,6 +2182,8 @@ const schema = gql`
   enum RuleEventType {
     PAGE_CREATED
     PAGE_UPDATED
+    LABEL_CREATED
+    HIGHLIGHT_CREATED
   }
 
   input SetRuleInput {
diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index 91249626b..8cc10ad0b 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -59,7 +59,7 @@ export const createHighlight = async (
 
   await pubsub.entityCreated<UpdateItemEvent>(
     EntityType.HIGHLIGHT,
-    { id: libraryItemId, highlights: [newHighlight] },
+    { id: libraryItemId, highlights: [newHighlight], userId },
     userId,
     libraryItemId
   )
@@ -107,7 +107,7 @@ export const mergeHighlights = async (
 
   await pubsub.entityCreated<UpdateItemEvent>(
     EntityType.HIGHLIGHT,
-    { id: libraryItemId, highlights: [newHighlight] },
+    { id: libraryItemId, highlights: [newHighlight], userId },
     userId,
     libraryItemId
   )
@@ -142,7 +142,7 @@ export const updateHighlight = async (
   const libraryItemId = updatedHighlight.libraryItem.id
   await pubsub.entityUpdated<UpdateItemEvent>(
     EntityType.HIGHLIGHT,
-    { id: libraryItemId, highlights: [highlight] },
+    { id: libraryItemId, highlights: [highlight], userId },
     userId,
     libraryItemId
   )
diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts
index ca0e02d64..1f05d14c1 100644
--- a/packages/api/src/services/labels.ts
+++ b/packages/api/src/services/labels.ts
@@ -146,7 +146,7 @@ export const saveLabelsInLibraryItem = async (
     // create pubsub event
     await pubsub.entityCreated<UpdateItemEvent>(
       EntityType.LABEL,
-      { id: libraryItemId, labels },
+      { id: libraryItemId, labels, userId },
       userId,
       libraryItemId
     )
@@ -217,7 +217,7 @@ export const saveLabelsInHighlight = async (
   // create pubsub event
   await pubsub.entityCreated<UpdateItemEvent>(
     EntityType.LABEL,
-    { id: libraryItemId, highlights: [{ id: highlightId, labels }] },
+    { id: libraryItemId, highlights: [{ id: highlightId, labels }], userId },
     userId,
     libraryItemId
   )
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index 83473104c..4ac000900 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -15,7 +15,7 @@ import { Highlight } from '../entity/highlight'
 import { Label } from '../entity/label'
 import { LibraryItem, LibraryItemState } from '../entity/library_item'
 import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql'
-import { createPubSubClient, EntityType } from '../pubsub'
+import { BaseEntityEvent, createPubSubClient, EntityType } from '../pubsub'
 import { redisDataSource } from '../redis_data_source'
 import {
   authTrx,
@@ -37,10 +37,13 @@ type IgnoredFields =
   | 'links'
   | 'textContentHash'
 export type ItemEvent = CreateItemEvent | UpdateItemEvent
-export type CreateItemEvent = Omit<DeepPartial<LibraryItem>, IgnoredFields>
-export type UpdateItemEvent = Omit<
-  QueryDeepPartialEntity<LibraryItem>,
-  IgnoredFields
+export type CreateItemEvent = Merge<
+  Omit<DeepPartial<LibraryItem>, IgnoredFields>,
+  BaseEntityEvent
+>
+export type UpdateItemEvent = Merge<
+  Omit<QueryDeepPartialEntity<LibraryItem>, IgnoredFields>,
+  BaseEntityEvent
 >
 
 export class RequiresSearchQueryError extends Error {
@@ -841,7 +844,7 @@ export const softDeleteLibraryItem = async (
     userId
   )
 
-  await pubsub.entityDeleted(EntityType.PAGE, id, userId)
+  await pubsub.entityDeleted(EntityType.ITEM, id, userId)
 
   return deletedLibraryItem
 }
@@ -883,13 +886,8 @@ export const updateLibraryItem = async (
   if (libraryItem.state === LibraryItemState.Succeeded) {
     // send create event if the item was created
     await pubsub.entityCreated<CreateItemEvent>(
-      EntityType.PAGE,
-      {
-        ...updatedLibraryItem,
-        originalContent: undefined,
-        readableContent: undefined,
-        feedContent: undefined,
-      },
+      EntityType.ITEM,
+      { ...updatedLibraryItem, userId },
       userId,
       id
     )
@@ -898,13 +896,8 @@ export const updateLibraryItem = async (
   }
 
   await pubsub.entityUpdated<UpdateItemEvent>(
-    EntityType.PAGE,
-    {
-      ...libraryItem,
-      originalContent: undefined,
-      readableContent: undefined,
-      feedContent: undefined,
-    },
+    EntityType.ITEM,
+    { ...libraryItem, id, userId },
     userId,
     id
   )
@@ -966,8 +959,8 @@ export const updateLibraryItemReadingProgress = async (
 
   const updatedItem = result[0][0]
   await pubsub.entityUpdated<UpdateItemEvent>(
-    EntityType.PAGE,
-    updatedItem,
+    EntityType.ITEM,
+    { ...updatedItem, id, userId },
     userId,
     id
   )
@@ -1067,13 +1060,8 @@ export const createOrUpdateLibraryItem = async (
   }
 
   await pubsub.entityCreated<CreateItemEvent>(
-    EntityType.PAGE,
-    {
-      ...newLibraryItem,
-      originalContent: undefined,
-      readableContent: undefined,
-      feedContent: undefined,
-    },
+    EntityType.ITEM,
+    { ...newLibraryItem, userId },
     userId,
     newLibraryItem.id
   )

From 88d6455222963e8819eef303702f2dc575a8b522 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 21 Mar 2024 19:51:31 +0800
Subject: [PATCH 03/35] default filter is in:all

---
 packages/api/src/jobs/trigger_rule.ts         | 21 +++++-------
 packages/api/src/pubsub.ts                    | 17 +---------
 packages/api/src/services/highlights.ts       | 17 +++++-----
 packages/api/src/services/labels.ts           | 16 +++------
 packages/api/src/services/library_item.ts     | 34 +++++++++----------
 .../networking/queries/useGetRulesQuery.tsx   |  3 ++
 packages/web/pages/settings/rules.tsx         | 34 +++++++++++++------
 7 files changed, 66 insertions(+), 76 deletions(-)

diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts
index 132218ebb..88cc65144 100644
--- a/packages/api/src/jobs/trigger_rule.ts
+++ b/packages/api/src/jobs/trigger_rule.ts
@@ -1,5 +1,5 @@
 import { LiqeQuery } from '@omnivore/liqe'
-import axios, { Method } from 'axios'
+import axios from 'axios'
 import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source'
 import { LibraryItem, LibraryItemState } from '../entity/library_item'
 import { Rule, RuleAction, RuleActionType, RuleEventType } from '../entity/rule'
@@ -89,24 +89,19 @@ const sendNotification = async (obj: RuleActionObj) => {
 }
 
 const sendToWebhook = async (obj: RuleActionObj) => {
-  const [url, method, contentType] = obj.action.params
-  const [type, action] = obj.ruleEventType.split('_')
+  const [url] = obj.action.params
 
-  const body = {
-    action,
-    userId: obj.userId,
-    [type]: obj.data,
+  const data = {
+    event: obj.ruleEventType,
+    data: obj.data,
   }
 
-  logger.info('triggering webhook', { url, method })
+  logger.info(`triggering webhook: ${url}`)
 
-  return axios.request({
-    url,
-    method: method as Method,
+  return axios.post(url, data, {
     headers: {
-      'Content-Type': contentType,
+      'Content-Type': 'application/json',
     },
-    data: body,
     timeout: 5000, // 5s
   })
 }
diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts
index d340f520b..55da4e7c9 100644
--- a/packages/api/src/pubsub.ts
+++ b/packages/api/src/pubsub.ts
@@ -7,7 +7,6 @@ import {
   enqueueExportItem,
   enqueueProcessYouTubeVideo,
   enqueueTriggerRuleJob,
-  enqueueWebhookJob,
 } from './utils/createTask'
 import { buildLogger } from './utils/logger'
 import { isYouTubeVideoURL } from './utils/youtube'
@@ -67,14 +66,7 @@ export const createPubSubClient = (): PubsubClient => {
         libraryItemIds: [libraryItemId],
       })
 
-      await enqueueWebhookJob({
-        userId,
-        type,
-        action: 'created',
-        data,
-      })
-
-      if (type === EntityType.PAGE) {
+      if (type === EntityType.ITEM) {
         // if (await findGrantedFeatureByName(FeatureName.AISummaries, userId)) {
         // await enqueueAISummarizeJob({
         //   userId,
@@ -113,13 +105,6 @@ export const createPubSubClient = (): PubsubClient => {
         userId,
         libraryItemIds: [libraryItemId],
       })
-
-      await enqueueWebhookJob({
-        userId,
-        type,
-        action: 'updated',
-        data,
-      })
     },
     entityDeleted: async (
       type: EntityType,
diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index 8cc10ad0b..624c5b2da 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -9,11 +9,12 @@ import { createPubSubClient, EntityType } from '../pubsub'
 import { authTrx } from '../repository'
 import { highlightRepository } from '../repository/highlight'
 import { enqueueUpdateHighlight } from '../utils/createTask'
-import { UpdateItemEvent } from './library_item'
+import { ItemEvent } from './library_item'
 
-type HighlightEvent = { id: string; pageId: string }
-type CreateHighlightEvent = DeepPartial<Highlight> & HighlightEvent
-type UpdateHighlightEvent = QueryDeepPartialEntity<Highlight> & HighlightEvent
+export type HighlightEvent = Omit<
+  DeepPartial<Highlight>,
+  'user' | 'userId' | 'sharedAt'
+>
 
 export const getHighlightLocation = (patch: string): number | undefined => {
   const dmp = new diff_match_patch()
@@ -57,7 +58,7 @@ export const createHighlight = async (
     userId
   )
 
-  await pubsub.entityCreated<UpdateItemEvent>(
+  await pubsub.entityCreated<ItemEvent>(
     EntityType.HIGHLIGHT,
     { id: libraryItemId, highlights: [newHighlight], userId },
     userId,
@@ -105,7 +106,7 @@ export const mergeHighlights = async (
     })
   })
 
-  await pubsub.entityCreated<UpdateItemEvent>(
+  await pubsub.entityCreated<ItemEvent>(
     EntityType.HIGHLIGHT,
     { id: libraryItemId, highlights: [newHighlight], userId },
     userId,
@@ -140,9 +141,9 @@ export const updateHighlight = async (
   })
 
   const libraryItemId = updatedHighlight.libraryItem.id
-  await pubsub.entityUpdated<UpdateItemEvent>(
+  await pubsub.entityUpdated<ItemEvent>(
     EntityType.HIGHLIGHT,
-    { id: libraryItemId, highlights: [highlight], userId },
+    { id: libraryItemId, highlights: [highlight], userId } as ItemEvent,
     userId,
     libraryItemId
   )
diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts
index 1f05d14c1..4a23f30c0 100644
--- a/packages/api/src/services/labels.ts
+++ b/packages/api/src/services/labels.ts
@@ -8,17 +8,9 @@ import { CreateLabelInput, labelRepository } from '../repository/label'
 import { bulkEnqueueUpdateLabels } from '../utils/createTask'
 import { logger } from '../utils/logger'
 import { findHighlightById } from './highlights'
-import { findLibraryItemIdsByLabelId, UpdateItemEvent } from './library_item'
+import { findLibraryItemIdsByLabelId, ItemEvent } from './library_item'
 
-type AddLabelsToLibraryItemEvent = {
-  pageId: string
-  labels: DeepPartial<Label>[]
-  source?: LabelSource
-}
-type AddLabelsToHighlightEvent = {
-  highlightId: string
-  labels: DeepPartial<Label>[]
-}
+export type LabelEvent = Omit<DeepPartial<Label>, 'description' | 'createdAt'>
 
 // const batchGetLabelsFromLinkIds = async (
 //   linkIds: readonly string[]
@@ -144,7 +136,7 @@ export const saveLabelsInLibraryItem = async (
 
   if (source === 'user') {
     // create pubsub event
-    await pubsub.entityCreated<UpdateItemEvent>(
+    await pubsub.entityCreated<ItemEvent>(
       EntityType.LABEL,
       { id: libraryItemId, labels, userId },
       userId,
@@ -215,7 +207,7 @@ export const saveLabelsInHighlight = async (
 
   const libraryItemId = highlight.libraryItemId
   // create pubsub event
-  await pubsub.entityCreated<UpdateItemEvent>(
+  await pubsub.entityCreated<ItemEvent>(
     EntityType.LABEL,
     { id: libraryItemId, highlights: [{ id: highlightId, labels }], userId },
     userId,
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index 4ac000900..b26ebd010 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -28,7 +28,8 @@ import { Merge } from '../util'
 import { setRecentlySavedItemInRedis } from '../utils/helpers'
 import { logger } from '../utils/logger'
 import { parseSearchQuery } from '../utils/search'
-import { addLabelsToLibraryItem } from './labels'
+import { HighlightEvent } from './highlights'
+import { addLabelsToLibraryItem, LabelEvent } from './labels'
 
 type IgnoredFields =
   | 'user'
@@ -36,15 +37,14 @@ type IgnoredFields =
   | 'previewContentType'
   | 'links'
   | 'textContentHash'
-export type ItemEvent = CreateItemEvent | UpdateItemEvent
-export type CreateItemEvent = Merge<
+type ItemBaseEvent = Merge<
   Omit<DeepPartial<LibraryItem>, IgnoredFields>,
-  BaseEntityEvent
->
-export type UpdateItemEvent = Merge<
-  Omit<QueryDeepPartialEntity<LibraryItem>, IgnoredFields>,
-  BaseEntityEvent
+  {
+    labels?: LabelEvent[]
+    highlights?: HighlightEvent[]
+  }
 >
+export type ItemEvent = Merge<ItemBaseEvent, BaseEntityEvent>
 
 export class RequiresSearchQueryError extends Error {
   constructor() {
@@ -885,7 +885,7 @@ export const updateLibraryItem = async (
 
   if (libraryItem.state === LibraryItemState.Succeeded) {
     // send create event if the item was created
-    await pubsub.entityCreated<CreateItemEvent>(
+    await pubsub.entityCreated<ItemEvent>(
       EntityType.ITEM,
       { ...updatedLibraryItem, userId },
       userId,
@@ -895,9 +895,9 @@ export const updateLibraryItem = async (
     return updatedLibraryItem
   }
 
-  await pubsub.entityUpdated<UpdateItemEvent>(
+  await pubsub.entityUpdated<ItemEvent>(
     EntityType.ITEM,
-    { ...libraryItem, id, userId },
+    { ...libraryItem, id, userId } as ItemEvent,
     userId,
     id
   )
@@ -958,7 +958,7 @@ export const updateLibraryItemReadingProgress = async (
   }
 
   const updatedItem = result[0][0]
-  await pubsub.entityUpdated<UpdateItemEvent>(
+  await pubsub.entityUpdated<ItemEvent>(
     EntityType.ITEM,
     { ...updatedItem, id, userId },
     userId,
@@ -1059,7 +1059,7 @@ export const createOrUpdateLibraryItem = async (
     return newLibraryItem
   }
 
-  await pubsub.entityCreated<CreateItemEvent>(
+  await pubsub.entityCreated<ItemEvent>(
     EntityType.ITEM,
     { ...newLibraryItem, userId },
     userId,
@@ -1422,7 +1422,7 @@ export const filterItemEvents = (
         }
       }
       case 'type': {
-        return event.itemType?.toString()?.toLowerCase() === lowercasedValue
+        return event.itemType?.toLowerCase() === lowercasedValue
       }
       case 'label': {
         const labels = event.labelNames as string[] | undefined
@@ -1497,7 +1497,7 @@ export const filterItemEvents = (
         // get camel case column name
         const key = camelCase(columnName) as 'subscription' | 'itemLanguage'
 
-        return event[key]?.toString()?.toLowerCase() === lowercasedValue
+        return event[key]?.toLowerCase() === lowercasedValue
       }
       // match filters
       case 'note':
@@ -1513,7 +1513,7 @@ export const filterItemEvents = (
         const keys = ['siteName', 'originalUrl'] as const
 
         return keys.some((key) => {
-          return event[key]?.toString()?.toLowerCase().includes(lowercasedValue)
+          return event[key]?.toLowerCase().includes(lowercasedValue)
         })
       }
       case 'includes': {
@@ -1522,7 +1522,7 @@ export const filterItemEvents = (
           throw new Error('Expected ids')
         }
 
-        return event.id && ids.includes(event.id.toString())
+        return event.id && ids.includes(event.id)
       }
       case 'recommendedby': {
         if (!event.recommenderNames) {
diff --git a/packages/web/lib/networking/queries/useGetRulesQuery.tsx b/packages/web/lib/networking/queries/useGetRulesQuery.tsx
index 2da718072..77b11154d 100644
--- a/packages/web/lib/networking/queries/useGetRulesQuery.tsx
+++ b/packages/web/lib/networking/queries/useGetRulesQuery.tsx
@@ -13,11 +13,14 @@ export enum RuleActionType {
   MarkAsRead = 'MARK_AS_READ',
   Delete = 'DELETE',
   SendNotification = 'SEND_NOTIFICATION',
+  Webhook = 'WEBHOOK',
 }
 
 export enum RuleEventType {
   PAGE_CREATED = 'PAGE_CREATED',
   PAGE_UPDATED = 'PAGE_UPDATED',
+  LABEL_CREATED = 'LABEL_CREATED',
+  HIGHLIGHT_CREATED = 'HIGHLIGHT_CREATED',
 }
 
 export interface Rule {
diff --git a/packages/web/pages/settings/rules.tsx b/packages/web/pages/settings/rules.tsx
index 5cf3e1636..7d5be493c 100644
--- a/packages/web/pages/settings/rules.tsx
+++ b/packages/web/pages/settings/rules.tsx
@@ -30,7 +30,7 @@ const CreateRuleModal = (props: CreateRuleModalProps): JSX.Element => {
 
   const onOk = async (values: any) => {
     const name = form.getFieldValue('name')
-    const filter = form.getFieldValue('filter')
+    const filter = form.getFieldValue('filter') || 'in:all'
     const eventTypes = form.getFieldValue('eventTypes')
     try {
       await setRuleMutation({
@@ -81,11 +81,7 @@ const CreateRuleModal = (props: CreateRuleModalProps): JSX.Element => {
           <Input />
         </Form.Item>
 
-        <Form.Item
-          label="Filter"
-          name="filter"
-          rules={[{ required: true, message: 'Please enter the rule filter' }]}
-        >
+        <Form.Item label="Filter" name="filter">
           <Input />
         </Form.Item>
 
@@ -131,8 +127,13 @@ const CreateActionModal = (props: CreateActionModalProps): JSX.Element => {
 
   const onOk = async (values: any) => {
     const actionType = form.getFieldValue('actionType') as RuleActionType
-    const params =
-      actionType == RuleActionType.AddLabel ? form.getFieldValue('labels') : []
+    let params = []
+    if (actionType == RuleActionType.AddLabel) {
+      params = form.getFieldValue('labels')
+    } else if (actionType == RuleActionType.Webhook) {
+      params = [form.getFieldValue('url')]
+    }
+
     if (props.rule) {
       await setRuleMutation({
         id: props.rule.id,
@@ -155,8 +156,9 @@ const CreateActionModal = (props: CreateActionModalProps): JSX.Element => {
     }
   }
 
-  const [actionType, setActionType] =
-    useState<RuleActionType | undefined>(undefined)
+  const [actionType, setActionType] = useState<RuleActionType | undefined>(
+    undefined
+  )
 
   return (
     <Modal
@@ -215,6 +217,18 @@ const CreateActionModal = (props: CreateActionModalProps): JSX.Element => {
             </Select>
           </Form.Item>
         )}
+
+        {actionType == RuleActionType.Webhook && (
+          <Form.Item
+            label="URL"
+            name="url"
+            rules={[
+              { required: true, message: 'Please key in your webhook url' },
+            ]}
+          >
+            <Input />
+          </Form.Item>
+        )}
       </Form>
     </Modal>
   )

From e2b3235046e5ae0b115a24eb6c45a55db5b7e712 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 10:45:20 +0800
Subject: [PATCH 04/35] delete unneccessary properties from webhook data

---
 packages/api/src/jobs/trigger_rule.ts     | 27 ++++------
 packages/api/src/pubsub.ts                | 30 +++++------
 packages/api/src/services/highlights.ts   | 24 ++++-----
 packages/api/src/services/labels.ts       | 26 ++++++---
 packages/api/src/services/library_item.ts | 65 ++++++++++-------------
 packages/api/src/utils/helpers.ts         |  5 +-
 6 files changed, 85 insertions(+), 92 deletions(-)

diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts
index 88cc65144..158df8168 100644
--- a/packages/api/src/jobs/trigger_rule.ts
+++ b/packages/api/src/jobs/trigger_rule.ts
@@ -18,14 +18,12 @@ import { logger } from '../utils/logger'
 import { parseSearchQuery } from '../utils/search'
 
 export interface TriggerRuleJobData {
-  libraryItemId: string
   userId: string
   ruleEventType: RuleEventType
   data: ItemEvent
 }
 
 interface RuleActionObj {
-  libraryItemId: string
   userId: string
   action: RuleAction
   data: ItemEvent | LibraryItem
@@ -39,21 +37,16 @@ const readingProgressDataSource = new ReadingProgressDataSource()
 const addLabels = async (obj: RuleActionObj) => {
   const labelIds = obj.action.params
 
-  return addLabelsToLibraryItem(
-    labelIds,
-    obj.libraryItemId,
-    obj.userId,
-    'system'
-  )
+  return addLabelsToLibraryItem(labelIds, obj.data.id, obj.userId, 'system')
 }
 
 const deleteLibraryItem = async (obj: RuleActionObj) => {
-  return softDeleteLibraryItem(obj.libraryItemId, obj.userId)
+  return softDeleteLibraryItem(obj.data.id, obj.userId)
 }
 
 const archivePage = async (obj: RuleActionObj) => {
   return updateLibraryItem(
-    obj.libraryItemId,
+    obj.data.id,
     { archivedAt: new Date(), state: LibraryItemState.Archived },
     obj.userId,
     undefined,
@@ -64,7 +57,7 @@ const archivePage = async (obj: RuleActionObj) => {
 const markPageAsRead = async (obj: RuleActionObj) => {
   return readingProgressDataSource.updateReadingProgress(
     obj.userId,
-    obj.libraryItemId,
+    obj.data.id,
     {
       readingProgressPercent: 100,
       readingProgressTopPercent: 100,
@@ -82,7 +75,7 @@ const sendNotification = async (obj: RuleActionObj) => {
   }
   const data = {
     folder: item.folder?.toString() || 'inbox',
-    libraryItemId: obj.libraryItemId,
+    libraryItemId: obj.data.id,
   }
 
   return sendPushNotifications(obj.userId, message, 'rule', data)
@@ -93,7 +86,9 @@ const sendToWebhook = async (obj: RuleActionObj) => {
 
   const data = {
     event: obj.ruleEventType,
-    data: obj.data,
+    item: obj.data,
+    userId: obj.userId,
+    timestamp: Date.now(),
   }
 
   logger.info(`triggering webhook: ${url}`)
@@ -129,7 +124,6 @@ const getRuleAction = (
 }
 
 const triggerActions = async (
-  libraryItemId: string,
   userId: string,
   rules: Rule[],
   data: ItemEvent,
@@ -184,7 +178,6 @@ const triggerActions = async (
       }
 
       const actionObj: RuleActionObj = {
-        libraryItemId,
         userId,
         action,
         data: results[0],
@@ -203,7 +196,7 @@ const triggerActions = async (
 }
 
 export const triggerRule = async (jobData: TriggerRuleJobData) => {
-  const { userId, ruleEventType, data, libraryItemId } = jobData
+  const { userId, ruleEventType, data } = jobData
 
   // get rules by calling api
   const rules = await findEnabledRules(userId, ruleEventType)
@@ -212,7 +205,7 @@ export const triggerRule = async (jobData: TriggerRuleJobData) => {
     return false
   }
 
-  await triggerActions(libraryItemId, userId, rules, data, ruleEventType)
+  await triggerActions(userId, rules, data, ruleEventType)
 
   return true
 }
diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts
index 55da4e7c9..5e761fe1b 100644
--- a/packages/api/src/pubsub.ts
+++ b/packages/api/src/pubsub.ts
@@ -11,7 +11,7 @@ import {
 import { buildLogger } from './utils/logger'
 import { isYouTubeVideoURL } from './utils/youtube'
 
-export type BaseEntityEvent = { id: string; userId: string }
+export type EntityEvent = { id: string }
 
 const logger = buildLogger('pubsub')
 
@@ -47,23 +47,21 @@ export const createPubSubClient = (): PubsubClient => {
         Buffer.from(JSON.stringify({ userId, email, name, username }))
       )
     },
-    entityCreated: async <T extends BaseEntityEvent>(
+    entityCreated: async <T extends EntityEvent>(
       type: EntityType,
       data: T,
-      userId: string,
-      libraryItemId: string
+      userId: string
     ): Promise<void> => {
       // queue trigger rule job
       await enqueueTriggerRuleJob({
         ruleEventType: `${type.toUpperCase()}_CREATED` as RuleEventType,
         data,
         userId,
-        libraryItemId,
       })
       // queue export item job
       await enqueueExportItem({
         userId,
-        libraryItemIds: [libraryItemId],
+        libraryItemIds: [data.id],
       })
 
       if (type === EntityType.ITEM) {
@@ -81,29 +79,27 @@ export const createPubSubClient = (): PubsubClient => {
         if (isItemWithURL(data) && isYouTubeVideoURL(data['originalUrl'])) {
           await enqueueProcessYouTubeVideo({
             userId,
-            libraryItemId,
+            libraryItemId: data.id,
           })
         }
       }
     },
-    entityUpdated: async <T extends BaseEntityEvent>(
+    entityUpdated: async <T extends EntityEvent>(
       type: EntityType,
       data: T,
-      userId: string,
-      libraryItemId: string
+      userId: string
     ): Promise<void> => {
       // queue trigger rule job
       await enqueueTriggerRuleJob({
         userId,
         ruleEventType: RuleEventType.PageUpdated,
-        libraryItemId,
         data,
       })
 
       // queue export item job
       await enqueueExportItem({
         userId,
-        libraryItemIds: [libraryItemId],
+        libraryItemIds: [data.id],
       })
     },
     entityDeleted: async (
@@ -144,17 +140,15 @@ export interface PubsubClient {
     name: string,
     username: string
   ) => Promise<void>
-  entityCreated: <T extends BaseEntityEvent>(
+  entityCreated: <T extends EntityEvent>(
     type: EntityType,
     data: T,
-    userId: string,
-    libraryItemId: string
+    userId: string
   ) => Promise<void>
-  entityUpdated: <T extends BaseEntityEvent>(
+  entityUpdated: <T extends EntityEvent>(
     type: EntityType,
     data: T,
-    userId: string,
-    libraryItemId: string
+    userId: string
   ) => Promise<void>
   entityDeleted: (type: EntityType, id: string, userId: string) => Promise<void>
   reportSubmitted(
diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index 624c5b2da..43d710d36 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -9,12 +9,12 @@ import { createPubSubClient, EntityType } from '../pubsub'
 import { authTrx } from '../repository'
 import { highlightRepository } from '../repository/highlight'
 import { enqueueUpdateHighlight } from '../utils/createTask'
+import { deepDelete } from '../utils/helpers'
 import { ItemEvent } from './library_item'
 
-export type HighlightEvent = Omit<
-  DeepPartial<Highlight>,
-  'user' | 'userId' | 'sharedAt'
->
+const columnToDelete = ['user', 'sharedAt'] as const
+type ColumnToDeleteType = typeof columnToDelete[number]
+export type HighlightEvent = Omit<DeepPartial<Highlight>, ColumnToDeleteType>
 
 export const getHighlightLocation = (patch: string): number | undefined => {
   const dmp = new diff_match_patch()
@@ -58,11 +58,11 @@ export const createHighlight = async (
     userId
   )
 
+  const cleanData = deepDelete(newHighlight, columnToDelete)
   await pubsub.entityCreated<ItemEvent>(
     EntityType.HIGHLIGHT,
-    { id: libraryItemId, highlights: [newHighlight], userId },
-    userId,
-    libraryItemId
+    { id: libraryItemId, highlights: [cleanData] },
+    userId
   )
 
   await enqueueUpdateHighlight({
@@ -108,9 +108,8 @@ export const mergeHighlights = async (
 
   await pubsub.entityCreated<ItemEvent>(
     EntityType.HIGHLIGHT,
-    { id: libraryItemId, highlights: [newHighlight], userId },
-    userId,
-    libraryItemId
+    { id: libraryItemId, highlights: [newHighlight] },
+    userId
   )
 
   await enqueueUpdateHighlight({
@@ -143,9 +142,8 @@ export const updateHighlight = async (
   const libraryItemId = updatedHighlight.libraryItem.id
   await pubsub.entityUpdated<ItemEvent>(
     EntityType.HIGHLIGHT,
-    { id: libraryItemId, highlights: [highlight], userId } as ItemEvent,
-    userId,
-    libraryItemId
+    { id: libraryItemId, highlights: [highlight] } as ItemEvent,
+    userId
   )
 
   await enqueueUpdateHighlight({
diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts
index 4a23f30c0..84ebc7c23 100644
--- a/packages/api/src/services/labels.ts
+++ b/packages/api/src/services/labels.ts
@@ -6,11 +6,14 @@ import { createPubSubClient, EntityType, PubsubClient } from '../pubsub'
 import { authTrx } from '../repository'
 import { CreateLabelInput, labelRepository } from '../repository/label'
 import { bulkEnqueueUpdateLabels } from '../utils/createTask'
+import { deepDelete } from '../utils/helpers'
 import { logger } from '../utils/logger'
 import { findHighlightById } from './highlights'
 import { findLibraryItemIdsByLabelId, ItemEvent } from './library_item'
 
-export type LabelEvent = Omit<DeepPartial<Label>, 'description' | 'createdAt'>
+const columnToDelete = ['description', 'createdAt'] as const
+type ColumnToDeleteType = typeof columnToDelete[number]
+export type LabelEvent = Omit<DeepPartial<Label>, ColumnToDeleteType>
 
 // const batchGetLabelsFromLinkIds = async (
 //   linkIds: readonly string[]
@@ -138,9 +141,11 @@ export const saveLabelsInLibraryItem = async (
     // create pubsub event
     await pubsub.entityCreated<ItemEvent>(
       EntityType.LABEL,
-      { id: libraryItemId, labels, userId },
-      userId,
-      libraryItemId
+      {
+        id: libraryItemId,
+        labels: labels.map((l) => deepDelete(l, columnToDelete)),
+      },
+      userId
     )
   }
 
@@ -209,9 +214,16 @@ export const saveLabelsInHighlight = async (
   // create pubsub event
   await pubsub.entityCreated<ItemEvent>(
     EntityType.LABEL,
-    { id: libraryItemId, highlights: [{ id: highlightId, labels }], userId },
-    userId,
-    libraryItemId
+    {
+      id: libraryItemId,
+      highlights: [
+        {
+          id: highlightId,
+          labels: labels.map((l) => deepDelete(l, columnToDelete)),
+        },
+      ],
+    },
+    userId
   )
 
   // update labels in library item
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index b26ebd010..4cc0806dd 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -15,7 +15,7 @@ import { Highlight } from '../entity/highlight'
 import { Label } from '../entity/label'
 import { LibraryItem, LibraryItemState } from '../entity/library_item'
 import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql'
-import { BaseEntityEvent, createPubSubClient, EntityType } from '../pubsub'
+import { createPubSubClient, EntityEvent, EntityType } from '../pubsub'
 import { redisDataSource } from '../redis_data_source'
 import {
   authTrx,
@@ -24,27 +24,29 @@ import {
   queryBuilderToRawSql,
 } from '../repository'
 import { libraryItemRepository } from '../repository/library_item'
-import { Merge } from '../util'
-import { setRecentlySavedItemInRedis } from '../utils/helpers'
+import { Merge, PickTuple } from '../util'
+import { deepDelete, setRecentlySavedItemInRedis } from '../utils/helpers'
 import { logger } from '../utils/logger'
 import { parseSearchQuery } from '../utils/search'
 import { HighlightEvent } from './highlights'
 import { addLabelsToLibraryItem, LabelEvent } from './labels'
 
-type IgnoredFields =
-  | 'user'
-  | 'uploadFile'
-  | 'previewContentType'
-  | 'links'
-  | 'textContentHash'
+const columnToDelete = [
+  'user',
+  'uploadFile',
+  'previewContentType',
+  'links',
+  'textContentHash',
+] as const
+type ColumnToDeleteType = typeof columnToDelete[number]
 type ItemBaseEvent = Merge<
-  Omit<DeepPartial<LibraryItem>, IgnoredFields>,
+  Omit<DeepPartial<LibraryItem>, ColumnToDeleteType>,
   {
     labels?: LabelEvent[]
     highlights?: HighlightEvent[]
   }
 >
-export type ItemEvent = Merge<ItemBaseEvent, BaseEntityEvent>
+export type ItemEvent = Merge<ItemBaseEvent, EntityEvent>
 
 export class RequiresSearchQueryError extends Error {
   constructor() {
@@ -884,22 +886,17 @@ export const updateLibraryItem = async (
   }
 
   if (libraryItem.state === LibraryItemState.Succeeded) {
+    const cleanedData = deepDelete(updatedLibraryItem, columnToDelete)
     // send create event if the item was created
-    await pubsub.entityCreated<ItemEvent>(
-      EntityType.ITEM,
-      { ...updatedLibraryItem, userId },
-      userId,
-      id
-    )
+    await pubsub.entityCreated<ItemEvent>(EntityType.ITEM, cleanedData, userId)
 
     return updatedLibraryItem
   }
 
   await pubsub.entityUpdated<ItemEvent>(
     EntityType.ITEM,
-    { ...libraryItem, id, userId } as ItemEvent,
-    userId,
-    id
+    { ...libraryItem, id } as ItemEvent,
+    userId
   )
 
   return updatedLibraryItem
@@ -958,12 +955,7 @@ export const updateLibraryItemReadingProgress = async (
   }
 
   const updatedItem = result[0][0]
-  await pubsub.entityUpdated<ItemEvent>(
-    EntityType.ITEM,
-    { ...updatedItem, id, userId },
-    userId,
-    id
-  )
+  await pubsub.entityUpdated<ItemEvent>(EntityType.ITEM, updatedItem, userId)
 
   return updatedItem
 }
@@ -1059,12 +1051,8 @@ export const createOrUpdateLibraryItem = async (
     return newLibraryItem
   }
 
-  await pubsub.entityCreated<ItemEvent>(
-    EntityType.ITEM,
-    { ...newLibraryItem, userId },
-    userId,
-    newLibraryItem.id
-  )
+  const cleanedData = deepDelete(newLibraryItem, columnToDelete)
+  await pubsub.entityCreated<ItemEvent>(EntityType.ITEM, cleanedData, userId)
 
   return newLibraryItem
 }
@@ -1346,7 +1334,8 @@ export const filterItemEvents = (
       subscription: /^subscription(s)?$/i,
     }
 
-    const matchingKeyword = Object.keys(keywordRegexMap).find((keyword) =>
+    const keys = Object.keys(keywordRegexMap)
+    const matchingKeyword = keys.find((keyword) =>
       value.match(keywordRegexMap[keyword])
     )
 
@@ -1354,7 +1343,9 @@ export const filterItemEvents = (
       throw new Error(`Unexpected keyword: ${value}`)
     }
 
-    const eventValue = event[matchingKeyword as keyof ItemEvent]
+    const eventValue = (event as PickTuple<ItemEvent, typeof keys>)[
+      matchingKeyword
+    ]
 
     return !eventValue || (Array.isArray(eventValue) && eventValue.length === 0)
   }
@@ -1484,8 +1475,10 @@ export const filterItemEvents = (
 
         const start = startDate ?? new Date(0)
         const end = endDate ?? new Date()
-        const key = `${field.name.toLowerCase()}At` as keyof ItemEvent
-        const eventValue = event[key] as Date
+        const key = `${field.name.toLowerCase()}At`
+        const eventValue = event[
+          key as 'readAt' | 'updatedAt' | 'publishedAt'
+        ] as Date
 
         return eventValue >= start && eventValue <= end
       }
diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts
index d2fec6520..785e5d5ef 100644
--- a/packages/api/src/utils/helpers.ts
+++ b/packages/api/src/utils/helpers.ts
@@ -367,7 +367,10 @@ export const cleanUrl = (url: string) => {
   })
 }
 
-export const deepDelete = <T, K extends keyof T>(obj: T, keys: K[]) => {
+export const deepDelete = <T, K extends keyof T>(
+  obj: T,
+  keys: readonly K[]
+) => {
   // make a copy of the object
   const copy = { ...obj }
 

From 58511049a252d651f53386675cbcc4ff7e683ced Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 10:49:26 +0800
Subject: [PATCH 05/35] rename LABEL_CREATED to LABEL_ATTACHED in webhook data

---
 packages/api/src/jobs/trigger_rule.ts | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts
index 158df8168..68d30e06b 100644
--- a/packages/api/src/jobs/trigger_rule.ts
+++ b/packages/api/src/jobs/trigger_rule.ts
@@ -84,8 +84,14 @@ const sendNotification = async (obj: RuleActionObj) => {
 const sendToWebhook = async (obj: RuleActionObj) => {
   const [url] = obj.action.params
 
+  let event = obj.ruleEventType.toString()
+  // rename LABEL_CREATED to LABEL_ATTACHED
+  if (obj.ruleEventType === RuleEventType.LabelCreated) {
+    event = 'LABEL_ATTACHED'
+  }
+
   const data = {
-    event: obj.ruleEventType,
+    event,
     item: obj.data,
     userId: obj.userId,
     timestamp: Date.now(),

From ad2e8ee002d0f31e325714cc6d54f2c7a71d1162 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 11:47:30 +0800
Subject: [PATCH 06/35] trigger export in rules

---
 packages/api/src/entity/rule.ts               |   1 +
 packages/api/src/jobs/trigger_rule.ts         |  69 +++++++++++
 packages/api/src/pubsub.ts                    |  14 +--
 packages/api/src/services/highlights.ts       |  28 ++++-
 .../src/services/integrations/integration.ts  |   5 +-
 .../api/src/services/integrations/notion.ts   | 114 ++++++++++--------
 .../api/src/services/integrations/readwise.ts |  14 +--
 packages/api/src/services/labels.ts           |  13 +-
 8 files changed, 182 insertions(+), 76 deletions(-)

diff --git a/packages/api/src/entity/rule.ts b/packages/api/src/entity/rule.ts
index 9ef6d797e..6dab77e60 100644
--- a/packages/api/src/entity/rule.ts
+++ b/packages/api/src/entity/rule.ts
@@ -16,6 +16,7 @@ export enum RuleActionType {
   MarkAsRead = 'MARK_AS_READ',
   SendNotification = 'SEND_NOTIFICATION',
   Webhook = 'WEBHOOK',
+  Export = 'EXPORT',
 }
 
 export enum RuleEventType {
diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts
index 68d30e06b..550f9cacf 100644
--- a/packages/api/src/jobs/trigger_rule.ts
+++ b/packages/api/src/jobs/trigger_rule.ts
@@ -1,8 +1,14 @@
 import { LiqeQuery } from '@omnivore/liqe'
 import axios from 'axios'
 import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source'
+import { IntegrationType } from '../entity/integration'
 import { LibraryItem, LibraryItemState } from '../entity/library_item'
 import { Rule, RuleAction, RuleActionType, RuleEventType } from '../entity/rule'
+import {
+  findIntegrations,
+  getIntegrationClient,
+  updateIntegration,
+} from '../services/integrations'
 import { addLabelsToLibraryItem } from '../services/labels'
 import {
   filterItemEvents,
@@ -107,6 +113,67 @@ const sendToWebhook = async (obj: RuleActionObj) => {
   })
 }
 
+const exportItem = async (obj: RuleActionObj) => {
+  const userId = obj.userId
+  const integrations = await findIntegrations(userId, {
+    enabled: true,
+    type: IntegrationType.Export,
+  })
+
+  if (integrations.length <= 0) {
+    return
+  }
+
+  await Promise.all(
+    integrations.map(async (integration) => {
+      try {
+        const logObject = {
+          userId,
+          integrationId: integration.id,
+        }
+        logger.info('exporting item...', logObject)
+
+        const client = getIntegrationClient(
+          integration.name,
+          integration.token,
+          integration
+        )
+
+        const synced = await client.export([obj.data])
+        if (!synced) {
+          logger.error('failed to export item', logObject)
+          return false
+        }
+
+        const syncedAt = new Date()
+        logger.info('updating integration...', {
+          ...logObject,
+          syncedAt,
+        })
+
+        // update integration syncedAt if successful
+        const updated = await updateIntegration(
+          integration.id,
+          {
+            syncedAt,
+          },
+          userId
+        )
+        logger.info('integration updated', {
+          ...logObject,
+          updated,
+        })
+      } catch (error) {
+        logger.error('failed to export item', {
+          userId,
+          integrationId: integration.id,
+          error,
+        })
+      }
+    })
+  )
+}
+
 const getRuleAction = (
   actionType: RuleActionType
 ): RuleActionFunc | undefined => {
@@ -123,6 +190,8 @@ const getRuleAction = (
       return sendNotification
     case RuleActionType.Webhook:
       return sendToWebhook
+    case RuleActionType.Export:
+      return exportItem
     default:
       logger.error('Unknown rule action type', actionType)
       return undefined
diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts
index 5e761fe1b..27babb13b 100644
--- a/packages/api/src/pubsub.ts
+++ b/packages/api/src/pubsub.ts
@@ -4,7 +4,6 @@ import { RuleEventType } from './entity/rule'
 import { env } from './env'
 import { ReportType } from './generated/graphql'
 import {
-  enqueueExportItem,
   enqueueProcessYouTubeVideo,
   enqueueTriggerRuleJob,
 } from './utils/createTask'
@@ -58,11 +57,6 @@ export const createPubSubClient = (): PubsubClient => {
         data,
         userId,
       })
-      // queue export item job
-      await enqueueExportItem({
-        userId,
-        libraryItemIds: [data.id],
-      })
 
       if (type === EntityType.ITEM) {
         // if (await findGrantedFeatureByName(FeatureName.AISummaries, userId)) {
@@ -92,15 +86,9 @@ export const createPubSubClient = (): PubsubClient => {
       // queue trigger rule job
       await enqueueTriggerRuleJob({
         userId,
-        ruleEventType: RuleEventType.PageUpdated,
+        ruleEventType: `${type.toUpperCase()}_UPDATED` as RuleEventType,
         data,
       })
-
-      // queue export item job
-      await enqueueExportItem({
-        userId,
-        libraryItemIds: [data.id],
-      })
     },
     entityDeleted: async (
       type: EntityType,
diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index 43d710d36..73a1bcb35 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -5,16 +5,20 @@ import { EntityLabel } from '../entity/entity_label'
 import { Highlight } from '../entity/highlight'
 import { Label } from '../entity/label'
 import { homePageURL } from '../env'
-import { createPubSubClient, EntityType } from '../pubsub'
+import { createPubSubClient, EntityEvent, EntityType } from '../pubsub'
 import { authTrx } from '../repository'
 import { highlightRepository } from '../repository/highlight'
+import { Merge } from '../util'
 import { enqueueUpdateHighlight } from '../utils/createTask'
 import { deepDelete } from '../utils/helpers'
 import { ItemEvent } from './library_item'
 
 const columnToDelete = ['user', 'sharedAt'] as const
 type ColumnToDeleteType = typeof columnToDelete[number]
-export type HighlightEvent = Omit<DeepPartial<Highlight>, ColumnToDeleteType>
+export type HighlightEvent = Merge<
+  Omit<DeepPartial<Highlight>, ColumnToDeleteType>,
+  EntityEvent
+>
 
 export const getHighlightLocation = (patch: string): number | undefined => {
   const dmp = new diff_match_patch()
@@ -51,6 +55,7 @@ export const createHighlight = async (
         where: { id: newHighlight.id },
         relations: {
           user: true,
+          libraryItem: true,
         },
       })
     },
@@ -61,7 +66,11 @@ export const createHighlight = async (
   const cleanData = deepDelete(newHighlight, columnToDelete)
   await pubsub.entityCreated<ItemEvent>(
     EntityType.HIGHLIGHT,
-    { id: libraryItemId, highlights: [cleanData] },
+    {
+      id: libraryItemId,
+      slug: newHighlight.libraryItem.slug,
+      highlights: [cleanData],
+    },
     userId
   )
 
@@ -102,13 +111,18 @@ export const mergeHighlights = async (
       where: { id: newHighlight.id },
       relations: {
         user: true,
+        libraryItem: true,
       },
     })
   })
 
   await pubsub.entityCreated<ItemEvent>(
     EntityType.HIGHLIGHT,
-    { id: libraryItemId, highlights: [newHighlight] },
+    {
+      id: libraryItemId,
+      slug: newHighlight.libraryItem.slug,
+      highlights: [newHighlight],
+    },
     userId
   )
 
@@ -142,7 +156,11 @@ export const updateHighlight = async (
   const libraryItemId = updatedHighlight.libraryItem.id
   await pubsub.entityUpdated<ItemEvent>(
     EntityType.HIGHLIGHT,
-    { id: libraryItemId, highlights: [highlight] } as ItemEvent,
+    {
+      id: libraryItemId,
+      slug: updatedHighlight.libraryItem.slug,
+      highlights: [highlight],
+    } as ItemEvent,
     userId
   )
 
diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts
index 44c95e680..613cae2c3 100644
--- a/packages/api/src/services/integrations/integration.ts
+++ b/packages/api/src/services/integrations/integration.ts
@@ -1,4 +1,5 @@
-import { LibraryItem, LibraryItemState } from '../../entity/library_item'
+import { LibraryItemState } from '../../entity/library_item'
+import { ItemEvent } from '../library_item'
 
 export interface RetrievedData {
   url: string
@@ -26,5 +27,5 @@ export interface IntegrationClient {
 
   auth(state: string): Promise<string>
 
-  export(items: LibraryItem[]): Promise<boolean>
+  export(items: ItemEvent[]): Promise<boolean>
 }
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index ef9b63319..c274f062a 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -1,11 +1,11 @@
 import { Client } from '@notionhq/client'
 import axios from 'axios'
 import { Integration } from '../../entity/integration'
-import { LibraryItem } from '../../entity/library_item'
 import { env } from '../../env'
 import { Merge } from '../../util'
 import { logger } from '../../utils/logger'
 import { getHighlightUrl } from '../highlights'
+import { ItemEvent } from '../library_item'
 import { IntegrationClient } from './integration'
 
 type AnnotationColor =
@@ -44,7 +44,7 @@ interface NotionPage {
     }
   }
   properties: {
-    Title: {
+    Title?: {
       title: [
         {
           text: {
@@ -53,25 +53,28 @@ interface NotionPage {
         }
       ]
     }
-    Author: {
+    Author?: {
       rich_text: Array<{
         text: {
           content: string
         }
       }>
     }
-    'Original URL': {
+    'Original URL'?: {
       url: string
     }
-    'Omnivore URL': {
+    'Omnivore ID': {
+      unique_id: string
+    }
+    'Omnivore URL'?: {
       url: string
     }
-    'Saved At': {
+    'Saved At'?: {
       date: {
         start: string
       }
     }
-    'Last Updated': {
+    'Last Updated'?: {
       date: {
         start: string
       }
@@ -177,7 +180,7 @@ export class NotionClient implements IntegrationClient {
   }
 
   private itemToNotionPage = (
-    item: LibraryItem,
+    item: ItemEvent,
     settings: Settings,
     lastSync?: Date | null
   ): NotionPage => {
@@ -200,44 +203,59 @@ export class NotionClient implements IntegrationClient {
           }
         : undefined,
       properties: {
-        Title: {
-          title: [
-            {
-              text: {
-                content: item.title,
+        Title: item.title
+          ? {
+              title: [
+                {
+                  text: {
+                    content: item.title,
+                  },
+                },
+              ],
+            }
+          : undefined,
+        Author: item.author
+          ? {
+              rich_text: [
+                {
+                  text: {
+                    content: item.author,
+                  },
+                },
+              ],
+            }
+          : undefined,
+        'Original URL': item.originalUrl
+          ? {
+              url: item.originalUrl,
+            }
+          : undefined,
+        'Omnivore ID': {
+          unique_id: item.id,
+        },
+        'Omnivore URL': item.slug
+          ? {
+              url: `${env.client.url}/me/${item.slug}`,
+            }
+          : undefined,
+        'Saved At': item.savedAt
+          ? {
+              date: {
+                start: (item.savedAt as Date).toISOString(),
               },
-            },
-          ],
-        },
-        Author: {
-          rich_text: [
-            {
-              text: {
-                content: item.author || 'unknown',
+            }
+          : undefined,
+        'Last Updated': item.updatedAt
+          ? {
+              date: {
+                start: (item.updatedAt as Date).toISOString(),
               },
-            },
-          ],
-        },
-        'Original URL': {
-          url: item.originalUrl,
-        },
-        'Omnivore URL': {
-          url: `${env.client.url}/me/${item.slug}`,
-        },
-        'Saved At': {
-          date: {
-            start: item.createdAt.toISOString(),
-          },
-        },
-        'Last Updated': {
-          date: {
-            start: item.updatedAt.toISOString(),
-          },
-        },
+            }
+          : undefined,
         Tags: item.labels
           ? {
               multi_select: item.labels.map((label) => ({
-                name: label.name,
+                name: label.name || '',
               })),
             }
           : undefined,
@@ -246,7 +264,9 @@ export class NotionClient implements IntegrationClient {
         settings.properties?.includes('highlights') && item.highlights
           ? item.highlights
               .filter(
-                (highlight) => !lastSync || highlight.updatedAt > lastSync // only new highlights
+                (highlight) =>
+                  !lastSync ||
+                  (highlight.updatedAt && highlight.updatedAt > lastSync) // only new highlights
               )
               .map((highlight) => ({
                 paragraph: {
@@ -255,7 +275,7 @@ export class NotionClient implements IntegrationClient {
                       text: {
                         content: highlight.quote || '',
                         link: {
-                          url: getHighlightUrl(item.slug, highlight.id),
+                          url: getHighlightUrl(item.slug!, highlight.id),
                         },
                       },
                       annotations: {
@@ -289,14 +309,14 @@ export class NotionClient implements IntegrationClient {
     await this.client.pages.create(page)
   }
 
-  private findPage = async (url: string, databaseId: string) => {
+  private findPage = async (id: string, databaseId: string) => {
     const response = await this.client.databases.query({
       database_id: databaseId,
       page_size: 1,
       filter: {
-        property: 'Omnivore URL',
+        property: 'Omnivore ID',
         url: {
-          equals: url,
+          equals: id,
         },
       },
     })
@@ -307,7 +327,7 @@ export class NotionClient implements IntegrationClient {
     return null
   }
 
-  export = async (items: LibraryItem[]): Promise<boolean> => {
+  export = async (items: ItemEvent[]): Promise<boolean> => {
     const settings = this.integrationData?.settings
     if (!this.integrationData || !settings) {
       logger.error('Notion integration data not found')
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index 0c204b630..b70ef4260 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -1,7 +1,7 @@
 import axios from 'axios'
-import { LibraryItem } from '../../entity/library_item'
 import { logger } from '../../utils/logger'
 import { getHighlightUrl } from '../highlights'
+import { ItemEvent } from '../library_item'
 import { IntegrationClient } from './integration'
 
 interface ReadwiseHighlight {
@@ -66,7 +66,7 @@ export class ReadwiseClient implements IntegrationClient {
     }
   }
 
-  export = async (items: LibraryItem[]): Promise<boolean> => {
+  export = async (items: ItemEvent[]): Promise<boolean> => {
     let result = true
 
     const highlights = items.flatMap(this._itemToReadwiseHighlight)
@@ -83,9 +83,7 @@ export class ReadwiseClient implements IntegrationClient {
     throw new Error('Method not implemented.')
   }
 
-  private _itemToReadwiseHighlight = (
-    item: LibraryItem
-  ): ReadwiseHighlight[] => {
+  private _itemToReadwiseHighlight = (item: ItemEvent): ReadwiseHighlight[] => {
     const category = item.siteName === 'Twitter' ? 'tweets' : 'articles'
     return item.highlights
       ?.map((highlight) => {
@@ -98,8 +96,10 @@ export class ReadwiseClient implements IntegrationClient {
           text: highlight.quote,
           title: item.title,
           author: item.author || undefined,
-          highlight_url: getHighlightUrl(item.slug, highlight.id),
-          highlighted_at: new Date(highlight.createdAt).toISOString(),
+          highlight_url: item.slug
+            ? getHighlightUrl(item.slug, highlight.id)
+            : undefined,
+          highlighted_at: (highlight.createdAt as Date).toISOString(),
           category,
           image_url: item.thumbnail || undefined,
           location_type: 'order',
diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts
index 84ebc7c23..c2f5c110e 100644
--- a/packages/api/src/services/labels.ts
+++ b/packages/api/src/services/labels.ts
@@ -2,9 +2,15 @@ import { DeepPartial, FindOptionsWhere, In } from 'typeorm'
 import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'
 import { EntityLabel, LabelSource } from '../entity/entity_label'
 import { Label } from '../entity/label'
-import { createPubSubClient, EntityType, PubsubClient } from '../pubsub'
+import {
+  createPubSubClient,
+  EntityEvent,
+  EntityType,
+  PubsubClient,
+} from '../pubsub'
 import { authTrx } from '../repository'
 import { CreateLabelInput, labelRepository } from '../repository/label'
+import { Merge } from '../util'
 import { bulkEnqueueUpdateLabels } from '../utils/createTask'
 import { deepDelete } from '../utils/helpers'
 import { logger } from '../utils/logger'
@@ -13,7 +19,10 @@ import { findLibraryItemIdsByLabelId, ItemEvent } from './library_item'
 
 const columnToDelete = ['description', 'createdAt'] as const
 type ColumnToDeleteType = typeof columnToDelete[number]
-export type LabelEvent = Omit<DeepPartial<Label>, ColumnToDeleteType>
+export type LabelEvent = Merge<
+  Omit<DeepPartial<Label>, ColumnToDeleteType>,
+  EntityEvent
+>
 
 // const batchGetLabelsFromLinkIds = async (
 //   linkIds: readonly string[]

From f35ac036ebceb81b492ad22717391bacd89ace82 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 12:17:26 +0800
Subject: [PATCH 07/35] add UI for export in rules

---
 packages/api/src/generated/graphql.ts         |  1 +
 packages/api/src/generated/schema.graphql     |  1 +
 packages/api/src/jobs/trigger_rule.ts         |  3 +++
 packages/api/src/schema.ts                    |  1 +
 .../api/src/services/integrations/notion.ts   |  9 +++++---
 .../networking/queries/useGetRulesQuery.tsx   |  1 +
 packages/web/pages/settings/rules.tsx         | 21 +++++++++++++++++++
 7 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index 3e3119db6..e5f8bb85a 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -2501,6 +2501,7 @@ export enum RuleActionType {
   AddLabel = 'ADD_LABEL',
   Archive = 'ARCHIVE',
   Delete = 'DELETE',
+  Export = 'EXPORT',
   MarkAsRead = 'MARK_AS_READ',
   SendNotification = 'SEND_NOTIFICATION',
   Webhook = 'WEBHOOK'
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 1782328b8..26684a5c0 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1877,6 +1877,7 @@ enum RuleActionType {
   ADD_LABEL
   ARCHIVE
   DELETE
+  EXPORT
   MARK_AS_READ
   SEND_NOTIFICATION
   WEBHOOK
diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts
index 550f9cacf..e7f1040cb 100644
--- a/packages/api/src/jobs/trigger_rule.ts
+++ b/packages/api/src/jobs/trigger_rule.ts
@@ -1,5 +1,6 @@
 import { LiqeQuery } from '@omnivore/liqe'
 import axios from 'axios'
+import { Any } from 'typeorm'
 import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source'
 import { IntegrationType } from '../entity/integration'
 import { LibraryItem, LibraryItemState } from '../entity/library_item'
@@ -115,7 +116,9 @@ const sendToWebhook = async (obj: RuleActionObj) => {
 
 const exportItem = async (obj: RuleActionObj) => {
   const userId = obj.userId
+  const integrationNames = obj.action.params
   const integrations = await findIntegrations(userId, {
+    name: Any(integrationNames.map((param) => param.toUpperCase())),
     enabled: true,
     type: IntegrationType.Export,
   })
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index d2279407e..c510e5a5c 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -2168,6 +2168,7 @@ const schema = gql`
     MARK_AS_READ
     SEND_NOTIFICATION
     WEBHOOK
+    EXPORT
   }
 
   type RulesError {
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index c274f062a..fb1667c56 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -64,7 +64,7 @@ interface NotionPage {
       url: string
     }
     'Omnivore ID': {
-      unique_id: string
+      url: string
     }
     'Omnivore URL'?: {
       url: string
@@ -231,7 +231,7 @@ export class NotionClient implements IntegrationClient {
             }
           : undefined,
         'Omnivore ID': {
-          unique_id: item.id,
+          url: item.id,
         },
         'Omnivore URL': item.slug
           ? {
@@ -275,7 +275,10 @@ export class NotionClient implements IntegrationClient {
                       text: {
                         content: highlight.quote || '',
                         link: {
-                          url: getHighlightUrl(item.slug!, highlight.id),
+                          url: getHighlightUrl(
+                            item.slug || item.id,
+                            highlight.id
+                          ),
                         },
                       },
                       annotations: {
diff --git a/packages/web/lib/networking/queries/useGetRulesQuery.tsx b/packages/web/lib/networking/queries/useGetRulesQuery.tsx
index 77b11154d..08dfd35ad 100644
--- a/packages/web/lib/networking/queries/useGetRulesQuery.tsx
+++ b/packages/web/lib/networking/queries/useGetRulesQuery.tsx
@@ -14,6 +14,7 @@ export enum RuleActionType {
   Delete = 'DELETE',
   SendNotification = 'SEND_NOTIFICATION',
   Webhook = 'WEBHOOK',
+  Export = 'EXPORT',
 }
 
 export enum RuleEventType {
diff --git a/packages/web/pages/settings/rules.tsx b/packages/web/pages/settings/rules.tsx
index 7d5be493c..9144114b6 100644
--- a/packages/web/pages/settings/rules.tsx
+++ b/packages/web/pages/settings/rules.tsx
@@ -132,6 +132,8 @@ const CreateActionModal = (props: CreateActionModalProps): JSX.Element => {
       params = form.getFieldValue('labels')
     } else if (actionType == RuleActionType.Webhook) {
       params = [form.getFieldValue('url')]
+    } else if (actionType == RuleActionType.Export) {
+      params = form.getFieldValue('integrations')
     }
 
     if (props.rule) {
@@ -229,6 +231,25 @@ const CreateActionModal = (props: CreateActionModalProps): JSX.Element => {
             <Input />
           </Form.Item>
         )}
+
+        {actionType == RuleActionType.Export && (
+          <Form.Item
+            label="Integrations"
+            name="integrations"
+            rules={[
+              { required: true, message: 'Please choose at least one integration' },
+            ]}
+          >
+            <Select mode="multiple">
+              <Select.Option key="notion" value="notion">
+                Notion
+              </Select.Option>
+              <Select.Option key="readwise" value="readwise">
+                Readwise
+              </Select.Option>
+            </Select>
+          </Form.Item>
+        )}
       </Form>
     </Modal>
   )

From ff78a4b2660836dc6a09a7eadd6d33eb22965245 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 15:55:45 +0800
Subject: [PATCH 08/35] fix readwise syncing error

---
 packages/api/src/jobs/trigger_rule.ts         | 16 +++---
 .../api/src/services/integrations/readwise.ts | 53 ++++++++++---------
 packages/web/pages/settings/rules.tsx         |  2 +-
 3 files changed, 37 insertions(+), 34 deletions(-)

diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts
index e7f1040cb..1ff163980 100644
--- a/packages/api/src/jobs/trigger_rule.ts
+++ b/packages/api/src/jobs/trigger_rule.ts
@@ -129,13 +129,14 @@ const exportItem = async (obj: RuleActionObj) => {
 
   await Promise.all(
     integrations.map(async (integration) => {
-      try {
-        const logObject = {
-          userId,
-          integrationId: integration.id,
-        }
-        logger.info('exporting item...', logObject)
+      const logObject = {
+        userId,
+        integrationId: integration.id,
+        name: integration.name,
+      }
+      logger.info('exporting item...', logObject)
 
+      try {
         const client = getIntegrationClient(
           integration.name,
           integration.token,
@@ -168,8 +169,7 @@ const exportItem = async (obj: RuleActionObj) => {
         })
       } catch (error) {
         logger.error('failed to export item', {
-          userId,
-          integrationId: integration.id,
+          ...logObject,
           error,
         })
       }
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index b70ef4260..b0e0e3459 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -1,6 +1,7 @@
 import axios from 'axios'
+import { HighlightType } from '../../entity/highlight'
 import { logger } from '../../utils/logger'
-import { getHighlightUrl } from '../highlights'
+import { getHighlightUrl, HighlightEvent } from '../highlights'
 import { ItemEvent } from '../library_item'
 import { IntegrationClient } from './integration'
 
@@ -84,31 +85,33 @@ export class ReadwiseClient implements IntegrationClient {
   }
 
   private _itemToReadwiseHighlight = (item: ItemEvent): ReadwiseHighlight[] => {
-    const category = item.siteName === 'Twitter' ? 'tweets' : 'articles'
-    return item.highlights
-      ?.map((highlight) => {
-        // filter out highlights that are not of type highlight or have no quote
-        if (highlight.highlightType !== 'HIGHLIGHT' || !highlight.quote) {
-          return undefined
-        }
+    const isHighlight = (
+      highlight: HighlightEvent
+    ): highlight is HighlightEvent & { quote: string } =>
+      highlight.highlightType === HighlightType.Highlight && !!highlight.quote
 
-        return {
-          text: highlight.quote,
-          title: item.title,
-          author: item.author || undefined,
-          highlight_url: item.slug
-            ? getHighlightUrl(item.slug, highlight.id)
-            : undefined,
-          highlighted_at: (highlight.createdAt as Date).toISOString(),
-          category,
-          image_url: item.thumbnail || undefined,
-          location_type: 'order',
-          note: highlight.annotation || undefined,
-          source_type: 'omnivore',
-          source_url: item.originalUrl,
-        }
-      })
-      .filter((highlight) => highlight !== undefined) as ReadwiseHighlight[]
+    const category = item.siteName === 'Twitter' ? 'tweets' : 'articles'
+
+    return item.highlights
+      ? item.highlights
+          // filter out highlights that are not of type highlight or have no quote
+          .filter(isHighlight)
+          .map((highlight) => ({
+            text: highlight.quote,
+            title: item.title,
+            author: item.author || undefined,
+            highlight_url: item.slug
+              ? getHighlightUrl(item.slug, highlight.id)
+              : undefined,
+            highlighted_at: (highlight.createdAt as Date).toISOString(),
+            category,
+            image_url: item.thumbnail || undefined,
+            location_type: 'order',
+            note: highlight.annotation || undefined,
+            source_type: 'omnivore',
+            source_url: item.originalUrl,
+          }))
+      : []
   }
 
   private _syncWithReadwise = async (
diff --git a/packages/web/pages/settings/rules.tsx b/packages/web/pages/settings/rules.tsx
index 9144114b6..db9c2fd69 100644
--- a/packages/web/pages/settings/rules.tsx
+++ b/packages/web/pages/settings/rules.tsx
@@ -104,7 +104,7 @@ const CreateRuleModal = (props: CreateRuleModalProps): JSX.Element => {
               const value = Object.values(RuleEventType)[index]
               return (
                 <Select.Option key={key} value={value}>
-                  {key}
+                  {key === 'LABEL_CREATED' ? 'LABEL_ATTACHED' : key}
                 </Select.Option>
               )
             })}

From 6954fffd240ff04498fe31c2101b4a582f71a2b1 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 16:28:14 +0800
Subject: [PATCH 09/35] fix readwise syncing error

---
 packages/api/src/services/highlights.ts       |  2 +-
 .../api/src/services/integrations/notion.ts   |  5 +++
 .../api/src/services/integrations/readwise.ts | 32 ++++++++++---------
 3 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index 73a1bcb35..469949bb1 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -13,7 +13,7 @@ import { enqueueUpdateHighlight } from '../utils/createTask'
 import { deepDelete } from '../utils/helpers'
 import { ItemEvent } from './library_item'
 
-const columnToDelete = ['user', 'sharedAt'] as const
+const columnToDelete = ['user', 'sharedAt', 'libraryItem'] as const
 type ColumnToDeleteType = typeof columnToDelete[number]
 export type HighlightEvent = Merge<
   Omit<DeepPartial<Highlight>, ColumnToDeleteType>,
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index fb1667c56..b68550e73 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -184,6 +184,11 @@ export class NotionClient implements IntegrationClient {
     settings: Settings,
     lastSync?: Date | null
   ): NotionPage => {
+    // logger.info('itemToNotionPage', {
+    //   highlights: item.highlights,
+    //   settings,
+    //   lastSync,
+    // })
     return {
       parent: {
         database_id: settings.parentDatabaseId,
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index b0e0e3459..dc17f3e87 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -96,21 +96,23 @@ export class ReadwiseClient implements IntegrationClient {
       ? item.highlights
           // filter out highlights that are not of type highlight or have no quote
           .filter(isHighlight)
-          .map((highlight) => ({
-            text: highlight.quote,
-            title: item.title,
-            author: item.author || undefined,
-            highlight_url: item.slug
-              ? getHighlightUrl(item.slug, highlight.id)
-              : undefined,
-            highlighted_at: (highlight.createdAt as Date).toISOString(),
-            category,
-            image_url: item.thumbnail || undefined,
-            location_type: 'order',
-            note: highlight.annotation || undefined,
-            source_type: 'omnivore',
-            source_url: item.originalUrl,
-          }))
+          .map((highlight) => {
+            return {
+              text: highlight.quote,
+              title: item.title,
+              author: item.author || undefined,
+              highlight_url: item.slug
+                ? getHighlightUrl(item.slug, highlight.id)
+                : undefined,
+              highlighted_at: highlight.createdAt as string | undefined,
+              category,
+              image_url: item.thumbnail || undefined,
+              location_type: 'order',
+              note: highlight.annotation || undefined,
+              source_type: 'omnivore',
+              source_url: item.originalUrl,
+            }
+          })
       : []
   }
 

From 0428da3b714394a34103ecde79d1b929221d8204 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 16:39:00 +0800
Subject: [PATCH 10/35] fix notion syncing error

---
 packages/api/src/services/integrations/notion.ts | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index b68550e73..0358d6732 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -184,11 +184,6 @@ export class NotionClient implements IntegrationClient {
     settings: Settings,
     lastSync?: Date | null
   ): NotionPage => {
-    // logger.info('itemToNotionPage', {
-    //   highlights: item.highlights,
-    //   settings,
-    //   lastSync,
-    // })
     return {
       parent: {
         database_id: settings.parentDatabaseId,
@@ -246,21 +241,21 @@ export class NotionClient implements IntegrationClient {
         'Saved At': item.savedAt
           ? {
               date: {
-                start: (item.savedAt as Date).toISOString(),
+                start: item.savedAt as string,
               },
             }
           : undefined,
         'Last Updated': item.updatedAt
           ? {
               date: {
-                start: (item.updatedAt as Date).toISOString(),
+                start: item.updatedAt as string,
               },
             }
           : undefined,
         Tags: item.labels
           ? {
               multi_select: item.labels.map((label) => ({
-                name: label.name || '',
+                name: label.name as string,
               })),
             }
           : undefined,

From d67aea6d8a5376e18f788331c721ccda73f0d675 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 17:16:33 +0800
Subject: [PATCH 11/35] deduplicate notion page by omnivore id

---
 packages/api/src/pubsub.ts                    |  8 ++--
 packages/api/src/services/highlights.ts       | 22 +++++----
 .../api/src/services/integrations/notion.ts   | 20 +++-----
 .../api/src/services/integrations/readwise.ts |  6 +--
 packages/api/src/services/labels.ts           | 46 +++++++++----------
 packages/api/src/services/library_item.ts     |  5 +-
 6 files changed, 51 insertions(+), 56 deletions(-)

diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts
index 27babb13b..4cb0116d4 100644
--- a/packages/api/src/pubsub.ts
+++ b/packages/api/src/pubsub.ts
@@ -115,10 +115,10 @@ export const createPubSubClient = (): PubsubClient => {
 }
 
 export enum EntityType {
-  ITEM = 'page',
-  HIGHLIGHT = 'highlight',
-  LABEL = 'label',
-  RSS_FEED = 'feed',
+  ITEM = 'PAGE',
+  HIGHLIGHT = 'HIGHLIGHT',
+  LABEL = 'LABEL',
+  RSS_FEED = 'FEED',
 }
 
 export interface PubsubClient {
diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index 469949bb1..57bb417b1 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -69,6 +69,8 @@ export const createHighlight = async (
     {
       id: libraryItemId,
       slug: newHighlight.libraryItem.slug,
+      originalUrl: newHighlight.libraryItem.originalUrl,
+      thumbnail: newHighlight.libraryItem.thumbnail,
       highlights: [cleanData],
     },
     userId
@@ -121,6 +123,8 @@ export const mergeHighlights = async (
     {
       id: libraryItemId,
       slug: newHighlight.libraryItem.slug,
+      originalUrl: newHighlight.libraryItem.originalUrl,
+      thumbnail: newHighlight.libraryItem.thumbnail,
       highlights: [newHighlight],
     },
     userId
@@ -154,15 +158,15 @@ export const updateHighlight = async (
   })
 
   const libraryItemId = updatedHighlight.libraryItem.id
-  await pubsub.entityUpdated<ItemEvent>(
-    EntityType.HIGHLIGHT,
-    {
-      id: libraryItemId,
-      slug: updatedHighlight.libraryItem.slug,
-      highlights: [highlight],
-    } as ItemEvent,
-    userId
-  )
+  // await pubsub.entityUpdated<ItemEvent>(
+  //   EntityType.HIGHLIGHT,
+  //   {
+  //     id: libraryItemId,
+  //     slug: updatedHighlight.libraryItem.slug,
+  //     highlights: [highlight],
+  //   } as ItemEvent,
+  //   userId
+  // )
 
   await enqueueUpdateHighlight({
     libraryItemId,
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 0358d6732..5b15639b7 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -63,10 +63,7 @@ interface NotionPage {
     'Original URL'?: {
       url: string
     }
-    'Omnivore ID': {
-      url: string
-    }
-    'Omnivore URL'?: {
+    'Omnivore URL': {
       url: string
     }
     'Saved At'?: {
@@ -230,14 +227,9 @@ export class NotionClient implements IntegrationClient {
               url: item.originalUrl,
             }
           : undefined,
-        'Omnivore ID': {
-          url: item.id,
+        'Omnivore URL': {
+          url: `${env.client.url}/me/${item.id}`,
         },
-        'Omnivore URL': item.slug
-          ? {
-              url: `${env.client.url}/me/${item.slug}`,
-            }
-          : undefined,
         'Saved At': item.savedAt
           ? {
               date: {
@@ -312,14 +304,14 @@ export class NotionClient implements IntegrationClient {
     await this.client.pages.create(page)
   }
 
-  private findPage = async (id: string, databaseId: string) => {
+  private findPage = async (url: string, databaseId: string) => {
     const response = await this.client.databases.query({
       database_id: databaseId,
       page_size: 1,
       filter: {
-        property: 'Omnivore ID',
+        property: 'Omnivore URL',
         url: {
-          equals: id,
+          equals: url,
         },
       },
     })
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index dc17f3e87..001820837 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -101,16 +101,14 @@ export class ReadwiseClient implements IntegrationClient {
               text: highlight.quote,
               title: item.title,
               author: item.author || undefined,
-              highlight_url: item.slug
-                ? getHighlightUrl(item.slug, highlight.id)
-                : undefined,
+              highlight_url: getHighlightUrl(item.slug as string, highlight.id),
               highlighted_at: highlight.createdAt as string | undefined,
               category,
               image_url: item.thumbnail || undefined,
               location_type: 'order',
               note: highlight.annotation || undefined,
               source_type: 'omnivore',
-              source_url: item.originalUrl,
+              source_url: item.originalUrl as string,
             }
           })
       : []
diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts
index c2f5c110e..f4987a4b8 100644
--- a/packages/api/src/services/labels.ts
+++ b/packages/api/src/services/labels.ts
@@ -13,8 +13,6 @@ import { CreateLabelInput, labelRepository } from '../repository/label'
 import { Merge } from '../util'
 import { bulkEnqueueUpdateLabels } from '../utils/createTask'
 import { deepDelete } from '../utils/helpers'
-import { logger } from '../utils/logger'
-import { findHighlightById } from './highlights'
 import { findLibraryItemIdsByLabelId, ItemEvent } from './library_item'
 
 const columnToDelete = ['description', 'createdAt'] as const
@@ -213,30 +211,30 @@ export const saveLabelsInHighlight = async (
     )
   })
 
-  const highlight = await findHighlightById(highlightId, userId)
-  if (!highlight) {
-    logger.error('Highlight not found', { highlightId, userId })
-    return
-  }
+  // const highlight = await findHighlightById(highlightId, userId)
+  // if (!highlight) {
+  //   logger.error('Highlight not found', { highlightId, userId })
+  //   return
+  // }
 
-  const libraryItemId = highlight.libraryItemId
-  // create pubsub event
-  await pubsub.entityCreated<ItemEvent>(
-    EntityType.LABEL,
-    {
-      id: libraryItemId,
-      highlights: [
-        {
-          id: highlightId,
-          labels: labels.map((l) => deepDelete(l, columnToDelete)),
-        },
-      ],
-    },
-    userId
-  )
+  // const libraryItemId = highlight.libraryItemId
+  // // create pubsub event
+  // await pubsub.entityCreated<ItemEvent>(
+  //   EntityType.LABEL,
+  //   {
+  //     id: libraryItemId,
+  //     highlights: [
+  //       {
+  //         id: highlightId,
+  //         labels: labels.map((l) => deepDelete(l, columnToDelete)),
+  //       },
+  //     ],
+  //   },
+  //   userId
+  // )
 
-  // update labels in library item
-  await bulkEnqueueUpdateLabels([{ libraryItemId, userId }])
+  // // update labels in library item
+  // await bulkEnqueueUpdateLabels([{ libraryItemId, userId }])
 }
 
 export const findLabelsByIds = async (
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index 4cc0806dd..4169ddc76 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -895,7 +895,10 @@ export const updateLibraryItem = async (
 
   await pubsub.entityUpdated<ItemEvent>(
     EntityType.ITEM,
-    { ...libraryItem, id } as ItemEvent,
+    {
+      ...libraryItem,
+      id,
+    } as ItemEvent,
     userId
   )
 

From 1b24a24f7c3eb8b0478b2d1fc30bd0c191601f1a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 17:20:37 +0800
Subject: [PATCH 12/35] remove slug from highlight event

---
 packages/api/src/services/highlights.ts            | 6 ++----
 packages/api/src/services/integrations/readwise.ts | 4 ++--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index 57bb417b1..92fedc303 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -68,9 +68,8 @@ export const createHighlight = async (
     EntityType.HIGHLIGHT,
     {
       id: libraryItemId,
-      slug: newHighlight.libraryItem.slug,
-      originalUrl: newHighlight.libraryItem.originalUrl,
-      thumbnail: newHighlight.libraryItem.thumbnail,
+      originalUrl: newHighlight.libraryItem.originalUrl, // for Readwise
+      thumbnail: newHighlight.libraryItem.thumbnail, // for Readwise
       highlights: [cleanData],
     },
     userId
@@ -122,7 +121,6 @@ export const mergeHighlights = async (
     EntityType.HIGHLIGHT,
     {
       id: libraryItemId,
-      slug: newHighlight.libraryItem.slug,
       originalUrl: newHighlight.libraryItem.originalUrl,
       thumbnail: newHighlight.libraryItem.thumbnail,
       highlights: [newHighlight],
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index 001820837..129144ea1 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -101,14 +101,14 @@ export class ReadwiseClient implements IntegrationClient {
               text: highlight.quote,
               title: item.title,
               author: item.author || undefined,
-              highlight_url: getHighlightUrl(item.slug as string, highlight.id),
+              highlight_url: getHighlightUrl(item.id, highlight.id),
               highlighted_at: highlight.createdAt as string | undefined,
               category,
               image_url: item.thumbnail || undefined,
               location_type: 'order',
               note: highlight.annotation || undefined,
               source_type: 'omnivore',
-              source_url: item.originalUrl as string,
+              source_url: item.originalUrl,
             }
           })
       : []

From 841a0026ff51f21361265d47d11e194e2fd08593 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 17:25:06 +0800
Subject: [PATCH 13/35] do not send page_update event for thumbnailer

---
 packages/api/src/jobs/find_thumbnail.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/api/src/jobs/find_thumbnail.ts b/packages/api/src/jobs/find_thumbnail.ts
index cf6caa0ee..63bae9b42 100644
--- a/packages/api/src/jobs/find_thumbnail.ts
+++ b/packages/api/src/jobs/find_thumbnail.ts
@@ -161,7 +161,9 @@ export const findThumbnail = async (data: Data) => {
       {
         thumbnail,
       },
-      userId
+      userId,
+      undefined,
+      true
     )
     logger.info(`thumbnail updated: ${thumbnail}`)
   }

From c2ab6c627e64a6b8778c14d246b4167fedb36b5c Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 22:54:19 +0800
Subject: [PATCH 14/35] show action param

---
 packages/web/pages/settings/rules.tsx | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/web/pages/settings/rules.tsx b/packages/web/pages/settings/rules.tsx
index db9c2fd69..97fa56cf5 100644
--- a/packages/web/pages/settings/rules.tsx
+++ b/packages/web/pages/settings/rules.tsx
@@ -241,10 +241,10 @@ const CreateActionModal = (props: CreateActionModalProps): JSX.Element => {
             ]}
           >
             <Select mode="multiple">
-              <Select.Option key="notion" value="notion">
+              <Select.Option key="Notion" value="NOTION">
                 Notion
               </Select.Option>
-              <Select.Option key="readwise" value="readwise">
+              <Select.Option key="Readwise" value="READWISE">
                 Readwise
               </Select.Option>
             </Select>
@@ -302,7 +302,7 @@ export default function Rules(): JSX.Element {
           })?.name ?? 'unknown'
         )
       }
-      return ''
+      return param
     },
     [labels]
   )

From a344e5cd6ae14e116f72c22821197be3a1a277fa Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 22 Mar 2024 23:12:26 +0800
Subject: [PATCH 15/35] drop unique constraint on filter column in rules table

---
 .../db/migrations/0170.do.drop_unique_key_on_rules.sql   | 9 +++++++++
 .../db/migrations/0170.undo.drop_unique_key_on_rules.sql | 9 +++++++++
 2 files changed, 18 insertions(+)
 create mode 100755 packages/db/migrations/0170.do.drop_unique_key_on_rules.sql
 create mode 100755 packages/db/migrations/0170.undo.drop_unique_key_on_rules.sql

diff --git a/packages/db/migrations/0170.do.drop_unique_key_on_rules.sql b/packages/db/migrations/0170.do.drop_unique_key_on_rules.sql
new file mode 100755
index 000000000..d79c6f6ba
--- /dev/null
+++ b/packages/db/migrations/0170.do.drop_unique_key_on_rules.sql
@@ -0,0 +1,9 @@
+-- Type: DO
+-- Name: drop_unique_key_on_rules
+-- Description: Drop unique constraint on rules table
+
+BEGIN;
+
+ALTER TABLE omnivore.rules DROP CONSTRAINT rules_user_id_filter_key;
+
+COMMIT;
diff --git a/packages/db/migrations/0170.undo.drop_unique_key_on_rules.sql b/packages/db/migrations/0170.undo.drop_unique_key_on_rules.sql
new file mode 100755
index 000000000..614ad017d
--- /dev/null
+++ b/packages/db/migrations/0170.undo.drop_unique_key_on_rules.sql
@@ -0,0 +1,9 @@
+-- Type: UNDO
+-- Name: drop_unique_key_on_rules
+-- Description: Drop unique constraint on rules table
+
+BEGIN;
+
+ALTER TABLE omnivore.rules ADD CONSTRAINT rules_user_id_filter_key UNIQUE (user_id, filter);
+
+COMMIT;

From 81f21141fed45dcf15b5e21d5f2fc2405099c4b4 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sat, 23 Mar 2024 13:56:56 +0800
Subject: [PATCH 16/35] optimize ts-node-dev

---
 packages/api/package.json                     | 10 ++---
 packages/api/src/entity/rule.ts               |  1 +
 packages/api/src/generated/graphql.ts         |  1 +
 packages/api/src/generated/schema.graphql     |  1 +
 packages/api/src/schema.ts                    |  1 +
 packages/api/src/services/highlights.ts       | 26 ++++++-----
 .../api/src/services/integrations/notion.ts   |  2 +-
 .../api/src/services/integrations/readwise.ts |  4 +-
 yarn.lock                                     | 44 ++-----------------
 9 files changed, 31 insertions(+), 59 deletions(-)

diff --git a/packages/api/package.json b/packages/api/package.json
index 3ca6cc60a..95aa34d84 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -3,16 +3,15 @@
   "version": "1.0.0",
   "license": "UNLICENSED",
   "scripts": {
-    "build": "tsc && yarn copy-files",
-    "dev": "ts-node-dev --files src/server.ts",
-    "dev_qp": "ts-node-dev --files src/queue-processor.ts",
+    "build": "tsc",
+    "dev": "ts-node-dev --debounce 100 --respawn --transpile-only src/server.ts",
+    "dev_qp": "ts-node-dev --debounce 100 --respawn --transpile-only src/queue-processor.ts",
     "start": "node dist/server.js",
     "start_queue_processor": "node dist/queue-processor.js",
     "lint": "eslint src --ext ts,js,tsx,jsx",
     "lint:fix": "eslint src --fix --ext ts,js,tsx,jsx",
     "test:typecheck": "tsc --noEmit",
-    "test": "nyc mocha -r ts-node/register --config mocha-config.json --timeout 10000",
-    "copy-files": "copyfiles -u 1 src/**/*.html dist/"
+    "test": "nyc mocha -r ts-node/register --config mocha-config.json --timeout 10000"
   },
   "dependencies": {
     "@bmatei/apollo-prometheus-exporter": "^3.0.0",
@@ -154,7 +153,6 @@
     "chai-as-promised": "^7.1.1",
     "chai-string": "^1.5.0",
     "circular-dependency-plugin": "^5.2.0",
-    "copyfiles": "^2.4.1",
     "mocha": "^9.0.1",
     "mocha-unfunk-reporter": "^0.4.0",
     "nock": "^13.2.4",
diff --git a/packages/api/src/entity/rule.ts b/packages/api/src/entity/rule.ts
index 6dab77e60..8e87a5cd9 100644
--- a/packages/api/src/entity/rule.ts
+++ b/packages/api/src/entity/rule.ts
@@ -24,6 +24,7 @@ export enum RuleEventType {
   PageUpdated = 'PAGE_UPDATED',
   LabelCreated = 'LABEL_CREATED',
   HighlightCreated = 'HIGHLIGHT_CREATED',
+  HighlightUpdated = 'HIGHLIGHT_UPDATED',
 }
 
 export interface RuleAction {
diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index e5f8bb85a..93afb8908 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -2509,6 +2509,7 @@ export enum RuleActionType {
 
 export enum RuleEventType {
   HighlightCreated = 'HIGHLIGHT_CREATED',
+  HighlightUpdated = 'HIGHLIGHT_UPDATED',
   LabelCreated = 'LABEL_CREATED',
   PageCreated = 'PAGE_CREATED',
   PageUpdated = 'PAGE_UPDATED'
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 26684a5c0..ba442c3a7 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1885,6 +1885,7 @@ enum RuleActionType {
 
 enum RuleEventType {
   HIGHLIGHT_CREATED
+  HIGHLIGHT_UPDATED
   LABEL_CREATED
   PAGE_CREATED
   PAGE_UPDATED
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index c510e5a5c..6b0c3cb01 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -2185,6 +2185,7 @@ const schema = gql`
     PAGE_UPDATED
     LABEL_CREATED
     HIGHLIGHT_CREATED
+    HIGHLIGHT_UPDATED
   }
 
   input SetRuleInput {
diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index 92fedc303..a127d5dfb 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -68,9 +68,12 @@ export const createHighlight = async (
     EntityType.HIGHLIGHT,
     {
       id: libraryItemId,
-      originalUrl: newHighlight.libraryItem.originalUrl, // for Readwise
-      thumbnail: newHighlight.libraryItem.thumbnail, // for Readwise
       highlights: [cleanData],
+      // for Readwise
+      originalUrl: newHighlight.libraryItem.originalUrl,
+      title: newHighlight.libraryItem.title,
+      author: newHighlight.libraryItem.author,
+      thumbnail: newHighlight.libraryItem.thumbnail,
     },
     userId
   )
@@ -122,6 +125,8 @@ export const mergeHighlights = async (
     {
       id: libraryItemId,
       originalUrl: newHighlight.libraryItem.originalUrl,
+      title: newHighlight.libraryItem.title,
+      author: newHighlight.libraryItem.author,
       thumbnail: newHighlight.libraryItem.thumbnail,
       highlights: [newHighlight],
     },
@@ -156,15 +161,14 @@ export const updateHighlight = async (
   })
 
   const libraryItemId = updatedHighlight.libraryItem.id
-  // await pubsub.entityUpdated<ItemEvent>(
-  //   EntityType.HIGHLIGHT,
-  //   {
-  //     id: libraryItemId,
-  //     slug: updatedHighlight.libraryItem.slug,
-  //     highlights: [highlight],
-  //   } as ItemEvent,
-  //   userId
-  // )
+  await pubsub.entityUpdated<ItemEvent>(
+    EntityType.HIGHLIGHT,
+    {
+      id: libraryItemId,
+      highlights: [highlight],
+    } as ItemEvent,
+    userId
+  )
 
   await enqueueUpdateHighlight({
     libraryItemId,
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 5b15639b7..d95bc893b 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -258,7 +258,7 @@ export class NotionClient implements IntegrationClient {
               .filter(
                 (highlight) =>
                   !lastSync ||
-                  (highlight.updatedAt && highlight.updatedAt > lastSync) // only new highlights
+                  new Date(highlight.updatedAt as string) > lastSync // only new highlights
               )
               .map((highlight) => ({
                 paragraph: {
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index 129144ea1..6a6209abc 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -102,7 +102,9 @@ export class ReadwiseClient implements IntegrationClient {
               title: item.title,
               author: item.author || undefined,
               highlight_url: getHighlightUrl(item.id, highlight.id),
-              highlighted_at: highlight.createdAt as string | undefined,
+              highlighted_at: highlight.createdAt
+                ? new Date(highlight.createdAt as string).toISOString()
+                : undefined,
               category,
               image_url: item.thumbnail || undefined,
               location_type: 'order',
diff --git a/yarn.lock b/yarn.lock
index 1324cd2a1..f31031bd2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -12914,19 +12914,6 @@ copy-to-clipboard@^3.3.1:
   dependencies:
     toggle-selection "^1.0.6"
 
-copyfiles@^2.4.1:
-  version "2.4.1"
-  resolved "https://registry.yarnpkg.com/copyfiles/-/copyfiles-2.4.1.tgz#d2dcff60aaad1015f09d0b66e7f0f1c5cd3c5da5"
-  integrity sha512-fereAvAvxDrQDOXybk3Qu3dPbOoKoysFMWtkY3mv5BsL8//OSZVL5DCLYqgRfY5cWirgRzlC+WSrxp6Bo3eNZg==
-  dependencies:
-    glob "^7.0.5"
-    minimatch "^3.0.3"
-    mkdirp "^1.0.4"
-    noms "0.0.0"
-    through2 "^2.0.1"
-    untildify "^4.0.0"
-    yargs "^16.1.0"
-
 core-js-compat@^3.20.2, core-js-compat@^3.21.0:
   version "3.21.1"
   resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.21.1.tgz#cac369f67c8d134ff8f9bd1623e3bc2c42068c82"
@@ -16833,7 +16820,7 @@ glob-to-regexp@^0.4.1:
   resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e"
   integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==
 
-glob@7, glob@^7.0.5:
+glob@7:
   version "7.2.3"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b"
   integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==
@@ -22431,7 +22418,7 @@ minimatch@5.0.1:
   dependencies:
     brace-expansion "^2.0.1"
 
-minimatch@^3.0.2, minimatch@^3.0.3, minimatch@^3.0.4, minimatch@^3.1.1, minimatch@^3.1.2:
+minimatch@^3.0.2, minimatch@^3.0.4, minimatch@^3.1.1, minimatch@^3.1.2:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
   integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==
@@ -23427,14 +23414,6 @@ nofilter@^3.1.0:
   resolved "https://registry.yarnpkg.com/nofilter/-/nofilter-3.1.0.tgz#c757ba68801d41ff930ba2ec55bab52ca184aa66"
   integrity sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g==
 
-noms@0.0.0:
-  version "0.0.0"
-  resolved "https://registry.yarnpkg.com/noms/-/noms-0.0.0.tgz#da8ebd9f3af9d6760919b27d9cdc8092a7332859"
-  integrity sha512-lNDU9VJaOPxUmXcLb+HQFeUgQQPtMI24Gt6hgfuMHRJgMRHMF/qZ4HJD3GDru4sSw9IQl2jPjAYnQrdIeLbwow==
-  dependencies:
-    inherits "^2.0.1"
-    readable-stream "~1.0.31"
-
 nopt@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/nopt/-/nopt-5.0.0.tgz#530942bb58a512fccafe53fe210f13a25355dc88"
@@ -27114,16 +27093,6 @@ readable-stream@^3.0.0, readable-stream@^3.0.2, readable-stream@^3.0.6, readable
     string_decoder "^1.1.1"
     util-deprecate "^1.0.1"
 
-readable-stream@~1.0.31:
-  version "1.0.34"
-  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c"
-  integrity sha512-ok1qVCJuRkNmvebYikljxJA/UEsKwLl2nI1OmaqAu4/UE+h0wKCHok4XkL/gvi39OacXvw59RJUOFUkDib2rHg==
-  dependencies:
-    core-util-is "~1.0.0"
-    inherits "~2.0.1"
-    isarray "0.0.1"
-    string_decoder "~0.10.x"
-
 readdir-scoped-modules@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309"
@@ -29252,11 +29221,6 @@ string_decoder@^1.0.0, string_decoder@^1.1.1:
   dependencies:
     safe-buffer "~5.2.0"
 
-string_decoder@~0.10.x:
-  version "0.10.31"
-  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
-  integrity sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==
-
 string_decoder@~1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
@@ -29868,7 +29832,7 @@ throttleit@^1.0.0:
   resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c"
   integrity sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw=
 
-through2@^2.0.0, through2@^2.0.1, through2@~2.0.0:
+through2@^2.0.0, through2@~2.0.0:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
   integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==
@@ -32209,7 +32173,7 @@ yargs-unparser@2.0.0:
     flat "^5.0.2"
     is-plain-obj "^2.1.0"
 
-yargs@16.2.0, yargs@^16.0.0, yargs@^16.1.0, yargs@^16.2.0:
+yargs@16.2.0, yargs@^16.0.0, yargs@^16.2.0:
   version "16.2.0"
   resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66"
   integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==

From 44c67e2ebfcdb426c08e8f2320a4763ce9a1ba76 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sat, 23 Mar 2024 15:52:52 +0800
Subject: [PATCH 17/35] remove babel and compile test files

---
 packages/api/.dockerignore          |  1 +
 packages/api/.eslintignore          |  2 --
 packages/api/mocha-config.json      |  3 ++-
 packages/api/package.json           |  7 +++---
 packages/api/test/babel-register.js |  3 ---
 packages/api/tsconfig.json          |  4 ++--
 tsconfig.json                       |  1 -
 yarn.lock                           | 37 ++++-------------------------
 8 files changed, 12 insertions(+), 46 deletions(-)
 delete mode 100644 packages/api/test/babel-register.js

diff --git a/packages/api/.dockerignore b/packages/api/.dockerignore
index 4534f100a..db2866f3c 100644
--- a/packages/api/.dockerignore
+++ b/packages/api/.dockerignore
@@ -6,3 +6,4 @@ Dockerfile
 .dockerignore
 *.yaml
 .secrets*.yaml
+test
diff --git a/packages/api/.eslintignore b/packages/api/.eslintignore
index 4a2dcf8e2..a784fd752 100644
--- a/packages/api/.eslintignore
+++ b/packages/api/.eslintignore
@@ -1,5 +1,3 @@
 node_modules/
 dist/
-readabilityjs/
 src/generated/
-test/resolvers/
diff --git a/packages/api/mocha-config.json b/packages/api/mocha-config.json
index 1309d5869..ebe512a91 100644
--- a/packages/api/mocha-config.json
+++ b/packages/api/mocha-config.json
@@ -2,5 +2,6 @@
   "extension": ["ts"],
   "spec": "test/**/*.test.ts",
   "reporter": "mocha-unfunk-reporter",
-  "require": ["test/babel-register.js", "test/global-setup.ts", "test/global-teardown.ts"]
+  "require": ["test/global-setup.ts", "test/global-teardown.ts"],
+  "timeout": 10000
 }
diff --git a/packages/api/package.json b/packages/api/package.json
index 95aa34d84..3b694314d 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -6,12 +6,12 @@
     "build": "tsc",
     "dev": "ts-node-dev --debounce 100 --respawn --transpile-only src/server.ts",
     "dev_qp": "ts-node-dev --debounce 100 --respawn --transpile-only src/queue-processor.ts",
-    "start": "node dist/server.js",
-    "start_queue_processor": "node dist/queue-processor.js",
+    "start": "node dist/src/server.js",
+    "start_queue_processor": "node dist/src/queue-processor.js",
     "lint": "eslint src --ext ts,js,tsx,jsx",
     "lint:fix": "eslint src --fix --ext ts,js,tsx,jsx",
     "test:typecheck": "tsc --noEmit",
-    "test": "nyc mocha -r ts-node/register --config mocha-config.json --timeout 10000"
+    "test": "nyc mocha -r ts-node/register --config mocha-config.json"
   },
   "dependencies": {
     "@bmatei/apollo-prometheus-exporter": "^3.0.0",
@@ -116,7 +116,6 @@
     "youtubei": "1.3.7"
   },
   "devDependencies": {
-    "@babel/register": "^7.14.5",
     "@istanbuljs/nyc-config-typescript": "^1.0.2",
     "@types/addressparser": "^1.0.1",
     "@types/analytics-node": "^3.1.7",
diff --git a/packages/api/test/babel-register.js b/packages/api/test/babel-register.js
deleted file mode 100644
index a6f65f60a..000000000
--- a/packages/api/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/api/tsconfig.json b/packages/api/tsconfig.json
index 7c8caecfd..bbc8a9aee 100644
--- a/packages/api/tsconfig.json
+++ b/packages/api/tsconfig.json
@@ -6,6 +6,6 @@
   "compilerOptions": {
     "outDir": "dist"
   },
-  "include": ["src", "test"],
-  "exclude": ["./src/generated", "./test"]
+  "include": ["src/**/*.ts", "test/**/*.ts"],
+  "exclude": ["./src/generated"]
 }
diff --git a/tsconfig.json b/tsconfig.json
index 3fd6afacb..9e833a858 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,6 +1,5 @@
 {
   "$schema": "https://json.schemastore.org/tsconfig",
-  "display": "Node 14",
 
   "compilerOptions": {
     "lib": ["es2020", "dom"],
diff --git a/yarn.lock b/yarn.lock
index f31031bd2..074fd4c76 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2233,17 +2233,6 @@
     pirates "^4.0.5"
     source-map-support "^0.5.16"
 
-"@babel/register@^7.14.5":
-  version "7.15.3"
-  resolved "https://registry.yarnpkg.com/@babel/register/-/register-7.15.3.tgz#6b40a549e06ec06c885b2ec42c3dd711f55fe752"
-  integrity sha512-mj4IY1ZJkorClxKTImccn4T81+UKTo4Ux0+OFSV9hME1ooqS9UV+pJ6BjD0qXPK4T3XW/KNa79XByjeEMZz+fw==
-  dependencies:
-    clone-deep "^4.0.1"
-    find-cache-dir "^2.0.0"
-    make-dir "^2.1.0"
-    pirates "^4.0.0"
-    source-map-support "^0.5.16"
-
 "@babel/runtime-corejs3@^7.10.2", "@babel/runtime-corejs3@^7.9.2":
   version "7.15.3"
   resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.15.3.tgz#28754263988198f2a928c09733ade2fb4d28089d"
@@ -11686,16 +11675,10 @@ camelcase@^6.0.0, camelcase@^6.2.0:
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.2.0.tgz#924af881c9d525ac9d87f40d964e5cea982a1809"
   integrity sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg==
 
-caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001251, caniuse-lite@^1.0.30001286, caniuse-lite@^1.0.30001317:
-  version "1.0.30001527"
-  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001527.tgz"
-  integrity sha512-YkJi7RwPgWtXVSgK4lG9AHH57nSzvvOp9MesgXmw4Q7n0C3H04L0foHqfxcmSAm5AcWb8dW9AYj2tR7/5GnddQ==
-
-caniuse-lite@^1.0.30001406:
-  version "1.0.30001554"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001554.tgz#ba80d88dff9acbc0cd4b7535fc30e0191c5e2e2a"
-  integrity sha512-A2E3U//MBwbJVzebddm1YfNp7Nud5Ip+IPn4BozBmn4KqVX7AvluoIDFWjsv5OkGnKUXQVmMSoMKLa3ScCblcQ==
-
+caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001251, caniuse-lite@^1.0.30001286, caniuse-lite@^1.0.30001317, caniuse-lite@^1.0.30001406:
+  version "1.0.30001600"
+  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001600.tgz"
+  integrity sha512-+2S9/2JFhYmYaDpZvo0lKkfvuKIglrx68MwOBqMGHhQsNkLjB5xtc/TGoEPs+MxjSyN/72qer2g97nzR641mOQ==
 capital-case@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669"
@@ -23333,11 +23316,6 @@ node-mailjet@^6.0.5:
     json-bigint "^1.0.0"
     url-join "^4.0.0"
 
-node-modules-regexp@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/node-modules-regexp/-/node-modules-regexp-1.0.0.tgz#8d9dbe28964a4ac5712e9131642107c71e90ec40"
-  integrity sha1-jZ2+KJZKSsVxLpExZCEHxx6Q7EA=
-
 node-plop@~0.26.2:
   version "0.26.2"
   resolved "https://registry.yarnpkg.com/node-plop/-/node-plop-0.26.2.tgz#c2523596dab4e28360e615b768b11b4d60d5b1b9"
@@ -25178,13 +25156,6 @@ pify@^4.0.1:
   resolved "https://registry.yarnpkg.com/pify/-/pify-4.0.1.tgz#4b2cd25c50d598735c50292224fd8c6df41e3231"
   integrity sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==
 
-pirates@^4.0.0:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.1.tgz#643a92caf894566f91b2b986d2c66950a8e2fb87"
-  integrity sha512-WuNqLTbMI3tmfef2TKxlQmAiLHKtFhlsCZnPIpuv2Ow0RDVO8lfy1Opf4NUzlMXLjPl+Men7AuVdX6TA+s+uGA==
-  dependencies:
-    node-modules-regexp "^1.0.0"
-
 pirates@^4.0.1, pirates@^4.0.4, pirates@^4.0.5:
   version "4.0.5"
   resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.5.tgz#feec352ea5c3268fb23a37c702ab1699f35a5f3b"

From 88a7e8d85bdd748bd256a3c65560e11b79b0d1b2 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sat, 23 Mar 2024 16:40:01 +0800
Subject: [PATCH 18/35] fix tests

---
 packages/api/.dockerignore                           |  1 -
 packages/api/.eslintrc                               | 10 +++++++++-
 packages/api/src/services/user.ts                    |  4 +++-
 packages/api/test/resolvers/features.test.ts         | 12 ++++++++----
 packages/content-fetch/mocha-config.json             |  5 ++---
 packages/content-fetch/test/babel-register.js        |  3 ---
 packages/content-handler/mocha-config.json           |  5 ++---
 packages/content-handler/test/babel-register.js      |  3 ---
 packages/import-handler/mocha-config.json            |  5 ++---
 packages/import-handler/test/babel-register.js       |  3 ---
 packages/import-handler/test/csv/csv.test.ts         |  4 ++--
 .../test/matter/matter_importer.test.ts              |  4 ++--
 packages/import-handler/test/util.ts                 |  4 ++--
 packages/inbound-email-handler/mocha-config.json     |  5 ++---
 .../inbound-email-handler/test/babel-register.js     |  3 ---
 packages/integration-handler/mocha-config.json       |  5 ++---
 packages/integration-handler/test/babel-register.js  |  3 ---
 packages/pdf-handler/mocha-config.json               |  5 ++---
 packages/pdf-handler/test/babel-register.js          |  3 ---
 packages/puppeteer-parse/mocha-config.json           |  5 ++---
 packages/puppeteer-parse/test/babel-register.js      |  3 ---
 packages/queue-manager/mocha-config.json             |  5 ++---
 packages/queue-manager/test/babel-register.js        |  3 ---
 packages/rss-handler/mocha-config.json               |  5 ++---
 packages/rss-handler/test/babel-register.js          |  3 ---
 packages/rule-handler/mocha-config.json              |  5 ++---
 packages/rule-handler/test/babel-register.js         |  3 ---
 packages/text-to-speech/mocha-config.json            |  5 ++---
 packages/text-to-speech/test/babel-register.js       |  3 ---
 packages/thumbnail-handler/mocha-config.json         |  5 ++---
 packages/thumbnail-handler/test/babel-register.js    |  3 ---
 31 files changed, 50 insertions(+), 85 deletions(-)
 delete mode 100644 packages/content-fetch/test/babel-register.js
 delete mode 100644 packages/content-handler/test/babel-register.js
 delete mode 100644 packages/import-handler/test/babel-register.js
 delete mode 100644 packages/inbound-email-handler/test/babel-register.js
 delete mode 100644 packages/integration-handler/test/babel-register.js
 delete mode 100644 packages/pdf-handler/test/babel-register.js
 delete mode 100644 packages/puppeteer-parse/test/babel-register.js
 delete mode 100644 packages/queue-manager/test/babel-register.js
 delete mode 100644 packages/rss-handler/test/babel-register.js
 delete mode 100644 packages/rule-handler/test/babel-register.js
 delete mode 100644 packages/text-to-speech/test/babel-register.js
 delete mode 100644 packages/thumbnail-handler/test/babel-register.js

diff --git a/packages/api/.dockerignore b/packages/api/.dockerignore
index db2866f3c..4534f100a 100644
--- a/packages/api/.dockerignore
+++ b/packages/api/.dockerignore
@@ -6,4 +6,3 @@ Dockerfile
 .dockerignore
 *.yaml
 .secrets*.yaml
-test
diff --git a/packages/api/.eslintrc b/packages/api/.eslintrc
index f2874a383..bc325835a 100644
--- a/packages/api/.eslintrc
+++ b/packages/api/.eslintrc
@@ -5,5 +5,13 @@
   },
   "rules": {
     "@typescript-eslint/no-unsafe-argument": 0
-  }
+  },
+  "overrides": [
+    {
+      "files": ["test/**/*.ts"],
+      "rules": {
+        "@typescript-eslint/no-unsafe-member-access": 0
+      }
+    }
+  ]
 }
diff --git a/packages/api/src/services/user.ts b/packages/api/src/services/user.ts
index fd62192a5..52efa08e9 100644
--- a/packages/api/src/services/user.ts
+++ b/packages/api/src/services/user.ts
@@ -64,7 +64,9 @@ export const findUsersById = async (ids: string[]): Promise<User[]> => {
   return userRepository.findBy({ id: In(ids) })
 }
 
-export const deleteUsers = async (criteria: FindOptionsWhere<User>) => {
+export const deleteUsers = async (
+  criteria: FindOptionsWhere<User> | string[]
+) => {
   return authTrx(
     async (t) => t.getRepository(User).delete(criteria),
     undefined,
diff --git a/packages/api/test/resolvers/features.test.ts b/packages/api/test/resolvers/features.test.ts
index 6fa3762e9..79fe126fc 100644
--- a/packages/api/test/resolvers/features.test.ts
+++ b/packages/api/test/resolvers/features.test.ts
@@ -5,8 +5,12 @@ import sinon, { SinonFakeTimers } from 'sinon'
 import { User } from '../../src/entity/user'
 import { env } from '../../src/env'
 import { userRepository } from '../../src/repository/user'
-import { createFeature, createFeatures, deleteFeature } from '../../src/services/features'
-import { deleteUser } from '../../src/services/user'
+import {
+  createFeature,
+  createFeatures,
+  deleteFeature,
+} from '../../src/services/features'
+import { deleteUser, deleteUsers } from '../../src/services/user'
 import { createTestUser } from '../db'
 import { graphqlRequest, request } from '../util'
 
@@ -21,7 +25,7 @@ describe('features resolvers', () => {
       .post('/local/debug/fake-user-login')
       .send({ fakeEmail: loginUser.email })
 
-    authToken = res.body.authToken
+    authToken = res.body.authToken as string
   })
 
   after(async () => {
@@ -124,7 +128,7 @@ describe('features resolvers', () => {
 
       after(async () => {
         // reset opt-in users
-        Promise.all(users.map((user) => deleteUser(user.id)))
+        await deleteUsers(users.map((user) => user.id))
         // reset feature
         await deleteFeature({ name: featureName })
       })
diff --git a/packages/content-fetch/mocha-config.json b/packages/content-fetch/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/content-fetch/mocha-config.json
+++ b/packages/content-fetch/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/content-fetch/test/babel-register.js b/packages/content-fetch/test/babel-register.js
deleted file mode 100644
index a6f65f60a..000000000
--- a/packages/content-fetch/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/content-handler/mocha-config.json b/packages/content-handler/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/content-handler/mocha-config.json
+++ b/packages/content-handler/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/content-handler/test/babel-register.js b/packages/content-handler/test/babel-register.js
deleted file mode 100644
index a6f65f60a..000000000
--- a/packages/content-handler/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/import-handler/mocha-config.json b/packages/import-handler/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/import-handler/mocha-config.json
+++ b/packages/import-handler/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/import-handler/test/babel-register.js b/packages/import-handler/test/babel-register.js
deleted file mode 100644
index 9e872798e..000000000
--- a/packages/import-handler/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default;
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] });
\ No newline at end of file
diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts
index 350231671..69487483d 100644
--- a/packages/import-handler/test/csv/csv.test.ts
+++ b/packages/import-handler/test/csv/csv.test.ts
@@ -12,8 +12,8 @@ chai.use(chaiString)
 describe('Test csv importer', () => {
   let stub: ImportContext
 
-  beforeEach(async () => {
-    stub = await stubImportCtx()
+  beforeEach(() => {
+    stub = stubImportCtx()
   })
 
   afterEach(async () => {
diff --git a/packages/import-handler/test/matter/matter_importer.test.ts b/packages/import-handler/test/matter/matter_importer.test.ts
index 94017e634..9fd777c37 100644
--- a/packages/import-handler/test/matter/matter_importer.test.ts
+++ b/packages/import-handler/test/matter/matter_importer.test.ts
@@ -17,7 +17,7 @@ describe('Load a simple _matter_history file', () => {
   it('should find the URL of each row', async () => {
     const urls: URL[] = []
     const stream = fs.createReadStream('./test/matter/data/_matter_history.csv')
-    const stub = await stubImportCtx()
+    const stub = stubImportCtx()
     stub.urlHandler = (ctx: ImportContext, url): Promise<void> => {
       urls.push(url)
       return Promise.resolve()
@@ -38,7 +38,7 @@ describe('Load archive file', () => {
   it('should find the URL of each row', async () => {
     const urls: URL[] = []
     const stream = fs.createReadStream('./test/matter/data/Archive.zip')
-    const stub = await stubImportCtx()
+    const stub = stubImportCtx()
     stub.contentHandler = (
       ctx: ImportContext,
       url: URL,
diff --git a/packages/import-handler/test/util.ts b/packages/import-handler/test/util.ts
index d90aaa92d..04f7f7117 100644
--- a/packages/import-handler/test/util.ts
+++ b/packages/import-handler/test/util.ts
@@ -2,8 +2,8 @@ import { Readability } from '@omnivore/readability'
 import { ArticleSavingRequestStatus, ImportContext } from '../src'
 import { createRedisClient } from '../src/redis'
 
-export const stubImportCtx = async (): Promise<ImportContext> => {
-  const redisClient = await createRedisClient(process.env.REDIS_URL)
+export const stubImportCtx = (): ImportContext => {
+  const redisClient = createRedisClient(process.env.REDIS_URL)
 
   return {
     userId: '',
diff --git a/packages/inbound-email-handler/mocha-config.json b/packages/inbound-email-handler/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/inbound-email-handler/mocha-config.json
+++ b/packages/inbound-email-handler/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/inbound-email-handler/test/babel-register.js b/packages/inbound-email-handler/test/babel-register.js
deleted file mode 100644
index a6f65f60a..000000000
--- a/packages/inbound-email-handler/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/integration-handler/mocha-config.json b/packages/integration-handler/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/integration-handler/mocha-config.json
+++ b/packages/integration-handler/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/integration-handler/test/babel-register.js b/packages/integration-handler/test/babel-register.js
deleted file mode 100644
index 9e872798e..000000000
--- a/packages/integration-handler/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default;
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] });
\ No newline at end of file
diff --git a/packages/pdf-handler/mocha-config.json b/packages/pdf-handler/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/pdf-handler/mocha-config.json
+++ b/packages/pdf-handler/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/pdf-handler/test/babel-register.js b/packages/pdf-handler/test/babel-register.js
deleted file mode 100644
index 9e872798e..000000000
--- a/packages/pdf-handler/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default;
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] });
\ No newline at end of file
diff --git a/packages/puppeteer-parse/mocha-config.json b/packages/puppeteer-parse/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/puppeteer-parse/mocha-config.json
+++ b/packages/puppeteer-parse/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/puppeteer-parse/test/babel-register.js b/packages/puppeteer-parse/test/babel-register.js
deleted file mode 100644
index a6f65f60a..000000000
--- a/packages/puppeteer-parse/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/queue-manager/mocha-config.json b/packages/queue-manager/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/queue-manager/mocha-config.json
+++ b/packages/queue-manager/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/queue-manager/test/babel-register.js b/packages/queue-manager/test/babel-register.js
deleted file mode 100644
index a6f65f60a..000000000
--- a/packages/queue-manager/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/rss-handler/mocha-config.json b/packages/rss-handler/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/rss-handler/mocha-config.json
+++ b/packages/rss-handler/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/rss-handler/test/babel-register.js b/packages/rss-handler/test/babel-register.js
deleted file mode 100644
index a6f65f60a..000000000
--- a/packages/rss-handler/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/rule-handler/mocha-config.json b/packages/rule-handler/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/rule-handler/mocha-config.json
+++ b/packages/rule-handler/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/rule-handler/test/babel-register.js b/packages/rule-handler/test/babel-register.js
deleted file mode 100644
index a6f65f60a..000000000
--- a/packages/rule-handler/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/text-to-speech/mocha-config.json b/packages/text-to-speech/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/text-to-speech/mocha-config.json
+++ b/packages/text-to-speech/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/text-to-speech/test/babel-register.js b/packages/text-to-speech/test/babel-register.js
deleted file mode 100644
index a6f65f60a..000000000
--- a/packages/text-to-speech/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
diff --git a/packages/thumbnail-handler/mocha-config.json b/packages/thumbnail-handler/mocha-config.json
index 44d1d24c1..5a2631f42 100644
--- a/packages/thumbnail-handler/mocha-config.json
+++ b/packages/thumbnail-handler/mocha-config.json
@@ -1,5 +1,4 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts",
-    "require": "test/babel-register.js"
-  }
\ No newline at end of file
+    "spec": "test/**/*.test.ts"
+  }
diff --git a/packages/thumbnail-handler/test/babel-register.js b/packages/thumbnail-handler/test/babel-register.js
deleted file mode 100644
index a6f65f60a..000000000
--- a/packages/thumbnail-handler/test/babel-register.js
+++ /dev/null
@@ -1,3 +0,0 @@
-const register = require('@babel/register').default
-
-register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })

From 0fab3c7904d8227b97b10c34f03fb457dd27ac9e Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sat, 23 Mar 2024 17:10:59 +0800
Subject: [PATCH 19/35] Add highlight_updated event

---
 packages/api/src/services/highlights.ts       | 10 ++++-
 .../api/src/services/integrations/notion.ts   |  6 ++-
 .../api/src/services/integrations/readwise.ts | 14 +++----
 .../networking/queries/useGetRulesQuery.tsx   | 42 ++++++++++---------
 4 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index a127d5dfb..b774b8e5d 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -165,7 +165,15 @@ export const updateHighlight = async (
     EntityType.HIGHLIGHT,
     {
       id: libraryItemId,
-      highlights: [highlight],
+      highlights: [
+        {
+          ...highlight,
+          id: highlightId,
+          updatedAt: new Date(),
+          quote: updatedHighlight.quote,
+          highlightType: updatedHighlight.highlightType,
+        },
+      ],
     } as ItemEvent,
     userId
   )
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index d95bc893b..9fbb6fe5a 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -1,5 +1,6 @@
 import { Client } from '@notionhq/client'
 import axios from 'axios'
+import { HighlightType } from '../../entity/highlight'
 import { Integration } from '../../entity/integration'
 import { env } from '../../env'
 import { Merge } from '../../util'
@@ -257,8 +258,9 @@ export class NotionClient implements IntegrationClient {
           ? item.highlights
               .filter(
                 (highlight) =>
-                  !lastSync ||
-                  new Date(highlight.updatedAt as string) > lastSync // only new highlights
+                  highlight.highlightType === HighlightType.Highlight &&
+                  (!lastSync ||
+                    new Date(highlight.updatedAt as string) > lastSync) // only new highlights
               )
               .map((highlight) => ({
                 paragraph: {
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index 6a6209abc..c43e28774 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -1,7 +1,7 @@
 import axios from 'axios'
 import { HighlightType } from '../../entity/highlight'
 import { logger } from '../../utils/logger'
-import { getHighlightUrl, HighlightEvent } from '../highlights'
+import { getHighlightUrl } from '../highlights'
 import { ItemEvent } from '../library_item'
 import { IntegrationClient } from './integration'
 
@@ -71,6 +71,7 @@ export class ReadwiseClient implements IntegrationClient {
     let result = true
 
     const highlights = items.flatMap(this._itemToReadwiseHighlight)
+    logger.info(`Exporting ${highlights.length} highlights to Readwise`)
 
     // If there are no highlights, we will skip the sync
     if (highlights.length > 0) {
@@ -85,20 +86,17 @@ export class ReadwiseClient implements IntegrationClient {
   }
 
   private _itemToReadwiseHighlight = (item: ItemEvent): ReadwiseHighlight[] => {
-    const isHighlight = (
-      highlight: HighlightEvent
-    ): highlight is HighlightEvent & { quote: string } =>
-      highlight.highlightType === HighlightType.Highlight && !!highlight.quote
-
     const category = item.siteName === 'Twitter' ? 'tweets' : 'articles'
 
     return item.highlights
       ? item.highlights
           // filter out highlights that are not of type highlight or have no quote
-          .filter(isHighlight)
+          .filter(
+            (highlight) => highlight.highlightType === HighlightType.Highlight
+          )
           .map((highlight) => {
             return {
-              text: highlight.quote,
+              text: highlight.quote || '',
               title: item.title,
               author: item.author || undefined,
               highlight_url: getHighlightUrl(item.id, highlight.id),
diff --git a/packages/web/lib/networking/queries/useGetRulesQuery.tsx b/packages/web/lib/networking/queries/useGetRulesQuery.tsx
index 08dfd35ad..39f68d39c 100644
--- a/packages/web/lib/networking/queries/useGetRulesQuery.tsx
+++ b/packages/web/lib/networking/queries/useGetRulesQuery.tsx
@@ -22,6 +22,7 @@ export enum RuleEventType {
   PAGE_UPDATED = 'PAGE_UPDATED',
   LABEL_CREATED = 'LABEL_CREATED',
   HIGHLIGHT_CREATED = 'HIGHLIGHT_CREATED',
+  HIGHLIGHT_UPDATED = 'HIGHLIGHT_UPDATED',
 }
 
 export interface Rule {
@@ -47,7 +48,8 @@ interface RulesQueryResponseData {
 }
 
 interface RulesData {
-  rules: unknown
+  rules: Rule[]
+  errorCodes?: string[]
 }
 
 export function useGetRulesQuery(): RulesQueryResponse {
@@ -78,27 +80,27 @@ export function useGetRulesQuery(): RulesQueryResponse {
   `
 
   const { data, mutate, isValidating } = useSWR(query, publicGqlFetcher)
-
-  try {
-    if (data) {
-      const result = data as RulesQueryResponseData
-      const rules = result.rules.rules as Rule[]
-
-      return {
-        isValidating,
-        rules: rules ?? [],
-        revalidate: () => {
-          mutate()
-        },
-      }
+  if (!data) {
+    return {
+      isValidating: false,
+      rules: [],
+      revalidate: () => {
+        mutate()
+      },
     }
-  } catch (error) {
-    console.log('error', error)
   }
+
+  const result = data as RulesQueryResponseData
+  const error = result.rules.errorCodes?.find(() => true)
+  if (error) {
+    throw error
+  }
+
   return {
-    isValidating: false,
-    rules: [],
-    // eslint-disable-next-line @typescript-eslint/no-empty-function
-    revalidate: () => {},
+    isValidating,
+    rules: result.rules.rules,
+    revalidate: () => {
+      mutate()
+    },
   }
 }

From b2cd43136f284af0adc3ad41d8b40f1cd393cebe Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sun, 24 Mar 2024 10:42:29 +0800
Subject: [PATCH 20/35] hide webhooks page

---
 packages/api/package.json                     |  4 +-
 packages/api/src/services/highlights.ts       |  4 +
 .../api/src/services/integrations/notion.ts   | 87 +++++++++--------
 .../api/src/services/integrations/readwise.ts |  5 +-
 packages/api/src/services/library_item.ts     |  3 +
 packages/web/pages/settings/integrations.tsx  | 93 +++++++++++--------
 .../pages/settings/integrations/notion.tsx    | 39 ++++----
 7 files changed, 126 insertions(+), 109 deletions(-)

diff --git a/packages/api/package.json b/packages/api/package.json
index 3b694314d..c3ec7bd1b 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -4,8 +4,8 @@
   "license": "UNLICENSED",
   "scripts": {
     "build": "tsc",
-    "dev": "ts-node-dev --debounce 100 --respawn --transpile-only src/server.ts",
-    "dev_qp": "ts-node-dev --debounce 100 --respawn --transpile-only src/queue-processor.ts",
+    "dev": "ts-node-dev --respawn --transpile-only src/server.ts",
+    "dev_qp": "ts-node-dev --respawn --transpile-only src/queue-processor.ts",
     "start": "node dist/src/server.js",
     "start_queue_processor": "node dist/src/queue-processor.js",
     "lint": "eslint src --ext ts,js,tsx,jsx",
diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index b774b8e5d..531ba640e 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -165,6 +165,10 @@ export const updateHighlight = async (
     EntityType.HIGHLIGHT,
     {
       id: libraryItemId,
+      originalUrl: updatedHighlight.libraryItem.originalUrl,
+      title: updatedHighlight.libraryItem.title,
+      author: updatedHighlight.libraryItem.author,
+      thumbnail: updatedHighlight.libraryItem.thumbnail,
       highlights: [
         {
           ...highlight,
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 9fbb6fe5a..150ec881e 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -6,7 +6,7 @@ import { env } from '../../env'
 import { Merge } from '../../util'
 import { logger } from '../../utils/logger'
 import { getHighlightUrl } from '../highlights'
-import { ItemEvent } from '../library_item'
+import { getItemUrl, ItemEvent } from '../library_item'
 import { IntegrationClient } from './integration'
 
 type AnnotationColor =
@@ -229,7 +229,7 @@ export class NotionClient implements IntegrationClient {
             }
           : undefined,
         'Omnivore URL': {
-          url: `${env.client.url}/me/${item.id}`,
+          url: getItemUrl(item.id),
         },
         'Saved At': item.savedAt
           ? {
@@ -253,52 +253,51 @@ export class NotionClient implements IntegrationClient {
             }
           : undefined,
       },
-      children:
-        settings.properties?.includes('highlights') && item.highlights
-          ? item.highlights
-              .filter(
-                (highlight) =>
-                  highlight.highlightType === HighlightType.Highlight &&
-                  (!lastSync ||
-                    new Date(highlight.updatedAt as string) > lastSync) // only new highlights
-              )
-              .map((highlight) => ({
-                paragraph: {
-                  rich_text: [
-                    {
-                      text: {
-                        content: highlight.quote || '',
-                        link: {
-                          url: getHighlightUrl(
-                            item.slug || item.id,
-                            highlight.id
-                          ),
-                        },
-                      },
-                      annotations: {
-                        code: true,
-                        color: highlight.color as AnnotationColor,
+      children: item.highlights
+        ? item.highlights
+            .filter(
+              (highlight) =>
+                highlight.highlightType === HighlightType.Highlight &&
+                (!lastSync ||
+                  new Date(highlight.updatedAt as string) > lastSync) // only new highlights
+            )
+            .map((highlight) => ({
+              paragraph: {
+                rich_text: [
+                  {
+                    text: {
+                      content: highlight.quote || '',
+                      link: {
+                        url: getHighlightUrl(
+                          item.slug || item.id,
+                          highlight.id
+                        ),
                       },
                     },
-                  ],
-                  children: highlight.annotation
-                    ? [
-                        {
-                          paragraph: {
-                            rich_text: [
-                              {
-                                text: {
-                                  content: highlight.annotation || '',
-                                },
+                    annotations: {
+                      code: true,
+                      color: highlight.color as AnnotationColor,
+                    },
+                  },
+                ],
+                children: highlight.annotation
+                  ? [
+                      {
+                        paragraph: {
+                          rich_text: [
+                            {
+                              text: {
+                                content: highlight.annotation || '',
                               },
-                            ],
-                          },
+                            },
+                          ],
                         },
-                      ]
-                    : undefined,
-                },
-              }))
-          : undefined,
+                      },
+                    ]
+                  : undefined,
+              },
+            }))
+        : undefined,
     }
   }
 
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index c43e28774..2a6ea2fd7 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -2,7 +2,7 @@ import axios from 'axios'
 import { HighlightType } from '../../entity/highlight'
 import { logger } from '../../utils/logger'
 import { getHighlightUrl } from '../highlights'
-import { ItemEvent } from '../library_item'
+import { getItemUrl, ItemEvent } from '../library_item'
 import { IntegrationClient } from './integration'
 
 interface ReadwiseHighlight {
@@ -71,7 +71,6 @@ export class ReadwiseClient implements IntegrationClient {
     let result = true
 
     const highlights = items.flatMap(this._itemToReadwiseHighlight)
-    logger.info(`Exporting ${highlights.length} highlights to Readwise`)
 
     // If there are no highlights, we will skip the sync
     if (highlights.length > 0) {
@@ -108,7 +107,7 @@ export class ReadwiseClient implements IntegrationClient {
               location_type: 'order',
               note: highlight.annotation || undefined,
               source_type: 'omnivore',
-              source_url: item.originalUrl,
+              source_url: getItemUrl(item.id),
             }
           })
       : []
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index 4169ddc76..a65961f74 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -14,6 +14,7 @@ import { EntityLabel } from '../entity/entity_label'
 import { Highlight } from '../entity/highlight'
 import { Label } from '../entity/label'
 import { LibraryItem, LibraryItemState } from '../entity/library_item'
+import { env } from '../env'
 import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql'
 import { createPubSubClient, EntityEvent, EntityType } from '../pubsub'
 import { redisDataSource } from '../redis_data_source'
@@ -138,6 +139,8 @@ interface Select {
 
 const readingProgressDataSource = new ReadingProgressDataSource()
 
+export const getItemUrl = (id: string) => `${env.client.url}/me/${id}`
+
 const markItemAsRead = async (libraryItemId: string, userId: string) => {
   return await readingProgressDataSource.updateReadingProgress(
     userId,
diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx
index 409f02ad3..57e3e6219 100644
--- a/packages/web/pages/settings/integrations.tsx
+++ b/packages/web/pages/settings/integrations.tsx
@@ -1,8 +1,8 @@
 import { styled } from '@stitches/react'
 import Image from 'next/image'
 import { useRouter } from 'next/router'
-import { DownloadSimple, Eye, Link, Spinner } from 'phosphor-react'
-import { useEffect, useMemo, useState } from 'react'
+import { DownloadSimple, Link, Spinner } from 'phosphor-react'
+import { useCallback, useEffect, useMemo, useState } from 'react'
 import { Toaster } from 'react-hot-toast'
 import { Button } from '../../components/elements/Button'
 import {
@@ -29,7 +29,6 @@ import {
   useGetIntegrationsQuery,
 } from '../../lib/networking/queries/useGetIntegrationsQuery'
 import { useGetViewerQuery } from '../../lib/networking/queries/useGetViewerQuery'
-import { useGetWebhooksQuery } from '../../lib/networking/queries/useGetWebhooksQuery'
 import { showErrorToast, showSuccessToast } from '../../lib/toastHelpers'
 // Styles
 const Header = styled(Box, {
@@ -78,7 +77,7 @@ export default function Integrations(): JSX.Element {
   const { viewerData } = useGetViewerQuery()
 
   const { integrations, revalidate } = useGetIntegrationsQuery()
-  const { webhooks } = useGetWebhooksQuery()
+  // const { webhooks } = useGetWebhooksQuery()
 
   const [integrationsArray, setIntegrationsArray] = useState(
     Array<integrationsCard>()
@@ -91,29 +90,38 @@ export default function Integrations(): JSX.Element {
   const pocketConnected = useMemo(() => {
     return integrations.find((i) => i.name == 'POCKET' && i.type == 'IMPORT')
   }, [integrations])
-  const isConnected = (name: string) => {
-    return integrations.find((i) => i.name == name)?.enabled
-  }
+  const isConnected = useCallback(
+    (name: string) => {
+      return integrations.find((i) => i.name == name)?.enabled
+    },
+    [integrations]
+  )
 
-  const deleteIntegration = async (id: string) => {
-    try {
-      await deleteIntegrationMutation(id)
-      revalidate()
-      showSuccessToast('Integration Removed')
-    } catch (err) {
-      showErrorToast('Error: ' + err)
-    }
-  }
+  const deleteIntegration = useCallback(
+    async (id: string) => {
+      try {
+        await deleteIntegrationMutation(id)
+        revalidate()
+        showSuccessToast('Integration Removed')
+      } catch (err) {
+        showErrorToast('Error: ' + err)
+      }
+    },
+    [revalidate]
+  )
 
-  const importFromIntegration = async (id: string) => {
-    try {
-      await importFromIntegrationMutation(id)
-      revalidate()
-      showSuccessToast('Import started')
-    } catch (err) {
-      showErrorToast('Error: ' + err)
-    }
-  }
+  const importFromIntegration = useCallback(
+    async (id: string) => {
+      try {
+        await importFromIntegrationMutation(id)
+        revalidate()
+        showSuccessToast('Import started')
+      } catch (err) {
+        showErrorToast('Error: ' + err)
+      }
+    },
+    [revalidate]
+  )
 
   const redirectToIntegration = (
     name: string,
@@ -178,7 +186,7 @@ export default function Integrations(): JSX.Element {
           token,
           name: 'NOTION',
           type: 'EXPORT',
-          enabled: false,
+          enabled: true,
         })
 
         showSuccessToast('Connected with Notion.')
@@ -201,7 +209,7 @@ export default function Integrations(): JSX.Element {
     if (router.query.code) {
       connectWithNotion()
     }
-  }, [router])
+  }, [importFromIntegration, pocketConnected, revalidate, router])
 
   useEffect(() => {
     const integrationsArray = [
@@ -269,17 +277,17 @@ export default function Integrations(): JSX.Element {
           ],
         },
       },
-      {
-        icon: '/static/icons/webhooks.svg',
-        title: 'Webhooks',
-        subText: `${webhooks.length} Webhooks`,
-        button: {
-          text: 'View Webhooks',
-          icon: <Eye size={16} weight={'bold'} />,
-          style: 'ctaWhite',
-          action: () => router.push('/settings/webhooks'),
-        },
-      },
+      // {
+      //   icon: '/static/icons/webhooks.svg',
+      //   title: 'Webhooks',
+      //   subText: `${webhooks.length} Webhooks`,
+      //   button: {
+      //     text: 'View Webhooks',
+      //     icon: <Eye size={16} weight={'bold'} />,
+      //     style: 'ctaWhite',
+      //     action: () => router.push('/settings/webhooks'),
+      //   },
+      // },
       {
         icon: '/static/icons/readwise.svg',
         title: 'Readwise',
@@ -317,7 +325,14 @@ export default function Integrations(): JSX.Element {
       })
 
     setIntegrationsArray(integrationsArray)
-  }, [pocketConnected, readwiseConnected, webhooks, integrations])
+  }, [
+    pocketConnected,
+    readwiseConnected,
+    integrations,
+    isConnected,
+    router,
+    deleteIntegration,
+  ])
 
   return (
     <SettingsLayout>
diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index 6a402deaa..769758c21 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -1,15 +1,5 @@
-import {
-  Button,
-  Checkbox,
-  Form,
-  FormProps,
-  Input,
-  message,
-  Space,
-  Spin,
-} from 'antd'
+import { Button, Form, FormProps, Input, message, Space, Spin } from 'antd'
 import 'antd/dist/antd.compact.css'
-import { CheckboxValueType } from 'antd/lib/checkbox/Group'
 import Image from 'next/image'
 import { useRouter } from 'next/router'
 import { useCallback, useEffect, useState } from 'react'
@@ -109,9 +99,9 @@ export default function Notion(): JSX.Element {
     console.log('Failed:', errorInfo)
   }
 
-  const onDataChange = (value: Array<CheckboxValueType>) => {
-    form.setFieldsValue({ properties: value.map((v) => v.toString()) })
-  }
+  // const onDataChange = (value: Array<CheckboxValueType>) => {
+  //   form.setFieldsValue({ properties: value.map((v) => v.toString()) })
+  // }
 
   const exportToNotion = useCallback(async () => {
     if (exporting) {
@@ -119,6 +109,11 @@ export default function Notion(): JSX.Element {
       return
     }
 
+    if (!notion.settings?.parentPageId) {
+      messageApi.error('Please set the Notion page id first.')
+      return
+    }
+
     try {
       const task = await exportToIntegrationMutation(notion.id)
       // long polling to check the status of the task in every 10 seconds
@@ -142,7 +137,7 @@ export default function Notion(): JSX.Element {
     } catch (error) {
       messageApi.error('There was an error exporting to Notion.')
     }
-  }, [exporting, messageApi, notion.id])
+  }, [exporting, messageApi, notion])
 
   return (
     <>
@@ -222,15 +217,17 @@ export default function Notion(): JSX.Element {
                 </Form.Item>
 
                 <Form.Item<FieldType>
-                  label="Properties to Export"
-                  name="properties"
+                  label="Notion Database Id"
+                  name="parentDatabaseId"
+                  hidden
                 >
-                  <Checkbox.Group onChange={onDataChange}>
-                    <Checkbox value="highlights">Highlights</Checkbox>
-                  </Checkbox.Group>
+                  <Input disabled />
                 </Form.Item>
 
-                <Form.Item>
+                <Form.Item
+                  wrapperCol={{ offset: 6 }}
+                  style={{ marginTop: '30px' }}
+                >
                   <Space>
                     <Button type="primary" htmlType="submit">
                       Save

From 4550bd42b5aec025feb8c31c70060326776697e9 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sun, 24 Mar 2024 11:32:25 +0800
Subject: [PATCH 21/35] go to integrations page if integration is not connected
 when adding rules

---
 packages/web/pages/settings/rules.tsx | 52 +++++++++++++++++++++------
 1 file changed, 41 insertions(+), 11 deletions(-)

diff --git a/packages/web/pages/settings/rules.tsx b/packages/web/pages/settings/rules.tsx
index 97fa56cf5..02247631d 100644
--- a/packages/web/pages/settings/rules.tsx
+++ b/packages/web/pages/settings/rules.tsx
@@ -1,5 +1,4 @@
 import { Button, Form, Input, Modal, Select, Space, Table, Tag } from 'antd'
-// import 'antd/dist/antd.dark.css'
 import 'antd/dist/antd.compact.css'
 import { useCallback, useMemo, useState } from 'react'
 import { Toaster } from 'react-hot-toast'
@@ -8,13 +7,14 @@ import { SettingsLayout } from '../../components/templates/SettingsLayout'
 import { Label } from '../../lib/networking/fragments/labelFragment'
 import { deleteRuleMutation } from '../../lib/networking/mutations/deleteRuleMutation'
 import { setRuleMutation } from '../../lib/networking/mutations/setRuleMutation'
+import { useGetIntegrationsQuery } from '../../lib/networking/queries/useGetIntegrationsQuery'
 import { useGetLabelsQuery } from '../../lib/networking/queries/useGetLabelsQuery'
 import {
   Rule,
   RuleAction,
   RuleActionType,
   RuleEventType,
-  useGetRulesQuery,
+  useGetRulesQuery
 } from '../../lib/networking/queries/useGetRulesQuery'
 import { applyStoredTheme } from '../../lib/themeUpdater'
 import { showErrorToast, showSuccessToast } from '../../lib/toastHelpers'
@@ -124,6 +124,13 @@ type CreateActionModalProps = {
 const CreateActionModal = (props: CreateActionModalProps): JSX.Element => {
   const [form] = Form.useForm()
   const { labels } = useGetLabelsQuery()
+  const { integrations } = useGetIntegrationsQuery()
+
+  const integrationOptions = ['Notion', 'Readwise']
+
+  const isIntegrationEnabled = (integration: string): boolean => {
+    return integrations.some((i) => i.name.toUpperCase() === integration.toUpperCase())
+  }
 
   const onOk = async (values: any) => {
     const actionType = form.getFieldValue('actionType') as RuleActionType
@@ -236,17 +243,39 @@ const CreateActionModal = (props: CreateActionModalProps): JSX.Element => {
           <Form.Item
             label="Integrations"
             name="integrations"
+            hasFeedback
             rules={[
-              { required: true, message: 'Please choose at least one integration' },
+              {
+                required: true,
+                message: 'Please choose at least one integration',
+              },
+              {
+                validator: (_, value: string[]) => {
+                  value.forEach((v) => {
+                    if (!isIntegrationEnabled(v)) {
+                      return Promise.reject(`Integration ${v} is not enabled`)
+                    }
+                  })
+
+                  return Promise.resolve()
+                },
+              },
             ]}
           >
             <Select mode="multiple">
-              <Select.Option key="Notion" value="NOTION">
-                Notion
-              </Select.Option>
-              <Select.Option key="Readwise" value="READWISE">
-                Readwise
-              </Select.Option>
+              {integrationOptions.map((integration) => {
+                return (
+                  <Select.Option key={integration} value={integration}>
+                    {isIntegrationEnabled(integration) ? (
+                      integration
+                    ) : (
+                      <Button type="link" href="/settings/integrations">
+                        Connect to {integration}
+                      </Button>
+                    )}
+                  </Select.Option>
+                )
+              })}
             </Select>
           </Form.Item>
         )}
@@ -259,8 +288,9 @@ export default function Rules(): JSX.Element {
   const { rules, revalidate } = useGetRulesQuery()
   const { labels } = useGetLabelsQuery()
   const [isCreateRuleModalOpen, setIsCreateRuleModalOpen] = useState(false)
-  const [createActionRule, setCreateActionRule] =
-    useState<Rule | undefined>(undefined)
+  const [createActionRule, setCreateActionRule] = useState<Rule | undefined>(
+    undefined
+  )
 
   const dataSource = useMemo(() => {
     return rules.map((rule: Rule) => {

From 5da5ff6da8382ea86b657f10c67441cfd585d2e1 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sun, 24 Mar 2024 11:36:21 +0800
Subject: [PATCH 22/35] prevent adding duplicate actions

---
 packages/web/pages/settings/rules.tsx | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/packages/web/pages/settings/rules.tsx b/packages/web/pages/settings/rules.tsx
index 02247631d..2b22592c8 100644
--- a/packages/web/pages/settings/rules.tsx
+++ b/packages/web/pages/settings/rules.tsx
@@ -30,7 +30,7 @@ const CreateRuleModal = (props: CreateRuleModalProps): JSX.Element => {
 
   const onOk = async (values: any) => {
     const name = form.getFieldValue('name')
-    const filter = form.getFieldValue('filter') || 'in:all'
+    const filter = form.getFieldValue('filter') || 'in:all' // default to all
     const eventTypes = form.getFieldValue('eventTypes')
     try {
       await setRuleMutation({
@@ -129,7 +129,9 @@ const CreateActionModal = (props: CreateActionModalProps): JSX.Element => {
   const integrationOptions = ['Notion', 'Readwise']
 
   const isIntegrationEnabled = (integration: string): boolean => {
-    return integrations.some((i) => i.name.toUpperCase() === integration.toUpperCase())
+    return integrations.some(
+      (i) => i.name.toUpperCase() === integration.toUpperCase()
+    )
   }
 
   const onOk = async (values: any) => {
@@ -144,6 +146,12 @@ const CreateActionModal = (props: CreateActionModalProps): JSX.Element => {
     }
 
     if (props.rule) {
+      // prevent adding duplicate actions
+      if (props.rule.actions.some((a) => a.type === actionType)) {
+        showErrorToast('Action already exists in the rule.')
+        return
+      }
+
       await setRuleMutation({
         id: props.rule.id,
         name: props.rule.name,

From 30fa6f8bd0cfb532d1dfa1f496e1b10c14a3a54f Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 25 Mar 2024 17:43:57 +0800
Subject: [PATCH 23/35] fix some state issue

---
 .../mutations/setIntegrationMutation.ts       |  11 +-
 packages/web/pages/settings/integrations.tsx  | 122 +++++++++---------
 2 files changed, 68 insertions(+), 65 deletions(-)

diff --git a/packages/web/lib/networking/mutations/setIntegrationMutation.ts b/packages/web/lib/networking/mutations/setIntegrationMutation.ts
index 0c0bc4b6d..78db04487 100644
--- a/packages/web/lib/networking/mutations/setIntegrationMutation.ts
+++ b/packages/web/lib/networking/mutations/setIntegrationMutation.ts
@@ -20,12 +20,12 @@ export type SetIntegrationInput = {
 }
 
 type SetIntegrationResult = {
-  setIntegration?: SetIntegrationData
+  setIntegration: SetIntegrationData
 }
 
 type SetIntegrationData = {
   integration: Integration
-  errorCodes?: unknown[]
+  errorCodes?: string[]
 }
 
 type Integration = {
@@ -40,7 +40,7 @@ type Integration = {
 
 export async function setIntegrationMutation(
   input: SetIntegrationInput
-): Promise<Integration | undefined> {
+): Promise<Integration> {
   const mutation = gql`
     mutation SetIntegration($input: SetIntegrationInput!) {
       setIntegration(input: $input) {
@@ -64,11 +64,10 @@ export async function setIntegrationMutation(
   `
 
   const data = (await gqlFetcher(mutation, { input })) as SetIntegrationResult
-  const output = data as any
-  const error = data.setIntegration?.errorCodes?.find(() => true)
+  const error = data.setIntegration.errorCodes?.find(() => true)
   if (error) {
     if (error === 'INVALID_TOKEN') throw 'Your token is invalid.'
     throw error
   }
-  return output.setIntegration?.integration
+  return data.setIntegration.integration
 }
diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx
index 57e3e6219..de4250f0f 100644
--- a/packages/web/pages/settings/integrations.tsx
+++ b/packages/web/pages/settings/integrations.tsx
@@ -2,18 +2,18 @@ import { styled } from '@stitches/react'
 import Image from 'next/image'
 import { useRouter } from 'next/router'
 import { DownloadSimple, Link, Spinner } from 'phosphor-react'
-import { useCallback, useEffect, useMemo, useState } from 'react'
+import { useCallback, useEffect, useState } from 'react'
 import { Toaster } from 'react-hot-toast'
 import { Button } from '../../components/elements/Button'
 import {
   Dropdown,
-  DropdownOption,
+  DropdownOption
 } from '../../components/elements/DropdownElements'
 import {
   Box,
   HStack,
   SpanBox,
-  VStack,
+  VStack
 } from '../../components/elements/LayoutPrimitives'
 import { SettingsLayout } from '../../components/templates/SettingsLayout'
 import { fetchEndpoint } from '../../lib/appConfig'
@@ -22,11 +22,11 @@ import { deleteIntegrationMutation } from '../../lib/networking/mutations/delete
 import { importFromIntegrationMutation } from '../../lib/networking/mutations/importFromIntegrationMutation'
 import {
   ImportItemState,
-  setIntegrationMutation,
+  setIntegrationMutation
 } from '../../lib/networking/mutations/setIntegrationMutation'
 import {
   Integration,
-  useGetIntegrationsQuery,
+  useGetIntegrationsQuery
 } from '../../lib/networking/queries/useGetIntegrationsQuery'
 import { useGetViewerQuery } from '../../lib/networking/queries/useGetViewerQuery'
 import { showErrorToast, showSuccessToast } from '../../lib/toastHelpers'
@@ -84,43 +84,32 @@ export default function Integrations(): JSX.Element {
   )
   const router = useRouter()
 
-  const readwiseConnected = useMemo(() => {
-    return integrations.find((i) => i.name == 'READWISE' && i.type == 'EXPORT')
-  }, [integrations])
-  const pocketConnected = useMemo(() => {
-    return integrations.find((i) => i.name == 'POCKET' && i.type == 'IMPORT')
-  }, [integrations])
-  const isConnected = useCallback(
+  const getIntegration = useCallback(
     (name: string) => {
-      return integrations.find((i) => i.name == name)?.enabled
+      return integrations.find((i) => i.name === name)
     },
     [integrations]
   )
 
-  const deleteIntegration = useCallback(
-    async (id: string) => {
-      try {
-        await deleteIntegrationMutation(id)
-        revalidate()
-        showSuccessToast('Integration Removed')
-      } catch (err) {
-        showErrorToast('Error: ' + err)
-      }
-    },
-    [revalidate]
-  )
+  const deleteIntegration = useCallback(async (id: string) => {
+    try {
+      await deleteIntegrationMutation(id)
+      showSuccessToast('Integration Removed')
+    } catch (err) {
+      showErrorToast('Error: ' + err)
+    }
+  }, [])
 
   const importFromIntegration = useCallback(
     async (id: string) => {
       try {
         await importFromIntegrationMutation(id)
-        revalidate()
         showSuccessToast('Import started')
       } catch (err) {
         showErrorToast('Error: ' + err)
       }
     },
-    [revalidate]
+    []
   )
 
   const redirectToIntegration = (
@@ -160,21 +149,17 @@ export default function Integrations(): JSX.Element {
           enabled: true,
           importItemState,
         })
-        if (result) {
-          revalidate()
-          showSuccessToast('Connected with Pocket.')
-          // start the import
-          await importFromIntegration(result.id)
-        } else {
-          showErrorToast('There was an error connecting to Pocket.')
-        }
+
+        showSuccessToast('Connected with Pocket.')
+        // start the import
+        await importFromIntegration(result.id)
       } catch (err) {
         showErrorToast(
           'There was an error connecting to Pocket. Please try again.',
           { duration: 5000 }
         )
       } finally {
-        router.push('/settings/integrations')
+        router.replace('/settings/integrations')
       }
     }
 
@@ -198,20 +183,30 @@ export default function Integrations(): JSX.Element {
           { duration: 5000 }
         )
 
-        router.push('/settings/integrations')
+        router.replace('/settings/integrations')
       }
     }
 
     if (!router.isReady) return
-    if (router.query.pocketToken && router.query.state && !pocketConnected) {
+
+    if (
+      router.query.pocketToken &&
+      router.query.state &&
+      !getIntegration('POCKET')
+    ) {
       connectToPocket()
     }
-    if (router.query.code) {
+
+    if (router.query.code && !getIntegration('NOTION')) {
       connectWithNotion()
     }
-  }, [importFromIntegration, pocketConnected, revalidate, router])
+  }, [importFromIntegration, getIntegration, revalidate, router])
 
   useEffect(() => {
+    const pocket = getIntegration('POCKET')
+    const readwise = getIntegration('READWISE')
+    const notion = getIntegration('NOTION')
+
     const integrationsArray = [
       {
         icon: '/static/icons/logseq.svg',
@@ -247,20 +242,20 @@ export default function Integrations(): JSX.Element {
         subText:
           'Pocket is a place to save articles, videos, and more. Our Pocket integration allows importing your Pocket library to Omnivore. Once connected we will asyncronously import all your Pocket articles into Omnivore, as this process is resource intensive it can take some time. You will receive an email when the process is completed. Limit 20k articles per import.',
         button: {
-          text: pocketConnected ? 'Disconnect' : 'Import',
-          icon: isImporting(pocketConnected) ? (
+          text: pocket ? 'Disconnect' : 'Import',
+          icon: isImporting(pocket) ? (
             <Spinner size={16} />
           ) : (
             <Link size={16} weight={'bold'} />
           ),
-          style: pocketConnected ? 'ctaWhite' : 'ctaDarkYellow',
+          style: pocket ? 'ctaWhite' : 'ctaDarkYellow',
           action: () => {
-            pocketConnected
-              ? deleteIntegration(pocketConnected.id)
+            pocket
+              ? deleteIntegration(pocket.id)
               : redirectToIntegration('pocket', ImportItemState.Unarchived)
           },
-          disabled: isImporting(pocketConnected),
-          isDropdown: !pocketConnected,
+          disabled: isImporting(pocket),
+          isDropdown: !pocket,
           dropdownOptions: [
             {
               text: 'Import All',
@@ -277,6 +272,22 @@ export default function Integrations(): JSX.Element {
           ],
         },
       },
+      {
+        icon: '/static/icons/notion.png',
+        title: 'Notion',
+        subText:
+          'Notion is an all-in-one workspace. Use our Notion integration to sync your Omnivore items to Notion.',
+        button: {
+          text: notion ? 'Settings' : 'Connect',
+          icon: <Link size={16} weight={'bold'} />,
+          style: notion ? 'ctaWhite' : 'ctaDarkYellow',
+          action: () => {
+            notion
+              ? router.push('/settings/integrations/notion')
+              : redirectToIntegration('NOTION')
+          },
+        },
+      },
       // {
       //   icon: '/static/icons/webhooks.svg',
       //   title: 'Webhooks',
@@ -294,12 +305,12 @@ export default function Integrations(): JSX.Element {
         subText:
           'Readwise makes it easy to revisit and learn from your ebook & article highlights. Use our Readwise integration to sync your highlights from Omnivore to Readwise.',
         button: {
-          text: readwiseConnected ? 'Remove' : 'Connect to Readwise',
+          text: readwise ? 'Remove' : 'Connect to Readwise',
           icon: <Link size={16} weight={'bold'} />,
-          style: readwiseConnected ? 'ctaWhite' : 'ctaDarkYellow',
+          style: readwise ? 'ctaWhite' : 'ctaDarkYellow',
           action: () => {
-            readwiseConnected
-              ? deleteIntegration(readwiseConnected.id)
+            readwise
+              ? deleteIntegration(readwise.id)
               : router.push('/settings/integrations/readwise')
           },
         },
@@ -325,14 +336,7 @@ export default function Integrations(): JSX.Element {
       })
 
     setIntegrationsArray(integrationsArray)
-  }, [
-    pocketConnected,
-    readwiseConnected,
-    integrations,
-    isConnected,
-    router,
-    deleteIntegration,
-  ])
+  }, [deleteIntegration, getIntegration, router])
 
   return (
     <SettingsLayout>

From 7f0ca62386dabe613e083ae17f2c7a884932328d Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 25 Mar 2024 17:56:20 +0800
Subject: [PATCH 24/35] fix some state for notion

---
 packages/web/pages/settings/integrations.tsx | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx
index de4250f0f..368f479a1 100644
--- a/packages/web/pages/settings/integrations.tsx
+++ b/packages/web/pages/settings/integrations.tsx
@@ -159,7 +159,7 @@ export default function Integrations(): JSX.Element {
           { duration: 5000 }
         )
       } finally {
-        router.replace('/settings/integrations')
+        router.push('/settings/integrations')
       }
     }
 
@@ -183,7 +183,7 @@ export default function Integrations(): JSX.Element {
           { duration: 5000 }
         )
 
-        router.replace('/settings/integrations')
+        router.push('/settings/integrations')
       }
     }
 
@@ -200,7 +200,7 @@ export default function Integrations(): JSX.Element {
     if (router.query.code && !getIntegration('NOTION')) {
       connectWithNotion()
     }
-  }, [importFromIntegration, getIntegration, revalidate, router])
+  }, [importFromIntegration, getIntegration, router])
 
   useEffect(() => {
     const pocket = getIntegration('POCKET')

From 592a759b8daab40562192ce2e7100282cc3836d3 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 25 Mar 2024 19:10:28 +0800
Subject: [PATCH 25/35] use callback

---
 packages/web/pages/settings/integrations.tsx | 71 +++++++++++---------
 1 file changed, 39 insertions(+), 32 deletions(-)

diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx
index 368f479a1..33e24c1ff 100644
--- a/packages/web/pages/settings/integrations.tsx
+++ b/packages/web/pages/settings/integrations.tsx
@@ -76,6 +76,7 @@ type integrationsCard = {
 export default function Integrations(): JSX.Element {
   const { viewerData } = useGetViewerQuery()
 
+
   const { integrations, revalidate } = useGetIntegrationsQuery()
   // const { webhooks } = useGetWebhooksQuery()
 
@@ -94,25 +95,24 @@ export default function Integrations(): JSX.Element {
   const deleteIntegration = useCallback(async (id: string) => {
     try {
       await deleteIntegrationMutation(id)
+      revalidate()
       showSuccessToast('Integration Removed')
     } catch (err) {
       showErrorToast('Error: ' + err)
     }
   }, [])
 
-  const importFromIntegration = useCallback(
-    async (id: string) => {
-      try {
-        await importFromIntegrationMutation(id)
-        showSuccessToast('Import started')
-      } catch (err) {
-        showErrorToast('Error: ' + err)
-      }
-    },
-    []
-  )
+  const importFromIntegration = async (id: string) => {
+    try {
+      await importFromIntegrationMutation(id)
+      revalidate()
+      showSuccessToast('Import started')
+    } catch (err) {
+      showErrorToast('Error: ' + err)
+    }
+  }
 
-  const redirectToIntegration = (
+  const redirectToIntegration = useCallback((
     name: string,
     importItemState?: ImportItemState
   ) => {
@@ -130,18 +130,20 @@ export default function Integrations(): JSX.Element {
     document.body.appendChild(form)
 
     form.submit()
-  }
+  }, [])
 
   const isImporting = (integration: Integration | undefined) => {
     return !!integration && !!integration.taskName
   }
 
   useEffect(() => {
-    const connectToPocket = async () => {
+    const connectToPocket = async (
+      token: string,
+      importItemState: ImportItemState
+    ) => {
+      router.push('/settings/integrations')
+
       try {
-        // get the token from query string
-        const token = router.query.pocketToken as string
-        const importItemState = router.query.state as ImportItemState
         const result = await setIntegrationMutation({
           token,
           name: 'POCKET',
@@ -150,7 +152,9 @@ export default function Integrations(): JSX.Element {
           importItemState,
         })
 
+        revalidate()
         showSuccessToast('Connected with Pocket.')
+
         // start the import
         await importFromIntegration(result.id)
       } catch (err) {
@@ -158,17 +162,15 @@ export default function Integrations(): JSX.Element {
           'There was an error connecting to Pocket. Please try again.',
           { duration: 5000 }
         )
-      } finally {
-        router.push('/settings/integrations')
       }
     }
 
-    const connectWithNotion = async () => {
+    const connectWithNotion = async (code: string) => {
+      router.push('/settings/integrations')
+
       try {
-        // get the token from query string
-        const token = router.query.code as string
         await setIntegrationMutation({
-          token,
+          token: code,
           name: 'NOTION',
           type: 'EXPORT',
           enabled: true,
@@ -182,8 +184,6 @@ export default function Integrations(): JSX.Element {
           'There was an error connecting to Notion. Please try again.',
           { duration: 5000 }
         )
-
-        router.push('/settings/integrations')
       }
     }
 
@@ -194,13 +194,20 @@ export default function Integrations(): JSX.Element {
       router.query.state &&
       !getIntegration('POCKET')
     ) {
-      connectToPocket()
+      // get the token from query string
+      const { pocketToken, state } = router.query as {
+        pocketToken: string
+        state: ImportItemState
+      }
+      connectToPocket(pocketToken, state)
     }
 
     if (router.query.code && !getIntegration('NOTION')) {
-      connectWithNotion()
+      // get the code from query string
+      const code = router.query.code as string
+      connectWithNotion(code)
     }
-  }, [importFromIntegration, getIntegration, router])
+  }, [getIntegration, router])
 
   useEffect(() => {
     const pocket = getIntegration('POCKET')
@@ -252,7 +259,7 @@ export default function Integrations(): JSX.Element {
           action: () => {
             pocket
               ? deleteIntegration(pocket.id)
-              : redirectToIntegration('pocket', ImportItemState.Unarchived)
+              : redirectToIntegration('POCKET', ImportItemState.Unarchived)
           },
           disabled: isImporting(pocket),
           isDropdown: !pocket,
@@ -260,13 +267,13 @@ export default function Integrations(): JSX.Element {
             {
               text: 'Import All',
               action: () => {
-                redirectToIntegration('pocket', ImportItemState.All)
+                redirectToIntegration('POCKET', ImportItemState.All)
               },
             },
             {
               text: 'Import Unarchived',
               action: () => {
-                redirectToIntegration('pocket', ImportItemState.Unarchived)
+                redirectToIntegration('POCKET', ImportItemState.Unarchived)
               },
             },
           ],
@@ -336,7 +343,7 @@ export default function Integrations(): JSX.Element {
       })
 
     setIntegrationsArray(integrationsArray)
-  }, [deleteIntegration, getIntegration, router])
+  }, [getIntegration, router])
 
   return (
     <SettingsLayout>

From 051e6a2b5604d0cf1238f38eb44543a37a40a119 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 25 Mar 2024 22:03:32 +0800
Subject: [PATCH 26/35] use callback

---
 packages/web/pages/settings/integrations/notion.tsx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index 769758c21..41c9c5292 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -41,13 +41,13 @@ export default function Notion(): JSX.Element {
     })
   }, [form, notion])
 
-  const deleteNotion = async () => {
+  const deleteNotion = useCallback(async () => {
     await deleteIntegrationMutation(notion.id)
     showSuccessToast('Notion integration disconnected successfully.')
 
     revalidate()
     router.push('/settings/integrations')
-  }
+  }, [notion.id, router])
 
   const updateNotion = async (values: FieldType) => {
     await setIntegrationMutation({

From 5e7a82562b3efef92d7d93d4e4bdf06cfb4d2e40 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 27 Mar 2024 16:53:29 +0800
Subject: [PATCH 27/35] resolve conflicts after rebasing

---
 packages/api/src/jobs/trigger_rule.ts        |  2 +-
 packages/api/src/services/highlights.ts      | 10 ++++-----
 packages/api/src/services/labels.ts          |  8 +++----
 packages/api/src/services/library_item.ts    | 22 +++++++++++--------
 packages/web/pages/settings/integrations.tsx | 23 ++++----------------
 5 files changed, 27 insertions(+), 38 deletions(-)

diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts
index 1ff163980..76fcbd3cc 100644
--- a/packages/api/src/jobs/trigger_rule.ts
+++ b/packages/api/src/jobs/trigger_rule.ts
@@ -230,7 +230,7 @@ const triggerActions = async (
         logger.info('Failed to filter items by metadata, running search query')
         const searchResult = await searchLibraryItems(
           {
-            query: `includes:${libraryItemId} AND (${rule.filter})`,
+            query: `includes:${data.id} AND (${rule.filter})`,
             size: 1,
           },
           userId
diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts
index 531ba640e..3893bdbc5 100644
--- a/packages/api/src/services/highlights.ts
+++ b/packages/api/src/services/highlights.ts
@@ -13,10 +13,10 @@ import { enqueueUpdateHighlight } from '../utils/createTask'
 import { deepDelete } from '../utils/helpers'
 import { ItemEvent } from './library_item'
 
-const columnToDelete = ['user', 'sharedAt', 'libraryItem'] as const
-type ColumnToDeleteType = typeof columnToDelete[number]
+const columnsToDelete = ['user', 'sharedAt', 'libraryItem'] as const
+type ColumnsToDeleteType = typeof columnsToDelete[number]
 export type HighlightEvent = Merge<
-  Omit<DeepPartial<Highlight>, ColumnToDeleteType>,
+  Omit<DeepPartial<Highlight>, ColumnsToDeleteType>,
   EntityEvent
 >
 
@@ -63,12 +63,12 @@ export const createHighlight = async (
     userId
   )
 
-  const cleanData = deepDelete(newHighlight, columnToDelete)
+  const data = deepDelete(newHighlight, columnsToDelete)
   await pubsub.entityCreated<ItemEvent>(
     EntityType.HIGHLIGHT,
     {
       id: libraryItemId,
-      highlights: [cleanData],
+      highlights: [data],
       // for Readwise
       originalUrl: newHighlight.libraryItem.originalUrl,
       title: newHighlight.libraryItem.title,
diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts
index f4987a4b8..dac0a3c94 100644
--- a/packages/api/src/services/labels.ts
+++ b/packages/api/src/services/labels.ts
@@ -15,10 +15,10 @@ import { bulkEnqueueUpdateLabels } from '../utils/createTask'
 import { deepDelete } from '../utils/helpers'
 import { findLibraryItemIdsByLabelId, ItemEvent } from './library_item'
 
-const columnToDelete = ['description', 'createdAt'] as const
-type ColumnToDeleteType = typeof columnToDelete[number]
+const columnsToDelete = ['description', 'createdAt'] as const
+type ColumnsToDeleteType = typeof columnsToDelete[number]
 export type LabelEvent = Merge<
-  Omit<DeepPartial<Label>, ColumnToDeleteType>,
+  Omit<DeepPartial<Label>, ColumnsToDeleteType>,
   EntityEvent
 >
 
@@ -150,7 +150,7 @@ export const saveLabelsInLibraryItem = async (
       EntityType.LABEL,
       {
         id: libraryItemId,
-        labels: labels.map((l) => deepDelete(l, columnToDelete)),
+        labels: labels.map((l) => deepDelete(l, columnsToDelete)),
       },
       userId
     )
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index a65961f74..fb8ae9c4f 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -32,16 +32,19 @@ import { parseSearchQuery } from '../utils/search'
 import { HighlightEvent } from './highlights'
 import { addLabelsToLibraryItem, LabelEvent } from './labels'
 
-const columnToDelete = [
+const columnsToDelete = [
   'user',
   'uploadFile',
   'previewContentType',
   'links',
   'textContentHash',
+  'readableContent',
+  'originalContent',
+  'feedContent',
 ] as const
-type ColumnToDeleteType = typeof columnToDelete[number]
+type ColumnsToDeleteType = typeof columnsToDelete[number]
 type ItemBaseEvent = Merge<
-  Omit<DeepPartial<LibraryItem>, ColumnToDeleteType>,
+  Omit<DeepPartial<LibraryItem>, ColumnsToDeleteType>,
   {
     labels?: LabelEvent[]
     highlights?: HighlightEvent[]
@@ -889,17 +892,18 @@ export const updateLibraryItem = async (
   }
 
   if (libraryItem.state === LibraryItemState.Succeeded) {
-    const cleanedData = deepDelete(updatedLibraryItem, columnToDelete)
+    const data = deepDelete(updatedLibraryItem, columnsToDelete)
     // send create event if the item was created
-    await pubsub.entityCreated<ItemEvent>(EntityType.ITEM, cleanedData, userId)
+    await pubsub.entityCreated<ItemEvent>(EntityType.ITEM, data, userId)
 
     return updatedLibraryItem
   }
 
+  const data = deepDelete(libraryItem, columnsToDelete)
   await pubsub.entityUpdated<ItemEvent>(
     EntityType.ITEM,
     {
-      ...libraryItem,
+      ...data,
       id,
     } as ItemEvent,
     userId
@@ -1057,8 +1061,8 @@ export const createOrUpdateLibraryItem = async (
     return newLibraryItem
   }
 
-  const cleanedData = deepDelete(newLibraryItem, columnToDelete)
-  await pubsub.entityCreated<ItemEvent>(EntityType.ITEM, cleanedData, userId)
+  const data = deepDelete(newLibraryItem, columnsToDelete)
+  await pubsub.entityCreated<ItemEvent>(EntityType.ITEM, data, userId)
 
   return newLibraryItem
 }
@@ -1575,7 +1579,7 @@ export const filterItemEvents = (
         }
       }
       default:
-        throw new Error(`Unexpected field: ${field.name}`)
+        throw new RequiresSearchQueryError()
     }
   }
 
diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx
index 33e24c1ff..c994580d2 100644
--- a/packages/web/pages/settings/integrations.tsx
+++ b/packages/web/pages/settings/integrations.tsx
@@ -279,22 +279,7 @@ export default function Integrations(): JSX.Element {
           ],
         },
       },
-      {
-        icon: '/static/icons/notion.png',
-        title: 'Notion',
-        subText:
-          'Notion is an all-in-one workspace. Use our Notion integration to sync your Omnivore items to Notion.',
-        button: {
-          text: notion ? 'Settings' : 'Connect',
-          icon: <Link size={16} weight={'bold'} />,
-          style: notion ? 'ctaWhite' : 'ctaDarkYellow',
-          action: () => {
-            notion
-              ? router.push('/settings/integrations/notion')
-              : redirectToIntegration('NOTION')
-          },
-        },
-      },
+
       // {
       //   icon: '/static/icons/webhooks.svg',
       //   title: 'Webhooks',
@@ -331,11 +316,11 @@ export default function Integrations(): JSX.Element {
         subText:
           'Notion is an all-in-one workspace. Use our Notion integration to sync your Omnivore items to Notion.',
         button: {
-          text: isConnected('NOTION') ? 'Settings' : 'Connect',
+          text: notion ? 'Settings' : 'Connect',
           icon: <Link size={16} weight={'bold'} />,
-          style: isConnected('NOTION') ? 'ctaWhite' : 'ctaDarkYellow',
+          style: notion ? 'ctaWhite' : 'ctaDarkYellow',
           action: () => {
-            isConnected('NOTION')
+            notion
               ? router.push('/settings/integrations/notion')
               : redirectToIntegration('NOTION')
           },

From 6361759eeff7ffe3b3817c9308953e66631fff9c Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 27 Mar 2024 18:13:47 +0800
Subject: [PATCH 28/35] wrap text in actions

---
 packages/web/pages/settings/integrations/notion.tsx | 3 ---
 packages/web/pages/settings/rules.tsx               | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index 41c9c5292..4035d3136 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -16,7 +16,6 @@ import {
 import { setIntegrationMutation } from '../../../lib/networking/mutations/setIntegrationMutation'
 import { apiFetcher } from '../../../lib/networking/networkHelpers'
 import { useGetIntegrationQuery } from '../../../lib/networking/queries/useGetIntegrationQuery'
-import { applyStoredTheme } from '../../../lib/themeUpdater'
 import { showSuccessToast } from '../../../lib/toastHelpers'
 
 type FieldType = {
@@ -25,8 +24,6 @@ type FieldType = {
 }
 
 export default function Notion(): JSX.Element {
-  applyStoredTheme()
-
   const router = useRouter()
   const { integration: notion, revalidate } = useGetIntegrationQuery('notion')
 
diff --git a/packages/web/pages/settings/rules.tsx b/packages/web/pages/settings/rules.tsx
index 2b22592c8..c4fd22288 100644
--- a/packages/web/pages/settings/rules.tsx
+++ b/packages/web/pages/settings/rules.tsx
@@ -377,7 +377,7 @@ export default function Rules(): JSX.Element {
           {row.actions.map((action: RuleAction, index: number) => {
             const color = action.type.length > 5 ? 'geekblue' : 'green'
             return (
-              <Tag color={color} key={index}>
+              <Tag color={color} key={index} style={{ whiteSpace: 'unset' }}>
                 {action.type}(
                 {action.params.map((param: string, index: number) => {
                   const paramString = stringForActionParam(action.type, param)

From ffed903bb93a1c9ae4164e83f32c8c921dfd530c Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 27 Mar 2024 18:50:08 +0800
Subject: [PATCH 29/35] add data migration script

---
 .../0171.do.migrate_webhooks_and_exporter.sql  | 18 ++++++++++++++++++
 ...0171.undo.migrate_webhooks_and_exporter.sql |  7 +++++++
 2 files changed, 25 insertions(+)
 create mode 100755 packages/db/migrations/0171.do.migrate_webhooks_and_exporter.sql
 create mode 100755 packages/db/migrations/0171.undo.migrate_webhooks_and_exporter.sql

diff --git a/packages/db/migrations/0171.do.migrate_webhooks_and_exporter.sql b/packages/db/migrations/0171.do.migrate_webhooks_and_exporter.sql
new file mode 100755
index 000000000..6c0b691dc
--- /dev/null
+++ b/packages/db/migrations/0171.do.migrate_webhooks_and_exporter.sql
@@ -0,0 +1,18 @@
+-- Type: DO
+-- Name: migrate_webhooks_and_exporter
+-- Description: Migrate data from webhooks and exporters to rules
+
+BEGIN;
+
+-- Migrate webhooks to rules
+INSERT INTO omnivore.rules (user_id, name, filter, actions, enabled, created_at, updated_at, event_types)
+SELECT user_id, 'webhook', 'in:all', jsonb_build_array(jsonb_build_object('type', 'WEBHOOK', 'params', jsonb_build_array(url))), enabled, created_at, updated_at, event_types
+FROM omnivore.webhooks;
+
+-- Migrate exporters to rules
+INSERT INTO omnivore.rules (user_id, name, filter, actions, enabled, created_at, updated_at, event_types)
+SELECT user_id, 'export', 'in:all', jsonb_build_array(jsonb_build_object('type', 'EXPORT', 'params', jsonb_build_array(name))), enabled, created_at, updated_at, '{PAGE_CREATED,PAGE_UPDATED,HIGHLIGHT_CREATED,LABEL_CREATED}'
+FROM omnivore.integrations
+WHERE type = 'EXPORT';
+
+COMMIT;
diff --git a/packages/db/migrations/0171.undo.migrate_webhooks_and_exporter.sql b/packages/db/migrations/0171.undo.migrate_webhooks_and_exporter.sql
new file mode 100755
index 000000000..1a5755bb4
--- /dev/null
+++ b/packages/db/migrations/0171.undo.migrate_webhooks_and_exporter.sql
@@ -0,0 +1,7 @@
+-- Type: UNDO
+-- Name: migrate_webhooks_and_exporter
+-- Description: Migrate data from webhooks and exporters to rules
+
+BEGIN;
+
+COMMIT;

From a1a3901ef689ba902df60371cf2a778d8e53846a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 27 Mar 2024 19:06:20 +0800
Subject: [PATCH 30/35] update integration names

---
 .../db/migrations/0171.do.migrate_webhooks_and_exporter.sql     | 2 +-
 packages/web/pages/settings/rules.tsx                           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/db/migrations/0171.do.migrate_webhooks_and_exporter.sql b/packages/db/migrations/0171.do.migrate_webhooks_and_exporter.sql
index 6c0b691dc..270c10fa7 100755
--- a/packages/db/migrations/0171.do.migrate_webhooks_and_exporter.sql
+++ b/packages/db/migrations/0171.do.migrate_webhooks_and_exporter.sql
@@ -11,7 +11,7 @@ FROM omnivore.webhooks;
 
 -- Migrate exporters to rules
 INSERT INTO omnivore.rules (user_id, name, filter, actions, enabled, created_at, updated_at, event_types)
-SELECT user_id, 'export', 'in:all', jsonb_build_array(jsonb_build_object('type', 'EXPORT', 'params', jsonb_build_array(name))), enabled, created_at, updated_at, '{PAGE_CREATED,PAGE_UPDATED,HIGHLIGHT_CREATED,LABEL_CREATED}'
+SELECT user_id, 'export', 'in:all', jsonb_build_array(jsonb_build_object('type', 'EXPORT', 'params', jsonb_build_array(name))), enabled, created_at, updated_at, '{PAGE_CREATED,PAGE_UPDATED,HIGHLIGHT_CREATED,HIGHLIGHT_UPDATED,LABEL_CREATED}'
 FROM omnivore.integrations
 WHERE type = 'EXPORT';
 
diff --git a/packages/web/pages/settings/rules.tsx b/packages/web/pages/settings/rules.tsx
index c4fd22288..3cd9441e5 100644
--- a/packages/web/pages/settings/rules.tsx
+++ b/packages/web/pages/settings/rules.tsx
@@ -126,7 +126,7 @@ const CreateActionModal = (props: CreateActionModalProps): JSX.Element => {
   const { labels } = useGetLabelsQuery()
   const { integrations } = useGetIntegrationsQuery()
 
-  const integrationOptions = ['Notion', 'Readwise']
+  const integrationOptions = ['NOTION', 'READWISE']
 
   const isIntegrationEnabled = (integration: string): boolean => {
     return integrations.some(

From 45a5382d7c43734aeb493365f74834ce636d851a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 27 Mar 2024 21:27:05 +0800
Subject: [PATCH 31/35] do not send bad_request error if db failure

---
 packages/api/src/resolvers/rules/index.ts | 55 +++++++++--------------
 1 file changed, 22 insertions(+), 33 deletions(-)

diff --git a/packages/api/src/resolvers/rules/index.ts b/packages/api/src/resolvers/rules/index.ts
index 0efa67750..996a11ba8 100644
--- a/packages/api/src/resolvers/rules/index.ts
+++ b/packages/api/src/resolvers/rules/index.ts
@@ -7,7 +7,6 @@ import {
   MutationSetRuleArgs,
   QueryRulesArgs,
   RulesError,
-  RulesErrorCode,
   RulesSuccess,
   SetRuleError,
   SetRuleErrorCode,
@@ -21,53 +20,43 @@ export const setRuleResolver = authorized<
   SetRuleSuccess,
   SetRuleError,
   MutationSetRuleArgs
->(async (_, { input }, { authTrx, uid, log }) => {
+>(async (_, { input }, { authTrx, uid }) => {
   try {
     // validate filter
     parseSearchQuery(input.filter)
-
-    const rule = await authTrx((t) =>
-      t.getRepository(Rule).save({
-        ...input,
-        id: input.id || undefined,
-        user: { id: uid },
-      })
-    )
-
-    return {
-      rule,
-    }
   } catch (error) {
-    log.error('Error setting rules', error)
-
     return {
       errorCodes: [SetRuleErrorCode.BadRequest],
     }
   }
+
+  const rule = await authTrx((t) =>
+    t.getRepository(Rule).save({
+      ...input,
+      id: input.id || undefined,
+      user: { id: uid },
+    })
+  )
+
+  return {
+    rule,
+  }
 })
 
 export const rulesResolver = authorized<
   RulesSuccess,
   RulesError,
   QueryRulesArgs
->(async (_, { enabled }, { authTrx, log, uid }) => {
-  try {
-    const rules = await authTrx((t) =>
-      t.getRepository(Rule).findBy({
-        user: { id: uid },
-        enabled: enabled === null ? undefined : enabled,
-      })
-    )
+>(async (_, { enabled }, { authTrx, uid }) => {
+  const rules = await authTrx((t) =>
+    t.getRepository(Rule).findBy({
+      user: { id: uid },
+      enabled: enabled === null ? undefined : enabled,
+    })
+  )
 
-    return {
-      rules,
-    }
-  } catch (error) {
-    log.error('Error getting rules', error)
-
-    return {
-      errorCodes: [RulesErrorCode.BadRequest],
-    }
+  return {
+    rules,
   }
 })
 

From 3983f2b3a252e6c72e6af5840e6d2074898daf88 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 4 Apr 2024 12:27:15 +0800
Subject: [PATCH 32/35] resolve conflicts after rebasing

---
 packages/web/pages/settings/integrations/notion.tsx | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index 4035d3136..1ce4f32d2 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -96,10 +96,6 @@ export default function Notion(): JSX.Element {
     console.log('Failed:', errorInfo)
   }
 
-  // const onDataChange = (value: Array<CheckboxValueType>) => {
-  //   form.setFieldsValue({ properties: value.map((v) => v.toString()) })
-  // }
-
   const exportToNotion = useCallback(async () => {
     if (exporting) {
       messageApi.warning('Exporting process is already running.')
@@ -213,14 +209,6 @@ export default function Notion(): JSX.Element {
                   <Input />
                 </Form.Item>
 
-                <Form.Item<FieldType>
-                  label="Notion Database Id"
-                  name="parentDatabaseId"
-                  hidden
-                >
-                  <Input disabled />
-                </Form.Item>
-
                 <Form.Item
                   wrapperCol={{ offset: 6 }}
                   style={{ marginTop: '30px' }}

From 8d11096142170e6876ab9a4b35ca4cf5cc39db34 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 4 Apr 2024 12:38:41 +0800
Subject: [PATCH 33/35] fix tests

---
 packages/api/test/services/library_item.test.ts | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/packages/api/test/services/library_item.test.ts b/packages/api/test/services/library_item.test.ts
index c567db29d..20935b63b 100644
--- a/packages/api/test/services/library_item.test.ts
+++ b/packages/api/test/services/library_item.test.ts
@@ -5,10 +5,11 @@ import { parseSearchQuery } from '../../src/utils/search'
 
 describe('filterItemEvents', () => {
   it('returns events if there are quotation marks in the subscription name', () => {
-    const query = 'subscription:"Best \\\"Omnivore\\\""'
+    const query = 'subscription:"Best \\"Omnivore\\""'
     const ast = parseSearchQuery(query)
     const events = [
       {
+        id: '1',
         subscription: 'Best "Omnivore"',
       },
     ]
@@ -21,6 +22,7 @@ describe('filterItemEvents', () => {
     const ast = parseSearchQuery(query)
     const events = [
       {
+        id: '1',
         subscription: 'Substack',
       },
     ]
@@ -33,6 +35,7 @@ describe('filterItemEvents', () => {
     const ast = parseSearchQuery(query)
     const events = [
       {
+        id: '1',
         siteName: 'YouTube',
       },
     ]
@@ -45,6 +48,7 @@ describe('filterItemEvents', () => {
     const ast = parseSearchQuery(query)
     const events = [
       {
+        id: '1',
         siteName: 'Der Standard',
       },
     ]
@@ -57,6 +61,7 @@ describe('filterItemEvents', () => {
     const ast = parseSearchQuery(query)
     const events = [
       {
+        id: '1',
         siteName: 'Stack Overflow',
         originalUrl: 'https://stackoverflow.com/questions/123',
       },
@@ -70,6 +75,7 @@ describe('filterItemEvents', () => {
     const ast = parseSearchQuery(query)
     const events = [
       {
+        id: '1',
         siteName: 'Stack Overflow',
         originalUrl: 'https://stackoverflow.com/questions/123',
       },
@@ -83,6 +89,7 @@ describe('filterItemEvents', () => {
     const ast = parseSearchQuery(query)
     const events = [
       {
+        id: '1',
         labelNames: ['foo'],
       },
     ]
@@ -91,10 +98,11 @@ describe('filterItemEvents', () => {
   })
 
   it('returns events if labels contain quotation marks', () => {
-    const query = 'label:"foo \\\"bar\\\""'
+    const query = 'label:"foo \\"bar\\""'
     const ast = parseSearchQuery(query)
     const events = [
       {
+        id: '1',
         labelNames: ['foo "bar"'],
       },
     ]
@@ -107,6 +115,7 @@ describe('filterItemEvents', () => {
     const ast = parseSearchQuery(query)
     const events = [
       {
+        id: '1',
         labelNames: ['foo bar'],
       },
     ]
@@ -119,6 +128,7 @@ describe('filterItemEvents', () => {
     const ast = parseSearchQuery(query)
     const events = [
       {
+        id: '1',
         labelNames: ['foo'],
       },
     ]
@@ -131,6 +141,7 @@ describe('filterItemEvents', () => {
     const ast = parseSearchQuery(query)
     const events = [
       {
+        id: '1',
         labelNames: ['foo', 'bar'],
       },
     ]

From 58708a507e9d622f9d1fbdfb0b5cf15a9d1c6ea3 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 4 Apr 2024 18:22:18 +0800
Subject: [PATCH 34/35] use old event format for the compatibility with the old
 webhooks

---
 packages/api/src/jobs/trigger_rule.ts | 29 ++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts
index 76fcbd3cc..dcc1c902f 100644
--- a/packages/api/src/jobs/trigger_rule.ts
+++ b/packages/api/src/jobs/trigger_rule.ts
@@ -91,17 +91,32 @@ const sendNotification = async (obj: RuleActionObj) => {
 const sendToWebhook = async (obj: RuleActionObj) => {
   const [url] = obj.action.params
 
-  let event = obj.ruleEventType.toString()
-  // rename LABEL_CREATED to LABEL_ATTACHED
-  if (obj.ruleEventType === RuleEventType.LabelCreated) {
-    event = 'LABEL_ATTACHED'
+  const [type, action] = obj.ruleEventType.toString().toLowerCase().split('_')
+
+  // use old event format for the compatibility with the old webhooks
+  let event
+  if (type === 'page') {
+    event = obj.data
+  } else if (type === 'label') {
+    event = {
+      labels: obj.data.labels,
+      pageId: obj.data.id,
+    }
+  } else {
+    if (!obj.data.highlights) {
+      return
+    }
+
+    event = {
+      ...obj.data.highlights[0],
+      pageId: obj.data.id,
+    }
   }
 
   const data = {
-    event,
-    item: obj.data,
+    action,
     userId: obj.userId,
-    timestamp: Date.now(),
+    [type]: event,
   }
 
   logger.info(`triggering webhook: ${url}`)

From 906b6c1ea5a6e38e2371e6048d737a1879ecc2e8 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 4 Apr 2024 18:53:10 +0800
Subject: [PATCH 35/35] fix web

---
 packages/api/src/services/integrations/notion.ts    | 2 +-
 packages/api/src/services/library_item.ts           | 2 +-
 packages/web/pages/settings/integrations/notion.tsx | 5 -----
 3 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 150ec881e..f391456f3 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -366,7 +366,7 @@ export class NotionClient implements IntegrationClient {
           }
 
           // create the page
-          return this.createPage(notionPage)
+          return await this.createPage(notionPage)
         } catch (error) {
           logger.error(error)
           return false
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index fb8ae9c4f..9adcde7b0 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -744,7 +744,7 @@ export const findRecentLibraryItems = async (
           'library_item.user_id = :userId AND library_item.state = :state',
           { userId, state: LibraryItemState.Succeeded }
         )
-        .orderBy('library_item.saved_at', 'DESC', 'NULLS LAST')
+        .orderBy('library_item.savedAt', 'DESC', 'NULLS LAST')
         .take(limit)
         .skip(offset)
         .getMany(),
diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index 1ce4f32d2..db15ec21c 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -102,11 +102,6 @@ export default function Notion(): JSX.Element {
       return
     }
 
-    if (!notion.settings?.parentPageId) {
-      messageApi.error('Please set the Notion page id first.')
-      return
-    }
-
     try {
       const task = await exportToIntegrationMutation(notion.id)
       // long polling to check the status of the task in every 10 seconds