remove unnecessary fields from item events

packages/api/src/pubsub.ts

@@ -3,6 +3,7 @@ import express from 'express'
 import { RuleEventType } from './entity/rule'
 import { env } from './env'
 import { ReportType } from './generated/graphql'
+import { FeatureName, findFeatureByName } from './services/features'
 import { Merge } from './util'
 import {
   enqueueAISummarizeJob,
@@ -11,13 +12,7 @@ import {
   enqueueTriggerRuleJob,
   enqueueWebhookJob,
 } from './utils/createTask'
-import { deepDelete } from './utils/helpers'
 import { buildLogger } from './utils/logger'
-import {
-  FeatureName,
-  findFeatureByName,
-  getFeatureName,
-} from './services/features'
 import { processYouTubeVideo } from './jobs/process-youtube-video'
 
 const logger = buildLogger('pubsub')
@@ -42,8 +37,6 @@ const isYouTubeVideoURL = (url: string | undefined): boolean => {
 }
 
 export const createPubSubClient = (): PubsubClient => {
-  const fieldsToDelete = ['user'] as const
-
   const publish = (topicName: string, msg: Buffer): Promise<void> => {
     if (env.dev.isLocal) {
       logger.info(`Publishing ${topicName}: ${msg.toString()}`)
@@ -93,11 +86,6 @@ export const createPubSubClient = (): PubsubClient => {
         libraryItemIds: [libraryItemId],
       })
 
-      const cleanData = deepDelete(
-        data as EntityData<T> & Record<typeof fieldsToDelete[number], unknown>,
-        [...fieldsToDelete]
-      )
-
       await enqueueWebhookJob({
         userId,
         type,
@@ -121,11 +109,6 @@ export const createPubSubClient = (): PubsubClient => {
           libraryItemId,
         })
       }
-
-      return publish(
-        'entityCreated',
-        Buffer.from(JSON.stringify({ type, userId, ...cleanData }))
-      )
     },
     entityUpdated: async <T extends Record<string, any>>(
       type: EntityType,
@@ -148,32 +131,20 @@ export const createPubSubClient = (): PubsubClient => {
         libraryItemIds: [libraryItemId],
       })
 
-      const cleanData = deepDelete(
-        data as EntityData<T> & Record<typeof fieldsToDelete[number], unknown>,
-        [...fieldsToDelete]
-      )
-
       await enqueueWebhookJob({
         userId,
         type,
         action: 'updated',
         data,
       })
-
-      return publish(
-        'entityUpdated',
-        Buffer.from(JSON.stringify({ type, userId, ...cleanData }))
-      )
     },
-    entityDeleted: (
+    entityDeleted: async (
       type: EntityType,
       id: string,
       userId: string
     ): Promise<void> => {
-      return publish(
-        'entityDeleted',
-        Buffer.from(JSON.stringify({ type, id, userId }))
-      )
+      logger.info(`entityDeleted: ${type} ${id} ${userId}`)
+      await Promise.resolve()
     },
     reportSubmitted: (
       submitterId: string,

packages/api/src/services/integrations/notion.ts

@@ -5,8 +5,8 @@ import { Integration } from '../../entity/integration'
 import { LibraryItem } from '../../entity/library_item'
 import { env } from '../../env'
 import { Merge } from '../../util'
-import { highlightUrl } from '../../utils/helpers'
 import { logger } from '../../utils/logger'
+import { getHighlightUrl } from '../highlights'
 import { IntegrationClient } from './integration'
 
 type AnnotationColor =

packages/api/src/services/integrations/readwise.ts

@@ -1,7 +1,7 @@
 import axios from 'axios'
 import { LibraryItem } from '../../entity/library_item'
-import { highlightUrl } from '../../utils/helpers'
 import { logger } from '../../utils/logger'
+import { getHighlightUrl } from '../highlights'
 import { IntegrationClient } from './integration'
 
 interface ReadwiseHighlight {
@@ -98,7 +98,7 @@ export class ReadwiseClient implements IntegrationClient {
           text: highlight.quote,
           title: item.title,
           author: item.author || undefined,
-          highlight_url: highlightUrl(item.slug, highlight.id),
+          highlight_url: getHighlightUrl(item.slug, highlight.id),
           highlighted_at: new Date(highlight.createdAt).toISOString(),
           category,
           image_url: item.thumbnail || undefined,

packages/api/src/services/library_item.ts

@@ -29,6 +29,26 @@ import { logger } from '../utils/logger'
 import { parseSearchQuery } from '../utils/search'
 import { addLabelsToLibraryItem } from './labels'
 
+type ItemEvent = { libraryItemId: string; userId: string }
+type IgnoredFields =
+  | 'user'
+  | 'uploadFile'
+  | 'labelNames'
+  | 'highlightAnnotations'
+  | 'previewContentType'
+  | 'links'
+  | 'recommenderNames'
+  | 'textContentHash'
+
+type CreateItemEvent = Merge<
+  Omit<DeepPartial<LibraryItem>, IgnoredFields>,
+  ItemEvent
+>
+type UpdateItemEvent = Merge<
+  Omit<QueryDeepPartialEntity<LibraryItem>, IgnoredFields>,
+  ItemEvent
+>
+
 enum ReadFilter {
   ALL = 'all',
   READ = 'read',
@@ -833,19 +853,32 @@ export const updateLibraryItem = async (
     userId
   )
 
-  if (skipPubSub) {
+  if (skipPubSub || libraryItem.state === LibraryItemState.Processing) {
     return updatedLibraryItem
   }
 
-  await pubsub.entityUpdated<QueryDeepPartialEntity<LibraryItem>>(
+  if (libraryItem.state === LibraryItemState.Succeeded) {
+    // send create event if the item was created
+    await pubsub.entityCreated<CreateItemEvent>(
+      EntityType.PAGE,
+      {
+        ...updatedLibraryItem,
+        libraryItemId: id,
+        userId,
+      },
+      userId
+    )
+
+    return updatedLibraryItem
+  }
+
+  await pubsub.entityUpdated<UpdateItemEvent>(
     EntityType.PAGE,
     {
       ...libraryItem,
       id,
       libraryItemId: id,
-      // don't send original content and readable content
-      originalContent: undefined,
-      readableContent: undefined,
+      userId,
     },
     userId
   )
@@ -999,14 +1032,12 @@ export const createOrUpdateLibraryItem = async (
     return newLibraryItem
   }
 
-  await pubsub.entityCreated<DeepPartial<LibraryItem>>(
+  await pubsub.entityCreated<CreateItemEvent>(
     EntityType.PAGE,
     {
       ...newLibraryItem,
       libraryItemId: newLibraryItem.id,
-      // don't send original content and readable content
-      originalContent: undefined,
-      readableContent: undefined,
+      userId,
     },
     userId
   )

packages/api/src/utils/helpers.ts

@@ -10,7 +10,6 @@ import { Highlight as HighlightData } from '../entity/highlight'
 import { LibraryItem, LibraryItemState } from '../entity/library_item'
 import { Recommendation as RecommendationData } from '../entity/recommendation'
 import { RegistrationType, User } from '../entity/user'
-import { env } from '../env'
 import {
   Article,
   ArticleSavingRequest,
@@ -404,6 +403,3 @@ export const setRecentlySavedItemInRedis = async (
     })
   }
 }
-
-export const highlightUrl = (slug: string, highlightId: string): string =>
-  `${env.client.url}/me/${slug}#${highlightId}`