From 208a5895ef165a45a6cfe1120343e99c15d313b4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 18 Mar 2024 11:06:24 +0800 Subject: [PATCH] remove unnecessary fields from item events --- packages/api/src/pubsub.ts | 37 ++------------ .../api/src/services/integrations/notion.ts | 2 +- .../api/src/services/integrations/readwise.ts | 4 +- packages/api/src/services/library_item.ts | 49 +++++++++++++++---- packages/api/src/utils/helpers.ts | 4 -- 5 files changed, 47 insertions(+), 49 deletions(-) diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index f13bfe5e4..53f95878a 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -3,6 +3,7 @@ import express from 'express' import { RuleEventType } from './entity/rule' import { env } from './env' import { ReportType } from './generated/graphql' +import { FeatureName, findFeatureByName } from './services/features' import { Merge } from './util' import { enqueueAISummarizeJob, @@ -11,13 +12,7 @@ import { enqueueTriggerRuleJob, enqueueWebhookJob, } from './utils/createTask' -import { deepDelete } from './utils/helpers' import { buildLogger } from './utils/logger' -import { - FeatureName, - findFeatureByName, - getFeatureName, -} from './services/features' import { processYouTubeVideo } from './jobs/process-youtube-video' const logger = buildLogger('pubsub') @@ -42,8 +37,6 @@ const isYouTubeVideoURL = (url: string | undefined): boolean => { } export const createPubSubClient = (): PubsubClient => { - const fieldsToDelete = ['user'] as const - const publish = (topicName: string, msg: Buffer): Promise => { if (env.dev.isLocal) { logger.info(`Publishing ${topicName}: ${msg.toString()}`) @@ -93,11 +86,6 @@ export const createPubSubClient = (): PubsubClient => { libraryItemIds: [libraryItemId], }) - const cleanData = deepDelete( - data as EntityData & Record, - [...fieldsToDelete] - ) - await enqueueWebhookJob({ userId, type, @@ -121,11 +109,6 @@ export const createPubSubClient = (): PubsubClient => { libraryItemId, }) } - - return publish( - 'entityCreated', - Buffer.from(JSON.stringify({ type, userId, ...cleanData })) - ) }, entityUpdated: async >( type: EntityType, @@ -148,32 +131,20 @@ export const createPubSubClient = (): PubsubClient => { libraryItemIds: [libraryItemId], }) - const cleanData = deepDelete( - data as EntityData & Record, - [...fieldsToDelete] - ) - await enqueueWebhookJob({ userId, type, action: 'updated', data, }) - - return publish( - 'entityUpdated', - Buffer.from(JSON.stringify({ type, userId, ...cleanData })) - ) }, - entityDeleted: ( + entityDeleted: async ( type: EntityType, id: string, userId: string ): Promise => { - return publish( - 'entityDeleted', - Buffer.from(JSON.stringify({ type, id, userId })) - ) + logger.info(`entityDeleted: ${type} ${id} ${userId}`) + await Promise.resolve() }, reportSubmitted: ( submitterId: string, diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 4d823e031..de517dca6 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -5,8 +5,8 @@ import { Integration } from '../../entity/integration' import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' import { Merge } from '../../util' -import { highlightUrl } from '../../utils/helpers' import { logger } from '../../utils/logger' +import { getHighlightUrl } from '../highlights' import { IntegrationClient } from './integration' type AnnotationColor = diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index dfc5b43db..0c204b630 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -1,7 +1,7 @@ import axios from 'axios' import { LibraryItem } from '../../entity/library_item' -import { highlightUrl } from '../../utils/helpers' import { logger } from '../../utils/logger' +import { getHighlightUrl } from '../highlights' import { IntegrationClient } from './integration' interface ReadwiseHighlight { @@ -98,7 +98,7 @@ export class ReadwiseClient implements IntegrationClient { text: highlight.quote, title: item.title, author: item.author || undefined, - highlight_url: highlightUrl(item.slug, highlight.id), + highlight_url: getHighlightUrl(item.slug, highlight.id), highlighted_at: new Date(highlight.createdAt).toISOString(), category, image_url: item.thumbnail || undefined, diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 52d44f7ff..7d13e2fa8 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -29,6 +29,26 @@ import { logger } from '../utils/logger' import { parseSearchQuery } from '../utils/search' import { addLabelsToLibraryItem } from './labels' +type ItemEvent = { libraryItemId: string; userId: string } +type IgnoredFields = + | 'user' + | 'uploadFile' + | 'labelNames' + | 'highlightAnnotations' + | 'previewContentType' + | 'links' + | 'recommenderNames' + | 'textContentHash' + +type CreateItemEvent = Merge< + Omit, IgnoredFields>, + ItemEvent +> +type UpdateItemEvent = Merge< + Omit, IgnoredFields>, + ItemEvent +> + enum ReadFilter { ALL = 'all', READ = 'read', @@ -833,19 +853,32 @@ export const updateLibraryItem = async ( userId ) - if (skipPubSub) { + if (skipPubSub || libraryItem.state === LibraryItemState.Processing) { return updatedLibraryItem } - await pubsub.entityUpdated>( + if (libraryItem.state === LibraryItemState.Succeeded) { + // send create event if the item was created + await pubsub.entityCreated( + EntityType.PAGE, + { + ...updatedLibraryItem, + libraryItemId: id, + userId, + }, + userId + ) + + return updatedLibraryItem + } + + await pubsub.entityUpdated( EntityType.PAGE, { ...libraryItem, id, libraryItemId: id, - // don't send original content and readable content - originalContent: undefined, - readableContent: undefined, + userId, }, userId ) @@ -999,14 +1032,12 @@ export const createOrUpdateLibraryItem = async ( return newLibraryItem } - await pubsub.entityCreated>( + await pubsub.entityCreated( EntityType.PAGE, { ...newLibraryItem, libraryItemId: newLibraryItem.id, - // don't send original content and readable content - originalContent: undefined, - readableContent: undefined, + userId, }, userId ) diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 9e3e18cd3..d2fec6520 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -10,7 +10,6 @@ import { Highlight as HighlightData } from '../entity/highlight' import { LibraryItem, LibraryItemState } from '../entity/library_item' import { Recommendation as RecommendationData } from '../entity/recommendation' import { RegistrationType, User } from '../entity/user' -import { env } from '../env' import { Article, ArticleSavingRequest, @@ -404,6 +403,3 @@ export const setRecentlySavedItemInRedis = async ( }) } } - -export const highlightUrl = (slug: string, highlightId: string): string => - `${env.client.url}/me/${slug}#${highlightId}`