diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts index 88cc65144..158df8168 100644 --- a/packages/api/src/jobs/trigger_rule.ts +++ b/packages/api/src/jobs/trigger_rule.ts @@ -18,14 +18,12 @@ import { logger } from '../utils/logger' import { parseSearchQuery } from '../utils/search' export interface TriggerRuleJobData { - libraryItemId: string userId: string ruleEventType: RuleEventType data: ItemEvent } interface RuleActionObj { - libraryItemId: string userId: string action: RuleAction data: ItemEvent | LibraryItem @@ -39,21 +37,16 @@ const readingProgressDataSource = new ReadingProgressDataSource() const addLabels = async (obj: RuleActionObj) => { const labelIds = obj.action.params - return addLabelsToLibraryItem( - labelIds, - obj.libraryItemId, - obj.userId, - 'system' - ) + return addLabelsToLibraryItem(labelIds, obj.data.id, obj.userId, 'system') } const deleteLibraryItem = async (obj: RuleActionObj) => { - return softDeleteLibraryItem(obj.libraryItemId, obj.userId) + return softDeleteLibraryItem(obj.data.id, obj.userId) } const archivePage = async (obj: RuleActionObj) => { return updateLibraryItem( - obj.libraryItemId, + obj.data.id, { archivedAt: new Date(), state: LibraryItemState.Archived }, obj.userId, undefined, @@ -64,7 +57,7 @@ const archivePage = async (obj: RuleActionObj) => { const markPageAsRead = async (obj: RuleActionObj) => { return readingProgressDataSource.updateReadingProgress( obj.userId, - obj.libraryItemId, + obj.data.id, { readingProgressPercent: 100, readingProgressTopPercent: 100, @@ -82,7 +75,7 @@ const sendNotification = async (obj: RuleActionObj) => { } const data = { folder: item.folder?.toString() || 'inbox', - libraryItemId: obj.libraryItemId, + libraryItemId: obj.data.id, } return sendPushNotifications(obj.userId, message, 'rule', data) @@ -93,7 +86,9 @@ const sendToWebhook = async (obj: RuleActionObj) => { const data = { event: obj.ruleEventType, - data: obj.data, + item: obj.data, + userId: obj.userId, + timestamp: Date.now(), } logger.info(`triggering webhook: ${url}`) @@ -129,7 +124,6 @@ const getRuleAction = ( } const triggerActions = async ( - libraryItemId: string, userId: string, rules: Rule[], data: ItemEvent, @@ -184,7 +178,6 @@ const triggerActions = async ( } const actionObj: RuleActionObj = { - libraryItemId, userId, action, data: results[0], @@ -203,7 +196,7 @@ const triggerActions = async ( } export const triggerRule = async (jobData: TriggerRuleJobData) => { - const { userId, ruleEventType, data, libraryItemId } = jobData + const { userId, ruleEventType, data } = jobData // get rules by calling api const rules = await findEnabledRules(userId, ruleEventType) @@ -212,7 +205,7 @@ export const triggerRule = async (jobData: TriggerRuleJobData) => { return false } - await triggerActions(libraryItemId, userId, rules, data, ruleEventType) + await triggerActions(userId, rules, data, ruleEventType) return true } diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index 55da4e7c9..5e761fe1b 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -11,7 +11,7 @@ import { import { buildLogger } from './utils/logger' import { isYouTubeVideoURL } from './utils/youtube' -export type BaseEntityEvent = { id: string; userId: string } +export type EntityEvent = { id: string } const logger = buildLogger('pubsub') @@ -47,23 +47,21 @@ export const createPubSubClient = (): PubsubClient => { Buffer.from(JSON.stringify({ userId, email, name, username })) ) }, - entityCreated: async ( + entityCreated: async ( type: EntityType, data: T, - userId: string, - libraryItemId: string + userId: string ): Promise => { // queue trigger rule job await enqueueTriggerRuleJob({ ruleEventType: `${type.toUpperCase()}_CREATED` as RuleEventType, data, userId, - libraryItemId, }) // queue export item job await enqueueExportItem({ userId, - libraryItemIds: [libraryItemId], + libraryItemIds: [data.id], }) if (type === EntityType.ITEM) { @@ -81,29 +79,27 @@ export const createPubSubClient = (): PubsubClient => { if (isItemWithURL(data) && isYouTubeVideoURL(data['originalUrl'])) { await enqueueProcessYouTubeVideo({ userId, - libraryItemId, + libraryItemId: data.id, }) } } }, - entityUpdated: async ( + entityUpdated: async ( type: EntityType, data: T, - userId: string, - libraryItemId: string + userId: string ): Promise => { // queue trigger rule job await enqueueTriggerRuleJob({ userId, ruleEventType: RuleEventType.PageUpdated, - libraryItemId, data, }) // queue export item job await enqueueExportItem({ userId, - libraryItemIds: [libraryItemId], + libraryItemIds: [data.id], }) }, entityDeleted: async ( @@ -144,17 +140,15 @@ export interface PubsubClient { name: string, username: string ) => Promise - entityCreated: ( + entityCreated: ( type: EntityType, data: T, - userId: string, - libraryItemId: string + userId: string ) => Promise - entityUpdated: ( + entityUpdated: ( type: EntityType, data: T, - userId: string, - libraryItemId: string + userId: string ) => Promise entityDeleted: (type: EntityType, id: string, userId: string) => Promise reportSubmitted( diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 624c5b2da..43d710d36 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -9,12 +9,12 @@ import { createPubSubClient, EntityType } from '../pubsub' import { authTrx } from '../repository' import { highlightRepository } from '../repository/highlight' import { enqueueUpdateHighlight } from '../utils/createTask' +import { deepDelete } from '../utils/helpers' import { ItemEvent } from './library_item' -export type HighlightEvent = Omit< - DeepPartial, - 'user' | 'userId' | 'sharedAt' -> +const columnToDelete = ['user', 'sharedAt'] as const +type ColumnToDeleteType = typeof columnToDelete[number] +export type HighlightEvent = Omit, ColumnToDeleteType> export const getHighlightLocation = (patch: string): number | undefined => { const dmp = new diff_match_patch() @@ -58,11 +58,11 @@ export const createHighlight = async ( userId ) + const cleanData = deepDelete(newHighlight, columnToDelete) await pubsub.entityCreated( EntityType.HIGHLIGHT, - { id: libraryItemId, highlights: [newHighlight], userId }, - userId, - libraryItemId + { id: libraryItemId, highlights: [cleanData] }, + userId ) await enqueueUpdateHighlight({ @@ -108,9 +108,8 @@ export const mergeHighlights = async ( await pubsub.entityCreated( EntityType.HIGHLIGHT, - { id: libraryItemId, highlights: [newHighlight], userId }, - userId, - libraryItemId + { id: libraryItemId, highlights: [newHighlight] }, + userId ) await enqueueUpdateHighlight({ @@ -143,9 +142,8 @@ export const updateHighlight = async ( const libraryItemId = updatedHighlight.libraryItem.id await pubsub.entityUpdated( EntityType.HIGHLIGHT, - { id: libraryItemId, highlights: [highlight], userId } as ItemEvent, - userId, - libraryItemId + { id: libraryItemId, highlights: [highlight] } as ItemEvent, + userId ) await enqueueUpdateHighlight({ diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index 4a23f30c0..84ebc7c23 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -6,11 +6,14 @@ import { createPubSubClient, EntityType, PubsubClient } from '../pubsub' import { authTrx } from '../repository' import { CreateLabelInput, labelRepository } from '../repository/label' import { bulkEnqueueUpdateLabels } from '../utils/createTask' +import { deepDelete } from '../utils/helpers' import { logger } from '../utils/logger' import { findHighlightById } from './highlights' import { findLibraryItemIdsByLabelId, ItemEvent } from './library_item' -export type LabelEvent = Omit, 'description' | 'createdAt'> +const columnToDelete = ['description', 'createdAt'] as const +type ColumnToDeleteType = typeof columnToDelete[number] +export type LabelEvent = Omit, ColumnToDeleteType> // const batchGetLabelsFromLinkIds = async ( // linkIds: readonly string[] @@ -138,9 +141,11 @@ export const saveLabelsInLibraryItem = async ( // create pubsub event await pubsub.entityCreated( EntityType.LABEL, - { id: libraryItemId, labels, userId }, - userId, - libraryItemId + { + id: libraryItemId, + labels: labels.map((l) => deepDelete(l, columnToDelete)), + }, + userId ) } @@ -209,9 +214,16 @@ export const saveLabelsInHighlight = async ( // create pubsub event await pubsub.entityCreated( EntityType.LABEL, - { id: libraryItemId, highlights: [{ id: highlightId, labels }], userId }, - userId, - libraryItemId + { + id: libraryItemId, + highlights: [ + { + id: highlightId, + labels: labels.map((l) => deepDelete(l, columnToDelete)), + }, + ], + }, + userId ) // update labels in library item diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index b26ebd010..4cc0806dd 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -15,7 +15,7 @@ import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' import { LibraryItem, LibraryItemState } from '../entity/library_item' import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql' -import { BaseEntityEvent, createPubSubClient, EntityType } from '../pubsub' +import { createPubSubClient, EntityEvent, EntityType } from '../pubsub' import { redisDataSource } from '../redis_data_source' import { authTrx, @@ -24,27 +24,29 @@ import { queryBuilderToRawSql, } from '../repository' import { libraryItemRepository } from '../repository/library_item' -import { Merge } from '../util' -import { setRecentlySavedItemInRedis } from '../utils/helpers' +import { Merge, PickTuple } from '../util' +import { deepDelete, setRecentlySavedItemInRedis } from '../utils/helpers' import { logger } from '../utils/logger' import { parseSearchQuery } from '../utils/search' import { HighlightEvent } from './highlights' import { addLabelsToLibraryItem, LabelEvent } from './labels' -type IgnoredFields = - | 'user' - | 'uploadFile' - | 'previewContentType' - | 'links' - | 'textContentHash' +const columnToDelete = [ + 'user', + 'uploadFile', + 'previewContentType', + 'links', + 'textContentHash', +] as const +type ColumnToDeleteType = typeof columnToDelete[number] type ItemBaseEvent = Merge< - Omit, IgnoredFields>, + Omit, ColumnToDeleteType>, { labels?: LabelEvent[] highlights?: HighlightEvent[] } > -export type ItemEvent = Merge +export type ItemEvent = Merge export class RequiresSearchQueryError extends Error { constructor() { @@ -884,22 +886,17 @@ export const updateLibraryItem = async ( } if (libraryItem.state === LibraryItemState.Succeeded) { + const cleanedData = deepDelete(updatedLibraryItem, columnToDelete) // send create event if the item was created - await pubsub.entityCreated( - EntityType.ITEM, - { ...updatedLibraryItem, userId }, - userId, - id - ) + await pubsub.entityCreated(EntityType.ITEM, cleanedData, userId) return updatedLibraryItem } await pubsub.entityUpdated( EntityType.ITEM, - { ...libraryItem, id, userId } as ItemEvent, - userId, - id + { ...libraryItem, id } as ItemEvent, + userId ) return updatedLibraryItem @@ -958,12 +955,7 @@ export const updateLibraryItemReadingProgress = async ( } const updatedItem = result[0][0] - await pubsub.entityUpdated( - EntityType.ITEM, - { ...updatedItem, id, userId }, - userId, - id - ) + await pubsub.entityUpdated(EntityType.ITEM, updatedItem, userId) return updatedItem } @@ -1059,12 +1051,8 @@ export const createOrUpdateLibraryItem = async ( return newLibraryItem } - await pubsub.entityCreated( - EntityType.ITEM, - { ...newLibraryItem, userId }, - userId, - newLibraryItem.id - ) + const cleanedData = deepDelete(newLibraryItem, columnToDelete) + await pubsub.entityCreated(EntityType.ITEM, cleanedData, userId) return newLibraryItem } @@ -1346,7 +1334,8 @@ export const filterItemEvents = ( subscription: /^subscription(s)?$/i, } - const matchingKeyword = Object.keys(keywordRegexMap).find((keyword) => + const keys = Object.keys(keywordRegexMap) + const matchingKeyword = keys.find((keyword) => value.match(keywordRegexMap[keyword]) ) @@ -1354,7 +1343,9 @@ export const filterItemEvents = ( throw new Error(`Unexpected keyword: ${value}`) } - const eventValue = event[matchingKeyword as keyof ItemEvent] + const eventValue = (event as PickTuple)[ + matchingKeyword + ] return !eventValue || (Array.isArray(eventValue) && eventValue.length === 0) } @@ -1484,8 +1475,10 @@ export const filterItemEvents = ( const start = startDate ?? new Date(0) const end = endDate ?? new Date() - const key = `${field.name.toLowerCase()}At` as keyof ItemEvent - const eventValue = event[key] as Date + const key = `${field.name.toLowerCase()}At` + const eventValue = event[ + key as 'readAt' | 'updatedAt' | 'publishedAt' + ] as Date return eventValue >= start && eventValue <= end } diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index d2fec6520..785e5d5ef 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -367,7 +367,10 @@ export const cleanUrl = (url: string) => { }) } -export const deepDelete = (obj: T, keys: K[]) => { +export const deepDelete = ( + obj: T, + keys: readonly K[] +) => { // make a copy of the object const copy = { ...obj }