prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

delete unneccessary properties from webhook data

Browse source
This commit is contained in:
Hongbo Wu 2024-03-22 10:45:20 +08:00
parent 88d6455222
commit e2b3235046
6 changed files with 85 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
packages/api/src/jobs/trigger_rule.ts
View file

@ -18,14 +18,12 @@ import { logger } from '../utils/logger'
import { parseSearchQuery } from '../utils/search'
export interface TriggerRuleJobData {
libraryItemId: string
userId: string
ruleEventType: RuleEventType
data: ItemEvent
}
interface RuleActionObj {
libraryItemId: string
userId: string
action: RuleAction
data: ItemEvent | LibraryItem
@ -39,21 +37,16 @@ const readingProgressDataSource = new ReadingProgressDataSource()
const addLabels = async (obj: RuleActionObj) => {
const labelIds = obj.action.params
return addLabelsToLibraryItem(
labelIds,
obj.libraryItemId,
obj.userId,
'system'
)
return addLabelsToLibraryItem(labelIds, obj.data.id, obj.userId, 'system')
}
const deleteLibraryItem = async (obj: RuleActionObj) => {
return softDeleteLibraryItem(obj.libraryItemId, obj.userId)
return softDeleteLibraryItem(obj.data.id, obj.userId)
}
const archivePage = async (obj: RuleActionObj) => {
return updateLibraryItem(
obj.libraryItemId,
obj.data.id,
{ archivedAt: new Date(), state: LibraryItemState.Archived },
obj.userId,
undefined,
@ -64,7 +57,7 @@ const archivePage = async (obj: RuleActionObj) => {
const markPageAsRead = async (obj: RuleActionObj) => {
return readingProgressDataSource.updateReadingProgress(
obj.userId,
obj.libraryItemId,
obj.data.id,
{
readingProgressPercent: 100,
readingProgressTopPercent: 100,
@ -82,7 +75,7 @@ const sendNotification = async (obj: RuleActionObj) => {
}
const data = {
folder: item.folder?.toString() || 'inbox',
libraryItemId: obj.libraryItemId,
libraryItemId: obj.data.id,
}
return sendPushNotifications(obj.userId, message, 'rule', data)
@ -93,7 +86,9 @@ const sendToWebhook = async (obj: RuleActionObj) => {
const data = {
event: obj.ruleEventType,
data: obj.data,
item: obj.data,
userId: obj.userId,
timestamp: Date.now(),
}
logger.info(`triggering webhook: ${url}`)
@ -129,7 +124,6 @@ const getRuleAction = (
}
const triggerActions = async (
libraryItemId: string,
userId: string,
rules: Rule[],
data: ItemEvent,
@ -184,7 +178,6 @@ const triggerActions = async (
}
const actionObj: RuleActionObj = {
libraryItemId,
userId,
action,
data: results[0],
@ -203,7 +196,7 @@ const triggerActions = async (
}
export const triggerRule = async (jobData: TriggerRuleJobData) => {
const { userId, ruleEventType, data, libraryItemId } = jobData
const { userId, ruleEventType, data } = jobData
// get rules by calling api
const rules = await findEnabledRules(userId, ruleEventType)
@ -212,7 +205,7 @@ export const triggerRule = async (jobData: TriggerRuleJobData) => {
return false
}
await triggerActions(libraryItemId, userId, rules, data, ruleEventType)
await triggerActions(userId, rules, data, ruleEventType)
return true
}

30
packages/api/src/pubsub.ts
View file

@ -11,7 +11,7 @@ import {
import { buildLogger } from './utils/logger'
import { isYouTubeVideoURL } from './utils/youtube'
export type BaseEntityEvent = { id: string; userId: string }
export type EntityEvent = { id: string }
const logger = buildLogger('pubsub')
@ -47,23 +47,21 @@ export const createPubSubClient = (): PubsubClient => {
Buffer.from(JSON.stringify({ userId, email, name, username }))
)
},
entityCreated: async <T extends BaseEntityEvent>(
entityCreated: async <T extends EntityEvent>(
type: EntityType,
data: T,
userId: string,
libraryItemId: string
userId: string
): Promise<void> => {
// queue trigger rule job
await enqueueTriggerRuleJob({
ruleEventType: `${type.toUpperCase()}_CREATED` as RuleEventType,
data,
userId,
libraryItemId,
})
// queue export item job
await enqueueExportItem({
userId,
libraryItemIds: [libraryItemId],
libraryItemIds: [data.id],
})
if (type === EntityType.ITEM) {
@ -81,29 +79,27 @@ export const createPubSubClient = (): PubsubClient => {
if (isItemWithURL(data) && isYouTubeVideoURL(data['originalUrl'])) {
await enqueueProcessYouTubeVideo({
userId,
libraryItemId,
libraryItemId: data.id,
})
}
}
},
entityUpdated: async <T extends BaseEntityEvent>(
entityUpdated: async <T extends EntityEvent>(
type: EntityType,
data: T,
userId: string,
libraryItemId: string
userId: string
): Promise<void> => {
// queue trigger rule job
await enqueueTriggerRuleJob({
userId,
ruleEventType: RuleEventType.PageUpdated,
libraryItemId,
data,
})
// queue export item job
await enqueueExportItem({
userId,
libraryItemIds: [libraryItemId],
libraryItemIds: [data.id],
})
},
entityDeleted: async (
@ -144,17 +140,15 @@ export interface PubsubClient {
name: string,
username: string
) => Promise<void>
entityCreated: <T extends BaseEntityEvent>(
entityCreated: <T extends EntityEvent>(
type: EntityType,
data: T,
userId: string,
libraryItemId: string
userId: string
) => Promise<void>
entityUpdated: <T extends BaseEntityEvent>(
entityUpdated: <T extends EntityEvent>(
type: EntityType,
data: T,
userId: string,
libraryItemId: string
userId: string
) => Promise<void>
entityDeleted: (type: EntityType, id: string, userId: string) => Promise<void>
reportSubmitted(

24
packages/api/src/services/highlights.ts
View file

@ -9,12 +9,12 @@ import { createPubSubClient, EntityType } from '../pubsub'
import { authTrx } from '../repository'
import { highlightRepository } from '../repository/highlight'
import { enqueueUpdateHighlight } from '../utils/createTask'
import { deepDelete } from '../utils/helpers'
import { ItemEvent } from './library_item'
export type HighlightEvent = Omit<
DeepPartial<Highlight>,
'user' | 'userId' | 'sharedAt'
>
const columnToDelete = ['user', 'sharedAt'] as const
type ColumnToDeleteType = typeof columnToDelete[number]
export type HighlightEvent = Omit<DeepPartial<Highlight>, ColumnToDeleteType>
export const getHighlightLocation = (patch: string): number | undefined => {
const dmp = new diff_match_patch()
@ -58,11 +58,11 @@ export const createHighlight = async (
userId
)
const cleanData = deepDelete(newHighlight, columnToDelete)
await pubsub.entityCreated<ItemEvent>(
EntityType.HIGHLIGHT,
{ id: libraryItemId, highlights: [newHighlight], userId },
userId,
libraryItemId
{ id: libraryItemId, highlights: [cleanData] },
userId
)
await enqueueUpdateHighlight({
@ -108,9 +108,8 @@ export const mergeHighlights = async (
await pubsub.entityCreated<ItemEvent>(
EntityType.HIGHLIGHT,
{ id: libraryItemId, highlights: [newHighlight], userId },
userId,
libraryItemId
{ id: libraryItemId, highlights: [newHighlight] },
userId
)
await enqueueUpdateHighlight({
@ -143,9 +142,8 @@ export const updateHighlight = async (
const libraryItemId = updatedHighlight.libraryItem.id
await pubsub.entityUpdated<ItemEvent>(
EntityType.HIGHLIGHT,
{ id: libraryItemId, highlights: [highlight], userId } as ItemEvent,
userId,
libraryItemId
{ id: libraryItemId, highlights: [highlight] } as ItemEvent,
userId
)
await enqueueUpdateHighlight({

26
packages/api/src/services/labels.ts
View file

@ -6,11 +6,14 @@ import { createPubSubClient, EntityType, PubsubClient } from '../pubsub'
import { authTrx } from '../repository'
import { CreateLabelInput, labelRepository } from '../repository/label'
import { bulkEnqueueUpdateLabels } from '../utils/createTask'
import { deepDelete } from '../utils/helpers'
import { logger } from '../utils/logger'
import { findHighlightById } from './highlights'
import { findLibraryItemIdsByLabelId, ItemEvent } from './library_item'
export type LabelEvent = Omit<DeepPartial<Label>, 'description' | 'createdAt'>
const columnToDelete = ['description', 'createdAt'] as const
type ColumnToDeleteType = typeof columnToDelete[number]
export type LabelEvent = Omit<DeepPartial<Label>, ColumnToDeleteType>
// const batchGetLabelsFromLinkIds = async (
// linkIds: readonly string[]
@ -138,9 +141,11 @@ export const saveLabelsInLibraryItem = async (
// create pubsub event
await pubsub.entityCreated<ItemEvent>(
EntityType.LABEL,
{ id: libraryItemId, labels, userId },
userId,
libraryItemId
{
id: libraryItemId,
labels: labels.map((l) => deepDelete(l, columnToDelete)),
},
userId
)
}
@ -209,9 +214,16 @@ export const saveLabelsInHighlight = async (
// create pubsub event
await pubsub.entityCreated<ItemEvent>(
EntityType.LABEL,
{ id: libraryItemId, highlights: [{ id: highlightId, labels }], userId },
userId,
libraryItemId
{
id: libraryItemId,
highlights: [
{
id: highlightId,
labels: labels.map((l) => deepDelete(l, columnToDelete)),
},
],
},
userId
)
// update labels in library item

65
packages/api/src/services/library_item.ts
View file

@ -15,7 +15,7 @@ import { Highlight } from '../entity/highlight'
import { Label } from '../entity/label'
import { LibraryItem, LibraryItemState } from '../entity/library_item'
import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql'
import { BaseEntityEvent, createPubSubClient, EntityType } from '../pubsub'
import { createPubSubClient, EntityEvent, EntityType } from '../pubsub'
import { redisDataSource } from '../redis_data_source'
import {
authTrx,
@ -24,27 +24,29 @@ import {
queryBuilderToRawSql,
} from '../repository'
import { libraryItemRepository } from '../repository/library_item'
import { Merge } from '../util'
import { setRecentlySavedItemInRedis } from '../utils/helpers'
import { Merge, PickTuple } from '../util'
import { deepDelete, setRecentlySavedItemInRedis } from '../utils/helpers'
import { logger } from '../utils/logger'
import { parseSearchQuery } from '../utils/search'
import { HighlightEvent } from './highlights'
import { addLabelsToLibraryItem, LabelEvent } from './labels'
type IgnoredFields =
| 'user'
| 'uploadFile'
| 'previewContentType'
| 'links'
| 'textContentHash'
const columnToDelete = [
'user',
'uploadFile',
'previewContentType',
'links',
'textContentHash',
] as const
type ColumnToDeleteType = typeof columnToDelete[number]
type ItemBaseEvent = Merge<
Omit<DeepPartial<LibraryItem>, IgnoredFields>,
Omit<DeepPartial<LibraryItem>, ColumnToDeleteType>,
{
labels?: LabelEvent[]
highlights?: HighlightEvent[]
}
>
export type ItemEvent = Merge<ItemBaseEvent, BaseEntityEvent>
export type ItemEvent = Merge<ItemBaseEvent, EntityEvent>
export class RequiresSearchQueryError extends Error {
constructor() {
@ -884,22 +886,17 @@ export const updateLibraryItem = async (
}
if (libraryItem.state === LibraryItemState.Succeeded) {
const cleanedData = deepDelete(updatedLibraryItem, columnToDelete)
// send create event if the item was created
await pubsub.entityCreated<ItemEvent>(
EntityType.ITEM,
{ ...updatedLibraryItem, userId },
userId,
id
)
await pubsub.entityCreated<ItemEvent>(EntityType.ITEM, cleanedData, userId)
return updatedLibraryItem
}
await pubsub.entityUpdated<ItemEvent>(
EntityType.ITEM,
{ ...libraryItem, id, userId } as ItemEvent,
userId,
id
{ ...libraryItem, id } as ItemEvent,
userId
)
return updatedLibraryItem
@ -958,12 +955,7 @@ export const updateLibraryItemReadingProgress = async (
}
const updatedItem = result[0][0]
await pubsub.entityUpdated<ItemEvent>(
EntityType.ITEM,
{ ...updatedItem, id, userId },
userId,
id
)
await pubsub.entityUpdated<ItemEvent>(EntityType.ITEM, updatedItem, userId)
return updatedItem
}
@ -1059,12 +1051,8 @@ export const createOrUpdateLibraryItem = async (
return newLibraryItem
}
await pubsub.entityCreated<ItemEvent>(
EntityType.ITEM,
{ ...newLibraryItem, userId },
userId,
newLibraryItem.id
)
const cleanedData = deepDelete(newLibraryItem, columnToDelete)
await pubsub.entityCreated<ItemEvent>(EntityType.ITEM, cleanedData, userId)
return newLibraryItem
}
@ -1346,7 +1334,8 @@ export const filterItemEvents = (
subscription: /^subscription(s)?$/i,
}
const matchingKeyword = Object.keys(keywordRegexMap).find((keyword) =>
const keys = Object.keys(keywordRegexMap)
const matchingKeyword = keys.find((keyword) =>
value.match(keywordRegexMap[keyword])
)
@ -1354,7 +1343,9 @@ export const filterItemEvents = (
throw new Error(`Unexpected keyword: ${value}`)
}
const eventValue = event[matchingKeyword as keyof ItemEvent]
const eventValue = (event as PickTuple<ItemEvent, typeof keys>)[
matchingKeyword
]
return !eventValue || (Array.isArray(eventValue) && eventValue.length === 0)
}
@ -1484,8 +1475,10 @@ export const filterItemEvents = (
const start = startDate ?? new Date(0)
const end = endDate ?? new Date()
const key = `${field.name.toLowerCase()}At` as keyof ItemEvent
const eventValue = event[key] as Date
const key = `${field.name.toLowerCase()}At`
const eventValue = event[
key as 'readAt' | 'updatedAt' | 'publishedAt'
] as Date
return eventValue >= start && eventValue <= end
}

5
packages/api/src/utils/helpers.ts
View file

@ -367,7 +367,10 @@ export const cleanUrl = (url: string) => {
})
}
export const deepDelete = <T, K extends keyof T>(obj: T, keys: K[]) => {
export const deepDelete = <T, K extends keyof T>(
obj: T,
keys: readonly K[]
) => {
// make a copy of the object
const copy = { ...obj }