diff --git a/packages/api/.eslintignore b/packages/api/.eslintignore index 4a2dcf8e2..a784fd752 100644 --- a/packages/api/.eslintignore +++ b/packages/api/.eslintignore @@ -1,5 +1,3 @@ node_modules/ dist/ -readabilityjs/ src/generated/ -test/resolvers/ diff --git a/packages/api/.eslintrc b/packages/api/.eslintrc index f2874a383..bc325835a 100644 --- a/packages/api/.eslintrc +++ b/packages/api/.eslintrc @@ -5,5 +5,13 @@ }, "rules": { "@typescript-eslint/no-unsafe-argument": 0 - } + }, + "overrides": [ + { + "files": ["test/**/*.ts"], + "rules": { + "@typescript-eslint/no-unsafe-member-access": 0 + } + } + ] } diff --git a/packages/api/mocha-config.json b/packages/api/mocha-config.json index 1309d5869..ebe512a91 100644 --- a/packages/api/mocha-config.json +++ b/packages/api/mocha-config.json @@ -2,5 +2,6 @@ "extension": ["ts"], "spec": "test/**/*.test.ts", "reporter": "mocha-unfunk-reporter", - "require": ["test/babel-register.js", "test/global-setup.ts", "test/global-teardown.ts"] + "require": ["test/global-setup.ts", "test/global-teardown.ts"], + "timeout": 10000 } diff --git a/packages/api/package.json b/packages/api/package.json index 3ca6cc60a..c3ec7bd1b 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -3,16 +3,15 @@ "version": "1.0.0", "license": "UNLICENSED", "scripts": { - "build": "tsc && yarn copy-files", - "dev": "ts-node-dev --files src/server.ts", - "dev_qp": "ts-node-dev --files src/queue-processor.ts", - "start": "node dist/server.js", - "start_queue_processor": "node dist/queue-processor.js", + "build": "tsc", + "dev": "ts-node-dev --respawn --transpile-only src/server.ts", + "dev_qp": "ts-node-dev --respawn --transpile-only src/queue-processor.ts", + "start": "node dist/src/server.js", + "start_queue_processor": "node dist/src/queue-processor.js", "lint": "eslint src --ext ts,js,tsx,jsx", "lint:fix": "eslint src --fix --ext ts,js,tsx,jsx", "test:typecheck": "tsc --noEmit", - "test": "nyc mocha -r ts-node/register --config mocha-config.json --timeout 10000", - "copy-files": "copyfiles -u 1 src/**/*.html dist/" + "test": "nyc mocha -r ts-node/register --config mocha-config.json" }, "dependencies": { "@bmatei/apollo-prometheus-exporter": "^3.0.0", @@ -117,7 +116,6 @@ "youtubei": "1.3.7" }, "devDependencies": { - "@babel/register": "^7.14.5", "@istanbuljs/nyc-config-typescript": "^1.0.2", "@types/addressparser": "^1.0.1", "@types/analytics-node": "^3.1.7", @@ -154,7 +152,6 @@ "chai-as-promised": "^7.1.1", "chai-string": "^1.5.0", "circular-dependency-plugin": "^5.2.0", - "copyfiles": "^2.4.1", "mocha": "^9.0.1", "mocha-unfunk-reporter": "^0.4.0", "nock": "^13.2.4", diff --git a/packages/api/src/entity/rule.ts b/packages/api/src/entity/rule.ts index 51c63b972..8e87a5cd9 100644 --- a/packages/api/src/entity/rule.ts +++ b/packages/api/src/entity/rule.ts @@ -15,11 +15,16 @@ export enum RuleActionType { Delete = 'DELETE', MarkAsRead = 'MARK_AS_READ', SendNotification = 'SEND_NOTIFICATION', + Webhook = 'WEBHOOK', + Export = 'EXPORT', } export enum RuleEventType { PageCreated = 'PAGE_CREATED', PageUpdated = 'PAGE_UPDATED', + LabelCreated = 'LABEL_CREATED', + HighlightCreated = 'HIGHLIGHT_CREATED', + HighlightUpdated = 'HIGHLIGHT_UPDATED', } export interface RuleAction { diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index ae5bcdbd8..93afb8908 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2501,11 +2501,16 @@ export enum RuleActionType { AddLabel = 'ADD_LABEL', Archive = 'ARCHIVE', Delete = 'DELETE', + Export = 'EXPORT', MarkAsRead = 'MARK_AS_READ', - SendNotification = 'SEND_NOTIFICATION' + SendNotification = 'SEND_NOTIFICATION', + Webhook = 'WEBHOOK' } export enum RuleEventType { + HighlightCreated = 'HIGHLIGHT_CREATED', + HighlightUpdated = 'HIGHLIGHT_UPDATED', + LabelCreated = 'LABEL_CREATED', PageCreated = 'PAGE_CREATED', PageUpdated = 'PAGE_UPDATED' } diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 94695e350..ba442c3a7 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1877,11 +1877,16 @@ enum RuleActionType { ADD_LABEL ARCHIVE DELETE + EXPORT MARK_AS_READ SEND_NOTIFICATION + WEBHOOK } enum RuleEventType { + HIGHLIGHT_CREATED + HIGHLIGHT_UPDATED + LABEL_CREATED PAGE_CREATED PAGE_UPDATED } diff --git a/packages/api/src/jobs/find_thumbnail.ts b/packages/api/src/jobs/find_thumbnail.ts index cf6caa0ee..63bae9b42 100644 --- a/packages/api/src/jobs/find_thumbnail.ts +++ b/packages/api/src/jobs/find_thumbnail.ts @@ -161,7 +161,9 @@ export const findThumbnail = async (data: Data) => { { thumbnail, }, - userId + userId, + undefined, + true ) logger.info(`thumbnail updated: ${thumbnail}`) } diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts index 596760295..dcc1c902f 100644 --- a/packages/api/src/jobs/trigger_rule.ts +++ b/packages/api/src/jobs/trigger_rule.ts @@ -1,7 +1,15 @@ import { LiqeQuery } from '@omnivore/liqe' +import axios from 'axios' +import { Any } from 'typeorm' import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source' +import { IntegrationType } from '../entity/integration' import { LibraryItem, LibraryItemState } from '../entity/library_item' import { Rule, RuleAction, RuleActionType, RuleEventType } from '../entity/rule' +import { + findIntegrations, + getIntegrationClient, + updateIntegration, +} from '../services/integrations' import { addLabelsToLibraryItem } from '../services/labels' import { filterItemEvents, @@ -17,17 +25,16 @@ import { logger } from '../utils/logger' import { parseSearchQuery } from '../utils/search' export interface TriggerRuleJobData { - libraryItemId: string userId: string ruleEventType: RuleEventType data: ItemEvent } interface RuleActionObj { - libraryItemId: string userId: string action: RuleAction data: ItemEvent | LibraryItem + ruleEventType: RuleEventType } type RuleActionFunc = (obj: RuleActionObj) => Promise @@ -37,21 +44,16 @@ const readingProgressDataSource = new ReadingProgressDataSource() const addLabels = async (obj: RuleActionObj) => { const labelIds = obj.action.params - return addLabelsToLibraryItem( - labelIds, - obj.libraryItemId, - obj.userId, - 'system' - ) + return addLabelsToLibraryItem(labelIds, obj.data.id, obj.userId, 'system') } const deleteLibraryItem = async (obj: RuleActionObj) => { - return softDeleteLibraryItem(obj.libraryItemId, obj.userId) + return softDeleteLibraryItem(obj.data.id, obj.userId) } const archivePage = async (obj: RuleActionObj) => { return updateLibraryItem( - obj.libraryItemId, + obj.data.id, { archivedAt: new Date(), state: LibraryItemState.Archived }, obj.userId, undefined, @@ -62,7 +64,7 @@ const archivePage = async (obj: RuleActionObj) => { const markPageAsRead = async (obj: RuleActionObj) => { return readingProgressDataSource.updateReadingProgress( obj.userId, - obj.libraryItemId, + obj.data.id, { readingProgressPercent: 100, readingProgressTopPercent: 100, @@ -80,13 +82,119 @@ const sendNotification = async (obj: RuleActionObj) => { } const data = { folder: item.folder?.toString() || 'inbox', - libraryItemId: obj.libraryItemId, + libraryItemId: obj.data.id, } return sendPushNotifications(obj.userId, message, 'rule', data) } -const getRuleAction = (actionType: RuleActionType): RuleActionFunc => { +const sendToWebhook = async (obj: RuleActionObj) => { + const [url] = obj.action.params + + const [type, action] = obj.ruleEventType.toString().toLowerCase().split('_') + + // use old event format for the compatibility with the old webhooks + let event + if (type === 'page') { + event = obj.data + } else if (type === 'label') { + event = { + labels: obj.data.labels, + pageId: obj.data.id, + } + } else { + if (!obj.data.highlights) { + return + } + + event = { + ...obj.data.highlights[0], + pageId: obj.data.id, + } + } + + const data = { + action, + userId: obj.userId, + [type]: event, + } + + logger.info(`triggering webhook: ${url}`) + + return axios.post(url, data, { + headers: { + 'Content-Type': 'application/json', + }, + timeout: 5000, // 5s + }) +} + +const exportItem = async (obj: RuleActionObj) => { + const userId = obj.userId + const integrationNames = obj.action.params + const integrations = await findIntegrations(userId, { + name: Any(integrationNames.map((param) => param.toUpperCase())), + enabled: true, + type: IntegrationType.Export, + }) + + if (integrations.length <= 0) { + return + } + + await Promise.all( + integrations.map(async (integration) => { + const logObject = { + userId, + integrationId: integration.id, + name: integration.name, + } + logger.info('exporting item...', logObject) + + try { + const client = getIntegrationClient( + integration.name, + integration.token, + integration + ) + + const synced = await client.export([obj.data]) + if (!synced) { + logger.error('failed to export item', logObject) + return false + } + + const syncedAt = new Date() + logger.info('updating integration...', { + ...logObject, + syncedAt, + }) + + // update integration syncedAt if successful + const updated = await updateIntegration( + integration.id, + { + syncedAt, + }, + userId + ) + logger.info('integration updated', { + ...logObject, + updated, + }) + } catch (error) { + logger.error('failed to export item', { + ...logObject, + error, + }) + } + }) + ) +} + +const getRuleAction = ( + actionType: RuleActionType +): RuleActionFunc | undefined => { switch (actionType) { case RuleActionType.AddLabel: return addLabels @@ -98,14 +206,21 @@ const getRuleAction = (actionType: RuleActionType): RuleActionFunc => { return markPageAsRead case RuleActionType.SendNotification: return sendNotification + case RuleActionType.Webhook: + return sendToWebhook + case RuleActionType.Export: + return exportItem + default: + logger.error('Unknown rule action type', actionType) + return undefined } } const triggerActions = async ( - libraryItemId: string, userId: string, rules: Rule[], - data: ItemEvent + data: ItemEvent, + ruleEventType: RuleEventType ) => { const actionPromises: Promise[] = [] @@ -130,7 +245,7 @@ const triggerActions = async ( logger.info('Failed to filter items by metadata, running search query') const searchResult = await searchLibraryItems( { - query: `includes:${libraryItemId} AND (${rule.filter})`, + query: `includes:${data.id} AND (${rule.filter})`, size: 1, }, userId @@ -150,11 +265,16 @@ const triggerActions = async ( for (const action of rule.actions) { const actionFunc = getRuleAction(action.type) + if (!actionFunc) { + logger.error('No action function found for action', action.type) + continue + } + const actionObj: RuleActionObj = { - libraryItemId, userId, action, data: results[0], + ruleEventType, } actionPromises.push(actionFunc(actionObj)) @@ -169,7 +289,7 @@ const triggerActions = async ( } export const triggerRule = async (jobData: TriggerRuleJobData) => { - const { userId, ruleEventType, data, libraryItemId } = jobData + const { userId, ruleEventType, data } = jobData // get rules by calling api const rules = await findEnabledRules(userId, ruleEventType) @@ -178,7 +298,7 @@ export const triggerRule = async (jobData: TriggerRuleJobData) => { return false } - await triggerActions(libraryItemId, userId, rules, data) + await triggerActions(userId, rules, data, ruleEventType) return true } diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index 4401cfad9..4cb0116d4 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -4,14 +4,14 @@ import { RuleEventType } from './entity/rule' import { env } from './env' import { ReportType } from './generated/graphql' import { - enqueueExportItem, enqueueProcessYouTubeVideo, enqueueTriggerRuleJob, - enqueueWebhookJob, } from './utils/createTask' import { buildLogger } from './utils/logger' import { isYouTubeVideoURL } from './utils/youtube' +export type EntityEvent = { id: string } + const logger = buildLogger('pubsub') const client = new PubSub() @@ -46,35 +46,19 @@ export const createPubSubClient = (): PubsubClient => { Buffer.from(JSON.stringify({ userId, email, name, username })) ) }, - entityCreated: async >( + entityCreated: async ( type: EntityType, data: T, - userId: string, - libraryItemId: string + userId: string ): Promise => { // queue trigger rule job - if (type === EntityType.PAGE) { - await enqueueTriggerRuleJob({ - userId, - ruleEventType: RuleEventType.PageCreated, - libraryItemId, - data, - }) - } - // queue export item job - await enqueueExportItem({ - userId, - libraryItemIds: [libraryItemId], - }) - - await enqueueWebhookJob({ - userId, - type, - action: 'created', + await enqueueTriggerRuleJob({ + ruleEventType: `${type.toUpperCase()}_CREATED` as RuleEventType, data, + userId, }) - if (type === EntityType.PAGE) { + if (type === EntityType.ITEM) { // if (await findGrantedFeatureByName(FeatureName.AISummaries, userId)) { // await enqueueAISummarizeJob({ // userId, @@ -89,36 +73,20 @@ export const createPubSubClient = (): PubsubClient => { if (isItemWithURL(data) && isYouTubeVideoURL(data['originalUrl'])) { await enqueueProcessYouTubeVideo({ userId, - libraryItemId, + libraryItemId: data.id, }) } } }, - entityUpdated: async >( + entityUpdated: async ( type: EntityType, data: T, - userId: string, - libraryItemId: string + userId: string ): Promise => { // queue trigger rule job - if (type === EntityType.PAGE) { - await enqueueTriggerRuleJob({ - userId, - ruleEventType: RuleEventType.PageUpdated, - libraryItemId, - data, - }) - } - // queue export item job - await enqueueExportItem({ + await enqueueTriggerRuleJob({ userId, - libraryItemIds: [libraryItemId], - }) - - await enqueueWebhookJob({ - userId, - type, - action: 'updated', + ruleEventType: `${type.toUpperCase()}_UPDATED` as RuleEventType, data, }) }, @@ -147,10 +115,10 @@ export const createPubSubClient = (): PubsubClient => { } export enum EntityType { - PAGE = 'page', - HIGHLIGHT = 'highlight', - LABEL = 'label', - RSS_FEED = 'feed', + ITEM = 'PAGE', + HIGHLIGHT = 'HIGHLIGHT', + LABEL = 'LABEL', + RSS_FEED = 'FEED', } export interface PubsubClient { @@ -160,17 +128,15 @@ export interface PubsubClient { name: string, username: string ) => Promise - entityCreated: >( + entityCreated: ( type: EntityType, data: T, - userId: string, - libraryItemId: string + userId: string ) => Promise - entityUpdated: >( + entityUpdated: ( type: EntityType, data: T, - userId: string, - libraryItemId: string + userId: string ) => Promise entityDeleted: (type: EntityType, id: string, userId: string) => Promise reportSubmitted( diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index fec1a9a82..502b06880 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -40,6 +40,7 @@ import { saveIntegration, updateIntegration, } from '../../services/integrations' +import { NotionClient } from '../../services/integrations/notion' import { analytics } from '../../utils/analytics' import { deleteTask, @@ -57,15 +58,14 @@ export const setIntegrationResolver = authorized< ...input, user: { id: uid }, id: input.id || undefined, - type: input.type || IntegrationType.Export, + type: input.type || undefined, syncedAt: input.syncedAt ? new Date(input.syncedAt) : undefined, importItemState: input.type === IntegrationType.Import ? input.importItemState || ImportItemState.Unarchived // default to unarchived : undefined, - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - settings: input.settings, } + if (input.id) { // Update const existingIntegration = await findIntegration({ id: input.id }, uid) @@ -96,6 +96,22 @@ export const setIntegrationResolver = authorized< if (integration.name.toLowerCase() === 'readwise') { // create a task to export all the items for readwise temporarily await enqueueExportToIntegration(integration.id, uid) + } else if ( + integration.name.toLowerCase() === 'notion' && + integration.settings + ) { + const settings = integration.settings as { parentDatabaseId?: string } + if (settings.parentDatabaseId) { + // update notion database properties + const notion = new NotionClient(integration.token, integration) + try { + await notion.updateDatabase(settings.parentDatabaseId) + } catch (error) { + return { + errorCodes: [SetIntegrationErrorCode.BadRequest], + } + } + } } analytics.capture({ diff --git a/packages/api/src/resolvers/rules/index.ts b/packages/api/src/resolvers/rules/index.ts index 0efa67750..996a11ba8 100644 --- a/packages/api/src/resolvers/rules/index.ts +++ b/packages/api/src/resolvers/rules/index.ts @@ -7,7 +7,6 @@ import { MutationSetRuleArgs, QueryRulesArgs, RulesError, - RulesErrorCode, RulesSuccess, SetRuleError, SetRuleErrorCode, @@ -21,53 +20,43 @@ export const setRuleResolver = authorized< SetRuleSuccess, SetRuleError, MutationSetRuleArgs ->(async (_, { input }, { authTrx, uid, log }) => { +>(async (_, { input }, { authTrx, uid }) => { try { // validate filter parseSearchQuery(input.filter) - - const rule = await authTrx((t) => - t.getRepository(Rule).save({ - ...input, - id: input.id || undefined, - user: { id: uid }, - }) - ) - - return { - rule, - } } catch (error) { - log.error('Error setting rules', error) - return { errorCodes: [SetRuleErrorCode.BadRequest], } } + + const rule = await authTrx((t) => + t.getRepository(Rule).save({ + ...input, + id: input.id || undefined, + user: { id: uid }, + }) + ) + + return { + rule, + } }) export const rulesResolver = authorized< RulesSuccess, RulesError, QueryRulesArgs ->(async (_, { enabled }, { authTrx, log, uid }) => { - try { - const rules = await authTrx((t) => - t.getRepository(Rule).findBy({ - user: { id: uid }, - enabled: enabled === null ? undefined : enabled, - }) - ) +>(async (_, { enabled }, { authTrx, uid }) => { + const rules = await authTrx((t) => + t.getRepository(Rule).findBy({ + user: { id: uid }, + enabled: enabled === null ? undefined : enabled, + }) + ) - return { - rules, - } - } catch (error) { - log.error('Error getting rules', error) - - return { - errorCodes: [RulesErrorCode.BadRequest], - } + return { + rules, } }) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e106514e4..6b0c3cb01 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2167,6 +2167,8 @@ const schema = gql` DELETE MARK_AS_READ SEND_NOTIFICATION + WEBHOOK + EXPORT } type RulesError { @@ -2181,6 +2183,9 @@ const schema = gql` enum RuleEventType { PAGE_CREATED PAGE_UPDATED + LABEL_CREATED + HIGHLIGHT_CREATED + HIGHLIGHT_UPDATED } input SetRuleInput { diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index babf4d4a4..3893bdbc5 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -5,14 +5,20 @@ import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' import { homePageURL } from '../env' -import { createPubSubClient, EntityType } from '../pubsub' +import { createPubSubClient, EntityEvent, EntityType } from '../pubsub' import { authTrx } from '../repository' import { highlightRepository } from '../repository/highlight' +import { Merge } from '../util' import { enqueueUpdateHighlight } from '../utils/createTask' +import { deepDelete } from '../utils/helpers' +import { ItemEvent } from './library_item' -type HighlightEvent = { id: string; pageId: string } -type CreateHighlightEvent = DeepPartial & HighlightEvent -type UpdateHighlightEvent = QueryDeepPartialEntity & HighlightEvent +const columnsToDelete = ['user', 'sharedAt', 'libraryItem'] as const +type ColumnsToDeleteType = typeof columnsToDelete[number] +export type HighlightEvent = Merge< + Omit, ColumnsToDeleteType>, + EntityEvent +> export const getHighlightLocation = (patch: string): number | undefined => { const dmp = new diff_match_patch() @@ -49,6 +55,7 @@ export const createHighlight = async ( where: { id: newHighlight.id }, relations: { user: true, + libraryItem: true, }, }) }, @@ -56,11 +63,19 @@ export const createHighlight = async ( userId ) - await pubsub.entityCreated( + const data = deepDelete(newHighlight, columnsToDelete) + await pubsub.entityCreated( EntityType.HIGHLIGHT, - { ...newHighlight, pageId: libraryItemId }, - userId, - libraryItemId + { + id: libraryItemId, + highlights: [data], + // for Readwise + originalUrl: newHighlight.libraryItem.originalUrl, + title: newHighlight.libraryItem.title, + author: newHighlight.libraryItem.author, + thumbnail: newHighlight.libraryItem.thumbnail, + }, + userId ) await enqueueUpdateHighlight({ @@ -100,15 +115,22 @@ export const mergeHighlights = async ( where: { id: newHighlight.id }, relations: { user: true, + libraryItem: true, }, }) }) - await pubsub.entityCreated( + await pubsub.entityCreated( EntityType.HIGHLIGHT, - { ...newHighlight, pageId: libraryItemId }, - userId, - libraryItemId + { + id: libraryItemId, + originalUrl: newHighlight.libraryItem.originalUrl, + title: newHighlight.libraryItem.title, + author: newHighlight.libraryItem.author, + thumbnail: newHighlight.libraryItem.thumbnail, + highlights: [newHighlight], + }, + userId ) await enqueueUpdateHighlight({ @@ -139,11 +161,25 @@ export const updateHighlight = async ( }) const libraryItemId = updatedHighlight.libraryItem.id - await pubsub.entityUpdated( + await pubsub.entityUpdated( EntityType.HIGHLIGHT, - { ...highlight, id: highlightId, pageId: libraryItemId }, - userId, - libraryItemId + { + id: libraryItemId, + originalUrl: updatedHighlight.libraryItem.originalUrl, + title: updatedHighlight.libraryItem.title, + author: updatedHighlight.libraryItem.author, + thumbnail: updatedHighlight.libraryItem.thumbnail, + highlights: [ + { + ...highlight, + id: highlightId, + updatedAt: new Date(), + quote: updatedHighlight.quote, + highlightType: updatedHighlight.highlightType, + }, + ], + } as ItemEvent, + userId ) await enqueueUpdateHighlight({ diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index 44c95e680..613cae2c3 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -1,4 +1,5 @@ -import { LibraryItem, LibraryItemState } from '../../entity/library_item' +import { LibraryItemState } from '../../entity/library_item' +import { ItemEvent } from '../library_item' export interface RetrievedData { url: string @@ -26,5 +27,5 @@ export interface IntegrationClient { auth(state: string): Promise - export(items: LibraryItem[]): Promise + export(items: ItemEvent[]): Promise } diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 7e891287d..f391456f3 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -1,12 +1,12 @@ import { Client } from '@notionhq/client' import axios from 'axios' -import { updateIntegration } from '.' +import { HighlightType } from '../../entity/highlight' import { Integration } from '../../entity/integration' -import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' import { Merge } from '../../util' import { logger } from '../../utils/logger' import { getHighlightUrl } from '../highlights' +import { getItemUrl, ItemEvent } from '../library_item' import { IntegrationClient } from './integration' type AnnotationColor = @@ -45,7 +45,7 @@ interface NotionPage { } } properties: { - Title: { + Title?: { title: [ { text: { @@ -54,25 +54,25 @@ interface NotionPage { } ] } - Author: { + Author?: { rich_text: Array<{ text: { content: string } }> } - 'Original URL': { + 'Original URL'?: { url: string } 'Omnivore URL': { url: string } - 'Saved At': { + 'Saved At'?: { date: { start: string } } - 'Last Updated': { + 'Last Updated'?: { date: { start: string } @@ -111,7 +111,7 @@ type Property = 'highlights' interface Settings { parentPageId: string parentDatabaseId: string - properties: Property[] + properties?: Property[] } export class NotionClient implements IntegrationClient { @@ -178,7 +178,7 @@ export class NotionClient implements IntegrationClient { } private itemToNotionPage = ( - item: LibraryItem, + item: ItemEvent, settings: Settings, lastSync?: Date | null ): NotionPage => { @@ -201,88 +201,103 @@ export class NotionClient implements IntegrationClient { } : undefined, properties: { - Title: { - title: [ - { - text: { - content: item.title, - }, - }, - ], - }, - Author: { - rich_text: [ - { - text: { - content: item.author || 'unknown', - }, - }, - ], - }, - 'Original URL': { - url: item.originalUrl, - }, + Title: item.title + ? { + title: [ + { + text: { + content: item.title, + }, + }, + ], + } + : undefined, + Author: item.author + ? { + rich_text: [ + { + text: { + content: item.author, + }, + }, + ], + } + : undefined, + 'Original URL': item.originalUrl + ? { + url: item.originalUrl, + } + : undefined, 'Omnivore URL': { - url: `${env.client.url}/me/${item.slug}`, - }, - 'Saved At': { - date: { - start: item.createdAt.toISOString(), - }, - }, - 'Last Updated': { - date: { - start: item.updatedAt.toISOString(), - }, + url: getItemUrl(item.id), }, + 'Saved At': item.savedAt + ? { + date: { + start: item.savedAt as string, + }, + } + : undefined, + 'Last Updated': item.updatedAt + ? { + date: { + start: item.updatedAt as string, + }, + } + : undefined, Tags: item.labels ? { multi_select: item.labels.map((label) => ({ - name: label.name, + name: label.name as string, })), } : undefined, }, - children: - settings.properties.includes('highlights') && item.highlights - ? item.highlights - .filter( - (highlight) => !lastSync || highlight.updatedAt > lastSync // only new highlights - ) - .map((highlight) => ({ - paragraph: { - rich_text: [ - { - text: { - content: highlight.quote || '', - link: { - url: getHighlightUrl(item.slug, highlight.id), - }, - }, - annotations: { - code: true, - color: highlight.color as AnnotationColor, + children: item.highlights + ? item.highlights + .filter( + (highlight) => + highlight.highlightType === HighlightType.Highlight && + (!lastSync || + new Date(highlight.updatedAt as string) > lastSync) // only new highlights + ) + .map((highlight) => ({ + paragraph: { + rich_text: [ + { + text: { + content: highlight.quote || '', + link: { + url: getHighlightUrl( + item.slug || item.id, + highlight.id + ), }, }, - ], - children: highlight.annotation - ? [ - { - paragraph: { - rich_text: [ - { - text: { - content: highlight.annotation || '', - }, + annotations: { + code: true, + color: highlight.color as AnnotationColor, + }, + }, + ], + children: highlight.annotation + ? [ + { + paragraph: { + rich_text: [ + { + text: { + content: highlight.annotation || '', }, - ], - }, + }, + ], }, - ] - : undefined, - }, - })) - : undefined, + }, + ] + : undefined, + }, + })) + : undefined, } } @@ -308,110 +323,99 @@ export class NotionClient implements IntegrationClient { return null } - export = async (items: LibraryItem[]): Promise => { + export = async (items: ItemEvent[]): Promise => { const settings = this.integrationData?.settings if (!this.integrationData || !settings) { logger.error('Notion integration data not found') return false } - const pageId = settings.parentPageId - if (!pageId) { - logger.error('Notion parent page id not found') - return false - } - - let databaseId = settings.parentDatabaseId + const databaseId = settings.parentDatabaseId if (!databaseId) { - // create a database for the items - const database = await this.client.databases.create({ - parent: { - page_id: pageId, - }, - title: [ - { - text: { - content: 'Library', - }, - }, - ], - description: [ - { - text: { - content: 'Library of saved items from Omnivore', - }, - }, - ], - properties: { - Title: { - title: {}, - }, - Author: { - rich_text: {}, - }, - 'Original URL': { - url: {}, - }, - 'Omnivore URL': { - url: {}, - }, - 'Saved At': { - date: {}, - }, - 'Last Updated': { - date: {}, - }, - Tags: { - multi_select: {}, - }, - }, - }) - - // save the database id - databaseId = database.id - settings.parentDatabaseId = databaseId - await updateIntegration( - this.integrationData.id, - { - settings, - }, - this.integrationData.user.id - ) + logger.error('Notion database id not found') + return false } await Promise.all( items.map(async (item) => { - const notionPage = this.itemToNotionPage( - item, - settings, - this.integrationData?.syncedAt - ) - const url = notionPage.properties['Omnivore URL'].url + try { + const notionPage = this.itemToNotionPage( + item, + settings, + this.integrationData?.syncedAt + ) + const url = notionPage.properties['Omnivore URL'].url - const existingPage = await this.findPage(url, databaseId) - if (existingPage) { - // update the page - await this.client.pages.update({ - page_id: existingPage.id, - properties: notionPage.properties, - }) - - // append the children incrementally - if (notionPage.children && notionPage.children.length > 0) { - await this.client.blocks.children.append({ - block_id: existingPage.id, - children: notionPage.children, + const existingPage = await this.findPage(url, databaseId) + if (existingPage) { + // update the page + await this.client.pages.update({ + page_id: existingPage.id, + properties: notionPage.properties, }) + + // append the children incrementally + if (notionPage.children && notionPage.children.length > 0) { + await this.client.blocks.children.append({ + block_id: existingPage.id, + children: notionPage.children, + }) + } + + return } - return + // create the page + return await this.createPage(notionPage) + } catch (error) { + logger.error(error) + return false } - - // create the page - return this.createPage(notionPage) }) ) return true } + + private findDatabase = async (databaseId: string) => { + return this.client.databases.retrieve({ + database_id: databaseId, + }) + } + + updateDatabase = async (databaseId: string) => { + const database = await this.findDatabase(databaseId) + // find the title property and update it + const titleProperty = Object.entries(database.properties).find( + ([, property]) => property.type === 'title' + ) + const title = titleProperty ? titleProperty[0] : 'Name' + + await this.client.databases.update({ + database_id: database.id, + properties: { + [title]: { + name: 'Title', + }, + Author: { + rich_text: {}, + }, + 'Original URL': { + url: {}, + }, + 'Omnivore URL': { + url: {}, + }, + 'Saved At': { + date: {}, + }, + 'Last Updated': { + date: {}, + }, + Tags: { + multi_select: {}, + }, + }, + }) + } } diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 0c204b630..2a6ea2fd7 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -1,7 +1,8 @@ import axios from 'axios' -import { LibraryItem } from '../../entity/library_item' +import { HighlightType } from '../../entity/highlight' import { logger } from '../../utils/logger' import { getHighlightUrl } from '../highlights' +import { getItemUrl, ItemEvent } from '../library_item' import { IntegrationClient } from './integration' interface ReadwiseHighlight { @@ -66,7 +67,7 @@ export class ReadwiseClient implements IntegrationClient { } } - export = async (items: LibraryItem[]): Promise => { + export = async (items: ItemEvent[]): Promise => { let result = true const highlights = items.flatMap(this._itemToReadwiseHighlight) @@ -83,32 +84,33 @@ export class ReadwiseClient implements IntegrationClient { throw new Error('Method not implemented.') } - private _itemToReadwiseHighlight = ( - item: LibraryItem - ): ReadwiseHighlight[] => { + private _itemToReadwiseHighlight = (item: ItemEvent): ReadwiseHighlight[] => { const category = item.siteName === 'Twitter' ? 'tweets' : 'articles' - return item.highlights - ?.map((highlight) => { - // filter out highlights that are not of type highlight or have no quote - if (highlight.highlightType !== 'HIGHLIGHT' || !highlight.quote) { - return undefined - } - return { - text: highlight.quote, - title: item.title, - author: item.author || undefined, - highlight_url: getHighlightUrl(item.slug, highlight.id), - highlighted_at: new Date(highlight.createdAt).toISOString(), - category, - image_url: item.thumbnail || undefined, - location_type: 'order', - note: highlight.annotation || undefined, - source_type: 'omnivore', - source_url: item.originalUrl, - } - }) - .filter((highlight) => highlight !== undefined) as ReadwiseHighlight[] + return item.highlights + ? item.highlights + // filter out highlights that are not of type highlight or have no quote + .filter( + (highlight) => highlight.highlightType === HighlightType.Highlight + ) + .map((highlight) => { + return { + text: highlight.quote || '', + title: item.title, + author: item.author || undefined, + highlight_url: getHighlightUrl(item.id, highlight.id), + highlighted_at: highlight.createdAt + ? new Date(highlight.createdAt as string).toISOString() + : undefined, + category, + image_url: item.thumbnail || undefined, + location_type: 'order', + note: highlight.annotation || undefined, + source_type: 'omnivore', + source_url: getItemUrl(item.id), + } + }) + : [] } private _syncWithReadwise = async ( diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index f4bae13b7..dac0a3c94 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -2,23 +2,25 @@ import { DeepPartial, FindOptionsWhere, In } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel, LabelSource } from '../entity/entity_label' import { Label } from '../entity/label' -import { createPubSubClient, EntityType, PubsubClient } from '../pubsub' +import { + createPubSubClient, + EntityEvent, + EntityType, + PubsubClient, +} from '../pubsub' import { authTrx } from '../repository' import { CreateLabelInput, labelRepository } from '../repository/label' +import { Merge } from '../util' import { bulkEnqueueUpdateLabels } from '../utils/createTask' -import { logger } from '../utils/logger' -import { findHighlightById } from './highlights' -import { findLibraryItemIdsByLabelId } from './library_item' +import { deepDelete } from '../utils/helpers' +import { findLibraryItemIdsByLabelId, ItemEvent } from './library_item' -type AddLabelsToLibraryItemEvent = { - pageId: string - labels: DeepPartial