diff --git a/packages/api/src/jobs/integration/export_all_items.ts b/packages/api/src/jobs/integration/export_all_items.ts index ad687d226..f3bf8950d 100644 --- a/packages/api/src/jobs/integration/export_all_items.ts +++ b/packages/api/src/jobs/integration/export_all_items.ts @@ -1,8 +1,11 @@ import { IntegrationType } from '../../entity/integration' -import { findIntegration, updateIntegration } from '../../services/integrations' +import { + findIntegration, + getIntegrationClient, + updateIntegration, +} from '../../services/integrations' import { findRecentLibraryItems } from '../../services/library_item' import { findActiveUser } from '../../services/user' -import { enqueueExportItem } from '../../utils/createTask' import { logger } from '../../utils/logger' export interface ExportAllItemsJobData { @@ -39,17 +42,23 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => { return } + const client = getIntegrationClient( + integration.name, + integration.token, + integration + ) + const maxItems = 100 const limit = 10 let offset = 0 - // get max 1000 most recent items from the database + // get max 100 most recent items from the database while (offset < maxItems) { const libraryItems = await findRecentLibraryItems(userId, limit, offset) if (libraryItems.length === 0) { logger.info('no library items found', { userId, }) - return + break } logger.info('enqueuing export item...', { @@ -58,24 +67,42 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => { integrationId, }) - await enqueueExportItem({ - userId, - libraryItemIds: libraryItems.map((item) => item.id), - integrationId, + const synced = await client.export(libraryItems) + if (!synced) { + logger.error('failed to export item', jobData) + continue + } + + const syncedAt = new Date() + logger.info('updating integration...', { + ...jobData, + syncedAt, + }) + + // update integration syncedAt if successful + const updated = await updateIntegration( + integration.id, + { + syncedAt, + }, + userId + ) + logger.info('integration updated', { + ...jobData, + updated, }) offset += libraryItems.length logger.info('exported items', { - userId, + ...jobData, offset, - integrationId, }) } logger.info('exported all items', { - userId, - integrationId, + ...jobData, + offset, }) // clear task name in integration diff --git a/packages/api/src/repository/library_item.ts b/packages/api/src/repository/library_item.ts index fd933faf2..9dd0b315e 100644 --- a/packages/api/src/repository/library_item.ts +++ b/packages/api/src/repository/library_item.ts @@ -105,3 +105,7 @@ export const libraryItemRepository = appDataSource ) as Promise }, }) + +export const metadataColumnsInItem = getColumns(libraryItemRepository).filter( + (column) => column !== 'readableContent' && column !== 'originalContent' +) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 7e891287d..f22f1b16c 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -1,6 +1,6 @@ import { Client } from '@notionhq/client' import axios from 'axios' -import { updateIntegration } from '.' +import { findIntegrationByName, updateIntegration } from '.' import { Integration } from '../../entity/integration' import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 9290b8ece..3e47d6d76 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -23,7 +23,10 @@ import { getRepository, queryBuilderToRawSql, } from '../repository' -import { libraryItemRepository } from '../repository/library_item' +import { + libraryItemRepository, + metadataColumnsInItem, +} from '../repository/library_item' import { Merge } from '../util' import { setRecentlySavedItemInRedis } from '../utils/helpers' import { logger } from '../utils/logger' @@ -720,10 +723,13 @@ export const findRecentLibraryItems = async ( async (tx) => tx .createQueryBuilder(LibraryItem, 'library_item') - .where('library_item.user_id = :userId', { userId }) - .andWhere('library_item.state = :state', { - state: LibraryItemState.Succeeded, - }) + .select(metadataColumnsInItem.map((column) => `library_item.${column}`)) + .leftJoinAndSelect('library_item.labels', 'labels') + .leftJoinAndSelect('library_item.highlights', 'highlights') + .where( + 'library_item.user_id = :userId AND library_item.state = :state', + { userId, state: LibraryItemState.Succeeded } + ) .orderBy('library_item.saved_at', 'DESC', 'NULLS LAST') .take(limit) .skip(offset)