export last 100 items in one job

packages/api/src/jobs/integration/export_all_items.ts

@@ -1,8 +1,11 @@
 import { IntegrationType } from '../../entity/integration'
-import { findIntegration, updateIntegration } from '../../services/integrations'
+import {
+  findIntegration,
+  getIntegrationClient,
+  updateIntegration,
+} from '../../services/integrations'
 import { findRecentLibraryItems } from '../../services/library_item'
 import { findActiveUser } from '../../services/user'
-import { enqueueExportItem } from '../../utils/createTask'
 import { logger } from '../../utils/logger'
 
 export interface ExportAllItemsJobData {
@@ -39,17 +42,23 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => {
     return
   }
 
+  const client = getIntegrationClient(
+    integration.name,
+    integration.token,
+    integration
+  )
+
   const maxItems = 100
   const limit = 10
   let offset = 0
-  // get max 1000 most recent items from the database
+  // get max 100 most recent items from the database
   while (offset < maxItems) {
     const libraryItems = await findRecentLibraryItems(userId, limit, offset)
     if (libraryItems.length === 0) {
       logger.info('no library items found', {
         userId,
       })
-      return
+      break
     }
 
     logger.info('enqueuing export item...', {
@@ -58,24 +67,42 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => {
       integrationId,
     })
 
-    await enqueueExportItem({
-      userId,
-      libraryItemIds: libraryItems.map((item) => item.id),
-      integrationId,
+    const synced = await client.export(libraryItems)
+    if (!synced) {
+      logger.error('failed to export item', jobData)
+      continue
+    }
+
+    const syncedAt = new Date()
+    logger.info('updating integration...', {
+      ...jobData,
+      syncedAt,
+    })
+
+    // update integration syncedAt if successful
+    const updated = await updateIntegration(
+      integration.id,
+      {
+        syncedAt,
+      },
+      userId
+    )
+    logger.info('integration updated', {
+      ...jobData,
+      updated,
     })
 
     offset += libraryItems.length
 
     logger.info('exported items', {
-      userId,
+      ...jobData,
       offset,
-      integrationId,
     })
   }
 
   logger.info('exported all items', {
-    userId,
-    integrationId,
+    ...jobData,
+    offset,
   })
 
   // clear task name in integration

packages/api/src/repository/library_item.ts

@@ -105,3 +105,7 @@ export const libraryItemRepository = appDataSource
       ) as Promise<LibraryItem[]>
     },
   })
+
+export const metadataColumnsInItem = getColumns(libraryItemRepository).filter(
+  (column) => column !== 'readableContent' && column !== 'originalContent'
+)

packages/api/src/services/integrations/notion.ts

@@ -1,6 +1,6 @@
 import { Client } from '@notionhq/client'
 import axios from 'axios'
-import { updateIntegration } from '.'
+import { findIntegrationByName, updateIntegration } from '.'
 import { Integration } from '../../entity/integration'
 import { LibraryItem } from '../../entity/library_item'
 import { env } from '../../env'

packages/api/src/services/library_item.ts

@@ -23,7 +23,10 @@ import {
   getRepository,
   queryBuilderToRawSql,
 } from '../repository'
-import { libraryItemRepository } from '../repository/library_item'
+import {
+  libraryItemRepository,
+  metadataColumnsInItem,
+} from '../repository/library_item'
 import { Merge } from '../util'
 import { setRecentlySavedItemInRedis } from '../utils/helpers'
 import { logger } from '../utils/logger'
@@ -720,10 +723,13 @@ export const findRecentLibraryItems = async (
     async (tx) =>
       tx
         .createQueryBuilder(LibraryItem, 'library_item')
-        .where('library_item.user_id = :userId', { userId })
-        .andWhere('library_item.state = :state', {
-          state: LibraryItemState.Succeeded,
-        })
+        .select(metadataColumnsInItem.map((column) => `library_item.${column}`))
+        .leftJoinAndSelect('library_item.labels', 'labels')
+        .leftJoinAndSelect('library_item.highlights', 'highlights')
+        .where(
+          'library_item.user_id = :userId AND library_item.state = :state',
+          { userId, state: LibraryItemState.Succeeded }
+        )
         .orderBy('library_item.saved_at', 'DESC', 'NULLS LAST')
         .take(limit)
         .skip(offset)