From e2d050288e4578db81021f6d8adcbac48ded40bd Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 16 Feb 2023 12:33:37 +0800
Subject: [PATCH] write the list of urls to a csv file and upload it to gcs

---
 .../src/services/integrations/integration.ts  |  4 +-
 .../api/src/services/integrations/pocket.ts   | 39 +++++++++----------
 2 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts
index aad49a369..1581a0842 100644
--- a/packages/api/src/services/integrations/integration.ts
+++ b/packages/api/src/services/integrations/integration.ts
@@ -13,7 +13,7 @@ export abstract class IntegrationService {
   ): Promise<boolean> => {
     return Promise.resolve(true)
   }
-  import = async (integration: Integration): Promise<void> => {
-    return Promise.resolve()
+  import = async (integration: Integration): Promise<number> => {
+    return Promise.resolve(0)
   }
 }
diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts
index 9ba79d3d7..6370ad986 100644
--- a/packages/api/src/services/integrations/pocket.ts
+++ b/packages/api/src/services/integrations/pocket.ts
@@ -2,7 +2,10 @@ import { IntegrationService } from './integration'
 import { Integration } from '../../entity/integration'
 import axios from 'axios'
 import { env } from '../../env'
-import { PubSub } from '@google-cloud/pubsub'
+import { DateTime } from 'luxon'
+import { uploadToBucket } from '../../utils/uploads'
+import { v4 as uuidv4 } from 'uuid'
+import { getRepository } from '../../entity/utils'
 
 interface PocketResponse {
   list: {
@@ -17,7 +20,6 @@ interface PocketItem {
 export class PocketIntegration extends IntegrationService {
   name = 'POCKET'
   POCKET_API_URL = 'https://getpocket.com/v3'
-  IMPORT_TOPIC = 'importURL'
 
   retrievePocketData = async (
     accessToken: string,
@@ -39,28 +41,25 @@ export class PocketIntegration extends IntegrationService {
     }
   }
 
-  import = async (integration: Integration): Promise<void> => {
+  import = async (integration: Integration): Promise<number> => {
     const syncAt = integration.syncedAt
       ? integration.syncedAt.getTime() / 1000
       : 0
     const pocketData = await this.retrievePocketData(integration.token, syncAt)
     const pocketItems = Object.values(pocketData.list)
-    // publish pocket items to queue
-    const client = new PubSub()
-    await Promise.all(
-      pocketItems.map((item) => {
-        return client
-          .topic(this.IMPORT_TOPIC)
-          .publishMessage({
-            data: JSON.stringify({
-              url: item.given_url,
-            }),
-          })
-          .catch((err) => {
-            console.log('error publishing to pubsub', err)
-            return undefined
-          })
-      })
-    )
+    // write the list of urls to a csv file and upload it to gcs
+    // path style: imports/<uid>/<date>/<type>-<uuid>.csv
+    const dateStr = DateTime.now().toISODate()
+    const fileUuid = uuidv4()
+    const fullPath = `imports/${integration.user.id}/${dateStr}/URL_LIST-${fileUuid}.csv`
+    const data = pocketItems.map((item) => item.given_url).join('\n')
+    await uploadToBucket(fullPath, Buffer.from(data, 'utf-8'), {
+      contentType: 'text/csv',
+    })
+    // update the integration's syncedAt
+    await getRepository(Integration).update(integration.id, {
+      syncedAt: new Date(),
+    })
+    return pocketItems.length
   }
 }