Merge pull request #2511 from omnivore-app/fix/pocket-importer

disable import button while importer is running

packages/api/src/generated/graphql.ts

@@ -972,6 +972,7 @@ export type Integration = {
   enabled: Scalars['Boolean'];
   id: Scalars['ID'];
   name: Scalars['String'];
+  taskName?: Maybe<Scalars['String']>;
   token: Scalars['String'];
   type: IntegrationType;
   updatedAt: Scalars['Date'];
@@ -4939,6 +4940,7 @@ export type IntegrationResolvers<ContextType = ResolverContext, ParentType exten
   enabled?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
   name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  taskName?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   token?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   type?: Resolver<ResolversTypes['IntegrationType'], ParentType, ContextType>;
   updatedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;

packages/api/src/generated/schema.graphql

@@ -864,6 +864,7 @@ type Integration {
   enabled: Boolean!
   id: ID!
   name: String!
+  taskName: String
   token: String!
   type: IntegrationType!
   updatedAt: Date!

packages/api/src/routers/svc/integrations.ts

@@ -211,52 +211,65 @@ export function integrationsServiceRouter() {
         integrationId: integration.id,
       })
 
-      // write the list of urls to a csv file and upload it to gcs
-      // path style: imports/<uid>/<date>/<type>-<uuid>.csv
-      const dateStr = DateTime.now().toISODate()
-      const fileUuid = uuidv4()
-      const fullPath = `imports/${userId}/${dateStr}/URL_LIST-${fileUuid}.csv`
-      // open a write_stream to the file
-      const file = createGCSFile(fullPath)
-      writeStream = file.createWriteStream({
-        contentType: 'text/csv',
-      })
-      // stringify the data and pipe it to the write_stream
-      const stringifier = stringify({
-        header: false,
-        columns: ['url', 'state', 'labels'],
-      })
-      stringifier.pipe(writeStream)
-
       let offset = 0
       const since = integration.syncedAt?.getTime() || 0
       let syncedAt = since
-      // eslint-disable-next-line no-constant-condition
-      while (true) {
-        // get pages from integration
-        const retrieved = await integrationService.retrieve({
-          token: integration.token,
-          since,
-          offset,
-        })
-        syncedAt = retrieved.since || Date.now()
 
-        const retrievedData = retrieved.data
-        if (retrievedData.length === 0) {
-          break
-        }
-        // write the list of urls, state and labels to the stream
-        retrievedData.forEach((row) => stringifier.write(row))
+      // get pages from integration
+      const retrieved = await integrationService.retrieve({
+        token: integration.token,
+        since,
+        offset,
+      })
+      syncedAt = retrieved.since || Date.now()
 
-        offset += retrievedData.length
-        console.debug('retrieved data', {
-          total: offset,
-          size: retrievedData.length,
+      let retrievedData = retrieved.data
+      // if there are pages to import
+      if (retrievedData.length > 0) {
+        // write the list of urls to a csv file and upload it to gcs
+        // path style: imports/<uid>/<date>/<type>-<uuid>.csv
+        const dateStr = DateTime.now().toISODate()
+        const fileUuid = uuidv4()
+        const fullPath = `imports/${userId}/${dateStr}/URL_LIST-${fileUuid}.csv`
+        // open a write_stream to the file
+        const file = createGCSFile(fullPath)
+        writeStream = file.createWriteStream({
+          contentType: 'text/csv',
         })
+        // stringify the data and pipe it to the write_stream
+        const stringifier = stringify({
+          header: false,
+          columns: ['url', 'state', 'labels'],
+        })
+        stringifier.pipe(writeStream)
+
+        // paginate api calls to the integration
+        do {
+          // write the list of urls, state and labels to the stream
+          retrievedData.forEach((row) => stringifier.write(row))
+
+          // get next pages from the integration
+          offset += retrievedData.length
+
+          const retrieved = await integrationService.retrieve({
+            token: integration.token,
+            since,
+            offset,
+          })
+          syncedAt = retrieved.since || Date.now()
+          retrievedData = retrieved.data
+
+          console.debug('retrieved data', {
+            total: offset,
+            size: retrievedData.length,
+          })
+        } while (retrievedData.length > 0)
       }
-      // update the integration's syncedAt
+
+      // update the integration's syncedAt and remove taskName
       await getRepository(Integration).update(integration.id, {
         syncedAt: new Date(syncedAt),
+        taskName: null,
       })
     } catch (err) {
       logger.error('import pages from integration failed', err)

packages/api/src/schema.ts

@@ -1940,6 +1940,7 @@ const schema = gql`
     enabled: Boolean!
     createdAt: Date!
     updatedAt: Date!
+    taskName: String
   }
 
   enum IntegrationType {

packages/api/src/utils/createTask.ts

@@ -497,7 +497,7 @@ export const enqueueImportFromIntegration = async (
           console.error(error)
         })
     }, 0)
-    return ''
+    return nanoid()
   }
 
   const createdTasks = await createHttpTaskWithToken({

packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx

@@ -10,6 +10,7 @@ export interface Integration {
   enabled: boolean
   createdAt: Date
   updatedAt: Date
+  taskName?: string
 }
 
 export type IntegrationType = 'EXPORT' | 'IMPORT'
@@ -41,6 +42,7 @@ export function useGetIntegrationsQuery(): IntegrationsQueryResponse {
             enabled
             createdAt
             updatedAt
+            taskName
           }
         }
         ... on IntegrationsError {

packages/web/pages/settings/integrations.tsx

@@ -1,7 +1,7 @@
 import { styled } from '@stitches/react'
 import Image from 'next/image'
 import { useRouter } from 'next/router'
-import { DownloadSimple, Eye, Link } from 'phosphor-react'
+import { DownloadSimple, Eye, Link, Spinner } from 'phosphor-react'
 import { useEffect, useMemo, useState } from 'react'
 import { Toaster } from 'react-hot-toast'
 import { Button } from '../../components/elements/Button'
@@ -16,7 +16,10 @@ import { fetchEndpoint } from '../../lib/appConfig'
 import { deleteIntegrationMutation } from '../../lib/networking/mutations/deleteIntegrationMutation'
 import { importFromIntegrationMutation } from '../../lib/networking/mutations/importFromIntegrationMutation'
 import { setIntegrationMutation } from '../../lib/networking/mutations/setIntegrationMutation'
-import { useGetIntegrationsQuery } from '../../lib/networking/queries/useGetIntegrationsQuery'
+import {
+  Integration,
+  useGetIntegrationsQuery,
+} from '../../lib/networking/queries/useGetIntegrationsQuery'
 import { useGetWebhooksQuery } from '../../lib/networking/queries/useGetWebhooksQuery'
 import { applyStoredTheme } from '../../lib/themeUpdater'
 import { showErrorToast, showSuccessToast } from '../../lib/toastHelpers'
@@ -53,6 +56,7 @@ type integrationsCard = {
     icon?: JSX.Element
     style: string
     action: () => void
+    disabled?: boolean
   }
 }
 export default function Integrations(): JSX.Element {
@@ -85,6 +89,7 @@ export default function Integrations(): JSX.Element {
   const importFromIntegration = async (id: string) => {
     try {
       await importFromIntegrationMutation(id)
+      revalidate()
       showSuccessToast('Import started')
     } catch (err) {
       showErrorToast('Error: ' + err)
@@ -100,6 +105,10 @@ export default function Integrations(): JSX.Element {
     form.submit()
   }
 
+  const isImporting = (integration: Integration | undefined) => {
+    return !!integration && !!integration.taskName
+  }
+
   useEffect(() => {
     const connectToPocket = async () => {
       try {
@@ -171,13 +180,18 @@ export default function Integrations(): JSX.Element {
           'Pocket is a place to save articles, videos, and more. Our Pocket integration allows importing your Pocket library to Omnivore. Once connected we will asyncronously import all your Pocket articles into Omnivore, as this process is resource intensive it can take some time. You will receive an email when the process is completed.',
         button: {
           text: pocketConnected ? 'Import' : 'Connect to Pocket',
-          icon: <Link size={16} weight={'bold'} />,
-          style: 'ctaDarkYellow',
+          icon: isImporting(pocketConnected) ? (
+            <Spinner size={16} />
+          ) : (
+            <Link size={16} weight={'bold'} />
+          ),
+          style: isImporting(pocketConnected) ? 'ctaWhite' : 'ctaDarkYellow',
           action: () => {
             pocketConnected
               ? importFromIntegration(pocketConnected.id)
               : redirectToPocket()
           },
+          disabled: isImporting(pocketConnected),
         },
       },
       {
@@ -294,6 +308,7 @@ export default function Integrations(): JSX.Element {
                       width: '100%',
                     }}
                     onClick={item.button.action}
+                    disabled={item.button.disabled}
                   >
                     {item.button.icon}
                     <SpanBox