diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index ad9738e17..92e724e37 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -972,6 +972,7 @@ export type Integration = { enabled: Scalars['Boolean']; id: Scalars['ID']; name: Scalars['String']; + taskName?: Maybe; token: Scalars['String']; type: IntegrationType; updatedAt: Scalars['Date']; @@ -4939,6 +4940,7 @@ export type IntegrationResolvers; id?: Resolver; name?: Resolver; + taskName?: Resolver, ParentType, ContextType>; token?: Resolver; type?: Resolver; updatedAt?: Resolver; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index d069781f2..2f967cf8e 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -864,6 +864,7 @@ type Integration { enabled: Boolean! id: ID! name: String! + taskName: String token: String! type: IntegrationType! updatedAt: Date! diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 2b115c5ff..9e31458bd 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -211,52 +211,65 @@ export function integrationsServiceRouter() { integrationId: integration.id, }) - // write the list of urls to a csv file and upload it to gcs - // path style: imports///-.csv - const dateStr = DateTime.now().toISODate() - const fileUuid = uuidv4() - const fullPath = `imports/${userId}/${dateStr}/URL_LIST-${fileUuid}.csv` - // open a write_stream to the file - const file = createGCSFile(fullPath) - writeStream = file.createWriteStream({ - contentType: 'text/csv', - }) - // stringify the data and pipe it to the write_stream - const stringifier = stringify({ - header: false, - columns: ['url', 'state', 'labels'], - }) - stringifier.pipe(writeStream) - let offset = 0 const since = integration.syncedAt?.getTime() || 0 let syncedAt = since - // eslint-disable-next-line no-constant-condition - while (true) { - // get pages from integration - const retrieved = await integrationService.retrieve({ - token: integration.token, - since, - offset, - }) - syncedAt = retrieved.since || Date.now() - const retrievedData = retrieved.data - if (retrievedData.length === 0) { - break - } - // write the list of urls, state and labels to the stream - retrievedData.forEach((row) => stringifier.write(row)) + // get pages from integration + const retrieved = await integrationService.retrieve({ + token: integration.token, + since, + offset, + }) + syncedAt = retrieved.since || Date.now() - offset += retrievedData.length - console.debug('retrieved data', { - total: offset, - size: retrievedData.length, + let retrievedData = retrieved.data + // if there are pages to import + if (retrievedData.length > 0) { + // write the list of urls to a csv file and upload it to gcs + // path style: imports///-.csv + const dateStr = DateTime.now().toISODate() + const fileUuid = uuidv4() + const fullPath = `imports/${userId}/${dateStr}/URL_LIST-${fileUuid}.csv` + // open a write_stream to the file + const file = createGCSFile(fullPath) + writeStream = file.createWriteStream({ + contentType: 'text/csv', }) + // stringify the data and pipe it to the write_stream + const stringifier = stringify({ + header: false, + columns: ['url', 'state', 'labels'], + }) + stringifier.pipe(writeStream) + + // paginate api calls to the integration + do { + // write the list of urls, state and labels to the stream + retrievedData.forEach((row) => stringifier.write(row)) + + // get next pages from the integration + offset += retrievedData.length + + const retrieved = await integrationService.retrieve({ + token: integration.token, + since, + offset, + }) + syncedAt = retrieved.since || Date.now() + retrievedData = retrieved.data + + console.debug('retrieved data', { + total: offset, + size: retrievedData.length, + }) + } while (retrievedData.length > 0) } - // update the integration's syncedAt + + // update the integration's syncedAt and remove taskName await getRepository(Integration).update(integration.id, { syncedAt: new Date(syncedAt), + taskName: null, }) } catch (err) { logger.error('import pages from integration failed', err) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 907c1558e..03deee21b 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1940,6 +1940,7 @@ const schema = gql` enabled: Boolean! createdAt: Date! updatedAt: Date! + taskName: String } enum IntegrationType { diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 26ec75c41..4db367f29 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -497,7 +497,7 @@ export const enqueueImportFromIntegration = async ( console.error(error) }) }, 0) - return '' + return nanoid() } const createdTasks = await createHttpTaskWithToken({ diff --git a/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx b/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx index 3541380d9..711079bae 100644 --- a/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx +++ b/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx @@ -10,6 +10,7 @@ export interface Integration { enabled: boolean createdAt: Date updatedAt: Date + taskName?: string } export type IntegrationType = 'EXPORT' | 'IMPORT' @@ -41,6 +42,7 @@ export function useGetIntegrationsQuery(): IntegrationsQueryResponse { enabled createdAt updatedAt + taskName } } ... on IntegrationsError { diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index 7d74f6e98..6f09a6c84 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -1,7 +1,7 @@ import { styled } from '@stitches/react' import Image from 'next/image' import { useRouter } from 'next/router' -import { DownloadSimple, Eye, Link } from 'phosphor-react' +import { DownloadSimple, Eye, Link, Spinner } from 'phosphor-react' import { useEffect, useMemo, useState } from 'react' import { Toaster } from 'react-hot-toast' import { Button } from '../../components/elements/Button' @@ -16,7 +16,10 @@ import { fetchEndpoint } from '../../lib/appConfig' import { deleteIntegrationMutation } from '../../lib/networking/mutations/deleteIntegrationMutation' import { importFromIntegrationMutation } from '../../lib/networking/mutations/importFromIntegrationMutation' import { setIntegrationMutation } from '../../lib/networking/mutations/setIntegrationMutation' -import { useGetIntegrationsQuery } from '../../lib/networking/queries/useGetIntegrationsQuery' +import { + Integration, + useGetIntegrationsQuery, +} from '../../lib/networking/queries/useGetIntegrationsQuery' import { useGetWebhooksQuery } from '../../lib/networking/queries/useGetWebhooksQuery' import { applyStoredTheme } from '../../lib/themeUpdater' import { showErrorToast, showSuccessToast } from '../../lib/toastHelpers' @@ -53,6 +56,7 @@ type integrationsCard = { icon?: JSX.Element style: string action: () => void + disabled?: boolean } } export default function Integrations(): JSX.Element { @@ -85,6 +89,7 @@ export default function Integrations(): JSX.Element { const importFromIntegration = async (id: string) => { try { await importFromIntegrationMutation(id) + revalidate() showSuccessToast('Import started') } catch (err) { showErrorToast('Error: ' + err) @@ -100,6 +105,10 @@ export default function Integrations(): JSX.Element { form.submit() } + const isImporting = (integration: Integration | undefined) => { + return !!integration && !!integration.taskName + } + useEffect(() => { const connectToPocket = async () => { try { @@ -171,13 +180,18 @@ export default function Integrations(): JSX.Element { 'Pocket is a place to save articles, videos, and more. Our Pocket integration allows importing your Pocket library to Omnivore. Once connected we will asyncronously import all your Pocket articles into Omnivore, as this process is resource intensive it can take some time. You will receive an email when the process is completed.', button: { text: pocketConnected ? 'Import' : 'Connect to Pocket', - icon: , - style: 'ctaDarkYellow', + icon: isImporting(pocketConnected) ? ( + + ) : ( + + ), + style: isImporting(pocketConnected) ? 'ctaWhite' : 'ctaDarkYellow', action: () => { pocketConnected ? importFromIntegration(pocketConnected.id) : redirectToPocket() }, + disabled: isImporting(pocketConnected), }, }, { @@ -294,6 +308,7 @@ export default function Integrations(): JSX.Element { width: '100%', }} onClick={item.button.action} + disabled={item.button.disabled} > {item.button.icon}