From d82eda1f32c7a38095f36197c0b30600c9d04a3c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 Nov 2023 16:37:14 +0800 Subject: [PATCH 1/4] allow importing all or unarchived only pocket items --- .../api/src/routers/integration_router.ts | 6 +- .../mutations/setIntegrationMutation.ts | 8 ++ packages/web/pages/settings/integrations.tsx | 133 ++++++++++++++---- 3 files changed, 119 insertions(+), 28 deletions(-) diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts index de46734db..cc6a520a7 100644 --- a/packages/api/src/routers/integration_router.ts +++ b/packages/api/src/routers/integration_router.ts @@ -23,6 +23,8 @@ export function integrationRouter() { const consumerKey = env.pocket.consumerKey const redirectUri = `${env.client.url}/settings/integrations` + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const state = req.body.state as string try { // make a POST request to Pocket to get a request token const response = await axios.post<{ code: string }>( @@ -41,7 +43,9 @@ export function integrationRouter() { const { code } = response.data // redirect the user to Pocket to authorize the request token res.redirect( - `https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}?pocketToken=${code}` + `https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}${encodeURIComponent( + `?pocketToken=${code}&state=${state}` + )}` ) } catch (error) { if (axios.isAxiosError(error)) { diff --git a/packages/web/lib/networking/mutations/setIntegrationMutation.ts b/packages/web/lib/networking/mutations/setIntegrationMutation.ts index bda8beaf0..66434ae7c 100644 --- a/packages/web/lib/networking/mutations/setIntegrationMutation.ts +++ b/packages/web/lib/networking/mutations/setIntegrationMutation.ts @@ -2,12 +2,20 @@ import { gql } from 'graphql-request' import { gqlFetcher } from '../networkHelpers' import { IntegrationType } from '../queries/useGetIntegrationsQuery' +export enum ImportItemState { + All = 'ALL', + Archived = 'ARCHIVED', + Unarchived = 'UNARCHIVED', + Unread = 'UNREAD' +} + export type SetIntegrationInput = { id?: string name: string type: IntegrationType token: string enabled: boolean + importItemState?: ImportItemState } type SetIntegrationResult = { diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index 33c316fcc..d04e863f8 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -5,6 +5,10 @@ import { DownloadSimple, Eye, Link, Spinner } from 'phosphor-react' import { useEffect, useMemo, useState } from 'react' import { Toaster } from 'react-hot-toast' import { Button } from '../../components/elements/Button' +import { + Dropdown, + DropdownOption, +} from '../../components/elements/DropdownElements' import { Box, HStack, @@ -15,7 +19,10 @@ import { SettingsLayout } from '../../components/templates/SettingsLayout' import { fetchEndpoint } from '../../lib/appConfig' import { deleteIntegrationMutation } from '../../lib/networking/mutations/deleteIntegrationMutation' import { importFromIntegrationMutation } from '../../lib/networking/mutations/importFromIntegrationMutation' -import { setIntegrationMutation } from '../../lib/networking/mutations/setIntegrationMutation' +import { + ImportItemState, + setIntegrationMutation, +} from '../../lib/networking/mutations/setIntegrationMutation' import { Integration, useGetIntegrationsQuery, @@ -47,6 +54,11 @@ interface Integrations { id: string } +interface DropdownOption { + text: string + action: () => void +} + type integrationsCard = { icon: string title: string @@ -57,6 +69,8 @@ type integrationsCard = { style: string action: () => void disabled?: boolean + isDropdown?: boolean + dropdownOptions?: DropdownOption[] } } export default function Integrations(): JSX.Element { @@ -96,11 +110,16 @@ export default function Integrations(): JSX.Element { } } - const redirectToPocket = () => { + const redirectToPocket = (importItemState: ImportItemState) => { // create a form and submit it to the backend const form = document.createElement('form') form.method = 'POST' form.action = `${fetchEndpoint}/integration/pocket/auth` + const input = document.createElement('input') + input.type = 'hidden' + input.name = 'state' + input.value = importItemState + form.appendChild(input) document.body.appendChild(form) form.submit() } @@ -114,11 +133,13 @@ export default function Integrations(): JSX.Element { try { // get the token from query string const token = router.query.pocketToken as string + const importItemState = router.query.state as ImportItemState const result = await setIntegrationMutation({ token, name: 'POCKET', type: 'IMPORT', enabled: true, + importItemState, }) if (result) { revalidate() @@ -138,7 +159,7 @@ export default function Integrations(): JSX.Element { } } if (!router.isReady) return - if (router.query.pocketToken && !pocketConnected) { + if (router.query.pocketToken && router.query.state && !pocketConnected) { connectToPocket() } }, [router]) @@ -179,7 +200,7 @@ export default function Integrations(): JSX.Element { subText: 'Pocket is a place to save articles, videos, and more. Our Pocket integration allows importing your Pocket library to Omnivore. Once connected we will asyncronously import all your Pocket articles into Omnivore, as this process is resource intensive it can take some time. You will receive an email when the process is completed. Limit 20k articles per import.', button: { - text: pocketConnected ? 'Import' : 'Connect to Pocket', + text: pocketConnected ? 'Disconnect' : 'Import', icon: isImporting(pocketConnected) ? ( ) : ( @@ -188,10 +209,25 @@ export default function Integrations(): JSX.Element { style: isImporting(pocketConnected) ? 'ctaWhite' : 'ctaDarkYellow', action: () => { pocketConnected - ? importFromIntegration(pocketConnected.id) - : redirectToPocket() + ? deleteIntegration(pocketConnected.id) + : redirectToPocket(ImportItemState.Unarchived) }, disabled: isImporting(pocketConnected), + isDropdown: !pocketConnected, + dropdownOptions: [ + { + text: 'Import All', + action: () => { + redirectToPocket(ImportItemState.All) + }, + }, + { + text: 'Import Unarchived', + action: () => { + redirectToPocket(ImportItemState.Unarchived) + }, + }, + ], }, }, { @@ -295,28 +331,71 @@ export default function Integrations(): JSX.Element {

{item.subText}

- + } > - {item.button.text} - - + {item.button.dropdownOptions?.map((option) => ( + + ))} + + ) : ( + + )} ) From 5614cb5580e6e42b1f82940dbeef44c8506c8a9d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 Nov 2023 16:40:01 +0800 Subject: [PATCH 2/4] remove task name once importer is done --- packages/integration-handler/src/index.ts | 3 ++- packages/integration-handler/src/integrations/index.ts | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/integration-handler/src/index.ts b/packages/integration-handler/src/index.ts index dabccce5e..081b909dc 100644 --- a/packages/integration-handler/src/index.ts +++ b/packages/integration-handler/src/index.ts @@ -252,7 +252,8 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( req.body.integrationName, claims.token, token, - 'IMPORT' + 'IMPORT', + null ) if (!result) { console.error('failed to update integration', { diff --git a/packages/integration-handler/src/integrations/index.ts b/packages/integration-handler/src/integrations/index.ts index 65c35a047..80cb3caff 100644 --- a/packages/integration-handler/src/integrations/index.ts +++ b/packages/integration-handler/src/integrations/index.ts @@ -31,7 +31,8 @@ export const updateIntegration = async ( name: string, integrationToken: string, token: string, - type: string + type: string, + taskName?: string | null ): Promise => { const requestData = JSON.stringify({ query: ` @@ -56,6 +57,7 @@ export const updateIntegration = async ( token: integrationToken, enabled: true, type, + taskName, }, }, }) From 06edefa1248e451c45f3fc28afb9f5f538075a65 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 Nov 2023 16:46:16 +0800 Subject: [PATCH 3/4] add debugging logs --- packages/integration-handler/src/index.ts | 29 ++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/integration-handler/src/index.ts b/packages/integration-handler/src/index.ts index 081b909dc..bb27136cd 100644 --- a/packages/integration-handler/src/index.ts +++ b/packages/integration-handler/src/index.ts @@ -189,7 +189,11 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( const since = syncedAt const state = req.body.state || State.UNARCHIVED // default to unarchived - console.log('importing pages from integration...') + console.log('importing pages from integration...', { + userId, + state, + since, + }) // get pages from integration const retrieved = await integrationClient.retrieve({ token: claims.token, @@ -198,10 +202,14 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( state, }) syncedAt = retrieved.since || Date.now() - - console.log('uploading items...') - let retrievedData = retrieved.data + + console.log('retrieved data', { + userId, + total: offset, + size: retrievedData.length, + }) + // if there are pages to import if (retrievedData.length > 0) { // write the list of urls to a csv file and upload it to gcs @@ -239,11 +247,16 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( retrievedData = retrieved.data console.log('retrieved data', { + userId, total: offset, size: retrievedData.length, }) - console.log('uploading integration...') + console.log('updating integration...', { + userId, + integrationId: req.body.integrationId, + syncedAt, + }) // update the integration's syncedAt and remove taskName const result = await updateIntegration( REST_BACKEND_ENDPOINT, @@ -257,6 +270,7 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( ) if (!result) { console.error('failed to update integration', { + userId, integrationId: req.body.integrationId, }) return res.status(400).send('Failed to update integration') @@ -266,7 +280,10 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( console.log('done') } catch (err) { - console.error('import pages from integration failed', err) + console.error('import pages from integration failed', { + userId: claims.uid, + err, + }) return res.status(500).send(err) } finally { console.log('closing write stream') From e31d651cade8c983664501b2cab78146c65f6e34 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 Nov 2023 16:56:46 +0800 Subject: [PATCH 4/4] change disconnect button color --- packages/web/pages/settings/integrations.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index d04e863f8..17314adc7 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -206,7 +206,7 @@ export default function Integrations(): JSX.Element { ) : ( ), - style: isImporting(pocketConnected) ? 'ctaWhite' : 'ctaDarkYellow', + style: pocketConnected ? 'ctaWhite' : 'ctaDarkYellow', action: () => { pocketConnected ? deleteIntegration(pocketConnected.id)