Merge pull request #3134 from omnivore-app/pocket-importer

improve pocket importer

packages/api/src/routers/integration_router.ts

@@ -23,6 +23,8 @@ export function integrationRouter() {
 
       const consumerKey = env.pocket.consumerKey
       const redirectUri = `${env.client.url}/settings/integrations`
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+      const state = req.body.state as string
       try {
         // make a POST request to Pocket to get a request token
         const response = await axios.post<{ code: string }>(
@@ -41,7 +43,9 @@ export function integrationRouter() {
         const { code } = response.data
         // redirect the user to Pocket to authorize the request token
         res.redirect(
-          `https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}?pocketToken=${code}`
+          `https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}${encodeURIComponent(
+            `?pocketToken=${code}&state=${state}`
+          )}`
         )
       } catch (error) {
         if (axios.isAxiosError(error)) {

packages/integration-handler/src/index.ts

@@ -189,7 +189,11 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction(
       const since = syncedAt
       const state = req.body.state || State.UNARCHIVED // default to unarchived
 
-      console.log('importing pages from integration...')
+      console.log('importing pages from integration...', {
+        userId,
+        state,
+        since,
+      })
       // get pages from integration
       const retrieved = await integrationClient.retrieve({
         token: claims.token,
@@ -198,10 +202,14 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction(
         state,
       })
       syncedAt = retrieved.since || Date.now()
-
-      console.log('uploading items...')
-
       let retrievedData = retrieved.data
+
+      console.log('retrieved data', {
+        userId,
+        total: offset,
+        size: retrievedData.length,
+      })
+
       // if there are pages to import
       if (retrievedData.length > 0) {
         // write the list of urls to a csv file and upload it to gcs
@@ -239,11 +247,16 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction(
           retrievedData = retrieved.data
 
           console.log('retrieved data', {
+            userId,
             total: offset,
             size: retrievedData.length,
           })
 
-          console.log('uploading integration...')
+          console.log('updating integration...', {
+            userId,
+            integrationId: req.body.integrationId,
+            syncedAt,
+          })
           // update the integration's syncedAt and remove taskName
           const result = await updateIntegration(
             REST_BACKEND_ENDPOINT,
@@ -252,10 +265,12 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction(
             req.body.integrationName,
             claims.token,
             token,
-            'IMPORT'
+            'IMPORT',
+            null
           )
           if (!result) {
             console.error('failed to update integration', {
+              userId,
               integrationId: req.body.integrationId,
             })
             return res.status(400).send('Failed to update integration')
@@ -265,7 +280,10 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction(
 
       console.log('done')
     } catch (err) {
-      console.error('import pages from integration failed', err)
+      console.error('import pages from integration failed', {
+        userId: claims.uid,
+        err,
+      })
       return res.status(500).send(err)
     } finally {
       console.log('closing write stream')

packages/integration-handler/src/integrations/index.ts

@@ -31,7 +31,8 @@ export const updateIntegration = async (
   name: string,
   integrationToken: string,
   token: string,
-  type: string
+  type: string,
+  taskName?: string | null
 ): Promise<boolean> => {
   const requestData = JSON.stringify({
     query: `
@@ -56,6 +57,7 @@ export const updateIntegration = async (
         token: integrationToken,
         enabled: true,
         type,
+        taskName,
       },
     },
   })

packages/web/lib/networking/mutations/setIntegrationMutation.ts

@@ -2,12 +2,20 @@ import { gql } from 'graphql-request'
 import { gqlFetcher } from '../networkHelpers'
 import { IntegrationType } from '../queries/useGetIntegrationsQuery'
 
+export enum ImportItemState {
+  All = 'ALL',
+  Archived = 'ARCHIVED',
+  Unarchived = 'UNARCHIVED',
+  Unread = 'UNREAD'
+}
+
 export type SetIntegrationInput = {
   id?: string
   name: string
   type: IntegrationType
   token: string
   enabled: boolean
+  importItemState?: ImportItemState
 }
 
 type SetIntegrationResult = {

packages/web/pages/settings/integrations.tsx

@@ -5,6 +5,10 @@ import { DownloadSimple, Eye, Link, Spinner } from 'phosphor-react'
 import { useEffect, useMemo, useState } from 'react'
 import { Toaster } from 'react-hot-toast'
 import { Button } from '../../components/elements/Button'
+import {
+  Dropdown,
+  DropdownOption,
+} from '../../components/elements/DropdownElements'
 import {
   Box,
   HStack,
@@ -15,7 +19,10 @@ import { SettingsLayout } from '../../components/templates/SettingsLayout'
 import { fetchEndpoint } from '../../lib/appConfig'
 import { deleteIntegrationMutation } from '../../lib/networking/mutations/deleteIntegrationMutation'
 import { importFromIntegrationMutation } from '../../lib/networking/mutations/importFromIntegrationMutation'
-import { setIntegrationMutation } from '../../lib/networking/mutations/setIntegrationMutation'
+import {
+  ImportItemState,
+  setIntegrationMutation,
+} from '../../lib/networking/mutations/setIntegrationMutation'
 import {
   Integration,
   useGetIntegrationsQuery,
@@ -47,6 +54,11 @@ interface Integrations {
   id: string
 }
 
+interface DropdownOption {
+  text: string
+  action: () => void
+}
+
 type integrationsCard = {
   icon: string
   title: string
@@ -57,6 +69,8 @@ type integrationsCard = {
     style: string
     action: () => void
     disabled?: boolean
+    isDropdown?: boolean
+    dropdownOptions?: DropdownOption[]
   }
 }
 export default function Integrations(): JSX.Element {
@@ -96,11 +110,16 @@ export default function Integrations(): JSX.Element {
     }
   }
 
-  const redirectToPocket = () => {
+  const redirectToPocket = (importItemState: ImportItemState) => {
     // create a form and submit it to the backend
     const form = document.createElement('form')
     form.method = 'POST'
     form.action = `${fetchEndpoint}/integration/pocket/auth`
+    const input = document.createElement('input')
+    input.type = 'hidden'
+    input.name = 'state'
+    input.value = importItemState
+    form.appendChild(input)
     document.body.appendChild(form)
     form.submit()
   }
@@ -114,11 +133,13 @@ export default function Integrations(): JSX.Element {
       try {
         // get the token from query string
         const token = router.query.pocketToken as string
+        const importItemState = router.query.state as ImportItemState
         const result = await setIntegrationMutation({
           token,
           name: 'POCKET',
           type: 'IMPORT',
           enabled: true,
+          importItemState,
         })
         if (result) {
           revalidate()
@@ -138,7 +159,7 @@ export default function Integrations(): JSX.Element {
       }
     }
     if (!router.isReady) return
-    if (router.query.pocketToken && !pocketConnected) {
+    if (router.query.pocketToken && router.query.state && !pocketConnected) {
       connectToPocket()
     }
   }, [router])
@@ -179,19 +200,34 @@ export default function Integrations(): JSX.Element {
         subText:
           'Pocket is a place to save articles, videos, and more. Our Pocket integration allows importing your Pocket library to Omnivore. Once connected we will asyncronously import all your Pocket articles into Omnivore, as this process is resource intensive it can take some time. You will receive an email when the process is completed. Limit 20k articles per import.',
         button: {
-          text: pocketConnected ? 'Import' : 'Connect to Pocket',
+          text: pocketConnected ? 'Disconnect' : 'Import',
           icon: isImporting(pocketConnected) ? (
             <Spinner size={16} />
           ) : (
             <Link size={16} weight={'bold'} />
           ),
-          style: isImporting(pocketConnected) ? 'ctaWhite' : 'ctaDarkYellow',
+          style: pocketConnected ? 'ctaWhite' : 'ctaDarkYellow',
           action: () => {
             pocketConnected
-              ? importFromIntegration(pocketConnected.id)
-              : redirectToPocket()
+              ? deleteIntegration(pocketConnected.id)
+              : redirectToPocket(ImportItemState.Unarchived)
           },
           disabled: isImporting(pocketConnected),
+          isDropdown: !pocketConnected,
+          dropdownOptions: [
+            {
+              text: 'Import All',
+              action: () => {
+                redirectToPocket(ImportItemState.All)
+              },
+            },
+            {
+              text: 'Import Unarchived',
+              action: () => {
+                redirectToPocket(ImportItemState.Unarchived)
+              },
+            },
+          ],
         },
       },
       {
@@ -295,28 +331,71 @@ export default function Integrations(): JSX.Element {
                   <p>{item.subText}</p>
                 </Box>
                 <HStack css={{ '@smDown': { width: '100%' } }}>
-                  <Button
-                    style={
-                      item.button.style === 'ctaDarkYellow'
-                        ? 'ctaDarkYellow'
-                        : 'ctaWhite'
-                    }
-                    css={{
-                      py: '10px',
-                      px: '14px',
-                      minWidth: '230px',
-                      width: '100%',
-                    }}
-                    onClick={item.button.action}
-                    disabled={item.button.disabled}
-                  >
-                    {item.button.icon}
-                    <SpanBox
-                      css={{ pl: '10px', fontWeight: '600', fontSize: '16px' }}
+                  {item.button.isDropdown ? (
+                    <Dropdown
+                      triggerElement={
+                        <Button
+                          style={
+                            item.button.style === 'ctaDarkYellow'
+                              ? 'ctaDarkYellow'
+                              : 'ctaWhite'
+                          }
+                          css={{
+                            py: '10px',
+                            px: '14px',
+                            minWidth: '230px',
+                            width: '100%',
+                          }}
+                        >
+                          {item.button.icon}
+                          <SpanBox
+                            css={{
+                              pl: '10px',
+                              fontWeight: '600',
+                              fontSize: '16px',
+                            }}
+                          >
+                            {item.button.text}
+                          </SpanBox>
+                        </Button>
+                      }
                     >
-                      {item.button.text}
-                    </SpanBox>
-                  </Button>
+                      {item.button.dropdownOptions?.map((option) => (
+                        <DropdownOption
+                          key={option.text}
+                          onSelect={option.action}
+                          title={option.text}
+                        ></DropdownOption>
+                      ))}
+                    </Dropdown>
+                  ) : (
+                    <Button
+                      style={
+                        item.button.style === 'ctaDarkYellow'
+                          ? 'ctaDarkYellow'
+                          : 'ctaWhite'
+                      }
+                      css={{
+                        py: '10px',
+                        px: '14px',
+                        minWidth: '230px',
+                        width: '100%',
+                      }}
+                      onClick={item.button.action}
+                      disabled={item.button.disabled}
+                    >
+                      {item.button.icon}
+                      <SpanBox
+                        css={{
+                          pl: '10px',
+                          fontWeight: '600',
+                          fontSize: '16px',
+                        }}
+                      >
+                        {item.button.text}
+                      </SpanBox>
+                    </Button>
+                  )}
                 </HStack>
               </HStack>
             )