Merge pull request #4483 from omnivore-app/jacksonh/export-direct-download

Export direct download links + progress updates

packages/api/src/jobs/export.ts

@@ -175,6 +175,12 @@ export const exportJob = async (jobData: ExportJobData) => {
       userId
     )
 
+    await saveExport(userId, {
+      id: exportId,
+      state: TaskState.Running,
+      totalItems: itemCount,
+    })
+
     logger.info(`exporting ${itemCount} items...`, {
       userId,
     })
@@ -226,6 +232,11 @@ export const exportJob = async (jobData: ExportJobData) => {
       // fetch data from the database
       const batchSize = 20
       for (cursor = 0; cursor < itemCount; cursor += batchSize) {
+        logger.info(`export extracting ${cursor} of ${itemCount}`, {
+          userId,
+          exportId,
+        })
+
         const items = await searchLibraryItems(
           {
             from: cursor,
@@ -242,6 +253,10 @@ export const exportJob = async (jobData: ExportJobData) => {
         // write data to the csv file
         if (size > 0) {
           await uploadToBucket(userId, items, cursor, size, archive)
+          await saveExport(userId, {
+            id: exportId,
+            processedItems: cursor,
+          })
         } else {
           break
         }
@@ -264,7 +279,7 @@ export const exportJob = async (jobData: ExportJobData) => {
     // generate a temporary signed url for the zip file
     const [signedUrl] = await file.getSignedUrl({
       action: 'read',
-      expires: Date.now() + 48 * 60 * 60 * 1000, // 48 hours
+      expires: Date.now() + 168 * 60 * 60 * 1000, // one week
     })
 
     logger.info('signed url for export:', {
@@ -275,6 +290,8 @@ export const exportJob = async (jobData: ExportJobData) => {
     await saveExport(userId, {
       id: exportId,
       state: TaskState.Succeeded,
+      signedUrl,
+      processedItems: itemCount,
     })
 
     const job = await sendExportJobEmail(userId, 'completed', signedUrl)

packages/api/src/routers/export_router.ts

@@ -3,8 +3,9 @@ import express, { Router } from 'express'
 import { TaskState } from '../generated/graphql'
 import { jobStateToTaskState } from '../queue-processor'
 import {
-  countExportsWithin24Hours,
+  countExportsWithin6Hours,
   countExportsWithinMinute,
+  findExports,
   saveExport,
 } from '../services/export'
 import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
@@ -42,9 +43,9 @@ export function exportRouter() {
         })
       }
 
-      const exportsWithin24Hours = await countExportsWithin24Hours(userId)
+      const exportsWithin24Hours = await countExportsWithin6Hours(userId)
       if (exportsWithin24Hours >= 3) {
-        logger.error('User has reached the limit of exports within 24 hours', {
+        logger.error('User has reached the limit of exports within 6 hours', {
           userId,
           exportsWithin24Hours,
         })
@@ -97,5 +98,37 @@ export function exportRouter() {
     }
   })
 
+  // eslint-disable-next-line @typescript-eslint/no-misused-promises
+  router.get('/list', cors<express.Request>(corsConfig), async (req, res) => {
+    const token = getTokenByRequest(req)
+    // get claims from token
+    const claims = await getClaimsByToken(token)
+    if (!claims) {
+      logger.error('Token not found')
+      return res.status(401).send({
+        error: 'UNAUTHORIZED',
+      })
+    }
+
+    // get user by uid from claims
+    const userId = claims.uid
+
+    try {
+      const exports = await findExports(userId)
+
+      res.send({
+        exports,
+      })
+    } catch (error) {
+      logger.error('Error fetching exports', {
+        userId,
+        error,
+      })
+      return res.status(500).send({
+        error: 'INTERNAL_ERROR',
+      })
+    }
+  })
+
   return router
 }

packages/api/src/services/export.ts

@@ -23,12 +23,12 @@ export const countExportsWithinMinute = async (
   })
 }
 
-export const countExportsWithin24Hours = async (
+export const countExportsWithin6Hours = async (
   userId: string
 ): Promise<number> => {
   return getRepository(Export).countBy({
     userId,
-    createdAt: MoreThan(new Date(Date.now() - 24 * 60 * 60 * 1000)),
+    createdAt: MoreThan(new Date(Date.now() - 6 * 60 * 60 * 1000)),
     state: In([TaskState.Pending, TaskState.Running, TaskState.Succeeded]),
   })
 }
@@ -42,3 +42,14 @@ export const findExportById = async (
     userId,
   })
 }
+
+export const findExports = async (userId: string): Promise<Export[] | null> => {
+  return getRepository(Export).find({
+    where: {
+      userId,
+    },
+    order: {
+      createdAt: 'DESC',
+    },
+  })
+}

packages/web/lib/networking/useCreateExport.tsx

@@ -1,11 +1,40 @@
+import { useQuery, useQueryClient } from '@tanstack/react-query'
 import { apiFetcher } from './networkHelpers'
+import { TaskState } from './mutations/exportToIntegrationMutation'
+
+type Export = {
+  id: string
+  state: TaskState
+  totalItems?: number
+  processedItems?: number
+  createdAt: string
+  signedUrl: string
+}
+
+type ExportsResponse = {
+  exports: Export[]
+}
 
 export const createExport = async (): Promise<boolean> => {
   try {
     const response = await apiFetcher(`/api/export/`)
+    console.log('RESPONSE:  ', response)
+    if ('error' in (response as any)) {
+      return false
+    }
     return true
   } catch (error) {
     console.log('error scheduling export. ')
     return false
   }
 }
+
+export function useGetExports() {
+  return useQuery({
+    queryKey: ['exports'],
+    queryFn: async () => {
+      const response = (await apiFetcher(`/api/export/list`)) as ExportsResponse
+      return response.exports
+    },
+  })
+}

packages/web/pages/settings/account.tsx

@@ -2,6 +2,7 @@ import { useCallback, useEffect, useMemo, useState } from 'react'
 import { Button } from '../../components/elements/Button'
 import {
   Box,
+  HStack,
   SpanBox,
   VStack,
 } from '../../components/elements/LayoutPrimitives'
@@ -27,7 +28,13 @@ import { useGetViewerQuery } from '../../lib/networking/queries/useGetViewerQuer
 import { useValidateUsernameQuery } from '../../lib/networking/queries/useValidateUsernameQuery'
 import { applyStoredTheme } from '../../lib/themeUpdater'
 import { showErrorToast, showSuccessToast } from '../../lib/toastHelpers'
-import { createExport } from '../../lib/networking/useCreateExport'
+import {
+  createExport,
+  useGetExports,
+} from '../../lib/networking/useCreateExport'
+import { TaskState } from '../../lib/networking/mutations/exportToIntegrationMutation'
+import { timeAgo } from '../../lib/textFormatting'
+import { Download, DownloadSimple } from '@phosphor-icons/react'
 
 const ACCOUNT_LIMIT = 50_000
 
@@ -475,6 +482,8 @@ export default function Account(): JSX.Element {
 }
 
 const ExportSection = (): JSX.Element => {
+  const { data: recentExports } = useGetExports()
+  console.log('recentExports: ', recentExports)
   const doExport = useCallback(async () => {
     const result = await createExport()
     if (result) {
@@ -502,6 +511,12 @@ const ExportSection = (): JSX.Element => {
         you should receive an email with a link to your data within an hour. The
         download link will be available for 24 hours.
       </StyledText>
+      <StyledText style="footnote" css={{ mt: '10px', mb: '20px' }}>
+        If you do not receive your completed export within 24hrs please contact{' '}
+        <a href="mailto:feedback@omnivore.app">
+          Contact&nbsp;us via&nbsp;email
+        </a>
+      </StyledText>
       <Button
         style="ctaDarkYellow"
         onClick={(event) => {
@@ -512,6 +527,51 @@ const ExportSection = (): JSX.Element => {
       >
         Export Data
       </Button>
+
+      {recentExports && (
+        <VStack css={{ width: '100% ', mt: '20px' }}>
+          <StyledLabel>Recent exports</StyledLabel>
+          {recentExports.map((item) => {
+            return (
+              <HStack
+                key={item.id}
+                css={{ width: '100%', height: '55px' }}
+                distribution="start"
+                alignment="center"
+              >
+                <SpanBox css={{ width: '180px' }} title={item.createdAt}>
+                  {timeAgo(item.createdAt)}
+                </SpanBox>
+                <SpanBox css={{ width: '180px' }}>{item.state}</SpanBox>
+                {item.totalItems && (
+                  <VStack css={{ width: '180px', height: '50px', pt: '12px' }}>
+                    <ProgressBar
+                      fillPercentage={
+                        ((item.processedItems ?? 0) / item.totalItems) * 100
+                      }
+                      fillColor={theme.colors.omnivoreCtaYellow.toString()}
+                      backgroundColor={theme.colors.grayText.toString()}
+                      borderRadius={'2px'}
+                    />
+                    <StyledText style="footnote" css={{ mt: '0px' }}>
+                      {`${item.processedItems ?? 0} of ${
+                        item.totalItems
+                      } items.`}
+                    </StyledText>
+                  </VStack>
+                )}
+                {item.signedUrl && (
+                  <SpanBox css={{ marginLeft: 'auto' }}>
+                    <a href={item.signedUrl} target="_blank" rel="noreferrer">
+                      Download
+                    </a>
+                  </SpanBox>
+                )}
+              </HStack>
+            )
+          })}
+        </VStack>
+      )}
     </VStack>
   )
 }