Merge pull request #2550 from omnivore-app/feat/web-library-uploader

Allow uploading via DnD on the library page

packages/web/components/templates/UploadModal.tsx

@@ -15,6 +15,11 @@ import { uploadFileRequestMutation } from '../../lib/networking/mutations/upload
 import axios from 'axios'
 import { File } from 'phosphor-react'
 import { showErrorToast } from '../../lib/toastHelpers'
+import {
+  UploadImportFileType,
+  uploadImportFileRequestMutation,
+} from '../../lib/networking/mutations/uploadImportFileMutation'
+import Papa from 'papaparse'
 
 const DragnDropContainer = styled('div', {
   width: '100%',
@@ -80,6 +85,14 @@ type UploadingFile = {
   progress: number
   status: 'inprogress' | 'success' | 'error'
   openUrl: string | undefined
+  contentType: string
+  message?: string
+}
+
+type UploadInfo = {
+  uploadSignedUrl?: string
+  requestId?: string
+  message?: string
 }
 
 export function UploadModal(props: UploadModalProps): JSX.Element {
@@ -106,19 +119,91 @@ export function UploadModal(props: UploadModalProps): JSX.Element {
     [dropzoneRef]
   )
 
+  const uploadSignedUrlForFile = async (
+    file: UploadingFile
+  ): Promise<UploadInfo> => {
+    switch (file.contentType) {
+      case 'text/csv': {
+        const { urlCount, invalidCount } = (await new Promise((resolve) => {
+          let urlCount = 0
+          let invalidCount = 0
+
+          Papa.parse(file.file, {
+            step: function (row, parser) {
+              console.log('row: ', row)
+              if (Array.isArray(row.data)) {
+                try {
+                  if (row.data[0].trim().length < 1) {
+                    return
+                  }
+
+                  const url = new URL(row.data[0])
+                  urlCount = urlCount + 1
+                } catch (err) {
+                  invalidCount = invalidCount + 1
+                }
+              }
+            },
+            complete: (results) => {
+              resolve({ urlCount, invalidCount })
+            },
+          })
+        })) as { urlCount: number; invalidCount: number }
+        const result = await uploadImportFileRequestMutation(
+          UploadImportFileType.URL_LIST,
+          file.contentType
+        )
+        return {
+          uploadSignedUrl: result?.uploadSignedUrl,
+          message:
+            invalidCount > 0
+              ? `Importing ${urlCount} URLs (${invalidCount} invalid)`
+              : `Importing ${urlCount} URLs`,
+        }
+      }
+      case 'application/zip': {
+        const result = await uploadImportFileRequestMutation(
+          UploadImportFileType.MATTER,
+          file.contentType
+        )
+        return {
+          uploadSignedUrl: result?.uploadSignedUrl,
+        }
+      }
+      case 'application/pdf':
+      case 'application/epub+zip': {
+        const request = await uploadFileRequestMutation({
+          // This will tell the backend not to save the URL
+          // and give it the local filename as the title.
+          url: `file://local/${file.id}/${file.file.path}`,
+          contentType: file.contentType,
+          createPageEntry: true,
+        })
+        return {
+          uploadSignedUrl: request?.uploadSignedUrl,
+          requestId: request?.createdPageId,
+        }
+      }
+    }
+    return {}
+  }
+
   const handleAcceptedFiles = useCallback(
     (acceptedFiles: any, event: DropEvent) => {
       setInDragOperation(false)
 
-      const addedFiles = acceptedFiles.map((file: { name: any }) => {
-        return {
-          id: uuidv4(),
-          file: file,
-          name: file.name,
-          progress: 0,
-          status: 'inprogress',
+      const addedFiles = acceptedFiles.map(
+        (file: { name: any; type: string }) => {
+          return {
+            id: uuidv4(),
+            file: file,
+            name: file.name,
+            progress: 0,
+            status: 'inprogress',
+            contentType: file.type,
+          }
         }
-      })
+      )
 
       const allFiles = [...uploadFiles, ...addedFiles]
 
@@ -127,22 +212,15 @@ export function UploadModal(props: UploadModalProps): JSX.Element {
         for (const file of addedFiles) {
           try {
             console.log('using content type: ', file.file.type)
-            const request = await uploadFileRequestMutation({
-              // This will tell the backend not to save the URL
-              // and give it the local filename as the title.
-              url: `file://local/${file.id}/${file.file.path}`,
-              contentType: file.file.type,
-              createPageEntry: true,
-            })
-
-            if (!request?.uploadSignedUrl) {
+            const uploadInfo = await uploadSignedUrlForFile(file)
+            if (!uploadInfo.uploadSignedUrl) {
               showErrorToast('No upload URL available')
               return
             }
 
             const uploadResult = await axios.request({
               method: 'PUT',
-              url: request?.uploadSignedUrl,
+              url: uploadInfo.uploadSignedUrl,
               data: file.file,
               withCredentials: false,
               headers: {
@@ -162,7 +240,10 @@ export function UploadModal(props: UploadModalProps): JSX.Element {
 
             file.progress = 100
             file.status = 'success'
-            file.openUrl = `/article/sr/${request.createdPageId}`
+            file.openUrl = uploadInfo.requestId
+              ? `/article/sr/${uploadInfo.requestId}`
+              : undefined
+            file.message = uploadInfo.message
 
             setUploadFiles([...allFiles])
           } catch (error) {
@@ -216,6 +297,8 @@ export function UploadModal(props: UploadModalProps): JSX.Element {
             preventDropOnDocument={true}
             noClick={true}
             accept={{
+              'text/csv': ['.csv'],
+              'application/zip': ['.zip'],
               'application/pdf': ['.pdf'],
               'application/epub+zip': ['.epub'],
             }}
@@ -280,7 +363,6 @@ export function UploadModal(props: UploadModalProps): JSX.Element {
                   </DragnDropStyle>
                   <VStack css={{ width: '100%', mt: '25px', gap: '5px' }}>
                     {uploadFiles.map((file) => {
-                      console.log('fileL ', file.name, file)
                       return (
                         <HStack
                           key={file.id}
@@ -318,6 +400,11 @@ export function UploadModal(props: UploadModalProps): JSX.Element {
                               {file.status == 'success' && file.openUrl && (
                                 <a href={file.openUrl}>Read Now</a>
                               )}
+                              {file.status == 'success' && !file.openUrl && (
+                                <span>
+                                  {file.message || 'Your import has started'}
+                                </span>
+                              )}
                               {file.status == 'error' && (
                                 <SpanBox css={{ color: 'red' }}>
                                   Error Uploading

packages/web/package.json

@@ -49,6 +49,7 @@
     "nanoid": "^3.1.29",
     "next": "^12.1.0",
     "node-html-markdown": "^1.3.0",
+    "papaparse": "^5.4.1",
     "phosphor-react": "^1.4.0",
     "pspdfkit": "^2022.2.3",
     "react": "^17.0.2",
@@ -89,6 +90,7 @@
     "@types/loadjs": "^4.0.1",
     "@types/lodash.debounce": "^4.0.6",
     "@types/markdown-it": "^12.2.3",
+    "@types/papaparse": "^5.3.7",
     "@types/react": "17.0.2",
     "@types/react-color": "^3.0.6",
     "@types/react-dom": "^17.0.2",

yarn.lock

@@ -8675,6 +8675,13 @@
   resolved "https://registry.yarnpkg.com/@types/overlayscrollbars/-/overlayscrollbars-1.12.1.tgz#fb637071b545834fb12aea94ee309a2ff4cdc0a8"
   integrity sha512-V25YHbSoKQN35UasHf0EKD9U2vcmexRSp78qa8UglxFH8H3D+adEa9zGZwrqpH4TdvqeMrgMqVqsLB4woAryrQ==
 
+"@types/papaparse@^5.3.7":
+  version "5.3.7"
+  resolved "https://registry.yarnpkg.com/@types/papaparse/-/papaparse-5.3.7.tgz#8d3bf9e62ac2897df596f49d9ca59a15451aa247"
+  integrity sha512-f2HKmlnPdCvS0WI33WtCs5GD7X1cxzzS/aduaxSu3I7TbhWlENjSPs6z5TaB9K0J+BH1jbmqTaM+ja5puis4wg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/parse-json@^4.0.0":
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0"
@@ -21927,6 +21934,11 @@ pako@~1.0.2, pako@~1.0.5:
   resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
   integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
 
+papaparse@^5.4.1:
+  version "5.4.1"
+  resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-5.4.1.tgz#f45c0f871853578bd3a30f92d96fdcfb6ebea127"
+  integrity sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==
+
 parallel-transform@^1.1.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/parallel-transform/-/parallel-transform-1.2.0.tgz#9049ca37d6cb2182c3b1d2c720be94d14a5814fc"