Merge pull request #2657 from omnivore-app/fix/importer

fix/importer

packages/import-handler/src/csv.ts

@@ -35,7 +35,8 @@ const parseState = (state: string): ArticleSavingRequestStatus => {
 }
 
 const parseDate = (date: string): Date => {
-  const parsedDate = new Date(date)
+  // date is unix timestamp in milliseconds
+  const parsedDate = new Date(parseInt(date, 10))
   if (isNaN(parsedDate.getTime())) {
     throw new Error('invalid date')
   }

packages/import-handler/src/index.ts

@@ -199,7 +199,9 @@ const urlHandler = async (
       'csv-importer',
       ctx.taskId,
       state,
-      labels && labels.length > 0 ? labels : undefined
+      labels && labels.length > 0 ? labels : undefined,
+      savedAt,
+      publishedAt
     )
     if (!result) {
       return Promise.reject('Failed to import url')

packages/import-handler/test/csv/csv.test.ts

@@ -56,18 +56,24 @@ describe('Test csv importer', () => {
         url: URL
         state?: ArticleSavingRequestStatus
         labels?: string[]
+        savedAt?: Date
+        publishedAt?: Date
       }[] = []
       const stream = fs.createReadStream('./test/csv/data/complex.csv')
       stub.urlHandler = (
         ctx: ImportContext,
         url,
         state,
-        labels
+        labels,
+        savedAt,
+        publishedAt
       ): Promise<void> => {
         results.push({
           url,
           state,
           labels,
+          savedAt,
+          publishedAt,
         })
         return Promise.resolve()
       }
@@ -80,16 +86,22 @@ describe('Test csv importer', () => {
           url: new URL('https://omnivore.app'),
           state: 'ARCHIVED',
           labels: ['test'],
+          savedAt: undefined,
+          publishedAt: undefined,
         },
         {
           url: new URL('https://google.com'),
-          state: 'SUCCEEDED',
           labels: ['test', 'development'],
+          state: undefined,
+          savedAt: undefined,
+          publishedAt: undefined,
         },
         {
           url: new URL('https://test.com'),
           state: 'SUCCEEDED',
           labels: ['test', 'development'],
+          savedAt: new Date(1692093633000),
+          publishedAt: new Date(1692093633000),
         },
       ])
     })

packages/import-handler/test/csv/data/complex.csv

@@ -1,4 +1,4 @@
 url,state,labels,saved_at,published_at
 "https://omnivore.app",ARCHIVED,"[""test""]",,
-"https://google.com",SUCCEEDED,"[""test"",""development""]",,
-https://test.com,SUCCEEDED,"[test, development]",,
+"https://google.com",,"[""test"",""development""]",,
+https://test.com,SUCCEEDED,"[test, development]",1692093633000,1692093633000

packages/web/pages/tools/import/file.tsx

@@ -1,5 +1,4 @@
 import 'antd/dist/antd.compact.css'
-import { ValidatorConfig } from 'csv-file-validator'
 import { ChangeEvent, useState } from 'react'
 import { SyncLoader } from 'react-spinners'
 import { Button } from '../../../components/elements/Button'
@@ -25,70 +24,6 @@ export default function ImportUploader(): JSX.Element {
   const [type, setType] = useState<UploadImportFileType>()
   const [uploadState, setUploadState] = useState<UploadState>('none')
 
-  const isUrlValid = (url: string | number | boolean) => {
-    if (typeof url !== 'string') {
-      return false
-    }
-
-    try {
-      new URL(url)
-      return true
-    } catch (e) {
-      return false
-    }
-  }
-
-  const isStateValid = (state: string | number | boolean) => {
-    if (typeof state !== 'string') {
-      return false
-    }
-
-    const validStates = ['SUCCEEDED', 'ARCHIVED']
-    return validStates.includes(state.toUpperCase())
-  }
-
-  const csvConfig: ValidatorConfig = {
-    headers: [
-      {
-        name: 'url',
-        inputName: 'url',
-        required: true,
-        unique: true,
-        validate: function (url) {
-          return isUrlValid(url)
-        },
-      },
-      {
-        name: 'state',
-        inputName: 'state',
-        required: false,
-        optional: true,
-        validate: function (state) {
-          return isStateValid(state)
-        },
-      },
-      {
-        name: 'labels',
-        inputName: 'labels',
-        required: false,
-        optional: true,
-        isArray: true,
-      },
-      {
-        name: 'saved_at',
-        inputName: 'saved_at',
-        required: false,
-        optional: true,
-      },
-      {
-        name: 'published_at',
-        inputName: 'published_at',
-        required: false,
-        optional: true,
-      },
-    ],
-  }
-
   const onFinish = (values: unknown) => {
     console.log(values)
   }

packages/web/utils/csvValidator.ts

@@ -39,6 +39,9 @@ const csvConfig: ValidatorConfig = {
       required: false,
       optional: true,
       validate: function (state) {
+        if (!state) {
+          return true
+        }
         return isStateValid(state)
       },
     },