From 8ee591943f8286f67bbb5249eb268ee4cbcfb25b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 15 Aug 2023 18:09:16 +0800 Subject: [PATCH 1/2] allow state to be empty --- packages/web/pages/tools/import/file.tsx | 66 +----------------------- packages/web/utils/csvValidator.ts | 3 ++ 2 files changed, 4 insertions(+), 65 deletions(-) diff --git a/packages/web/pages/tools/import/file.tsx b/packages/web/pages/tools/import/file.tsx index e8a353dcf..67f4cf70a 100644 --- a/packages/web/pages/tools/import/file.tsx +++ b/packages/web/pages/tools/import/file.tsx @@ -1,5 +1,4 @@ import 'antd/dist/antd.compact.css' -import { ValidatorConfig } from 'csv-file-validator' import { ChangeEvent, useState } from 'react' import { SyncLoader } from 'react-spinners' import { Button } from '../../../components/elements/Button' @@ -25,70 +24,6 @@ export default function ImportUploader(): JSX.Element { const [type, setType] = useState() const [uploadState, setUploadState] = useState('none') - const isUrlValid = (url: string | number | boolean) => { - if (typeof url !== 'string') { - return false - } - - try { - new URL(url) - return true - } catch (e) { - return false - } - } - - const isStateValid = (state: string | number | boolean) => { - if (typeof state !== 'string') { - return false - } - - const validStates = ['SUCCEEDED', 'ARCHIVED'] - return validStates.includes(state.toUpperCase()) - } - - const csvConfig: ValidatorConfig = { - headers: [ - { - name: 'url', - inputName: 'url', - required: true, - unique: true, - validate: function (url) { - return isUrlValid(url) - }, - }, - { - name: 'state', - inputName: 'state', - required: false, - optional: true, - validate: function (state) { - return isStateValid(state) - }, - }, - { - name: 'labels', - inputName: 'labels', - required: false, - optional: true, - isArray: true, - }, - { - name: 'saved_at', - inputName: 'saved_at', - required: false, - optional: true, - }, - { - name: 'published_at', - inputName: 'published_at', - required: false, - optional: true, - }, - ], - } - const onFinish = (values: unknown) => { console.log(values) } @@ -117,6 +52,7 @@ export default function ImportUploader(): JSX.Element { // validate csv file try { const csvData = await validateCsvFile(file) + console.log(csvData.data) if (csvData.inValidData.length > 0) { setErrorMessage(csvData.inValidData[0].message) setUploadState('none') diff --git a/packages/web/utils/csvValidator.ts b/packages/web/utils/csvValidator.ts index e7c26bb69..7ba863734 100644 --- a/packages/web/utils/csvValidator.ts +++ b/packages/web/utils/csvValidator.ts @@ -39,6 +39,9 @@ const csvConfig: ValidatorConfig = { required: false, optional: true, validate: function (state) { + if (!state) { + return true + } return isStateValid(state) }, }, From 9ee87e2e522afc46294db808bd344e1a5fa79417 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 15 Aug 2023 18:44:30 +0800 Subject: [PATCH 2/2] convert type of date in the csv to int before parsing date --- packages/import-handler/src/csv.ts | 3 ++- packages/import-handler/src/index.ts | 4 +++- packages/import-handler/test/csv/csv.test.ts | 16 ++++++++++++++-- .../import-handler/test/csv/data/complex.csv | 4 ++-- packages/web/pages/tools/import/file.tsx | 1 - 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/packages/import-handler/src/csv.ts b/packages/import-handler/src/csv.ts index e25ef9735..d04852c75 100644 --- a/packages/import-handler/src/csv.ts +++ b/packages/import-handler/src/csv.ts @@ -35,7 +35,8 @@ const parseState = (state: string): ArticleSavingRequestStatus => { } const parseDate = (date: string): Date => { - const parsedDate = new Date(date) + // date is unix timestamp in milliseconds + const parsedDate = new Date(parseInt(date, 10)) if (isNaN(parsedDate.getTime())) { throw new Error('invalid date') } diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 91e0ce9d4..9fa090f78 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -199,7 +199,9 @@ const urlHandler = async ( 'csv-importer', ctx.taskId, state, - labels && labels.length > 0 ? labels : undefined + labels && labels.length > 0 ? labels : undefined, + savedAt, + publishedAt ) if (!result) { return Promise.reject('Failed to import url') diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index 4de6cb433..350231671 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -56,18 +56,24 @@ describe('Test csv importer', () => { url: URL state?: ArticleSavingRequestStatus labels?: string[] + savedAt?: Date + publishedAt?: Date }[] = [] const stream = fs.createReadStream('./test/csv/data/complex.csv') stub.urlHandler = ( ctx: ImportContext, url, state, - labels + labels, + savedAt, + publishedAt ): Promise => { results.push({ url, state, labels, + savedAt, + publishedAt, }) return Promise.resolve() } @@ -80,16 +86,22 @@ describe('Test csv importer', () => { url: new URL('https://omnivore.app'), state: 'ARCHIVED', labels: ['test'], + savedAt: undefined, + publishedAt: undefined, }, { url: new URL('https://google.com'), - state: 'SUCCEEDED', labels: ['test', 'development'], + state: undefined, + savedAt: undefined, + publishedAt: undefined, }, { url: new URL('https://test.com'), state: 'SUCCEEDED', labels: ['test', 'development'], + savedAt: new Date(1692093633000), + publishedAt: new Date(1692093633000), }, ]) }) diff --git a/packages/import-handler/test/csv/data/complex.csv b/packages/import-handler/test/csv/data/complex.csv index 651b6b0f2..e310fb302 100644 --- a/packages/import-handler/test/csv/data/complex.csv +++ b/packages/import-handler/test/csv/data/complex.csv @@ -1,4 +1,4 @@ url,state,labels,saved_at,published_at "https://omnivore.app",ARCHIVED,"[""test""]",, -"https://google.com",SUCCEEDED,"[""test"",""development""]",, -https://test.com,SUCCEEDED,"[test, development]",, +"https://google.com",,"[""test"",""development""]",, +https://test.com,SUCCEEDED,"[test, development]",1692093633000,1692093633000 diff --git a/packages/web/pages/tools/import/file.tsx b/packages/web/pages/tools/import/file.tsx index 67f4cf70a..2ea4495d2 100644 --- a/packages/web/pages/tools/import/file.tsx +++ b/packages/web/pages/tools/import/file.tsx @@ -52,7 +52,6 @@ export default function ImportUploader(): JSX.Element { // validate csv file try { const csvData = await validateCsvFile(file) - console.log(csvData.data) if (csvData.inValidData.length > 0) { setErrorMessage(csvData.inValidData[0].message) setUploadState('none')