From 2423bff11c8b119d976970b171a7b860584e5d24 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 16 May 2023 10:27:55 +0800 Subject: [PATCH 1/3] Fix parsing error on empty array of labels in the csv file --- packages/import-handler/src/csv.ts | 11 +++++++++-- packages/import-handler/test/csv/csv.test.ts | 7 ++++++- packages/import-handler/test/csv/data/complex.csv | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/import-handler/src/csv.ts b/packages/import-handler/src/csv.ts index 66835127c..1d382d8db 100644 --- a/packages/import-handler/src/csv.ts +++ b/packages/import-handler/src/csv.ts @@ -14,8 +14,15 @@ export const importCsv = async (ctx: ImportContext, stream: Stream) => { try { const url = new URL(row[0]) const state = row.length > 1 ? row[1] : undefined - // labels follows format: "[label1, label2]" - const labels = row.length > 2 ? row[2].slice(1, -1).split(',') : undefined + // labels follows format: "[label1,label2]" or "[]" + const labels = + row.length > 2 + ? (row[2] as string) + .slice(1, -1) + .split(',') + .map((l) => l.trim()) + .filter((l) => l !== '') + : undefined await ctx.urlHandler(ctx, url, state, labels) ctx.countImported += 1 } catch (error) { diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index 0c447b60b..5e1c2cef3 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -68,7 +68,7 @@ describe('Load a complex CSV file', () => { await importCsv(stub, stream) expect(stub.countFailed).to.equal(0) - expect(stub.countImported).to.equal(2) + expect(stub.countImported).to.equal(3) expect(results).to.eql([ { url: new URL('https://omnivore.app'), @@ -80,6 +80,11 @@ describe('Load a complex CSV file', () => { state: 'SUCCEEDED', labels: ['test', 'development'], }, + { + url: new URL('https://test.com'), + state: 'SUCCEEDED', + labels: [], + }, ]) }) }) diff --git a/packages/import-handler/test/csv/data/complex.csv b/packages/import-handler/test/csv/data/complex.csv index c215da398..90e72956b 100644 --- a/packages/import-handler/test/csv/data/complex.csv +++ b/packages/import-handler/test/csv/data/complex.csv @@ -1,2 +1,3 @@ "https://omnivore.app",ARCHIVED,"[test]" "https://google.com",SUCCEEDED,"[test,development]" +https://test.com,SUCCEEDED,"[]" From e14080a05fd9ccaa10030b714bfa38b075b97aea Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 16 May 2023 11:04:22 +0800 Subject: [PATCH 2/3] Set labels = undefined if pocket tags is undefined --- packages/api/src/services/integrations/pocket.ts | 10 ++++++---- packages/import-handler/src/csv.ts | 2 +- packages/import-handler/test/csv/csv.test.ts | 2 +- packages/import-handler/test/csv/data/complex.csv | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 0b25928ab..c2a269cbe 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -1,11 +1,11 @@ +import axios from 'axios' +import { ArticleSavingRequestStatus } from '../../elastic/types' +import { env } from '../../env' import { IntegrationService, RetrievedResult, RetrieveRequest, } from './integration' -import axios from 'axios' -import { env } from '../../env' -import { ArticleSavingRequestStatus } from '../../elastic/types' interface PocketResponse { status: number // 1 if success @@ -130,7 +130,9 @@ export class PocketIntegration extends IntegrationService { } const data = pocketItems.map((item) => ({ url: item.given_url, - labels: Object.values(item.tags ?? {}).map((tag) => tag.tag), + labels: item.tags + ? Object.values(item.tags).map((tag) => tag.tag) + : undefined, state: statusToState[item.status], })) return { diff --git a/packages/import-handler/src/csv.ts b/packages/import-handler/src/csv.ts index 1d382d8db..c73c0dd59 100644 --- a/packages/import-handler/src/csv.ts +++ b/packages/import-handler/src/csv.ts @@ -14,7 +14,7 @@ export const importCsv = async (ctx: ImportContext, stream: Stream) => { try { const url = new URL(row[0]) const state = row.length > 1 ? row[1] : undefined - // labels follows format: "[label1,label2]" or "[]" + // labels follows format: "[label1,label2]" const labels = row.length > 2 ? (row[2] as string) diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index 5e1c2cef3..0a8ea2e25 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -83,7 +83,7 @@ describe('Load a complex CSV file', () => { { url: new URL('https://test.com'), state: 'SUCCEEDED', - labels: [], + labels: ['test', 'development'], }, ]) }) diff --git a/packages/import-handler/test/csv/data/complex.csv b/packages/import-handler/test/csv/data/complex.csv index 90e72956b..fa4ce8b83 100644 --- a/packages/import-handler/test/csv/data/complex.csv +++ b/packages/import-handler/test/csv/data/complex.csv @@ -1,3 +1,3 @@ "https://omnivore.app",ARCHIVED,"[test]" "https://google.com",SUCCEEDED,"[test,development]" -https://test.com,SUCCEEDED,"[]" +https://test.com,SUCCEEDED,"[test, development]" From 201666532398a9a86994ddbe5eb7ce9ce3fc8190 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 16 May 2023 12:23:10 +0800 Subject: [PATCH 3/3] Set labels = undefined if pocket tags is empty array --- packages/import-handler/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 9998d7d01..89b469ad1 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -153,7 +153,7 @@ const urlHandler = async ( url, 'csv-importer', state, - labels + labels && labels.length > 0 ? labels : undefined ) if (!result) { return Promise.reject('Failed to import url')