prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

stringify csv before uploading

Browse source
This commit is contained in:
Hongbo Wu 2023-06-14 12:50:06 +08:00
parent fcf9766efc
commit 11cf9f7c28
4 changed files with 29 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/api/package.json
View file

@ -49,6 +49,7 @@
"cookie": "^0.5.0",
"cookie-parser": "^1.4.5",
"cors": "^2.8.5",
"csv-stringify": "^6.4.0",
"dataloader": "^2.0.0",
"diff-match-patch": "^1.0.5",
"dompurify": "^2.0.17",
@ -107,6 +108,7 @@
"@types/chai-string": "^1.4.2",
"@types/cookie": "^0.4.0",
"@types/cookie-parser": "^1.4.2",
"@types/csv-stringify": "^3.1.0",
"@types/diff-match-patch": "^1.0.32",
"@types/dompurify": "^2.0.4",
"@types/express": "^4.17.7",

13
packages/api/src/routers/svc/integrations.ts
View file

@ -1,6 +1,7 @@
/* eslint-disable @typescript-eslint/no-misused-promises */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
import { stringify } from 'csv-stringify'
import express from 'express'
import { DateTime } from 'luxon'
import { v4 as uuidv4 } from 'uuid'
@ -220,6 +221,12 @@ export function integrationsServiceRouter() {
writeStream = file.createWriteStream({
contentType: 'text/csv',
})
// stringify the data and pipe it to the write_stream
const stringifier = stringify({
header: false,
columns: ['url', 'state', 'labels'],
})
stringifier.pipe(writeStream)
let hasMore = true
let offset = 0
@ -236,11 +243,7 @@ export function integrationsServiceRouter() {
break
}
// write the list of urls, state and labels to the stream
const csvData = retrievedData.map((page) => {
const { url, state, labels } = page
return [url, state, `"[${labels?.join(',') || ''}]"`].join(',')
})
writeStream.write(csvData.join('\n'))
retrievedData.forEach((row) => stringifier.write(row))
hasMore = !!retrieved.hasMore
offset += retrievedData.length

8
packages/api/test/routers/integrations.test.ts
View file

@ -369,8 +369,13 @@ describe('Integrations routers', () => {
complete: 1,
list: {
'123': {
given_url: 'https://omnivore.app/pocket-import-test',
given_url: 'https://omnivore.app/pocket-import-test,test',
state: '0',
tags: {
'1234': {
tag: 'test',
},
},
},
},
since: Date.now() / 1000,
@ -387,6 +392,7 @@ describe('Integrations routers', () => {
after(async () => {
sinon.restore()
nock.cleanAll()
await deleteTestIntegrations(user.id, [integration.id])
})

12
yarn.lock
View file

@ -8111,6 +8111,13 @@
resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.12.tgz#6b2c510a7ad7039e98e7b8d3d6598f4359e5c080"
integrity sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==
"@types/csv-stringify@^3.1.0":
version "3.1.0"
resolved "https://registry.yarnpkg.com/@types/csv-stringify/-/csv-stringify-3.1.0.tgz#4c172ef462740e584a5bfe66ea78b67759f7bb32"
integrity sha512-jNRWx49wIc9UjJXukCaQt8iZRjyzDiEC1CGAAIZsydECWl5xM9oq4pSc5+Jhl4oATrRr+eGA9Vf0y9duDbKAvg==
dependencies:
csv-stringify "*"
"@types/debug@^4.0.0", "@types/debug@^4.1.0":
version "4.1.7"
resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82"
@ -12689,6 +12696,11 @@ csv-parser@^3.0.0:
dependencies:
minimist "^1.2.0"
csv-stringify@*, csv-stringify@^6.4.0:
version "6.4.0"
resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.4.0.tgz#6d006dca9194700e44f9fbc541bee8bbbd4f459c"
integrity sha512-HQsw0QXiN5fdlO+R8/JzCZnR3Fqp8E87YVnhHlaPtNGJjt6ffbV0LpOkieIb1x6V1+xt878IYq77SpXHWAqKkA==
cyclist@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"