prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

validate csv file in the importer

Browse source
This commit is contained in:
Hongbo Wu 2023-08-14 13:38:36 +08:00
parent 41f43ed5e1
commit ba2d34c1af
1 changed files with 21 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
packages/import-handler/src/csv.ts
View file

@ -5,7 +5,7 @@
import { parse } from '@fast-csv/parse'
import { Stream } from 'stream'
import { ImportContext } from '.'
import { ArticleSavingRequestStatus, ImportContext } from '.'
import { createMetrics, ImportStatus, updateMetrics } from './metrics'
const parseLabels = (labels: string): string[] => {
@ -24,17 +24,33 @@ const parseLabels = (labels: string): string[] => {
}
}
const parseState = (state: string): ArticleSavingRequestStatus => {
const validStates = ['SUCCEEDED', 'ARCHIVED']
// validate state
if (!validStates.includes(state.toUpperCase())) {
throw new Error('invalid state')
}
return state as ArticleSavingRequestStatus
}
export const importCsv = async (ctx: ImportContext, stream: Stream) => {
// create metrics in redis
await createMetrics(ctx.redisClient, ctx.userId, ctx.taskId, 'csv-importer')
const parser = parse()
const parser = parse({
headers: true,
discardUnmappedColumns: true,
objectMode: true,
ignoreEmpty: true,
trim: true,
})
stream.pipe(parser)
for await (const row of parser) {
try {
const url = new URL(row[0])
const state = row.length > 1 && row[1] ? row[1] : undefined
const labels = row.length > 2 ? parseLabels(row[2]) : undefined
const url = new URL(row['url'])
const state = row['state'] ? parseState(row['state']) : undefined
const labels = row['labels'] ? parseLabels(row['labels']) : undefined
// update total counter
await updateMetrics(