prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Updates to handle new import file format

Browse source
This commit is contained in:
Jackson Harper 2023-01-03 20:55:53 +08:00
parent f939786c2a
commit 97981f9a7b
1 changed files with 33 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
packages/import-handler/src/index.ts
View file

@ -4,8 +4,10 @@ import {
} from '@google-cloud/functions-framework/build/src/functions'
import { Storage } from '@google-cloud/storage'
import { PubSub } from '@google-cloud/pubsub'
import { importCsv } from './csv'
import { importCsv, UrlHandler } from './csv'
import * as path from 'path'
import { importMatterHistory } from './matterHistory'
import { Stream } from 'node:stream'
const pubsub = new PubSub()
const storage = new Storage()
@ -17,6 +19,11 @@ interface StorageEventData {
contentType: string
}
type importHandlerFunc = (
stream: Stream,
handler: UrlHandler
) => Promise<number>
const shouldHandle = (data: StorageEventData, ctx: CloudFunctionsContext) => {
console.log('deciding to handle', ctx, data)
if (ctx.eventType !== 'google.storage.object.finalize') {
@ -45,6 +52,17 @@ const importURL = async (
})
}
const handlerForFile = (name: string): importHandlerFunc | undefined => {
const fileName = path.parse(name).name
if (fileName.startsWith('MATTER')) {
return importMatterHistory
} else if (fileName.startsWith('URL_LIST')) {
return importCsv
}
return undefined
}
export const importHandler: EventFunction = async (event, context) => {
const data = event as StorageEventData
const ctx = context as CloudFunctionsContext
@ -57,10 +75,21 @@ export const importHandler: EventFunction = async (event, context) => {
.file(data.name)
.createReadStream()
await importCsv(stream, async (url): Promise<void> => {
const handler = handlerForFile(data.name)
if (!handler) {
console.log('no handler for file:', data.name)
return
}
await handler(stream, async (url): Promise<void> => {
try {
// Imports are stored in the format imports/<import id>/<user id>.extension
const userId = path.parse(data.name).name
// Imports are stored in the format imports/<user id>/<type>-<uuid>.csv
const group = path.parse(data.name).name.match(/(?<=-).*/gi)
if (!group || group.length < 1) {
console.log('could not match file pattern: ', data.name)
return
}
const userId = [...group][0]
const result = await importURL(userId, url, 'csv-importer')
console.log('import url result', result)
} catch (err) {