prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add savedAt and publishedAt to the csv file

Browse source
This commit is contained in:
Hongbo Wu 2023-08-14 13:49:06 +08:00
parent fb845152d9
commit d25f19207b
2 changed files with 26 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
packages/import-handler/src/csv.ts
View file

@ -34,6 +34,15 @@ const parseState = (state: string): ArticleSavingRequestStatus => {
return state as ArticleSavingRequestStatus
}
const parseDate = (date: string): Date => {
const parsedDate = new Date(date)
if (isNaN(parsedDate.getTime())) {
throw new Error('invalid date')
}
return parsedDate
}
export const importCsv = async (ctx: ImportContext, stream: Stream) => {
// create metrics in redis
await createMetrics(ctx.redisClient, ctx.userId, ctx.taskId, 'csv-importer')
@ -51,6 +60,10 @@ export const importCsv = async (ctx: ImportContext, stream: Stream) => {
const url = new URL(row['url'])
const state = row['state'] ? parseState(row['state']) : undefined
const labels = row['labels'] ? parseLabels(row['labels']) : undefined
const savedAt = row['saved_at'] ? parseDate(row['saved_at']) : undefined
const publishedAt = row['published_at']
? parseDate(row['published_at'])
: undefined
// update total counter
await updateMetrics(
@ -60,7 +73,7 @@ export const importCsv = async (ctx: ImportContext, stream: Stream) => {
ImportStatus.TOTAL
)
await ctx.urlHandler(ctx, url, state, labels)
await ctx.urlHandler(ctx, url, state, labels, savedAt, publishedAt)
ctx.countImported += 1
// update started counter
@ -77,7 +90,7 @@ export const importCsv = async (ctx: ImportContext, stream: Stream) => {
break
}
} catch (error) {
console.log('invalid url', row, error)
console.log('invalid data', row, error)
ctx.countFailed += 1
// update invalid counter

14
packages/import-handler/src/index.ts
View file

@ -41,7 +41,9 @@ export type UrlHandler = (
ctx: ImportContext,
url: URL,
state?: ArticleSavingRequestStatus,
labels?: string[]
labels?: string[],
savedAt?: Date,
publishedAt?: Date
) => Promise<void>
export type ContentHandler = (
ctx: ImportContext,
@ -98,7 +100,9 @@ const importURL = async (
source: string,
taskId: string,
state?: ArticleSavingRequestStatus,
labels?: string[]
labels?: string[],
savedAt?: Date,
publishedAt?: Date
): Promise<string | undefined> => {
return createCloudTask(CONTENT_FETCH_URL, {
userId,
@ -110,6 +114,8 @@ const importURL = async (
return { name: l }
}),
taskId,
savedAt,
publishedAt,
})
}
@ -181,7 +187,9 @@ const urlHandler = async (
ctx: ImportContext,
url: URL,
state?: ArticleSavingRequestStatus,
labels?: string[]
labels?: string[],
savedAt?: Date,
publishedAt?: Date
): Promise<void> => {
try {
// Imports are stored in the format imports/<user id>/<type>-<uuid>.csv