From 11538347af3d33ebeb4da19bcac9123f35ed3a9d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 17 Apr 2023 10:24:46 +0800 Subject: [PATCH 1/3] Fix incorrect successful and failed counter of imported urls in the importer --- packages/import-handler/src/csv.ts | 1 - packages/import-handler/src/index.ts | 25 +++++++++++-------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/packages/import-handler/src/csv.ts b/packages/import-handler/src/csv.ts index 66835127c..80dd9c038 100644 --- a/packages/import-handler/src/csv.ts +++ b/packages/import-handler/src/csv.ts @@ -17,7 +17,6 @@ export const importCsv = async (ctx: ImportContext, stream: Stream) => { // labels follows format: "[label1, label2]" const labels = row.length > 2 ? row[2].slice(1, -1).split(',') : undefined await ctx.urlHandler(ctx, url, state, labels) - ctx.countImported += 1 } catch (error) { console.log('invalid url', row, error) ctx.countFailed += 1 diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 557a57a4a..1b80314fb 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -1,17 +1,15 @@ import { Storage } from '@google-cloud/storage' -import { importCsv } from './csv' -import * as path from 'path' -import { importMatterArchive } from './matterHistory' -import { Stream } from 'node:stream' -import { v4 as uuid } from 'uuid' -import { CONTENT_FETCH_URL, createCloudTask, emailUserUrl } from './task' - -import axios from 'axios' -import { promisify } from 'util' -import * as jwt from 'jsonwebtoken' import { Readability } from '@omnivore/readability' - import * as Sentry from '@sentry/serverless' +import axios from 'axios' +import * as jwt from 'jsonwebtoken' +import { Stream } from 'node:stream' +import * as path from 'path' +import { promisify } from 'util' +import { v4 as uuid } from 'uuid' +import { importCsv } from './csv' +import { importMatterArchive } from './matterHistory' +import { CONTENT_FETCH_URL, createCloudTask, emailUserUrl } from './task' export enum ArticleSavingRequestStatus { Failed = 'FAILED', @@ -157,11 +155,10 @@ const urlHandler = async ( state, labels ) - if (result) { - ctx.countImported += 1 - } + result ? ctx.countImported++ : ctx.countFailed++ } catch (err) { console.log('error importing url', err) + ctx.countFailed += 1 } } From a1e782e39d8f4d794b370b0d0002aff25b73a217 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 17 Apr 2023 17:14:40 +0800 Subject: [PATCH 2/3] Fix tests --- packages/import-handler/src/csv.ts | 1 + packages/import-handler/src/index.ts | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/import-handler/src/csv.ts b/packages/import-handler/src/csv.ts index 80dd9c038..c827ed8ec 100644 --- a/packages/import-handler/src/csv.ts +++ b/packages/import-handler/src/csv.ts @@ -17,6 +17,7 @@ export const importCsv = async (ctx: ImportContext, stream: Stream) => { // labels follows format: "[label1, label2]" const labels = row.length > 2 ? row[2].slice(1, -1).split(',') : undefined await ctx.urlHandler(ctx, url, state, labels) + ctx.countImported++ } catch (error) { console.log('invalid url', row, error) ctx.countFailed += 1 diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 1b80314fb..9998d7d01 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -155,10 +155,12 @@ const urlHandler = async ( state, labels ) - result ? ctx.countImported++ : ctx.countFailed++ + if (!result) { + return Promise.reject('Failed to import url') + } } catch (err) { console.log('error importing url', err) - ctx.countFailed += 1 + throw err } } From c03d59c3c99729e147e2dd0ed7eac64e3c1ab17e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 17 Apr 2023 17:20:14 +0800 Subject: [PATCH 3/3] Add test case for failures --- packages/import-handler/src/csv.ts | 2 +- packages/import-handler/test/csv/csv.test.ts | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/import-handler/src/csv.ts b/packages/import-handler/src/csv.ts index c827ed8ec..66835127c 100644 --- a/packages/import-handler/src/csv.ts +++ b/packages/import-handler/src/csv.ts @@ -17,7 +17,7 @@ export const importCsv = async (ctx: ImportContext, stream: Stream) => { // labels follows format: "[label1, label2]" const labels = row.length > 2 ? row[2].slice(1, -1).split(',') : undefined await ctx.urlHandler(ctx, url, state, labels) - ctx.countImported++ + ctx.countImported += 1 } catch (error) { console.log('invalid url', row, error) ctx.countFailed += 1 diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index e87d5aa10..0c447b60b 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -1,10 +1,10 @@ -import 'mocha' import * as chai from 'chai' import { expect } from 'chai' import chaiString from 'chai-string' import * as fs from 'fs' -import { importCsv } from '../../src/csv' +import 'mocha' import { ArticleSavingRequestStatus, ImportContext } from '../../src' +import { importCsv } from '../../src/csv' import { stubImportCtx } from '../util' chai.use(chaiString) @@ -27,6 +27,20 @@ describe('Load a simple CSV file', () => { new URL('https://google.com'), ]) }) + + it('increments the failed count when the URL is invalid', async () => { + const urls: URL[] = [] + const stream = fs.createReadStream('./test/csv/data/simple.csv') + const stub = stubImportCtx() + stub.urlHandler = (ctx: ImportContext, url): Promise => { + urls.push(url) + return Promise.reject('Failed to import url') + } + + await importCsv(stub, stream) + expect(stub.countFailed).to.equal(2) + expect(stub.countImported).to.equal(0) + }) }) describe('Load a complex CSV file', () => {