diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 557a57a4a..9998d7d01 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -1,17 +1,15 @@ import { Storage } from '@google-cloud/storage' -import { importCsv } from './csv' -import * as path from 'path' -import { importMatterArchive } from './matterHistory' -import { Stream } from 'node:stream' -import { v4 as uuid } from 'uuid' -import { CONTENT_FETCH_URL, createCloudTask, emailUserUrl } from './task' - -import axios from 'axios' -import { promisify } from 'util' -import * as jwt from 'jsonwebtoken' import { Readability } from '@omnivore/readability' - import * as Sentry from '@sentry/serverless' +import axios from 'axios' +import * as jwt from 'jsonwebtoken' +import { Stream } from 'node:stream' +import * as path from 'path' +import { promisify } from 'util' +import { v4 as uuid } from 'uuid' +import { importCsv } from './csv' +import { importMatterArchive } from './matterHistory' +import { CONTENT_FETCH_URL, createCloudTask, emailUserUrl } from './task' export enum ArticleSavingRequestStatus { Failed = 'FAILED', @@ -157,11 +155,12 @@ const urlHandler = async ( state, labels ) - if (result) { - ctx.countImported += 1 + if (!result) { + return Promise.reject('Failed to import url') } } catch (err) { console.log('error importing url', err) + throw err } } diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index e87d5aa10..0c447b60b 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -1,10 +1,10 @@ -import 'mocha' import * as chai from 'chai' import { expect } from 'chai' import chaiString from 'chai-string' import * as fs from 'fs' -import { importCsv } from '../../src/csv' +import 'mocha' import { ArticleSavingRequestStatus, ImportContext } from '../../src' +import { importCsv } from '../../src/csv' import { stubImportCtx } from '../util' chai.use(chaiString) @@ -27,6 +27,20 @@ describe('Load a simple CSV file', () => { new URL('https://google.com'), ]) }) + + it('increments the failed count when the URL is invalid', async () => { + const urls: URL[] = [] + const stream = fs.createReadStream('./test/csv/data/simple.csv') + const stub = stubImportCtx() + stub.urlHandler = (ctx: ImportContext, url): Promise => { + urls.push(url) + return Promise.reject('Failed to import url') + } + + await importCsv(stub, stream) + expect(stub.countFailed).to.equal(2) + expect(stub.countImported).to.equal(0) + }) }) describe('Load a complex CSV file', () => {