Merge pull request #2059 from omnivore-app/fix/importer-counter

Fix incorrect successful and failed counter of imported urls in the importer

packages/import-handler/src/index.ts

@@ -1,17 +1,15 @@
 import { Storage } from '@google-cloud/storage'
-import { importCsv } from './csv'
-import * as path from 'path'
-import { importMatterArchive } from './matterHistory'
-import { Stream } from 'node:stream'
-import { v4 as uuid } from 'uuid'
-import { CONTENT_FETCH_URL, createCloudTask, emailUserUrl } from './task'
-
-import axios from 'axios'
-import { promisify } from 'util'
-import * as jwt from 'jsonwebtoken'
 import { Readability } from '@omnivore/readability'
-
 import * as Sentry from '@sentry/serverless'
+import axios from 'axios'
+import * as jwt from 'jsonwebtoken'
+import { Stream } from 'node:stream'
+import * as path from 'path'
+import { promisify } from 'util'
+import { v4 as uuid } from 'uuid'
+import { importCsv } from './csv'
+import { importMatterArchive } from './matterHistory'
+import { CONTENT_FETCH_URL, createCloudTask, emailUserUrl } from './task'
 
 export enum ArticleSavingRequestStatus {
   Failed = 'FAILED',
@@ -157,11 +155,12 @@ const urlHandler = async (
       state,
       labels
     )
-    if (result) {
-      ctx.countImported += 1
+    if (!result) {
+      return Promise.reject('Failed to import url')
     }
   } catch (err) {
     console.log('error importing url', err)
+    throw err
   }
 }
 

packages/import-handler/test/csv/csv.test.ts

@@ -1,10 +1,10 @@
-import 'mocha'
 import * as chai from 'chai'
 import { expect } from 'chai'
 import chaiString from 'chai-string'
 import * as fs from 'fs'
-import { importCsv } from '../../src/csv'
+import 'mocha'
 import { ArticleSavingRequestStatus, ImportContext } from '../../src'
+import { importCsv } from '../../src/csv'
 import { stubImportCtx } from '../util'
 
 chai.use(chaiString)
@@ -27,6 +27,20 @@ describe('Load a simple CSV file', () => {
       new URL('https://google.com'),
     ])
   })
+
+  it('increments the failed count when the URL is invalid', async () => {
+    const urls: URL[] = []
+    const stream = fs.createReadStream('./test/csv/data/simple.csv')
+    const stub = stubImportCtx()
+    stub.urlHandler = (ctx: ImportContext, url): Promise<void> => {
+      urls.push(url)
+      return Promise.reject('Failed to import url')
+    }
+
+    await importCsv(stub, stream)
+    expect(stub.countFailed).to.equal(2)
+    expect(stub.countImported).to.equal(0)
+  })
 })
 
 describe('Load a complex CSV file', () => {