diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 1475e8e39..bd28840be 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -10,7 +10,7 @@ const IMPORTER_METRICS_COLLECTOR_URL = env.queue.importerMetricsUrl const JWT_SECRET = env.server.jwtSecret const REST_BACKEND_ENDPOINT = `${env.server.internalApiUrl}/api` -const MAX_ATTEMPTS = 1 +const MAX_ATTEMPTS = 2 const REQUEST_TIMEOUT = 30000 // 30 seconds interface Data { @@ -410,11 +410,20 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { isImported = !!readabilityResult isSaved = true } catch (e) { - console.error('error while saving page', e) + if (e instanceof Error) { + console.error('error while saving page', e.message) + } else { + console.error('error while saving page', 'unknown error') + } + throw e } finally { - const isFinalized = isSaved || attemptsMade >= MAX_ATTEMPTS - if (taskId && isFinalized) { + const lastAttempt = attemptsMade === MAX_ATTEMPTS - 1 + if (lastAttempt) { + console.log('last attempt reached', data.url) + } + + if (taskId && (isSaved || lastAttempt)) { // send import status to update the metrics for importer await sendImportStatusUpdate(userId, taskId, isImported) } diff --git a/packages/content-fetch/src/job.ts b/packages/content-fetch/src/job.ts index 201911ef7..b03ed9469 100644 --- a/packages/content-fetch/src/job.ts +++ b/packages/content-fetch/src/job.ts @@ -35,10 +35,10 @@ const getPriority = (job: savePageJob): number => { const getAttempts = (job: savePageJob): number => { if (job.isRss || job.isImport) { // we don't want to retry rss or import jobs - return 0 + return 1 } - return 1 + return 2 } const getOpts = (job: savePageJob): BulkJobOptions => { diff --git a/packages/content-fetch/src/request_handler.ts b/packages/content-fetch/src/request_handler.ts index 06aa0f9e9..9672344d3 100644 --- a/packages/content-fetch/src/request_handler.ts +++ b/packages/content-fetch/src/request_handler.ts @@ -137,12 +137,8 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => { const cacheResult = await cacheFetchResult(fetchResult) console.log('cacheFetchResult result', cacheResult) - const queueResult = await queueSavePageJob(savePageJobs) - console.log('queueSavePageJob result', queueResult) - if (!queueResult) { - logRecord.error = 'error while queueing save page job' - return res.sendStatus(500) - } + const jobs = await queueSavePageJob(savePageJobs) + console.log('save-page jobs queued', jobs.length) } catch (error) { if (error instanceof Error) { logRecord.error = error.message