prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

reduce logs

Browse source
This commit is contained in:
Hongbo Wu 2024-01-19 18:01:09 +08:00
parent 4340286b89
commit 356ad38678
3 changed files with 17 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
packages/api/src/jobs/save_page.ts
View file

@ -10,7 +10,7 @@ const IMPORTER_METRICS_COLLECTOR_URL = env.queue.importerMetricsUrl
const JWT_SECRET = env.server.jwtSecret
const REST_BACKEND_ENDPOINT = `${env.server.internalApiUrl}/api`
const MAX_ATTEMPTS = 1
const MAX_ATTEMPTS = 2
const REQUEST_TIMEOUT = 30000 // 30 seconds
interface Data {
@ -410,11 +410,20 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
isImported = !!readabilityResult
isSaved = true
} catch (e) {
console.error('error while saving page', e)
if (e instanceof Error) {
console.error('error while saving page', e.message)
} else {
console.error('error while saving page', 'unknown error')
}
throw e
} finally {
const isFinalized = isSaved || attemptsMade >= MAX_ATTEMPTS
if (taskId && isFinalized) {
const lastAttempt = attemptsMade === MAX_ATTEMPTS - 1
if (lastAttempt) {
console.log('last attempt reached', data.url)
}
if (taskId && (isSaved || lastAttempt)) {
// send import status to update the metrics for importer
await sendImportStatusUpdate(userId, taskId, isImported)
}

4
packages/content-fetch/src/job.ts
View file

@ -35,10 +35,10 @@ const getPriority = (job: savePageJob): number => {
const getAttempts = (job: savePageJob): number => {
if (job.isRss || job.isImport) {
// we don't want to retry rss or import jobs
return 0
return 1
}
return 1
return 2
}
const getOpts = (job: savePageJob): BulkJobOptions => {

8
packages/content-fetch/src/request_handler.ts
View file

@ -137,12 +137,8 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => {
const cacheResult = await cacheFetchResult(fetchResult)
console.log('cacheFetchResult result', cacheResult)
const queueResult = await queueSavePageJob(savePageJobs)
console.log('queueSavePageJob result', queueResult)
if (!queueResult) {
logRecord.error = 'error while queueing save page job'
return res.sendStatus(500)
}
const jobs = await queueSavePageJob(savePageJobs)
console.log('save-page jobs queued', jobs.length)
} catch (error) {
if (error instanceof Error) {
logRecord.error = error.message