prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix not timeout if scroll more than 5 seconds

Browse source
This commit is contained in:
Hongbo Wu 2023-02-13 22:10:56 +08:00
parent 69486a8527
commit 69b87078a5
1 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
packages/puppeteer-parse/index.js
View file

@ -7,7 +7,6 @@ const { encode } = require("urlsafe-base64");
const crypto = require("crypto");
const Url = require('url');
// const puppeteer = require('puppeteer-extra');
const axios = require('axios');
const jwt = require('jsonwebtoken');
const { promisify } = require('util');
@ -26,9 +25,10 @@ puppeteer.use(StealthPlugin());
// Add adblocker plugin to block all ads and trackers (saves bandwidth)
const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker');
const createDOMPurify = require("dompurify");
puppeteer.use(AdblockerPlugin({ blockTrackers: true }));
const createDOMPurify = require("dompurify");
const storage = new Storage();
const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : [];
const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.env.PREVIEW_IMAGE_BUCKET) : undefined;
@ -296,7 +296,7 @@ async function fetchContent(req, res) {
if (contentType === 'application/pdf') {
const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId);
const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId);
await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId);
} else {
if (!content || !title) {
const result = await retrieveHtml(page, logRecord);
@ -557,7 +557,7 @@ async function retrieveHtml(page, logRecord) {
}
})();
}),
await page.waitForTimeout(5000),
page.waitForTimeout(5000),
]);
logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart };