prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

upgrade puppeteer-core

Browse source
This commit is contained in:
Hongbo Wu 2024-07-09 18:06:42 +08:00
parent 0899b8fc8f
commit c0f45e2411
4 changed files with 97 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
packages/puppeteer-parse/package.json
View file

@ -13,12 +13,13 @@
"axios": "^1.4.0",
"crypto": "^1.0.1",
"dompurify": "^2.4.1",
"fingerprint-generator": "^2.1.52",
"fingerprint-injector": "^2.1.52",
"linkedom": "^0.14.9",
"puppeteer-core": "^22.8.0",
"puppeteer-extra": "^3.3.4",
"puppeteer-extra-plugin-adblocker": "^2.13.5",
"puppeteer-extra-plugin-stealth": "^2.11.1",
"puppeteer-core": "^22.12.1",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-adblocker": "^2.13.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"urlsafe-base64": "^1.0.0"
},
"devDependencies": {

9
packages/puppeteer-parse/src/browser.ts
View file

@ -1,9 +1,9 @@
import { Browser } from 'puppeteer-core'
import puppeteer from 'puppeteer-extra'
import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker'
import StealthPlugin from 'puppeteer-extra-plugin-stealth'
// import StealthPlugin from 'puppeteer-extra-plugin-stealth'
puppeteer.use(StealthPlugin())
// puppeteer.use(StealthPlugin())
puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
let browserInstance: Browser | null = null
@ -44,9 +44,10 @@ export const getBrowser = async (): Promise<Browser> => {
'--disable-dev-shm-usage',
'--no-first-run',
'--disable-background-networking',
// use SwANGLE for WebGL
// '--disable-gpu',
// '--disable-software-rasterizer',
'--use-gl=angle',
'--use-angle=swiftshader',
'--use-angle=swiftshader-webgl',
],
defaultViewport: {
deviceScaleFactor: 1,

38
packages/puppeteer-parse/src/index.ts
View file

@ -2,7 +2,8 @@
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { preHandleContent } from '@omnivore/content-handler'
import axios from 'axios'
import { newInjectedPage } from 'fingerprint-injector'
import { FingerprintGenerator } from 'fingerprint-generator'
import { FingerprintInjector } from 'fingerprint-injector'
import { parseHTML } from 'linkedom'
import path from 'path'
import { Page, Protocol } from 'puppeteer-core'
@ -238,20 +239,31 @@ async function retrievePage(
}
const browser = await getBrowser()
const page = (await newInjectedPage(browser, {
// constraints for the generated fingerprint
fingerprintOptions: {
devices: ['desktop'],
operatingSystems: ['linux'],
browsers: ['chrome'],
locales: [locale || 'en-US'],
screen: {
maxWidth: 1920,
maxHeight: 1080,
const fingerprintGenerator = new FingerprintGenerator({
devices: ['desktop'],
operatingSystems: ['linux'],
browsers: ['chrome'],
locales: ['en-US'],
})
const fingerprintWithHeaders = fingerprintGenerator.getFingerprint()
const fingerprint = {
fingerprint: {
...fingerprintWithHeaders.fingerprint,
videoCard: {
vendor: 'Google Inc. (Intel Open Source Technology Center)',
renderer:
'ANGLE (Intel Open Source Technology Center, Mesa DRI Intel(R) HD Graphics 4400 (HSW GT2), OpenGL 4.5 (Core Profile) Mesa 21.2.0-devel (git-fb586a8 2021-06-28 focal-oibaf-ppa))',
},
mockWebRTC: true,
},
})) as Page
headers: fingerprintWithHeaders.headers,
}
console.log('fingerprint', fingerprint)
const page = await browser.newPage()
// Attach fingerprint to page
const fingerprintInjector = new FingerprintInjector()
await fingerprintInjector.attachFingerprintToPuppeteer(page, fingerprint)
// Puppeteer fails during download of PDf files,
// so record the failure and use those items

63
yarn.lock
View file

@ -12127,6 +12127,15 @@ chromium-bidi@0.5.19:
urlpattern-polyfill "10.0.0"
zod "3.22.4"
chromium-bidi@0.5.24:
version "0.5.24"
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-0.5.24.tgz#0fe672fa30b1e6bcc63ae818442b21c41849c435"
integrity sha512-5xQNN2SVBdZv4TxeMLaI+PelrnZsHDhn8h2JtyriLr+0qHcZS8BMuo93qN6J1VmtmrgYP+rmcLHcbpnA8QJh+w==
dependencies:
mitt "3.0.1"
urlpattern-polyfill "10.0.0"
zod "3.23.8"
chunkd@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/chunkd/-/chunkd-2.0.1.tgz#49cd1d7b06992dc4f7fccd962fe2a101ee7da920"
@ -13571,6 +13580,13 @@ debug@^3.0.0, debug@^3.1.0, debug@^3.1.1, debug@^3.2.7:
dependencies:
ms "^2.1.1"
debug@^4.3.5:
version "4.3.5"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e"
integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==
dependencies:
ms "2.1.2"
debuglog@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
@ -13938,6 +13954,11 @@ devtools-protocol@0.0.1273771:
resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1273771.tgz#46aeb5db41417e2c2ad3d8367c598c975290b1a5"
integrity sha512-QDbb27xcTVReQQW/GHJsdQqGKwYBE7re7gxehj467kKP2DKuYBUj6i2k5LRiAC66J1yZG/9gsxooz/s9pcm0Og==
devtools-protocol@0.0.1299070:
version "0.0.1299070"
resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1299070.tgz#b3e4cf0b678a46f0f907ae6e07e03ad3a53c00df"
integrity sha512-+qtL3eX50qsJ7c+qVyagqi7AWMoQCBGNfoyJZMwm/NSXVqLYbuitrWEEIzxfUmTNy7//Xe8yhMmQ+elj3uAqSg==
dezalgo@1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.3.tgz#7f742de066fc748bc8db820569dddce49bf0d456"
@ -26175,6 +26196,17 @@ pupa@^2.1.1:
dependencies:
escape-goat "^2.0.0"
puppeteer-core@^22.12.1:
version "22.12.1"
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-22.12.1.tgz#4dacc2e9ab127ef534a4bb4793d33f9424a5b48d"
integrity sha512-XmqeDPVdC5/3nGJys1jbgeoZ02wP0WV1GBlPtr/ULRbGXJFuqgXMcKQ3eeNtFpBzGRbpeoCGWHge1ZWKWl0Exw==
dependencies:
"@puppeteer/browsers" "2.2.3"
chromium-bidi "0.5.24"
debug "^4.3.5"
devtools-protocol "0.0.1299070"
ws "^8.17.1"
puppeteer-core@^22.8.0:
version "22.8.0"
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-22.8.0.tgz#82c0e7ebf62ba5f34404394034e313b82014de5f"
@ -26196,7 +26228,17 @@ puppeteer-extra-plugin-adblocker@^2.13.5:
node-fetch "^2.6.0"
puppeteer-extra-plugin "^3.2.2"
puppeteer-extra-plugin-stealth@^2.11.1:
puppeteer-extra-plugin-adblocker@^2.13.6:
version "2.13.6"
resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.13.6.tgz#99828243579b59ed81e8b1da23d16a3a0dbae557"
integrity sha512-AftgnUZ1rg2RPe9RpX6rkYAxEohwp3iFeGIyjsAuTaIiw4VLZqOb1LSY8/S60vAxpeat60fbCajxoUetmLy4Dw==
dependencies:
"@cliqz/adblocker-puppeteer" "1.23.8"
debug "^4.1.1"
node-fetch "^2.6.0"
puppeteer-extra-plugin "^3.2.3"
puppeteer-extra-plugin-stealth@^2.11.1, puppeteer-extra-plugin-stealth@^2.11.2:
version "2.11.2"
resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz#bd3f5a1781cac8a98c983d148086585a84fcc8f1"
integrity sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ==
@ -26243,6 +26285,15 @@ puppeteer-extra@^3.3.4:
debug "^4.1.1"
deepmerge "^4.2.2"
puppeteer-extra@^3.3.6:
version "3.3.6"
resolved "https://registry.yarnpkg.com/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz#fc16ff396aae52664842da9a557ea8fa51eaa8b7"
integrity sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A==
dependencies:
"@types/debug" "^4.1.0"
debug "^4.1.1"
deepmerge "^4.2.2"
q@^1.5.1:
version "1.5.1"
resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
@ -32248,6 +32299,11 @@ ws@^7.5.6:
resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591"
integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==
ws@^8.17.1:
version "8.18.0"
resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc"
integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==
ws@^8.2.3, ws@^8.3.0, ws@^8.4.2:
version "8.8.1"
resolved "https://registry.yarnpkg.com/ws/-/ws-8.8.1.tgz#5dbad0feb7ade8ecc99b830c1d77c913d4955ff0"
@ -32518,6 +32574,11 @@ zod@3.22.4, zod@^3.22.3, zod@^3.22.4:
resolved "https://registry.yarnpkg.com/zod/-/zod-3.22.4.tgz#f31c3a9386f61b1f228af56faa9255e845cf3fff"
integrity sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==
zod@3.23.8:
version "3.23.8"
resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d"
integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==
zwitch@^1.0.0:
version "1.0.5"
resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-1.0.5.tgz#d11d7381ffed16b742f6af7b3f223d5cd9fe9920"