diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 1eed768c4..27cdcd550 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -13,12 +13,13 @@ "axios": "^1.4.0", "crypto": "^1.0.1", "dompurify": "^2.4.1", + "fingerprint-generator": "^2.1.52", "fingerprint-injector": "^2.1.52", "linkedom": "^0.14.9", - "puppeteer-core": "^22.8.0", - "puppeteer-extra": "^3.3.4", - "puppeteer-extra-plugin-adblocker": "^2.13.5", - "puppeteer-extra-plugin-stealth": "^2.11.1", + "puppeteer-core": "^22.12.1", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-adblocker": "^2.13.6", + "puppeteer-extra-plugin-stealth": "^2.11.2", "urlsafe-base64": "^1.0.0" }, "devDependencies": { diff --git a/packages/puppeteer-parse/src/browser.ts b/packages/puppeteer-parse/src/browser.ts index b200dca03..3c66a7a95 100644 --- a/packages/puppeteer-parse/src/browser.ts +++ b/packages/puppeteer-parse/src/browser.ts @@ -1,9 +1,9 @@ import { Browser } from 'puppeteer-core' import puppeteer from 'puppeteer-extra' import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker' -import StealthPlugin from 'puppeteer-extra-plugin-stealth' +// import StealthPlugin from 'puppeteer-extra-plugin-stealth' -puppeteer.use(StealthPlugin()) +// puppeteer.use(StealthPlugin()) puppeteer.use(AdblockerPlugin({ blockTrackers: true })) let browserInstance: Browser | null = null @@ -44,9 +44,10 @@ export const getBrowser = async (): Promise => { '--disable-dev-shm-usage', '--no-first-run', '--disable-background-networking', - // use SwANGLE for WebGL + // '--disable-gpu', + // '--disable-software-rasterizer', '--use-gl=angle', - '--use-angle=swiftshader', + '--use-angle=swiftshader-webgl', ], defaultViewport: { deviceScaleFactor: 1, diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index 812502409..7d467cbc2 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -2,7 +2,8 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { preHandleContent } from '@omnivore/content-handler' import axios from 'axios' -import { newInjectedPage } from 'fingerprint-injector' +import { FingerprintGenerator } from 'fingerprint-generator' +import { FingerprintInjector } from 'fingerprint-injector' import { parseHTML } from 'linkedom' import path from 'path' import { Page, Protocol } from 'puppeteer-core' @@ -238,20 +239,31 @@ async function retrievePage( } const browser = await getBrowser() - const page = (await newInjectedPage(browser, { - // constraints for the generated fingerprint - fingerprintOptions: { - devices: ['desktop'], - operatingSystems: ['linux'], - browsers: ['chrome'], - locales: [locale || 'en-US'], - screen: { - maxWidth: 1920, - maxHeight: 1080, + + const fingerprintGenerator = new FingerprintGenerator({ + devices: ['desktop'], + operatingSystems: ['linux'], + browsers: ['chrome'], + locales: ['en-US'], + }) + const fingerprintWithHeaders = fingerprintGenerator.getFingerprint() + const fingerprint = { + fingerprint: { + ...fingerprintWithHeaders.fingerprint, + videoCard: { + vendor: 'Google Inc. (Intel Open Source Technology Center)', + renderer: + 'ANGLE (Intel Open Source Technology Center, Mesa DRI Intel(R) HD Graphics 4400 (HSW GT2), OpenGL 4.5 (Core Profile) Mesa 21.2.0-devel (git-fb586a8 2021-06-28 focal-oibaf-ppa))', }, - mockWebRTC: true, }, - })) as Page + headers: fingerprintWithHeaders.headers, + } + console.log('fingerprint', fingerprint) + const page = await browser.newPage() + + // Attach fingerprint to page + const fingerprintInjector = new FingerprintInjector() + await fingerprintInjector.attachFingerprintToPuppeteer(page, fingerprint) // Puppeteer fails during download of PDf files, // so record the failure and use those items diff --git a/yarn.lock b/yarn.lock index 100158cc5..befba1956 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12127,6 +12127,15 @@ chromium-bidi@0.5.19: urlpattern-polyfill "10.0.0" zod "3.22.4" +chromium-bidi@0.5.24: + version "0.5.24" + resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-0.5.24.tgz#0fe672fa30b1e6bcc63ae818442b21c41849c435" + integrity sha512-5xQNN2SVBdZv4TxeMLaI+PelrnZsHDhn8h2JtyriLr+0qHcZS8BMuo93qN6J1VmtmrgYP+rmcLHcbpnA8QJh+w== + dependencies: + mitt "3.0.1" + urlpattern-polyfill "10.0.0" + zod "3.23.8" + chunkd@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/chunkd/-/chunkd-2.0.1.tgz#49cd1d7b06992dc4f7fccd962fe2a101ee7da920" @@ -13571,6 +13580,13 @@ debug@^3.0.0, debug@^3.1.0, debug@^3.1.1, debug@^3.2.7: dependencies: ms "^2.1.1" +debug@^4.3.5: + version "4.3.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e" + integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg== + dependencies: + ms "2.1.2" + debuglog@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" @@ -13938,6 +13954,11 @@ devtools-protocol@0.0.1273771: resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1273771.tgz#46aeb5db41417e2c2ad3d8367c598c975290b1a5" integrity sha512-QDbb27xcTVReQQW/GHJsdQqGKwYBE7re7gxehj467kKP2DKuYBUj6i2k5LRiAC66J1yZG/9gsxooz/s9pcm0Og== +devtools-protocol@0.0.1299070: + version "0.0.1299070" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1299070.tgz#b3e4cf0b678a46f0f907ae6e07e03ad3a53c00df" + integrity sha512-+qtL3eX50qsJ7c+qVyagqi7AWMoQCBGNfoyJZMwm/NSXVqLYbuitrWEEIzxfUmTNy7//Xe8yhMmQ+elj3uAqSg== + dezalgo@1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.3.tgz#7f742de066fc748bc8db820569dddce49bf0d456" @@ -26175,6 +26196,17 @@ pupa@^2.1.1: dependencies: escape-goat "^2.0.0" +puppeteer-core@^22.12.1: + version "22.12.1" + resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-22.12.1.tgz#4dacc2e9ab127ef534a4bb4793d33f9424a5b48d" + integrity sha512-XmqeDPVdC5/3nGJys1jbgeoZ02wP0WV1GBlPtr/ULRbGXJFuqgXMcKQ3eeNtFpBzGRbpeoCGWHge1ZWKWl0Exw== + dependencies: + "@puppeteer/browsers" "2.2.3" + chromium-bidi "0.5.24" + debug "^4.3.5" + devtools-protocol "0.0.1299070" + ws "^8.17.1" + puppeteer-core@^22.8.0: version "22.8.0" resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-22.8.0.tgz#82c0e7ebf62ba5f34404394034e313b82014de5f" @@ -26196,7 +26228,17 @@ puppeteer-extra-plugin-adblocker@^2.13.5: node-fetch "^2.6.0" puppeteer-extra-plugin "^3.2.2" -puppeteer-extra-plugin-stealth@^2.11.1: +puppeteer-extra-plugin-adblocker@^2.13.6: + version "2.13.6" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.13.6.tgz#99828243579b59ed81e8b1da23d16a3a0dbae557" + integrity sha512-AftgnUZ1rg2RPe9RpX6rkYAxEohwp3iFeGIyjsAuTaIiw4VLZqOb1LSY8/S60vAxpeat60fbCajxoUetmLy4Dw== + dependencies: + "@cliqz/adblocker-puppeteer" "1.23.8" + debug "^4.1.1" + node-fetch "^2.6.0" + puppeteer-extra-plugin "^3.2.3" + +puppeteer-extra-plugin-stealth@^2.11.1, puppeteer-extra-plugin-stealth@^2.11.2: version "2.11.2" resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz#bd3f5a1781cac8a98c983d148086585a84fcc8f1" integrity sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ== @@ -26243,6 +26285,15 @@ puppeteer-extra@^3.3.4: debug "^4.1.1" deepmerge "^4.2.2" +puppeteer-extra@^3.3.6: + version "3.3.6" + resolved "https://registry.yarnpkg.com/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz#fc16ff396aae52664842da9a557ea8fa51eaa8b7" + integrity sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A== + dependencies: + "@types/debug" "^4.1.0" + debug "^4.1.1" + deepmerge "^4.2.2" + q@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" @@ -32248,6 +32299,11 @@ ws@^7.5.6: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== +ws@^8.17.1: + version "8.18.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc" + integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw== + ws@^8.2.3, ws@^8.3.0, ws@^8.4.2: version "8.8.1" resolved "https://registry.yarnpkg.com/ws/-/ws-8.8.1.tgz#5dbad0feb7ade8ecc99b830c1d77c913d4955ff0" @@ -32518,6 +32574,11 @@ zod@3.22.4, zod@^3.22.3, zod@^3.22.4: resolved "https://registry.yarnpkg.com/zod/-/zod-3.22.4.tgz#f31c3a9386f61b1f228af56faa9255e845cf3fff" integrity sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg== +zod@3.23.8: + version "3.23.8" + resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d" + integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g== + zwitch@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-1.0.5.tgz#d11d7381ffed16b742f6af7b3f223d5cd9fe9920"