From 81fbaf98078b8bdb97f1e3af255cf6937a40d917 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 5 Jul 2024 18:52:20 +0800 Subject: [PATCH 01/14] inject fingerprint --- packages/puppeteer-parse/package.json | 1 + packages/puppeteer-parse/src/index.ts | 39 +++++----- yarn.lock | 108 +++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 21 deletions(-) diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 4ed611ab4..1eed768c4 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -13,6 +13,7 @@ "axios": "^1.4.0", "crypto": "^1.0.1", "dompurify": "^2.4.1", + "fingerprint-injector": "^2.1.52", "linkedom": "^0.14.9", "puppeteer-core": "^22.8.0", "puppeteer-extra": "^3.3.4", diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index 4d66460b2..f7c02326f 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -2,9 +2,10 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { preHandleContent } from '@omnivore/content-handler' import axios from 'axios' +import { newInjectedPage } from 'fingerprint-injector' import { parseHTML } from 'linkedom' import path from 'path' -import { BrowserContext, Page, Protocol } from 'puppeteer-core' +import { Page, Protocol } from 'puppeteer-core' import { getBrowser } from './browser' const DESKTOP_USER_AGENT = @@ -80,8 +81,7 @@ export const fetchContent = async ( } console.log(`content-fetch request`, logRecord) - let context: BrowserContext | undefined, - page: Page | undefined, + let page: Page | undefined, title: string | undefined, content: string | undefined, contentType: string | undefined @@ -120,9 +120,6 @@ export const fetchContent = async ( locale, timezone ) - if (result && result.context) { - context = result.context - } if (result && result.page) { page = result.page } @@ -168,11 +165,11 @@ export const fetchContent = async ( throw e } finally { - // close browser context if it was opened - if (context) { - console.info('closing context...', url) - await context.close() - console.info('context closed', url) + // close browser page if it was opened + if (page) { + console.info('closing page...', url) + await page.close() + console.info('page closed', url) } console.info(`content-fetch result`, logRecord) @@ -241,16 +238,21 @@ async function retrievePage( } const browser = await getBrowser() - // create a new incognito browser context - const context = await browser.createBrowserContext() + const page = (await newInjectedPage(browser, { + // constraints for the generated fingerprint + fingerprintOptions: { + devices: ['desktop'], + operatingSystems: ['windows'], + browsers: ['firefox'], + mockWebRTC: true, + locales: [locale || 'en-US'], + }, + })) as Page // Puppeteer fails during download of PDf files, // so record the failure and use those items let lastPdfUrl - let page try { - page = await context.newPage() - if (!enableJavascriptForUrl(url)) { await page.setJavaScriptEnabled(false) } @@ -371,17 +373,16 @@ async function retrievePage( logRecord.finalUrl = finalUrl logRecord.contentType = contentType - return { context, page, finalUrl, contentType } + return { page, finalUrl, contentType } } catch (error) { if (lastPdfUrl) { return { - context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf', } } - await context.close() + await page.close() throw error } } diff --git a/yarn.lock b/yarn.lock index 071cdbc7a..100158cc5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6141,6 +6141,11 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ== +"@sindresorhus/is@^4.2.0": + version "4.6.0" + resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" + integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== + "@sinonjs/commons@^1", "@sinonjs/commons@^1.3.0", "@sinonjs/commons@^1.4.0", "@sinonjs/commons@^1.6.0", "@sinonjs/commons@^1.7.0", "@sinonjs/commons@^1.8.3": version "1.8.3" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.3.tgz#3802ddd21a50a949b6721ddd72da36e67e7f1b2d" @@ -9521,6 +9526,11 @@ addressparser@^1.0.1: resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746" integrity sha512-aQX7AISOMM7HFE0iZ3+YnD07oIeJqWGVnJ+ZIKaBZAk03ftmVYVqsGas/rbXKR21n4D/hKCSHypvcyOkds/xzg== +adm-zip@^0.5.9: + version "0.5.14" + resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.14.tgz#2c557c0bf12af4311cf6d32970f4060cf8133b2a" + integrity sha512-DnyqqifT4Jrcvb8USYjp6FHtBpEIz1mnXu6pTRHZ0RL69LbQYiO+0lDFg5+OKA7U29oWSs3a/i8fhn8ZcceIWg== + afinn-165-financialmarketnews@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/afinn-165-financialmarketnews/-/afinn-165-financialmarketnews-3.0.0.tgz#cf422577775bf94f9bc156f3f001a1f29338c3d8" @@ -11374,6 +11384,16 @@ browserslist@^4.17.5: node-releases "^2.0.1" picocolors "^1.0.0" +browserslist@^4.21.1: + version "4.23.1" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.1.tgz#ce4af0534b3d37db5c1a4ca98b9080f985041e96" + integrity sha512-TUfofFo/KsK/bWZ9TWQ5O26tsWW4Uhmt8IYklbnUa70udB6P2wA7w7o4PY4muaEPBQaAX+CEnmmIA41NVHtPVw== + dependencies: + caniuse-lite "^1.0.30001629" + electron-to-chromium "^1.4.796" + node-releases "^2.0.14" + update-browserslist-db "^1.0.16" + bser@2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/bser/-/bser-2.1.1.tgz#e6787da20ece9d07998533cfd9de6f5c38f4bc05" @@ -11677,7 +11697,7 @@ call-me-maybe@^1.0.1: resolved "https://registry.yarnpkg.com/call-me-maybe/-/call-me-maybe-1.0.1.tgz#26d208ea89e37b5cbde60250a15f031c16a4d66b" integrity sha1-JtII6onje1y95gJQoV8DHBak1ms= -callsites@^3.0.0: +callsites@^3.0.0, callsites@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== @@ -11747,6 +11767,11 @@ caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001251, caniuse-lite@^1.0.300012 resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001600.tgz" integrity sha512-+2S9/2JFhYmYaDpZvo0lKkfvuKIglrx68MwOBqMGHhQsNkLjB5xtc/TGoEPs+MxjSyN/72qer2g97nzR641mOQ== +caniuse-lite@^1.0.30001629: + version "1.0.30001640" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001640.tgz#32c467d4bf1f1a0faa63fc793c2ba81169e7652f" + integrity sha512-lA4VMpW0PSUrFnkmVuEKBUovSWKhj7puyCg8StBChgu298N1AtuF1sKWEvfDuimSEDbhlb/KqPKC3fs1HbuQUA== + capital-case@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669" @@ -14221,7 +14246,7 @@ dot-prop@^5.1.0, dot-prop@^5.2.0: dependencies: is-obj "^2.0.0" -dot-prop@^6.0.0: +dot-prop@^6.0.0, dot-prop@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-6.0.1.tgz#fc26b3cf142b9e59b74dbd39ed66ce620c681083" integrity sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA== @@ -14380,6 +14405,11 @@ electron-to-chromium@^1.4.17: resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.68.tgz#d79447b6bd1bec9183f166bb33d4bef0d5e4e568" integrity sha512-cId+QwWrV8R1UawO6b9BR1hnkJ4EJPCPAr4h315vliHUtVUJDk39Sg1PMNnaWKfj5x+93ssjeJ9LKL6r8LaMiA== +electron-to-chromium@^1.4.796: + version "1.4.816" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.816.tgz#3624649d1e7fde5cdbadf59d31a524245d8ee85f" + integrity sha512-EKH5X5oqC6hLmiS7/vYtZHZFTNdhsYG5NVPRN6Yn0kQHNBlT59+xSM8HBy66P5fxWpKgZbPqb+diC64ng295Jw== + electron-to-chromium@^1.4.84: version "1.4.89" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.89.tgz#33c06592812a17a7131873f4596579084ce33ff8" @@ -14854,6 +14884,11 @@ escalade@^3.1.1: resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== +escalade@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" + integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== + escape-goat@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675" @@ -16111,6 +16146,23 @@ fined@^1.0.1: object.pick "^1.2.0" parse-filepath "^1.0.1" +fingerprint-generator@^2.1.52: + version "2.1.52" + resolved "https://registry.yarnpkg.com/fingerprint-generator/-/fingerprint-generator-2.1.52.tgz#af40cb4f5b33a0a0173aeaa952f2b3b00bfce638" + integrity sha512-ZdXUn/qIB4vI7pDze5aXidjoFwLdEfbBNj6+3oHzXcgwxzEfCOfNe3wW5NRZDJKgxF40R7TSOA7noBAAehSLgQ== + dependencies: + generative-bayesian-network "^2.1.52" + header-generator "^2.1.52" + tslib "^2.4.0" + +fingerprint-injector@^2.1.52: + version "2.1.52" + resolved "https://registry.yarnpkg.com/fingerprint-injector/-/fingerprint-injector-2.1.52.tgz#d45cea9802f7a47c87fa1875dde05c61777cbb70" + integrity sha512-Sx+ykblqEP/P6nPRIE+C5CUNEfFpMZ3M/r5NDxOkSCTQVdfXXxlFx/UKOQNorvuJxryrtek4T0FvcB/KUbQfCQ== + dependencies: + fingerprint-generator "^2.1.52" + tslib "^2.4.0" + firebase-admin@^11.5.0: version "11.11.0" resolved "https://registry.yarnpkg.com/firebase-admin/-/firebase-admin-11.11.0.tgz#3d6df5dfbcf85dc1c6c4302f8aee4f7c82171725" @@ -16633,6 +16685,14 @@ gcp-metadata@^6.1.0: gaxios "^6.0.0" json-bigint "^1.0.0" +generative-bayesian-network@^2.1.52: + version "2.1.52" + resolved "https://registry.yarnpkg.com/generative-bayesian-network/-/generative-bayesian-network-2.1.52.tgz#0d8aa6dd14558bf88fb999feeff8c86d9e60322a" + integrity sha512-8fYemN+uiVPCjoodQX4HUH8RLDqiQeGfemlWO9yR6SqIh/6BsrW52M0YTSafsH0615BhulRy5BR2uKAqLTJ22A== + dependencies: + adm-zip "^0.5.9" + tslib "^2.4.0" + generic-pool@3.9.0: version "3.9.0" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" @@ -17668,6 +17728,16 @@ header-case@^2.0.4: capital-case "^1.0.4" tslib "^2.0.3" +header-generator@^2.1.52: + version "2.1.52" + resolved "https://registry.yarnpkg.com/header-generator/-/header-generator-2.1.52.tgz#1560fc3a2f2f65ed1d777a1660fb0c4459bf4827" + integrity sha512-2roqbZdd0hc7Bx+6BIQaHaCaSdnTXCnqayFbS8dpj53hmkQAXbSwiuTpfyAY1vePiaKweH6vDYhbtGOW+NmTmw== + dependencies: + browserslist "^4.21.1" + generative-bayesian-network "^2.1.52" + ow "^0.28.1" + tslib "^2.4.0" + heap-js@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/heap-js/-/heap-js-2.2.0.tgz#f4418874cd2aedc2cf3a7492d579afe23b627c5d" @@ -23441,6 +23511,11 @@ node-releases@^2.0.1, node-releases@^2.0.2: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.2.tgz#7139fe71e2f4f11b47d4d2986aaf8c48699e0c01" integrity sha512-XxYDdcQ6eKqp/YjI+tb2C5WM2LgjnZrfYg4vgQt49EK268b6gYCHsBLrK2qvJo4FmCtqmKezb0WZFK4fkrZNsg== +node-releases@^2.0.14: + version "2.0.14" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b" + integrity sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw== + nodemailer@^6.7.3: version "6.7.3" resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.3.tgz#b73f9a81b9c8fa8acb4ea14b608f5e725ea8e018" @@ -24422,6 +24497,17 @@ overlayscrollbars@^1.13.1: resolved "https://registry.yarnpkg.com/overlayscrollbars/-/overlayscrollbars-1.13.1.tgz#0b840a88737f43a946b9d87875a2f9e421d0338a" integrity sha512-gIQfzgGgu1wy80EB4/6DaJGHMEGmizq27xHIESrzXq0Y/J0Ay1P3DWk6tuVmEPIZH15zaBlxeEJOqdJKmowHCQ== +ow@^0.28.1: + version "0.28.2" + resolved "https://registry.yarnpkg.com/ow/-/ow-0.28.2.tgz#782b28102124e665c49ec7725e2066a129acf6bf" + integrity sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q== + dependencies: + "@sindresorhus/is" "^4.2.0" + callsites "^3.1.0" + dot-prop "^6.0.1" + lodash.isequal "^4.5.0" + vali-date "^1.0.0" + p-all@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-all/-/p-all-2.1.0.tgz#91419be56b7dee8fe4c5db875d55e0da084244a0" @@ -25207,6 +25293,11 @@ picocolors@^1.0.0: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== +picocolors@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1" + integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew== + picomatch@^2.0.4, picomatch@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" @@ -31012,6 +31103,14 @@ upath@^1.1.1: resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894" integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg== +update-browserslist-db@^1.0.16: + version "1.1.0" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz#7ca61c0d8650766090728046e416a8cde682859e" + integrity sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ== + dependencies: + escalade "^3.1.2" + picocolors "^1.0.1" + update-notifier@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9" @@ -31292,6 +31391,11 @@ v8flags@^2.0.10: dependencies: user-home "^1.1.1" +vali-date@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/vali-date/-/vali-date-1.0.0.tgz#1b904a59609fb328ef078138420934f6b86709a6" + integrity sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg== + valid-url@^1.0.9: version "1.0.9" resolved "https://registry.yarnpkg.com/valid-url/-/valid-url-1.0.9.tgz#1c14479b40f1397a75782f115e4086447433a200" From 728059c6f86d669e47fb42bf419a8fe41d859327 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 5 Jul 2024 19:05:36 +0800 Subject: [PATCH 02/14] do not cache some urls --- packages/content-fetch/src/request_handler.ts | 17 +++++++++++++++++ packages/puppeteer-parse/src/index.ts | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/content-fetch/src/request_handler.ts b/packages/content-fetch/src/request_handler.ts index b1aa2aa1f..269130634 100644 --- a/packages/content-fetch/src/request_handler.ts +++ b/packages/content-fetch/src/request_handler.ts @@ -62,6 +62,13 @@ const storage = process.env.GCS_UPLOAD_SA_KEY_FILE_PATH : new Storage() const bucketName = process.env.GCS_UPLOAD_BUCKET || 'omnivore-files' +const NO_CACHE_URLS = ['https://deviceandbrowserinfo.com/are_you_a_bot'] + +const isUrlCacheable = (cacheKey: string) => { + const url = cacheKey.split(':')[1] + return !NO_CACHE_URLS.includes(url) +} + const uploadToBucket = async (filePath: string, data: string) => { await storage .bucket(bucketName) @@ -97,6 +104,11 @@ export const cacheFetchResult = async ( key: string, fetchResult: FetchResult ) => { + if (!isUrlCacheable(key)) { + console.info('url is not cacheable', key) + return undefined + } + // cache the fetch result for 24 hours const ttl = 24 * 60 * 60 const value = JSON.stringify(fetchResult) @@ -107,6 +119,11 @@ const getCachedFetchResult = async ( redisDataSource: RedisDataSource, key: string ): Promise => { + if (!isUrlCacheable(key)) { + console.info('url is not cacheable', key) + return undefined + } + const result = await redisDataSource.cacheClient.get(key) if (!result) { console.info('fetch result is not cached', key) diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index f7c02326f..576368dec 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -243,7 +243,7 @@ async function retrievePage( fingerprintOptions: { devices: ['desktop'], operatingSystems: ['windows'], - browsers: ['firefox'], + browsers: ['chrome'], mockWebRTC: true, locales: [locale || 'en-US'], }, From dd0120237442b3cbf4d2cbe4b3b244be9b7813d4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 5 Jul 2024 19:46:18 +0800 Subject: [PATCH 03/14] do not cache some urls --- packages/content-fetch/src/request_handler.ts | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/packages/content-fetch/src/request_handler.ts b/packages/content-fetch/src/request_handler.ts index 269130634..bbb98f8d2 100644 --- a/packages/content-fetch/src/request_handler.ts +++ b/packages/content-fetch/src/request_handler.ts @@ -64,11 +64,6 @@ const bucketName = process.env.GCS_UPLOAD_BUCKET || 'omnivore-files' const NO_CACHE_URLS = ['https://deviceandbrowserinfo.com/are_you_a_bot'] -const isUrlCacheable = (cacheKey: string) => { - const url = cacheKey.split(':')[1] - return !NO_CACHE_URLS.includes(url) -} - const uploadToBucket = async (filePath: string, data: string) => { await storage .bucket(bucketName) @@ -104,11 +99,6 @@ export const cacheFetchResult = async ( key: string, fetchResult: FetchResult ) => { - if (!isUrlCacheable(key)) { - console.info('url is not cacheable', key) - return undefined - } - // cache the fetch result for 24 hours const ttl = 24 * 60 * 60 const value = JSON.stringify(fetchResult) @@ -119,11 +109,6 @@ const getCachedFetchResult = async ( redisDataSource: RedisDataSource, key: string ): Promise => { - if (!isUrlCacheable(key)) { - console.info('url is not cacheable', key) - return undefined - } - const result = await redisDataSource.cacheClient.get(key) if (!result) { console.info('fetch result is not cached', key) @@ -215,7 +200,7 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => { fetchResult = await fetchContent(url, locale, timezone) console.log('content has been fetched') - if (fetchResult.content) { + if (fetchResult.content && !NO_CACHE_URLS.includes(url)) { const cacheResult = await cacheFetchResult( redisDataSource, key, From 2c15c21bf19b6654e1be69cfecce6442dc6230bc Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 8 Jul 2024 18:59:53 +0800 Subject: [PATCH 04/14] remove user-agent --- packages/puppeteer-parse/src/index.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index 576368dec..b7b2af23e 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -244,8 +244,11 @@ async function retrievePage( devices: ['desktop'], operatingSystems: ['windows'], browsers: ['chrome'], - mockWebRTC: true, locales: [locale || 'en-US'], + screen: { + maxWidth: 1920, + maxHeight: 1080, + }, }, })) as Page @@ -256,12 +259,12 @@ async function retrievePage( if (!enableJavascriptForUrl(url)) { await page.setJavaScriptEnabled(false) } - await page.setUserAgent(userAgentForUrl(url)) + // await page.setUserAgent(userAgentForUrl(url)) - // set locale for the page - if (locale) { - await page.setExtraHTTPHeaders({ 'Accept-Language': locale }) - } + // // set locale for the page + // if (locale) { + // await page.setExtraHTTPHeaders({ 'Accept-Language': locale }) + // } // set timezone for the page if (timezone) { From c75cbb39d6c285cd934c987922d12a38edb7e42e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 9 Jul 2024 14:11:31 +0800 Subject: [PATCH 05/14] injecting webgl fingerprint --- packages/content-fetch/src/request_handler.ts | 5 ++++- packages/puppeteer-parse/src/browser.ts | 3 +-- packages/puppeteer-parse/src/index.ts | 14 +++++++------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/content-fetch/src/request_handler.ts b/packages/content-fetch/src/request_handler.ts index bbb98f8d2..81fc04de2 100644 --- a/packages/content-fetch/src/request_handler.ts +++ b/packages/content-fetch/src/request_handler.ts @@ -62,7 +62,10 @@ const storage = process.env.GCS_UPLOAD_SA_KEY_FILE_PATH : new Storage() const bucketName = process.env.GCS_UPLOAD_BUCKET || 'omnivore-files' -const NO_CACHE_URLS = ['https://deviceandbrowserinfo.com/are_you_a_bot'] +const NO_CACHE_URLS = [ + 'https://deviceandbrowserinfo.com/are_you_a_bot', + 'https://deviceandbrowserinfo.com/info_device', +] const uploadToBucket = async (filePath: string, data: string) => { await storage diff --git a/packages/puppeteer-parse/src/browser.ts b/packages/puppeteer-parse/src/browser.ts index b121a5dfa..ab6568759 100644 --- a/packages/puppeteer-parse/src/browser.ts +++ b/packages/puppeteer-parse/src/browser.ts @@ -34,7 +34,6 @@ export const getBrowser = async (): Promise => { '--disk-cache-size=33554432', '--enable-features=SharedArrayBuffer', '--hide-scrollbars', - '--disable-gpu', '--mute-audio', '--no-default-browser-check', '--no-pings', @@ -56,7 +55,7 @@ export const getBrowser = async (): Promise => { width: 1920, }, executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, + headless: process.env.LAUNCH_HEADLESS === 'true', timeout: 10_000, // 10 seconds dumpio: true, // show console logs in the terminal })) as Browser diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index b7b2af23e..812502409 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -242,13 +242,14 @@ async function retrievePage( // constraints for the generated fingerprint fingerprintOptions: { devices: ['desktop'], - operatingSystems: ['windows'], + operatingSystems: ['linux'], browsers: ['chrome'], locales: [locale || 'en-US'], screen: { maxWidth: 1920, maxHeight: 1080, }, + mockWebRTC: true, }, })) as Page @@ -259,12 +260,11 @@ async function retrievePage( if (!enableJavascriptForUrl(url)) { await page.setJavaScriptEnabled(false) } - // await page.setUserAgent(userAgentForUrl(url)) - // // set locale for the page - // if (locale) { - // await page.setExtraHTTPHeaders({ 'Accept-Language': locale }) - // } + // set locale for the page + if (locale) { + await page.setExtraHTTPHeaders({ 'Accept-Language': locale }) + } // set timezone for the page if (timezone) { @@ -364,7 +364,7 @@ async function retrievePage( const response = await page.goto(url, { timeout: 30 * 1000, - waitUntil: ['networkidle2'], + waitUntil: ['networkidle0'], }) if (!response) { throw new Error('No response from page') From 0899b8fc8fd66fc866526f8b379810c3d162c798 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 9 Jul 2024 14:31:51 +0800 Subject: [PATCH 06/14] use swAngle --- packages/puppeteer-parse/src/browser.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/puppeteer-parse/src/browser.ts b/packages/puppeteer-parse/src/browser.ts index ab6568759..b200dca03 100644 --- a/packages/puppeteer-parse/src/browser.ts +++ b/packages/puppeteer-parse/src/browser.ts @@ -44,7 +44,9 @@ export const getBrowser = async (): Promise => { '--disable-dev-shm-usage', '--no-first-run', '--disable-background-networking', - '--use-gl=swiftshader', + // use SwANGLE for WebGL + '--use-gl=angle', + '--use-angle=swiftshader', ], defaultViewport: { deviceScaleFactor: 1, From c0f45e2411da517188509ca47c189f15edd72de4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 9 Jul 2024 18:06:42 +0800 Subject: [PATCH 07/14] upgrade puppeteer-core --- packages/puppeteer-parse/package.json | 9 ++-- packages/puppeteer-parse/src/browser.ts | 9 ++-- packages/puppeteer-parse/src/index.ts | 38 ++++++++++----- yarn.lock | 63 ++++++++++++++++++++++++- 4 files changed, 97 insertions(+), 22 deletions(-) diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 1eed768c4..27cdcd550 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -13,12 +13,13 @@ "axios": "^1.4.0", "crypto": "^1.0.1", "dompurify": "^2.4.1", + "fingerprint-generator": "^2.1.52", "fingerprint-injector": "^2.1.52", "linkedom": "^0.14.9", - "puppeteer-core": "^22.8.0", - "puppeteer-extra": "^3.3.4", - "puppeteer-extra-plugin-adblocker": "^2.13.5", - "puppeteer-extra-plugin-stealth": "^2.11.1", + "puppeteer-core": "^22.12.1", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-adblocker": "^2.13.6", + "puppeteer-extra-plugin-stealth": "^2.11.2", "urlsafe-base64": "^1.0.0" }, "devDependencies": { diff --git a/packages/puppeteer-parse/src/browser.ts b/packages/puppeteer-parse/src/browser.ts index b200dca03..3c66a7a95 100644 --- a/packages/puppeteer-parse/src/browser.ts +++ b/packages/puppeteer-parse/src/browser.ts @@ -1,9 +1,9 @@ import { Browser } from 'puppeteer-core' import puppeteer from 'puppeteer-extra' import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker' -import StealthPlugin from 'puppeteer-extra-plugin-stealth' +// import StealthPlugin from 'puppeteer-extra-plugin-stealth' -puppeteer.use(StealthPlugin()) +// puppeteer.use(StealthPlugin()) puppeteer.use(AdblockerPlugin({ blockTrackers: true })) let browserInstance: Browser | null = null @@ -44,9 +44,10 @@ export const getBrowser = async (): Promise => { '--disable-dev-shm-usage', '--no-first-run', '--disable-background-networking', - // use SwANGLE for WebGL + // '--disable-gpu', + // '--disable-software-rasterizer', '--use-gl=angle', - '--use-angle=swiftshader', + '--use-angle=swiftshader-webgl', ], defaultViewport: { deviceScaleFactor: 1, diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index 812502409..7d467cbc2 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -2,7 +2,8 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { preHandleContent } from '@omnivore/content-handler' import axios from 'axios' -import { newInjectedPage } from 'fingerprint-injector' +import { FingerprintGenerator } from 'fingerprint-generator' +import { FingerprintInjector } from 'fingerprint-injector' import { parseHTML } from 'linkedom' import path from 'path' import { Page, Protocol } from 'puppeteer-core' @@ -238,20 +239,31 @@ async function retrievePage( } const browser = await getBrowser() - const page = (await newInjectedPage(browser, { - // constraints for the generated fingerprint - fingerprintOptions: { - devices: ['desktop'], - operatingSystems: ['linux'], - browsers: ['chrome'], - locales: [locale || 'en-US'], - screen: { - maxWidth: 1920, - maxHeight: 1080, + + const fingerprintGenerator = new FingerprintGenerator({ + devices: ['desktop'], + operatingSystems: ['linux'], + browsers: ['chrome'], + locales: ['en-US'], + }) + const fingerprintWithHeaders = fingerprintGenerator.getFingerprint() + const fingerprint = { + fingerprint: { + ...fingerprintWithHeaders.fingerprint, + videoCard: { + vendor: 'Google Inc. (Intel Open Source Technology Center)', + renderer: + 'ANGLE (Intel Open Source Technology Center, Mesa DRI Intel(R) HD Graphics 4400 (HSW GT2), OpenGL 4.5 (Core Profile) Mesa 21.2.0-devel (git-fb586a8 2021-06-28 focal-oibaf-ppa))', }, - mockWebRTC: true, }, - })) as Page + headers: fingerprintWithHeaders.headers, + } + console.log('fingerprint', fingerprint) + const page = await browser.newPage() + + // Attach fingerprint to page + const fingerprintInjector = new FingerprintInjector() + await fingerprintInjector.attachFingerprintToPuppeteer(page, fingerprint) // Puppeteer fails during download of PDf files, // so record the failure and use those items diff --git a/yarn.lock b/yarn.lock index 100158cc5..befba1956 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12127,6 +12127,15 @@ chromium-bidi@0.5.19: urlpattern-polyfill "10.0.0" zod "3.22.4" +chromium-bidi@0.5.24: + version "0.5.24" + resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-0.5.24.tgz#0fe672fa30b1e6bcc63ae818442b21c41849c435" + integrity sha512-5xQNN2SVBdZv4TxeMLaI+PelrnZsHDhn8h2JtyriLr+0qHcZS8BMuo93qN6J1VmtmrgYP+rmcLHcbpnA8QJh+w== + dependencies: + mitt "3.0.1" + urlpattern-polyfill "10.0.0" + zod "3.23.8" + chunkd@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/chunkd/-/chunkd-2.0.1.tgz#49cd1d7b06992dc4f7fccd962fe2a101ee7da920" @@ -13571,6 +13580,13 @@ debug@^3.0.0, debug@^3.1.0, debug@^3.1.1, debug@^3.2.7: dependencies: ms "^2.1.1" +debug@^4.3.5: + version "4.3.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e" + integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg== + dependencies: + ms "2.1.2" + debuglog@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" @@ -13938,6 +13954,11 @@ devtools-protocol@0.0.1273771: resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1273771.tgz#46aeb5db41417e2c2ad3d8367c598c975290b1a5" integrity sha512-QDbb27xcTVReQQW/GHJsdQqGKwYBE7re7gxehj467kKP2DKuYBUj6i2k5LRiAC66J1yZG/9gsxooz/s9pcm0Og== +devtools-protocol@0.0.1299070: + version "0.0.1299070" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1299070.tgz#b3e4cf0b678a46f0f907ae6e07e03ad3a53c00df" + integrity sha512-+qtL3eX50qsJ7c+qVyagqi7AWMoQCBGNfoyJZMwm/NSXVqLYbuitrWEEIzxfUmTNy7//Xe8yhMmQ+elj3uAqSg== + dezalgo@1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.3.tgz#7f742de066fc748bc8db820569dddce49bf0d456" @@ -26175,6 +26196,17 @@ pupa@^2.1.1: dependencies: escape-goat "^2.0.0" +puppeteer-core@^22.12.1: + version "22.12.1" + resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-22.12.1.tgz#4dacc2e9ab127ef534a4bb4793d33f9424a5b48d" + integrity sha512-XmqeDPVdC5/3nGJys1jbgeoZ02wP0WV1GBlPtr/ULRbGXJFuqgXMcKQ3eeNtFpBzGRbpeoCGWHge1ZWKWl0Exw== + dependencies: + "@puppeteer/browsers" "2.2.3" + chromium-bidi "0.5.24" + debug "^4.3.5" + devtools-protocol "0.0.1299070" + ws "^8.17.1" + puppeteer-core@^22.8.0: version "22.8.0" resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-22.8.0.tgz#82c0e7ebf62ba5f34404394034e313b82014de5f" @@ -26196,7 +26228,17 @@ puppeteer-extra-plugin-adblocker@^2.13.5: node-fetch "^2.6.0" puppeteer-extra-plugin "^3.2.2" -puppeteer-extra-plugin-stealth@^2.11.1: +puppeteer-extra-plugin-adblocker@^2.13.6: + version "2.13.6" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.13.6.tgz#99828243579b59ed81e8b1da23d16a3a0dbae557" + integrity sha512-AftgnUZ1rg2RPe9RpX6rkYAxEohwp3iFeGIyjsAuTaIiw4VLZqOb1LSY8/S60vAxpeat60fbCajxoUetmLy4Dw== + dependencies: + "@cliqz/adblocker-puppeteer" "1.23.8" + debug "^4.1.1" + node-fetch "^2.6.0" + puppeteer-extra-plugin "^3.2.3" + +puppeteer-extra-plugin-stealth@^2.11.1, puppeteer-extra-plugin-stealth@^2.11.2: version "2.11.2" resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz#bd3f5a1781cac8a98c983d148086585a84fcc8f1" integrity sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ== @@ -26243,6 +26285,15 @@ puppeteer-extra@^3.3.4: debug "^4.1.1" deepmerge "^4.2.2" +puppeteer-extra@^3.3.6: + version "3.3.6" + resolved "https://registry.yarnpkg.com/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz#fc16ff396aae52664842da9a557ea8fa51eaa8b7" + integrity sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A== + dependencies: + "@types/debug" "^4.1.0" + debug "^4.1.1" + deepmerge "^4.2.2" + q@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" @@ -32248,6 +32299,11 @@ ws@^7.5.6: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== +ws@^8.17.1: + version "8.18.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc" + integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw== + ws@^8.2.3, ws@^8.3.0, ws@^8.4.2: version "8.8.1" resolved "https://registry.yarnpkg.com/ws/-/ws-8.8.1.tgz#5dbad0feb7ade8ecc99b830c1d77c913d4955ff0" @@ -32518,6 +32574,11 @@ zod@3.22.4, zod@^3.22.3, zod@^3.22.4: resolved "https://registry.yarnpkg.com/zod/-/zod-3.22.4.tgz#f31c3a9386f61b1f228af56faa9255e845cf3fff" integrity sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg== +zod@3.23.8: + version "3.23.8" + resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d" + integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g== + zwitch@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-1.0.5.tgz#d11d7381ffed16b742f6af7b3f223d5cd9fe9920" From cd83199eb367e2072c38568d2b43444f380a2c7f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 9 Jul 2024 18:18:11 +0800 Subject: [PATCH 08/14] remove hardcode gpu vendor as it does not work --- packages/puppeteer-parse/src/browser.ts | 8 +++---- packages/puppeteer-parse/src/index.ts | 29 ++++++++++++++----------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/packages/puppeteer-parse/src/browser.ts b/packages/puppeteer-parse/src/browser.ts index 3c66a7a95..b7dc9e43f 100644 --- a/packages/puppeteer-parse/src/browser.ts +++ b/packages/puppeteer-parse/src/browser.ts @@ -44,10 +44,10 @@ export const getBrowser = async (): Promise => { '--disable-dev-shm-usage', '--no-first-run', '--disable-background-networking', - // '--disable-gpu', - // '--disable-software-rasterizer', - '--use-gl=angle', - '--use-angle=swiftshader-webgl', + '--disable-gpu', + '--disable-software-rasterizer', + // '--use-gl=angle', + // '--use-angle=swiftshader-webgl', ], defaultViewport: { deviceScaleFactor: 1, diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index 7d467cbc2..fe78c5e9b 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -247,23 +247,26 @@ async function retrievePage( locales: ['en-US'], }) const fingerprintWithHeaders = fingerprintGenerator.getFingerprint() - const fingerprint = { - fingerprint: { - ...fingerprintWithHeaders.fingerprint, - videoCard: { - vendor: 'Google Inc. (Intel Open Source Technology Center)', - renderer: - 'ANGLE (Intel Open Source Technology Center, Mesa DRI Intel(R) HD Graphics 4400 (HSW GT2), OpenGL 4.5 (Core Profile) Mesa 21.2.0-devel (git-fb586a8 2021-06-28 focal-oibaf-ppa))', - }, - }, - headers: fingerprintWithHeaders.headers, - } - console.log('fingerprint', fingerprint) + // const fingerprint = { + // fingerprint: { + // ...fingerprintWithHeaders.fingerprint, + // videoCard: { + // vendor: 'Google Inc. (Intel Open Source Technology Center)', + // renderer: + // 'ANGLE (Intel Open Source Technology Center, Mesa DRI Intel(R) HD Graphics 4400 (HSW GT2), OpenGL 4.5 (Core Profile) Mesa 21.2.0-devel (git-fb586a8 2021-06-28 focal-oibaf-ppa))', + // }, + // }, + // headers: fingerprintWithHeaders.headers, + // } + console.log('fingerprint', fingerprintWithHeaders) const page = await browser.newPage() // Attach fingerprint to page const fingerprintInjector = new FingerprintInjector() - await fingerprintInjector.attachFingerprintToPuppeteer(page, fingerprint) + await fingerprintInjector.attachFingerprintToPuppeteer( + page, + fingerprintWithHeaders + ) // Puppeteer fails during download of PDf files, // so record the failure and use those items From 73e180f43dd1b7866252657bdb8f8409b3e92341 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 9 Jul 2024 19:16:21 +0800 Subject: [PATCH 09/14] add more dependencies to docker container --- packages/content-fetch/Dockerfile | 23 +++++++++++++-- packages/puppeteer-parse/src/browser.ts | 8 +++--- packages/puppeteer-parse/src/index.ts | 38 ++++++------------------- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/packages/content-fetch/Dockerfile b/packages/content-fetch/Dockerfile index b87362ee2..ec5884570 100644 --- a/packages/content-fetch/Dockerfile +++ b/packages/content-fetch/Dockerfile @@ -1,7 +1,7 @@ FROM node:18.16 LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore" -# Installs latest Chromium (92) package. +# Installs latest Chromium package. RUN apt-get update && apt-get install -y \ chromium \ ca-certificates \ @@ -9,7 +9,26 @@ RUN apt-get update && apt-get install -y \ yarn \ g++ \ make \ - python3 + python3 \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libatspi2.0-0 \ + libcups2 \ + libdbus-1-3 \ + libdrm2 \ + libgbm1 \ + libgtk-3-0 \ + libnspr4 \ + libnss3 \ + libu2f-udev \ + libvulkan1 \ + libwayland-client0 \ + libxcomposite1 \ + libxdamage1 \ + libxfixes3 \ + libxkbcommon0 \ + libxrandr2 WORKDIR /app diff --git a/packages/puppeteer-parse/src/browser.ts b/packages/puppeteer-parse/src/browser.ts index b7dc9e43f..9584badd7 100644 --- a/packages/puppeteer-parse/src/browser.ts +++ b/packages/puppeteer-parse/src/browser.ts @@ -1,9 +1,9 @@ import { Browser } from 'puppeteer-core' import puppeteer from 'puppeteer-extra' import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker' -// import StealthPlugin from 'puppeteer-extra-plugin-stealth' +import StealthPlugin from 'puppeteer-extra-plugin-stealth' -// puppeteer.use(StealthPlugin()) +puppeteer.use(StealthPlugin()) puppeteer.use(AdblockerPlugin({ blockTrackers: true })) let browserInstance: Browser | null = null @@ -46,8 +46,8 @@ export const getBrowser = async (): Promise => { '--disable-background-networking', '--disable-gpu', '--disable-software-rasterizer', - // '--use-gl=angle', - // '--use-angle=swiftshader-webgl', + '--use-gl=angle', + '--use-angle=swiftshader', ], defaultViewport: { deviceScaleFactor: 1, diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index fe78c5e9b..c35b1d268 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -2,8 +2,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { preHandleContent } from '@omnivore/content-handler' import axios from 'axios' -import { FingerprintGenerator } from 'fingerprint-generator' -import { FingerprintInjector } from 'fingerprint-injector' +import { newInjectedPage } from 'fingerprint-injector' import { parseHTML } from 'linkedom' import path from 'path' import { Page, Protocol } from 'puppeteer-core' @@ -240,33 +239,14 @@ async function retrievePage( const browser = await getBrowser() - const fingerprintGenerator = new FingerprintGenerator({ - devices: ['desktop'], - operatingSystems: ['linux'], - browsers: ['chrome'], - locales: ['en-US'], - }) - const fingerprintWithHeaders = fingerprintGenerator.getFingerprint() - // const fingerprint = { - // fingerprint: { - // ...fingerprintWithHeaders.fingerprint, - // videoCard: { - // vendor: 'Google Inc. (Intel Open Source Technology Center)', - // renderer: - // 'ANGLE (Intel Open Source Technology Center, Mesa DRI Intel(R) HD Graphics 4400 (HSW GT2), OpenGL 4.5 (Core Profile) Mesa 21.2.0-devel (git-fb586a8 2021-06-28 focal-oibaf-ppa))', - // }, - // }, - // headers: fingerprintWithHeaders.headers, - // } - console.log('fingerprint', fingerprintWithHeaders) - const page = await browser.newPage() - - // Attach fingerprint to page - const fingerprintInjector = new FingerprintInjector() - await fingerprintInjector.attachFingerprintToPuppeteer( - page, - fingerprintWithHeaders - ) + const page = (await newInjectedPage(browser, { + fingerprintOptions: { + devices: ['desktop'], + operatingSystems: ['linux'], + browsers: ['chrome'], + locales: ['en-US'], + }, + })) as Page // Puppeteer fails during download of PDf files, // so record the failure and use those items From 75338f592764ae34104f7e992b1c93829af30474 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 10 Jul 2024 14:43:47 +0800 Subject: [PATCH 10/14] bypass cloudflare captcha --- packages/content-fetch/Dockerfile | 21 +---- packages/puppeteer-parse/package.json | 2 - packages/puppeteer-parse/src/browser.ts | 9 +- packages/puppeteer-parse/src/index.ts | 35 +------- yarn.lock | 108 +----------------------- 5 files changed, 9 insertions(+), 166 deletions(-) diff --git a/packages/content-fetch/Dockerfile b/packages/content-fetch/Dockerfile index ec5884570..52824edf6 100644 --- a/packages/content-fetch/Dockerfile +++ b/packages/content-fetch/Dockerfile @@ -9,26 +9,7 @@ RUN apt-get update && apt-get install -y \ yarn \ g++ \ make \ - python3 \ - libasound2 \ - libatk-bridge2.0-0 \ - libatk1.0-0 \ - libatspi2.0-0 \ - libcups2 \ - libdbus-1-3 \ - libdrm2 \ - libgbm1 \ - libgtk-3-0 \ - libnspr4 \ - libnss3 \ - libu2f-udev \ - libvulkan1 \ - libwayland-client0 \ - libxcomposite1 \ - libxdamage1 \ - libxfixes3 \ - libxkbcommon0 \ - libxrandr2 + python3 WORKDIR /app diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 27cdcd550..14637c232 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -13,8 +13,6 @@ "axios": "^1.4.0", "crypto": "^1.0.1", "dompurify": "^2.4.1", - "fingerprint-generator": "^2.1.52", - "fingerprint-injector": "^2.1.52", "linkedom": "^0.14.9", "puppeteer-core": "^22.12.1", "puppeteer-extra": "^3.3.6", diff --git a/packages/puppeteer-parse/src/browser.ts b/packages/puppeteer-parse/src/browser.ts index 9584badd7..b856a106f 100644 --- a/packages/puppeteer-parse/src/browser.ts +++ b/packages/puppeteer-parse/src/browser.ts @@ -1,4 +1,4 @@ -import { Browser } from 'puppeteer-core' +import { Browser, Target } from 'puppeteer-core' import puppeteer from 'puppeteer-extra' import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker' import StealthPlugin from 'puppeteer-extra-plugin-stealth' @@ -26,12 +26,9 @@ export const getBrowser = async (): Promise => { '--autoplay-policy=user-gesture-required', '--disable-component-update', '--disable-domain-reliability', - '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process', '--disable-print-preview', '--disable-setuid-sandbox', - '--disable-site-isolation-trials', '--disable-speech-api', - '--disk-cache-size=33554432', '--enable-features=SharedArrayBuffer', '--hide-scrollbars', '--mute-audio', @@ -39,15 +36,12 @@ export const getBrowser = async (): Promise => { '--no-pings', '--no-sandbox', '--no-zygote', - '--window-size=1920,1080', '--disable-extensions', '--disable-dev-shm-usage', '--no-first-run', '--disable-background-networking', '--disable-gpu', '--disable-software-rasterizer', - '--use-gl=angle', - '--use-angle=swiftshader', ], defaultViewport: { deviceScaleFactor: 1, @@ -61,6 +55,7 @@ export const getBrowser = async (): Promise => { headless: process.env.LAUNCH_HEADLESS === 'true', timeout: 10_000, // 10 seconds dumpio: true, // show console logs in the terminal + targetFilter: (target: Target) => target.type() !== 'other', })) as Browser const version = await browserInstance.version() diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index c35b1d268..ff61c2cb1 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -2,17 +2,11 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { preHandleContent } from '@omnivore/content-handler' import axios from 'axios' -import { newInjectedPage } from 'fingerprint-injector' import { parseHTML } from 'linkedom' import path from 'path' import { Page, Protocol } from 'puppeteer-core' import { getBrowser } from './browser' -const DESKTOP_USER_AGENT = - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const NON_BOT_DESKTOP_USER_AGENT = - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] const NON_SCRIPT_HOSTS = ['medium.com', 'fastcompany.com', 'fortelabs.com'] const ALLOWED_CONTENT_TYPES = [ @@ -21,21 +15,6 @@ const ALLOWED_CONTENT_TYPES = [ 'text/plain', 'application/pdf', ] -const REQUEST_TIMEOUT = 30000 - -const userAgentForUrl = (url: string) => { - try { - const u = new URL(url) - for (const host of NON_BOT_HOSTS) { - if (u.hostname.endsWith(host)) { - return NON_BOT_DESKTOP_USER_AGENT - } - } - } catch (e) { - console.log('error getting user agent for url', url, e) - } - return DESKTOP_USER_AGENT -} const fetchContentWithScrapingBee = async (url: string) => { const response = await axios.get('https://app.scrapingbee.com/api/v1', { @@ -46,7 +25,7 @@ const fetchContentWithScrapingBee = async (url: string) => { premium_proxy: 'true', country_code: 'us', }, - timeout: REQUEST_TIMEOUT, + timeout: 10_000, }) const dom = parseHTML(response.data).document @@ -238,15 +217,7 @@ async function retrievePage( } const browser = await getBrowser() - - const page = (await newInjectedPage(browser, { - fingerprintOptions: { - devices: ['desktop'], - operatingSystems: ['linux'], - browsers: ['chrome'], - locales: ['en-US'], - }, - })) as Page + const page = await browser.newPage() // Puppeteer fails during download of PDf files, // so record the failure and use those items @@ -365,6 +336,8 @@ async function retrievePage( throw new Error('No response from page') } + await page.waitForSelector('body') + const finalUrl = response.url() const contentType = response.headers()['content-type'] diff --git a/yarn.lock b/yarn.lock index befba1956..1e580e2d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6141,11 +6141,6 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ== -"@sindresorhus/is@^4.2.0": - version "4.6.0" - resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" - integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== - "@sinonjs/commons@^1", "@sinonjs/commons@^1.3.0", "@sinonjs/commons@^1.4.0", "@sinonjs/commons@^1.6.0", "@sinonjs/commons@^1.7.0", "@sinonjs/commons@^1.8.3": version "1.8.3" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.3.tgz#3802ddd21a50a949b6721ddd72da36e67e7f1b2d" @@ -9526,11 +9521,6 @@ addressparser@^1.0.1: resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746" integrity sha512-aQX7AISOMM7HFE0iZ3+YnD07oIeJqWGVnJ+ZIKaBZAk03ftmVYVqsGas/rbXKR21n4D/hKCSHypvcyOkds/xzg== -adm-zip@^0.5.9: - version "0.5.14" - resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.14.tgz#2c557c0bf12af4311cf6d32970f4060cf8133b2a" - integrity sha512-DnyqqifT4Jrcvb8USYjp6FHtBpEIz1mnXu6pTRHZ0RL69LbQYiO+0lDFg5+OKA7U29oWSs3a/i8fhn8ZcceIWg== - afinn-165-financialmarketnews@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/afinn-165-financialmarketnews/-/afinn-165-financialmarketnews-3.0.0.tgz#cf422577775bf94f9bc156f3f001a1f29338c3d8" @@ -11384,16 +11374,6 @@ browserslist@^4.17.5: node-releases "^2.0.1" picocolors "^1.0.0" -browserslist@^4.21.1: - version "4.23.1" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.1.tgz#ce4af0534b3d37db5c1a4ca98b9080f985041e96" - integrity sha512-TUfofFo/KsK/bWZ9TWQ5O26tsWW4Uhmt8IYklbnUa70udB6P2wA7w7o4PY4muaEPBQaAX+CEnmmIA41NVHtPVw== - dependencies: - caniuse-lite "^1.0.30001629" - electron-to-chromium "^1.4.796" - node-releases "^2.0.14" - update-browserslist-db "^1.0.16" - bser@2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/bser/-/bser-2.1.1.tgz#e6787da20ece9d07998533cfd9de6f5c38f4bc05" @@ -11697,7 +11677,7 @@ call-me-maybe@^1.0.1: resolved "https://registry.yarnpkg.com/call-me-maybe/-/call-me-maybe-1.0.1.tgz#26d208ea89e37b5cbde60250a15f031c16a4d66b" integrity sha1-JtII6onje1y95gJQoV8DHBak1ms= -callsites@^3.0.0, callsites@^3.1.0: +callsites@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== @@ -11767,11 +11747,6 @@ caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001251, caniuse-lite@^1.0.300012 resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001600.tgz" integrity sha512-+2S9/2JFhYmYaDpZvo0lKkfvuKIglrx68MwOBqMGHhQsNkLjB5xtc/TGoEPs+MxjSyN/72qer2g97nzR641mOQ== -caniuse-lite@^1.0.30001629: - version "1.0.30001640" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001640.tgz#32c467d4bf1f1a0faa63fc793c2ba81169e7652f" - integrity sha512-lA4VMpW0PSUrFnkmVuEKBUovSWKhj7puyCg8StBChgu298N1AtuF1sKWEvfDuimSEDbhlb/KqPKC3fs1HbuQUA== - capital-case@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669" @@ -14267,7 +14242,7 @@ dot-prop@^5.1.0, dot-prop@^5.2.0: dependencies: is-obj "^2.0.0" -dot-prop@^6.0.0, dot-prop@^6.0.1: +dot-prop@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-6.0.1.tgz#fc26b3cf142b9e59b74dbd39ed66ce620c681083" integrity sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA== @@ -14426,11 +14401,6 @@ electron-to-chromium@^1.4.17: resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.68.tgz#d79447b6bd1bec9183f166bb33d4bef0d5e4e568" integrity sha512-cId+QwWrV8R1UawO6b9BR1hnkJ4EJPCPAr4h315vliHUtVUJDk39Sg1PMNnaWKfj5x+93ssjeJ9LKL6r8LaMiA== -electron-to-chromium@^1.4.796: - version "1.4.816" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.816.tgz#3624649d1e7fde5cdbadf59d31a524245d8ee85f" - integrity sha512-EKH5X5oqC6hLmiS7/vYtZHZFTNdhsYG5NVPRN6Yn0kQHNBlT59+xSM8HBy66P5fxWpKgZbPqb+diC64ng295Jw== - electron-to-chromium@^1.4.84: version "1.4.89" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.89.tgz#33c06592812a17a7131873f4596579084ce33ff8" @@ -14905,11 +14875,6 @@ escalade@^3.1.1: resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== -escalade@^3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" - integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== - escape-goat@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675" @@ -16167,23 +16132,6 @@ fined@^1.0.1: object.pick "^1.2.0" parse-filepath "^1.0.1" -fingerprint-generator@^2.1.52: - version "2.1.52" - resolved "https://registry.yarnpkg.com/fingerprint-generator/-/fingerprint-generator-2.1.52.tgz#af40cb4f5b33a0a0173aeaa952f2b3b00bfce638" - integrity sha512-ZdXUn/qIB4vI7pDze5aXidjoFwLdEfbBNj6+3oHzXcgwxzEfCOfNe3wW5NRZDJKgxF40R7TSOA7noBAAehSLgQ== - dependencies: - generative-bayesian-network "^2.1.52" - header-generator "^2.1.52" - tslib "^2.4.0" - -fingerprint-injector@^2.1.52: - version "2.1.52" - resolved "https://registry.yarnpkg.com/fingerprint-injector/-/fingerprint-injector-2.1.52.tgz#d45cea9802f7a47c87fa1875dde05c61777cbb70" - integrity sha512-Sx+ykblqEP/P6nPRIE+C5CUNEfFpMZ3M/r5NDxOkSCTQVdfXXxlFx/UKOQNorvuJxryrtek4T0FvcB/KUbQfCQ== - dependencies: - fingerprint-generator "^2.1.52" - tslib "^2.4.0" - firebase-admin@^11.5.0: version "11.11.0" resolved "https://registry.yarnpkg.com/firebase-admin/-/firebase-admin-11.11.0.tgz#3d6df5dfbcf85dc1c6c4302f8aee4f7c82171725" @@ -16706,14 +16654,6 @@ gcp-metadata@^6.1.0: gaxios "^6.0.0" json-bigint "^1.0.0" -generative-bayesian-network@^2.1.52: - version "2.1.52" - resolved "https://registry.yarnpkg.com/generative-bayesian-network/-/generative-bayesian-network-2.1.52.tgz#0d8aa6dd14558bf88fb999feeff8c86d9e60322a" - integrity sha512-8fYemN+uiVPCjoodQX4HUH8RLDqiQeGfemlWO9yR6SqIh/6BsrW52M0YTSafsH0615BhulRy5BR2uKAqLTJ22A== - dependencies: - adm-zip "^0.5.9" - tslib "^2.4.0" - generic-pool@3.9.0: version "3.9.0" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" @@ -17749,16 +17689,6 @@ header-case@^2.0.4: capital-case "^1.0.4" tslib "^2.0.3" -header-generator@^2.1.52: - version "2.1.52" - resolved "https://registry.yarnpkg.com/header-generator/-/header-generator-2.1.52.tgz#1560fc3a2f2f65ed1d777a1660fb0c4459bf4827" - integrity sha512-2roqbZdd0hc7Bx+6BIQaHaCaSdnTXCnqayFbS8dpj53hmkQAXbSwiuTpfyAY1vePiaKweH6vDYhbtGOW+NmTmw== - dependencies: - browserslist "^4.21.1" - generative-bayesian-network "^2.1.52" - ow "^0.28.1" - tslib "^2.4.0" - heap-js@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/heap-js/-/heap-js-2.2.0.tgz#f4418874cd2aedc2cf3a7492d579afe23b627c5d" @@ -23532,11 +23462,6 @@ node-releases@^2.0.1, node-releases@^2.0.2: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.2.tgz#7139fe71e2f4f11b47d4d2986aaf8c48699e0c01" integrity sha512-XxYDdcQ6eKqp/YjI+tb2C5WM2LgjnZrfYg4vgQt49EK268b6gYCHsBLrK2qvJo4FmCtqmKezb0WZFK4fkrZNsg== -node-releases@^2.0.14: - version "2.0.14" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b" - integrity sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw== - nodemailer@^6.7.3: version "6.7.3" resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.3.tgz#b73f9a81b9c8fa8acb4ea14b608f5e725ea8e018" @@ -24518,17 +24443,6 @@ overlayscrollbars@^1.13.1: resolved "https://registry.yarnpkg.com/overlayscrollbars/-/overlayscrollbars-1.13.1.tgz#0b840a88737f43a946b9d87875a2f9e421d0338a" integrity sha512-gIQfzgGgu1wy80EB4/6DaJGHMEGmizq27xHIESrzXq0Y/J0Ay1P3DWk6tuVmEPIZH15zaBlxeEJOqdJKmowHCQ== -ow@^0.28.1: - version "0.28.2" - resolved "https://registry.yarnpkg.com/ow/-/ow-0.28.2.tgz#782b28102124e665c49ec7725e2066a129acf6bf" - integrity sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q== - dependencies: - "@sindresorhus/is" "^4.2.0" - callsites "^3.1.0" - dot-prop "^6.0.1" - lodash.isequal "^4.5.0" - vali-date "^1.0.0" - p-all@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-all/-/p-all-2.1.0.tgz#91419be56b7dee8fe4c5db875d55e0da084244a0" @@ -25314,11 +25228,6 @@ picocolors@^1.0.0: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== -picocolors@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1" - integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew== - picomatch@^2.0.4, picomatch@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" @@ -31154,14 +31063,6 @@ upath@^1.1.1: resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894" integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg== -update-browserslist-db@^1.0.16: - version "1.1.0" - resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz#7ca61c0d8650766090728046e416a8cde682859e" - integrity sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ== - dependencies: - escalade "^3.1.2" - picocolors "^1.0.1" - update-notifier@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9" @@ -31442,11 +31343,6 @@ v8flags@^2.0.10: dependencies: user-home "^1.1.1" -vali-date@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/vali-date/-/vali-date-1.0.0.tgz#1b904a59609fb328ef078138420934f6b86709a6" - integrity sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg== - valid-url@^1.0.9: version "1.0.9" resolved "https://registry.yarnpkg.com/valid-url/-/valid-url-1.0.9.tgz#1c14479b40f1397a75782f115e4086447433a200" From 431c4cc0983026a74e7c81a3d610651929f8f535 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 10 Jul 2024 15:13:16 +0800 Subject: [PATCH 11/14] use old headless mode for better performance --- packages/puppeteer-parse/src/browser.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/puppeteer-parse/src/browser.ts b/packages/puppeteer-parse/src/browser.ts index b856a106f..4f27ab94c 100644 --- a/packages/puppeteer-parse/src/browser.ts +++ b/packages/puppeteer-parse/src/browser.ts @@ -52,10 +52,11 @@ export const getBrowser = async (): Promise => { width: 1920, }, executablePath: process.env.CHROMIUM_PATH, - headless: process.env.LAUNCH_HEADLESS === 'true', + headless: process.env.HEADLESS_MODE, timeout: 10_000, // 10 seconds dumpio: true, // show console logs in the terminal - targetFilter: (target: Target) => target.type() !== 'other', + targetFilter: (target: Target) => + target.type() !== 'other' || !!target.url(), })) as Browser const version = await browserInstance.version() From e61da03ed46ff03159ae6314f83e69c228ab026d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 10 Jul 2024 16:01:39 +0800 Subject: [PATCH 12/14] wait until body is fetched --- packages/puppeteer-parse/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index ff61c2cb1..457ac504a 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -336,8 +336,6 @@ async function retrievePage( throw new Error('No response from page') } - await page.waitForSelector('body') - const finalUrl = response.url() const contentType = response.headers()['content-type'] @@ -364,6 +362,8 @@ async function retrieveHtml(page: Page, logRecord: Record) { title = await page.title() logRecord.title = title + await page.waitForSelector('body') + const pageScrollingStart = Date.now() /* scroll with a 5 seconds timeout */ try { From dfc0ce0e54bb754ce36d06a5732d1c8a78894462 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 10 Jul 2024 17:20:04 +0800 Subject: [PATCH 13/14] create a context for each new page --- packages/puppeteer-parse/src/browser.ts | 4 +++- packages/puppeteer-parse/src/index.ts | 24 +++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/puppeteer-parse/src/browser.ts b/packages/puppeteer-parse/src/browser.ts index 4f27ab94c..c89e92fa0 100644 --- a/packages/puppeteer-parse/src/browser.ts +++ b/packages/puppeteer-parse/src/browser.ts @@ -52,9 +52,11 @@ export const getBrowser = async (): Promise => { width: 1920, }, executablePath: process.env.CHROMIUM_PATH, - headless: process.env.HEADLESS_MODE, + // run in shell mode if headless + headless: process.env.LAUNCH_HEADLESS === 'true' ? 'shell' : false, timeout: 10_000, // 10 seconds dumpio: true, // show console logs in the terminal + // filter out targets targetFilter: (target: Target) => target.type() !== 'other' || !!target.url(), })) as Browser diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index 457ac504a..78031e58c 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -4,7 +4,7 @@ import { preHandleContent } from '@omnivore/content-handler' import axios from 'axios' import { parseHTML } from 'linkedom' import path from 'path' -import { Page, Protocol } from 'puppeteer-core' +import { BrowserContext, Page, Protocol } from 'puppeteer-core' import { getBrowser } from './browser' const NON_SCRIPT_HOSTS = ['medium.com', 'fastcompany.com', 'fortelabs.com'] @@ -63,7 +63,8 @@ export const fetchContent = async ( let page: Page | undefined, title: string | undefined, content: string | undefined, - contentType: string | undefined + contentType: string | undefined, + context: BrowserContext | undefined try { url = getUrl(url) @@ -99,6 +100,9 @@ export const fetchContent = async ( locale, timezone ) + if (result && result.context) { + context = result.context + } if (result && result.page) { page = result.page } @@ -144,10 +148,10 @@ export const fetchContent = async ( throw e } finally { - // close browser page if it was opened - if (page) { + // close browser context if it was created + if (context) { console.info('closing page...', url) - await page.close() + await context.close() console.info('page closed', url) } @@ -217,12 +221,14 @@ async function retrievePage( } const browser = await getBrowser() - const page = await browser.newPage() + const context = await browser.createBrowserContext() // Puppeteer fails during download of PDf files, // so record the failure and use those items let lastPdfUrl try { + const page = await context.newPage() + if (!enableJavascriptForUrl(url)) { await page.setJavaScriptEnabled(false) } @@ -342,16 +348,16 @@ async function retrievePage( logRecord.finalUrl = finalUrl logRecord.contentType = contentType - return { page, finalUrl, contentType } + return { page, finalUrl, contentType, context } } catch (error) { if (lastPdfUrl) { return { - page, + context, finalUrl: lastPdfUrl, contentType: 'application/pdf', } } - await page.close() + await context.close() throw error } } From 8e5439fa7b4b9d9f1492cf16fb3569024ae97475 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 10 Jul 2024 17:59:14 +0800 Subject: [PATCH 14/14] generate random user-agent when fetching rss feed --- packages/api/package.json | 1 + packages/api/src/jobs/rss/refreshFeed.ts | 8 +- .../api/src/resolvers/discover_feeds/add.ts | 4 +- .../api/src/resolvers/subscriptions/index.ts | 4 +- packages/api/src/utils/helpers.ts | 5 + packages/api/src/utils/parser.ts | 23 ++-- yarn.lock | 100 +++++++++++++++++- 7 files changed, 123 insertions(+), 22 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 505362610..089294a75 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -68,6 +68,7 @@ "express-prom-bundle": "^7.0.0", "express-rate-limit": "^6.3.0", "fast-safe-stringify": "^2.1.1", + "fingerprint-generator": "^2.1.52", "firebase-admin": "^11.5.0", "googleapis": "^125.0.0", "graphql": "^15.3.0", diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index dadd0908c..78069a118 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -18,6 +18,7 @@ import createHttpTaskWithToken from '../../utils/createTask' import { cleanUrl } from '../../utils/helpers' import { createThumbnailProxyUrl } from '../../utils/imageproxy' import { logger } from '../../utils/logger' +import { rssParserConfig } from '../../utils/parser' import { RSSRefreshContext } from './refreshAllFeeds' type FolderType = 'following' | 'inbox' @@ -181,15 +182,10 @@ const getThumbnail = (item: RssFeedItem) => { export const fetchAndChecksum = async (url: string) => { try { const response = await axios.get(url, { + ...rssParserConfig(), responseType: 'arraybuffer', timeout: 60_000, maxRedirects: 10, - headers: { - 'User-Agent': - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - Accept: - 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml, text/html;q=0.4', - }, }) const hash = crypto.createHash('sha256') diff --git a/packages/api/src/resolvers/discover_feeds/add.ts b/packages/api/src/resolvers/discover_feeds/add.ts index 54bb79df4..34eb23419 100644 --- a/packages/api/src/resolvers/discover_feeds/add.ts +++ b/packages/api/src/resolvers/discover_feeds/add.ts @@ -14,7 +14,7 @@ import { MutationAddDiscoverFeedArgs, } from '../../generated/graphql' import { authorized } from '../../utils/gql-utils' -import { RSS_PARSER_CONFIG } from '../../utils/parser' +import { rssParserConfig } from '../../utils/parser' const parser = new XMLParser({ ignoreAttributes: false, @@ -93,7 +93,7 @@ const addNewSubscription = async ( userId: string ): Promise => { // First things first, we need to validate that this is an actual RSS or ATOM feed. - const response = await axios.get(url, RSS_PARSER_CONFIG) + const response = await axios.get(url, rssParserConfig()) const content = response.data const contentType = response.headers['content-type'] diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index fc8d4d55a..e9a004721 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -52,7 +52,7 @@ import { analytics } from '../../utils/analytics' import { enqueueRssFeedFetch } from '../../utils/createTask' import { authorized } from '../../utils/gql-utils' import { getAbsoluteUrl, keysToCamelCase } from '../../utils/helpers' -import { parseFeed, parseOpml, RSS_PARSER_CONFIG } from '../../utils/parser' +import { parseFeed, parseOpml, rssParserConfig } from '../../utils/parser' type PartialSubscription = Omit @@ -442,7 +442,7 @@ export const scanFeedsResolver = authorized< try { // fetch page content and parse feeds - const response = await axios.get(url, RSS_PARSER_CONFIG) + const response = await axios.get(url, rssParserConfig()) const content = response.data as string // check if the content is html or xml const contentType = response.headers['Content-Type'] diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index d35261f13..4b750ed90 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -2,6 +2,7 @@ import languages from '@cospired/i18n-iso-languages' import { countWords } from 'alfaaz' import crypto from 'crypto' +import { FingerprintGenerator } from 'fingerprint-generator' import Redis from 'ioredis' import { parseHTML } from 'linkedom' import normalizeUrl from 'normalize-url' @@ -286,3 +287,7 @@ export const getClientFromUserAgent = (userAgent: string): string => { export const lanaugeToCode = (language: string): string => languages.getAlpha2Code(language, 'en') || 'en' + +export const generateFingerprint = () => { + return new FingerprintGenerator().getFingerprint() +} diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 182600aff..943bb4465 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -26,6 +26,7 @@ import { env } from '../env' import { PageType, PreparedDocumentInput } from '../generated/graphql' import { userRepository } from '../repository/user' import { ArticleFormat } from '../resolvers/article' +import { generateFingerprint } from './helpers' import { EmbeddedHighlightData, findEmbeddedHighlight, @@ -77,15 +78,17 @@ const DOM_PURIFY_CONFIG = { const ARTICLE_PREFIX = 'omnivore:' export const FAKE_URL_PREFIX = 'https://omnivore.app/no_url?q=' -export const RSS_PARSER_CONFIG = { - timeout: 20000, // 20 seconds - headers: { - // some rss feeds require user agent - 'User-Agent': - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - Accept: - 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4, text/html;q=0.2', - }, +export const rssParserConfig = () => { + const fingerprint = generateFingerprint() + + return { + headers: { + 'user-agent': fingerprint.headers['user-agent'], + accept: + 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4, text/html;q=0.2', + }, + timeout: 20000, // 20 seconds + } } /** Hook that prevents DOMPurify from removing youtube iframes */ @@ -836,7 +839,7 @@ export const parseFeed = async ( } } - const parser = new Parser(RSS_PARSER_CONFIG) + const parser = new Parser(rssParserConfig()) const feed = content ? await parser.parseString(content) diff --git a/yarn.lock b/yarn.lock index 1e580e2d8..71643dd71 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6141,6 +6141,11 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ== +"@sindresorhus/is@^4.2.0": + version "4.6.0" + resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" + integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== + "@sinonjs/commons@^1", "@sinonjs/commons@^1.3.0", "@sinonjs/commons@^1.4.0", "@sinonjs/commons@^1.6.0", "@sinonjs/commons@^1.7.0", "@sinonjs/commons@^1.8.3": version "1.8.3" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.3.tgz#3802ddd21a50a949b6721ddd72da36e67e7f1b2d" @@ -9521,6 +9526,11 @@ addressparser@^1.0.1: resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746" integrity sha512-aQX7AISOMM7HFE0iZ3+YnD07oIeJqWGVnJ+ZIKaBZAk03ftmVYVqsGas/rbXKR21n4D/hKCSHypvcyOkds/xzg== +adm-zip@^0.5.9: + version "0.5.14" + resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.14.tgz#2c557c0bf12af4311cf6d32970f4060cf8133b2a" + integrity sha512-DnyqqifT4Jrcvb8USYjp6FHtBpEIz1mnXu6pTRHZ0RL69LbQYiO+0lDFg5+OKA7U29oWSs3a/i8fhn8ZcceIWg== + afinn-165-financialmarketnews@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/afinn-165-financialmarketnews/-/afinn-165-financialmarketnews-3.0.0.tgz#cf422577775bf94f9bc156f3f001a1f29338c3d8" @@ -11374,6 +11384,16 @@ browserslist@^4.17.5: node-releases "^2.0.1" picocolors "^1.0.0" +browserslist@^4.21.1: + version "4.23.2" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.2.tgz#244fe803641f1c19c28c48c4b6ec9736eb3d32ed" + integrity sha512-qkqSyistMYdxAcw+CzbZwlBy8AGmS/eEWs+sEV5TnLRGDOL+C5M2EnH6tlZyg0YoAxGJAFKh61En9BR941GnHA== + dependencies: + caniuse-lite "^1.0.30001640" + electron-to-chromium "^1.4.820" + node-releases "^2.0.14" + update-browserslist-db "^1.1.0" + bser@2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/bser/-/bser-2.1.1.tgz#e6787da20ece9d07998533cfd9de6f5c38f4bc05" @@ -11677,7 +11697,7 @@ call-me-maybe@^1.0.1: resolved "https://registry.yarnpkg.com/call-me-maybe/-/call-me-maybe-1.0.1.tgz#26d208ea89e37b5cbde60250a15f031c16a4d66b" integrity sha1-JtII6onje1y95gJQoV8DHBak1ms= -callsites@^3.0.0: +callsites@^3.0.0, callsites@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== @@ -11747,6 +11767,11 @@ caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001251, caniuse-lite@^1.0.300012 resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001600.tgz" integrity sha512-+2S9/2JFhYmYaDpZvo0lKkfvuKIglrx68MwOBqMGHhQsNkLjB5xtc/TGoEPs+MxjSyN/72qer2g97nzR641mOQ== +caniuse-lite@^1.0.30001640: + version "1.0.30001641" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001641.tgz#3572862cd18befae3f637f2a1101cc033c6782ac" + integrity sha512-Phv5thgl67bHYo1TtMY/MurjkHhV4EDaCosezRXgZ8jzA/Ub+wjxAvbGvjoFENStinwi5kCyOYV3mi5tOGykwA== + capital-case@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669" @@ -14242,7 +14267,7 @@ dot-prop@^5.1.0, dot-prop@^5.2.0: dependencies: is-obj "^2.0.0" -dot-prop@^6.0.0: +dot-prop@^6.0.0, dot-prop@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-6.0.1.tgz#fc26b3cf142b9e59b74dbd39ed66ce620c681083" integrity sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA== @@ -14401,6 +14426,11 @@ electron-to-chromium@^1.4.17: resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.68.tgz#d79447b6bd1bec9183f166bb33d4bef0d5e4e568" integrity sha512-cId+QwWrV8R1UawO6b9BR1hnkJ4EJPCPAr4h315vliHUtVUJDk39Sg1PMNnaWKfj5x+93ssjeJ9LKL6r8LaMiA== +electron-to-chromium@^1.4.820: + version "1.4.823" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.823.tgz#38587f7aa55bed14930f04091dfc65c39a3d8bd7" + integrity sha512-4h+oPeAiGQOHFyUJOqpoEcPj/xxlicxBzOErVeYVMMmAiXUXsGpsFd0QXBMaUUbnD8hhSfLf9uw+MlsoIA7j5w== + electron-to-chromium@^1.4.84: version "1.4.89" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.89.tgz#33c06592812a17a7131873f4596579084ce33ff8" @@ -14875,6 +14905,11 @@ escalade@^3.1.1: resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== +escalade@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" + integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== + escape-goat@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675" @@ -16132,6 +16167,15 @@ fined@^1.0.1: object.pick "^1.2.0" parse-filepath "^1.0.1" +fingerprint-generator@^2.1.52: + version "2.1.52" + resolved "https://registry.yarnpkg.com/fingerprint-generator/-/fingerprint-generator-2.1.52.tgz#af40cb4f5b33a0a0173aeaa952f2b3b00bfce638" + integrity sha512-ZdXUn/qIB4vI7pDze5aXidjoFwLdEfbBNj6+3oHzXcgwxzEfCOfNe3wW5NRZDJKgxF40R7TSOA7noBAAehSLgQ== + dependencies: + generative-bayesian-network "^2.1.52" + header-generator "^2.1.52" + tslib "^2.4.0" + firebase-admin@^11.5.0: version "11.11.0" resolved "https://registry.yarnpkg.com/firebase-admin/-/firebase-admin-11.11.0.tgz#3d6df5dfbcf85dc1c6c4302f8aee4f7c82171725" @@ -16654,6 +16698,14 @@ gcp-metadata@^6.1.0: gaxios "^6.0.0" json-bigint "^1.0.0" +generative-bayesian-network@^2.1.52: + version "2.1.52" + resolved "https://registry.yarnpkg.com/generative-bayesian-network/-/generative-bayesian-network-2.1.52.tgz#0d8aa6dd14558bf88fb999feeff8c86d9e60322a" + integrity sha512-8fYemN+uiVPCjoodQX4HUH8RLDqiQeGfemlWO9yR6SqIh/6BsrW52M0YTSafsH0615BhulRy5BR2uKAqLTJ22A== + dependencies: + adm-zip "^0.5.9" + tslib "^2.4.0" + generic-pool@3.9.0: version "3.9.0" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" @@ -17689,6 +17741,16 @@ header-case@^2.0.4: capital-case "^1.0.4" tslib "^2.0.3" +header-generator@^2.1.52: + version "2.1.52" + resolved "https://registry.yarnpkg.com/header-generator/-/header-generator-2.1.52.tgz#1560fc3a2f2f65ed1d777a1660fb0c4459bf4827" + integrity sha512-2roqbZdd0hc7Bx+6BIQaHaCaSdnTXCnqayFbS8dpj53hmkQAXbSwiuTpfyAY1vePiaKweH6vDYhbtGOW+NmTmw== + dependencies: + browserslist "^4.21.1" + generative-bayesian-network "^2.1.52" + ow "^0.28.1" + tslib "^2.4.0" + heap-js@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/heap-js/-/heap-js-2.2.0.tgz#f4418874cd2aedc2cf3a7492d579afe23b627c5d" @@ -23462,6 +23524,11 @@ node-releases@^2.0.1, node-releases@^2.0.2: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.2.tgz#7139fe71e2f4f11b47d4d2986aaf8c48699e0c01" integrity sha512-XxYDdcQ6eKqp/YjI+tb2C5WM2LgjnZrfYg4vgQt49EK268b6gYCHsBLrK2qvJo4FmCtqmKezb0WZFK4fkrZNsg== +node-releases@^2.0.14: + version "2.0.14" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b" + integrity sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw== + nodemailer@^6.7.3: version "6.7.3" resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.3.tgz#b73f9a81b9c8fa8acb4ea14b608f5e725ea8e018" @@ -24443,6 +24510,17 @@ overlayscrollbars@^1.13.1: resolved "https://registry.yarnpkg.com/overlayscrollbars/-/overlayscrollbars-1.13.1.tgz#0b840a88737f43a946b9d87875a2f9e421d0338a" integrity sha512-gIQfzgGgu1wy80EB4/6DaJGHMEGmizq27xHIESrzXq0Y/J0Ay1P3DWk6tuVmEPIZH15zaBlxeEJOqdJKmowHCQ== +ow@^0.28.1: + version "0.28.2" + resolved "https://registry.yarnpkg.com/ow/-/ow-0.28.2.tgz#782b28102124e665c49ec7725e2066a129acf6bf" + integrity sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q== + dependencies: + "@sindresorhus/is" "^4.2.0" + callsites "^3.1.0" + dot-prop "^6.0.1" + lodash.isequal "^4.5.0" + vali-date "^1.0.0" + p-all@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-all/-/p-all-2.1.0.tgz#91419be56b7dee8fe4c5db875d55e0da084244a0" @@ -25228,6 +25306,11 @@ picocolors@^1.0.0: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== +picocolors@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1" + integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew== + picomatch@^2.0.4, picomatch@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" @@ -31063,6 +31146,14 @@ upath@^1.1.1: resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894" integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg== +update-browserslist-db@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz#7ca61c0d8650766090728046e416a8cde682859e" + integrity sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ== + dependencies: + escalade "^3.1.2" + picocolors "^1.0.1" + update-notifier@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9" @@ -31343,6 +31434,11 @@ v8flags@^2.0.10: dependencies: user-home "^1.1.1" +vali-date@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/vali-date/-/vali-date-1.0.0.tgz#1b904a59609fb328ef078138420934f6b86709a6" + integrity sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg== + valid-url@^1.0.9: version "1.0.9" resolved "https://registry.yarnpkg.com/valid-url/-/valid-url-1.0.9.tgz#1c14479b40f1397a75782f115e4086447433a200"