Merge pull request #4155 from omnivore-app/fingerprint

inject fingerprint
2026-03-11 08:54:26 +00:00 · 2024-07-11 14:24:07 +08:00 · 2024-07-11 14:24:07 +08:00 · 3d791a7866
commit 3d791a7866
parent 57d546bff9 8e5439fa7b
12 changed files with 216 additions and 71 deletions
--- a/packages/api/package.json
+++ b/packages/api/package.json
@ -68,6 +68,7 @@
    "express-prom-bundle": "^7.0.0",
    "express-rate-limit": "^6.3.0",
    "fast-safe-stringify": "^2.1.1",
+    "fingerprint-generator": "^2.1.52",
    "firebase-admin": "^11.5.0",
    "googleapis": "^125.0.0",
    "graphql": "^15.3.0",
--- a/packages/api/src/jobs/rss/refreshFeed.ts
+++ b/packages/api/src/jobs/rss/refreshFeed.ts
@ -18,6 +18,7 @@ import createHttpTaskWithToken from '../../utils/createTask'
 import { cleanUrl } from '../../utils/helpers'
 import { createThumbnailProxyUrl } from '../../utils/imageproxy'
 import { logger } from '../../utils/logger'
+import { rssParserConfig } from '../../utils/parser'
 import { RSSRefreshContext } from './refreshAllFeeds'

 type FolderType = 'following' | 'inbox'
@ -181,15 +182,10 @@ const getThumbnail = (item: RssFeedItem) => {
 export const fetchAndChecksum = async (url: string) => {
  try {
    const response = await axios.get(url, {
+      ...rssParserConfig(),
      responseType: 'arraybuffer',
      timeout: 60_000,
      maxRedirects: 10,
-      headers: {
-        'User-Agent':
-          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
-        Accept:
-          'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml, text/html;q=0.4',
-      },
    })

    const hash = crypto.createHash('sha256')
--- a/packages/api/src/resolvers/discover_feeds/add.ts
+++ b/packages/api/src/resolvers/discover_feeds/add.ts
@ -14,7 +14,7 @@ import {
  MutationAddDiscoverFeedArgs,
 } from '../../generated/graphql'
 import { authorized } from '../../utils/gql-utils'
-import { RSS_PARSER_CONFIG } from '../../utils/parser'
+import { rssParserConfig } from '../../utils/parser'

 const parser = new XMLParser({
  ignoreAttributes: false,
@ -93,7 +93,7 @@ const addNewSubscription = async (
  userId: string
 ): Promise<AddDiscoverFeedSuccess | AddDiscoverFeedError> => {
  // First things first, we need to validate that this is an actual RSS or ATOM feed.
-  const response = await axios.get(url, RSS_PARSER_CONFIG)
+  const response = await axios.get(url, rssParserConfig())
  const content = response.data

  const contentType = response.headers['content-type']
--- a/packages/api/src/resolvers/subscriptions/index.ts
+++ b/packages/api/src/resolvers/subscriptions/index.ts
@ -52,7 +52,7 @@ import { analytics } from '../../utils/analytics'
 import { enqueueRssFeedFetch } from '../../utils/createTask'
 import { authorized } from '../../utils/gql-utils'
 import { getAbsoluteUrl, keysToCamelCase } from '../../utils/helpers'
-import { parseFeed, parseOpml, RSS_PARSER_CONFIG } from '../../utils/parser'
+import { parseFeed, parseOpml, rssParserConfig } from '../../utils/parser'

 type PartialSubscription = Omit<Subscription, 'newsletterEmail'>

@ -442,7 +442,7 @@ export const scanFeedsResolver = authorized<

  try {
    // fetch page content and parse feeds
-    const response = await axios.get(url, RSS_PARSER_CONFIG)
+    const response = await axios.get(url, rssParserConfig())
    const content = response.data as string
    // check if the content is html or xml
    const contentType = response.headers['Content-Type']
--- a/packages/api/src/utils/helpers.ts
+++ b/packages/api/src/utils/helpers.ts
@ -2,6 +2,7 @@
 import languages from '@cospired/i18n-iso-languages'
 import { countWords } from 'alfaaz'
 import crypto from 'crypto'
+import { FingerprintGenerator } from 'fingerprint-generator'
 import Redis from 'ioredis'
 import { parseHTML } from 'linkedom'
 import normalizeUrl from 'normalize-url'
@ -286,3 +287,7 @@ export const getClientFromUserAgent = (userAgent: string): string => {

 export const lanaugeToCode = (language: string): string =>
  languages.getAlpha2Code(language, 'en') || 'en'
+
+export const generateFingerprint = () => {
+  return new FingerprintGenerator().getFingerprint()
+}
--- a/packages/api/src/utils/parser.ts
+++ b/packages/api/src/utils/parser.ts
@ -26,6 +26,7 @@ import { env } from '../env'
 import { PageType, PreparedDocumentInput } from '../generated/graphql'
 import { userRepository } from '../repository/user'
 import { ArticleFormat } from '../resolvers/article'
+import { generateFingerprint } from './helpers'
 import {
  EmbeddedHighlightData,
  findEmbeddedHighlight,
@ -77,15 +78,17 @@ const DOM_PURIFY_CONFIG = {
 const ARTICLE_PREFIX = 'omnivore:'

 export const FAKE_URL_PREFIX = 'https://omnivore.app/no_url?q='
-export const RSS_PARSER_CONFIG = {
-  timeout: 20000, // 20 seconds
-  headers: {
-    // some rss feeds require user agent
-    'User-Agent':
-      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
-    Accept:
-      'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4, text/html;q=0.2',
-  },
+export const rssParserConfig = () => {
+  const fingerprint = generateFingerprint()
+
+  return {
+    headers: {
+      'user-agent': fingerprint.headers['user-agent'],
+      accept:
+        'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4, text/html;q=0.2',
+    },
+    timeout: 20000, // 20 seconds
+  }
 }

 /** Hook that prevents DOMPurify from removing youtube iframes */
@ -836,7 +839,7 @@ export const parseFeed = async (
      }
    }

-    const parser = new Parser(RSS_PARSER_CONFIG)
+    const parser = new Parser(rssParserConfig())

    const feed = content
      ? await parser.parseString(content)
--- a/packages/content-fetch/Dockerfile
+++ b/packages/content-fetch/Dockerfile
@ -1,7 +1,7 @@
 FROM node:18.16
 LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"

-# Installs latest Chromium (92) package.
+# Installs latest Chromium package.
 RUN apt-get update && apt-get install -y \
    chromium \
    ca-certificates \
--- a/packages/content-fetch/src/request_handler.ts
+++ b/packages/content-fetch/src/request_handler.ts
@ -62,6 +62,11 @@ const storage = process.env.GCS_UPLOAD_SA_KEY_FILE_PATH
  : new Storage()
 const bucketName = process.env.GCS_UPLOAD_BUCKET || 'omnivore-files'

+const NO_CACHE_URLS = [
+  'https://deviceandbrowserinfo.com/are_you_a_bot',
+  'https://deviceandbrowserinfo.com/info_device',
+]
+
 const uploadToBucket = async (filePath: string, data: string) => {
  await storage
    .bucket(bucketName)
@ -198,7 +203,7 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => {
      fetchResult = await fetchContent(url, locale, timezone)
      console.log('content has been fetched')

-      if (fetchResult.content) {
+      if (fetchResult.content && !NO_CACHE_URLS.includes(url)) {
        const cacheResult = await cacheFetchResult(
          redisDataSource,
          key,
--- a/packages/puppeteer-parse/package.json
+++ b/packages/puppeteer-parse/package.json
@ -14,10 +14,10 @@
    "crypto": "^1.0.1",
    "dompurify": "^2.4.1",
    "linkedom": "^0.14.9",
-    "puppeteer-core": "^22.8.0",
-    "puppeteer-extra": "^3.3.4",
-    "puppeteer-extra-plugin-adblocker": "^2.13.5",
-    "puppeteer-extra-plugin-stealth": "^2.11.1",
+    "puppeteer-core": "^22.12.1",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-adblocker": "^2.13.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
    "urlsafe-base64": "^1.0.0"
  },
  "devDependencies": {
--- a/packages/puppeteer-parse/src/browser.ts
+++ b/packages/puppeteer-parse/src/browser.ts
@ -1,4 +1,4 @@
-import { Browser } from 'puppeteer-core'
+import { Browser, Target } from 'puppeteer-core'
 import puppeteer from 'puppeteer-extra'
 import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker'
 import StealthPlugin from 'puppeteer-extra-plugin-stealth'
@ -26,26 +26,22 @@ export const getBrowser = async (): Promise<Browser> => {
      '--autoplay-policy=user-gesture-required',
      '--disable-component-update',
      '--disable-domain-reliability',
-      '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process',
      '--disable-print-preview',
      '--disable-setuid-sandbox',
-      '--disable-site-isolation-trials',
      '--disable-speech-api',
-      '--disk-cache-size=33554432',
      '--enable-features=SharedArrayBuffer',
      '--hide-scrollbars',
-      '--disable-gpu',
      '--mute-audio',
      '--no-default-browser-check',
      '--no-pings',
      '--no-sandbox',
      '--no-zygote',
-      '--window-size=1920,1080',
      '--disable-extensions',
      '--disable-dev-shm-usage',
      '--no-first-run',
      '--disable-background-networking',
-      '--use-gl=swiftshader',
+      '--disable-gpu',
+      '--disable-software-rasterizer',
    ],
    defaultViewport: {
      deviceScaleFactor: 1,
@ -56,9 +52,13 @@ export const getBrowser = async (): Promise<Browser> => {
      width: 1920,
    },
    executablePath: process.env.CHROMIUM_PATH,
-    headless: !!process.env.LAUNCH_HEADLESS,
+    // run in shell mode if headless
+    headless: process.env.LAUNCH_HEADLESS === 'true' ? 'shell' : false,
    timeout: 10_000, // 10 seconds
    dumpio: true, // show console logs in the terminal
+    // filter out targets
+    targetFilter: (target: Target) =>
+      target.type() !== 'other' || !!target.url(),
  })) as Browser

  const version = await browserInstance.version()
--- a/packages/puppeteer-parse/src/index.ts
+++ b/packages/puppeteer-parse/src/index.ts
@ -7,11 +7,6 @@ import path from 'path'
 import { BrowserContext, Page, Protocol } from 'puppeteer-core'
 import { getBrowser } from './browser'

-const DESKTOP_USER_AGENT =
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
-const NON_BOT_DESKTOP_USER_AGENT =
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
-const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com']
 const NON_SCRIPT_HOSTS = ['medium.com', 'fastcompany.com', 'fortelabs.com']

 const ALLOWED_CONTENT_TYPES = [
@ -20,21 +15,6 @@ const ALLOWED_CONTENT_TYPES = [
  'text/plain',
  'application/pdf',
 ]
-const REQUEST_TIMEOUT = 30000
-
-const userAgentForUrl = (url: string) => {
-  try {
-    const u = new URL(url)
-    for (const host of NON_BOT_HOSTS) {
-      if (u.hostname.endsWith(host)) {
-        return NON_BOT_DESKTOP_USER_AGENT
-      }
-    }
-  } catch (e) {
-    console.log('error getting user agent for url', url, e)
-  }
-  return DESKTOP_USER_AGENT
-}

 const fetchContentWithScrapingBee = async (url: string) => {
  const response = await axios.get('https://app.scrapingbee.com/api/v1', {
@ -45,7 +25,7 @@ const fetchContentWithScrapingBee = async (url: string) => {
      premium_proxy: 'true',
      country_code: 'us',
    },
-    timeout: REQUEST_TIMEOUT,
+    timeout: 10_000,
  })

  const dom = parseHTML(response.data).document
@ -80,11 +60,11 @@ export const fetchContent = async (
  }
  console.log(`content-fetch request`, logRecord)

-  let context: BrowserContext | undefined,
-    page: Page | undefined,
+  let page: Page | undefined,
    title: string | undefined,
    content: string | undefined,
-    contentType: string | undefined
+    contentType: string | undefined,
+    context: BrowserContext | undefined

  try {
    url = getUrl(url)
@ -168,11 +148,11 @@ export const fetchContent = async (

    throw e
  } finally {
-    // close browser context if it was opened
+    // close browser context if it was created
    if (context) {
-      console.info('closing context...', url)
+      console.info('closing page...', url)
      await context.close()
-      console.info('context closed', url)
+      console.info('page closed', url)
    }

    console.info(`content-fetch result`, logRecord)
@ -241,20 +221,17 @@ async function retrievePage(
  }

  const browser = await getBrowser()
-  // create a new incognito browser context
  const context = await browser.createBrowserContext()

  // Puppeteer fails during download of PDf files,
  // so record the failure and use those items
  let lastPdfUrl
-  let page
  try {
-    page = await context.newPage()
+    const page = await context.newPage()

    if (!enableJavascriptForUrl(url)) {
      await page.setJavaScriptEnabled(false)
    }
-    await page.setUserAgent(userAgentForUrl(url))

    // set locale for the page
    if (locale) {
@ -359,7 +336,7 @@ async function retrievePage(

    const response = await page.goto(url, {
      timeout: 30 * 1000,
-      waitUntil: ['networkidle2'],
+      waitUntil: ['networkidle0'],
    })
    if (!response) {
      throw new Error('No response from page')
@ -371,12 +348,11 @@ async function retrievePage(
    logRecord.finalUrl = finalUrl
    logRecord.contentType = contentType

-    return { context, page, finalUrl, contentType }
+    return { page, finalUrl, contentType, context }
  } catch (error) {
    if (lastPdfUrl) {
      return {
        context,
-        page,
        finalUrl: lastPdfUrl,
        contentType: 'application/pdf',
      }
@ -392,6 +368,8 @@ async function retrieveHtml(page: Page, logRecord: Record<string, any>) {
    title = await page.title()
    logRecord.title = title

+    await page.waitForSelector('body')
+
    const pageScrollingStart = Date.now()
    /* scroll with a 5 seconds timeout */
    try {
--- a/yarn.lock
+++ b/yarn.lock
@ -6141,6 +6141,11 @@
  resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea"
  integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==

+"@sindresorhus/is@^4.2.0":
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f"
+  integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==
+
 "@sinonjs/commons@^1", "@sinonjs/commons@^1.3.0", "@sinonjs/commons@^1.4.0", "@sinonjs/commons@^1.6.0", "@sinonjs/commons@^1.7.0", "@sinonjs/commons@^1.8.3":
  version "1.8.3"
  resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.3.tgz#3802ddd21a50a949b6721ddd72da36e67e7f1b2d"
@ -9521,6 +9526,11 @@ addressparser@^1.0.1:
  resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746"
  integrity sha512-aQX7AISOMM7HFE0iZ3+YnD07oIeJqWGVnJ+ZIKaBZAk03ftmVYVqsGas/rbXKR21n4D/hKCSHypvcyOkds/xzg==

+adm-zip@^0.5.9:
+  version "0.5.14"
+  resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.14.tgz#2c557c0bf12af4311cf6d32970f4060cf8133b2a"
+  integrity sha512-DnyqqifT4Jrcvb8USYjp6FHtBpEIz1mnXu6pTRHZ0RL69LbQYiO+0lDFg5+OKA7U29oWSs3a/i8fhn8ZcceIWg==
+
 afinn-165-financialmarketnews@^3.0.0:
  version "3.0.0"
  resolved "https://registry.yarnpkg.com/afinn-165-financialmarketnews/-/afinn-165-financialmarketnews-3.0.0.tgz#cf422577775bf94f9bc156f3f001a1f29338c3d8"
@ -11374,6 +11384,16 @@ browserslist@^4.17.5:
    node-releases "^2.0.1"
    picocolors "^1.0.0"

+browserslist@^4.21.1:
+  version "4.23.2"
+  resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.2.tgz#244fe803641f1c19c28c48c4b6ec9736eb3d32ed"
+  integrity sha512-qkqSyistMYdxAcw+CzbZwlBy8AGmS/eEWs+sEV5TnLRGDOL+C5M2EnH6tlZyg0YoAxGJAFKh61En9BR941GnHA==
+  dependencies:
+    caniuse-lite "^1.0.30001640"
+    electron-to-chromium "^1.4.820"
+    node-releases "^2.0.14"
+    update-browserslist-db "^1.1.0"
+
 bser@2.1.1:
  version "2.1.1"
  resolved "https://registry.yarnpkg.com/bser/-/bser-2.1.1.tgz#e6787da20ece9d07998533cfd9de6f5c38f4bc05"
@ -11677,7 +11697,7 @@ call-me-maybe@^1.0.1:
  resolved "https://registry.yarnpkg.com/call-me-maybe/-/call-me-maybe-1.0.1.tgz#26d208ea89e37b5cbde60250a15f031c16a4d66b"
  integrity sha1-JtII6onje1y95gJQoV8DHBak1ms=

-callsites@^3.0.0:
+callsites@^3.0.0, callsites@^3.1.0:
  version "3.1.0"
  resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73"
  integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==
@ -11747,6 +11767,11 @@ caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001251, caniuse-lite@^1.0.300012
  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001600.tgz"
  integrity sha512-+2S9/2JFhYmYaDpZvo0lKkfvuKIglrx68MwOBqMGHhQsNkLjB5xtc/TGoEPs+MxjSyN/72qer2g97nzR641mOQ==

+caniuse-lite@^1.0.30001640:
+  version "1.0.30001641"
+  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001641.tgz#3572862cd18befae3f637f2a1101cc033c6782ac"
+  integrity sha512-Phv5thgl67bHYo1TtMY/MurjkHhV4EDaCosezRXgZ8jzA/Ub+wjxAvbGvjoFENStinwi5kCyOYV3mi5tOGykwA==
+
 capital-case@^1.0.4:
  version "1.0.4"
  resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669"
@ -12102,6 +12127,15 @@ chromium-bidi@0.5.19:
    urlpattern-polyfill "10.0.0"
    zod "3.22.4"

+chromium-bidi@0.5.24:
+  version "0.5.24"
+  resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-0.5.24.tgz#0fe672fa30b1e6bcc63ae818442b21c41849c435"
+  integrity sha512-5xQNN2SVBdZv4TxeMLaI+PelrnZsHDhn8h2JtyriLr+0qHcZS8BMuo93qN6J1VmtmrgYP+rmcLHcbpnA8QJh+w==
+  dependencies:
+    mitt "3.0.1"
+    urlpattern-polyfill "10.0.0"
+    zod "3.23.8"
+
 chunkd@^2.0.1:
  version "2.0.1"
  resolved "https://registry.yarnpkg.com/chunkd/-/chunkd-2.0.1.tgz#49cd1d7b06992dc4f7fccd962fe2a101ee7da920"
@ -13546,6 +13580,13 @@ debug@^3.0.0, debug@^3.1.0, debug@^3.1.1, debug@^3.2.7:
  dependencies:
    ms "^2.1.1"

+debug@^4.3.5:
+  version "4.3.5"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e"
+  integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==
+  dependencies:
+    ms "2.1.2"
+
 debuglog@^1.0.1:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
@ -13913,6 +13954,11 @@ devtools-protocol@0.0.1273771:
  resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1273771.tgz#46aeb5db41417e2c2ad3d8367c598c975290b1a5"
  integrity sha512-QDbb27xcTVReQQW/GHJsdQqGKwYBE7re7gxehj467kKP2DKuYBUj6i2k5LRiAC66J1yZG/9gsxooz/s9pcm0Og==

+devtools-protocol@0.0.1299070:
+  version "0.0.1299070"
+  resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1299070.tgz#b3e4cf0b678a46f0f907ae6e07e03ad3a53c00df"
+  integrity sha512-+qtL3eX50qsJ7c+qVyagqi7AWMoQCBGNfoyJZMwm/NSXVqLYbuitrWEEIzxfUmTNy7//Xe8yhMmQ+elj3uAqSg==
+
 dezalgo@1.0.3:
  version "1.0.3"
  resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.3.tgz#7f742de066fc748bc8db820569dddce49bf0d456"
@ -14221,7 +14267,7 @@ dot-prop@^5.1.0, dot-prop@^5.2.0:
  dependencies:
    is-obj "^2.0.0"

-dot-prop@^6.0.0:
+dot-prop@^6.0.0, dot-prop@^6.0.1:
  version "6.0.1"
  resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-6.0.1.tgz#fc26b3cf142b9e59b74dbd39ed66ce620c681083"
  integrity sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==
@ -14380,6 +14426,11 @@ electron-to-chromium@^1.4.17:
  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.68.tgz#d79447b6bd1bec9183f166bb33d4bef0d5e4e568"
  integrity sha512-cId+QwWrV8R1UawO6b9BR1hnkJ4EJPCPAr4h315vliHUtVUJDk39Sg1PMNnaWKfj5x+93ssjeJ9LKL6r8LaMiA==

+electron-to-chromium@^1.4.820:
+  version "1.4.823"
+  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.823.tgz#38587f7aa55bed14930f04091dfc65c39a3d8bd7"
+  integrity sha512-4h+oPeAiGQOHFyUJOqpoEcPj/xxlicxBzOErVeYVMMmAiXUXsGpsFd0QXBMaUUbnD8hhSfLf9uw+MlsoIA7j5w==
+
 electron-to-chromium@^1.4.84:
  version "1.4.89"
  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.89.tgz#33c06592812a17a7131873f4596579084ce33ff8"
@ -14854,6 +14905,11 @@ escalade@^3.1.1:
  resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
  integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==

+escalade@^3.1.2:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27"
+  integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==
+
 escape-goat@^2.0.0:
  version "2.1.1"
  resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675"
@ -16111,6 +16167,15 @@ fined@^1.0.1:
    object.pick "^1.2.0"
    parse-filepath "^1.0.1"

+fingerprint-generator@^2.1.52:
+  version "2.1.52"
+  resolved "https://registry.yarnpkg.com/fingerprint-generator/-/fingerprint-generator-2.1.52.tgz#af40cb4f5b33a0a0173aeaa952f2b3b00bfce638"
+  integrity sha512-ZdXUn/qIB4vI7pDze5aXidjoFwLdEfbBNj6+3oHzXcgwxzEfCOfNe3wW5NRZDJKgxF40R7TSOA7noBAAehSLgQ==
+  dependencies:
+    generative-bayesian-network "^2.1.52"
+    header-generator "^2.1.52"
+    tslib "^2.4.0"
+
 firebase-admin@^11.5.0:
  version "11.11.0"
  resolved "https://registry.yarnpkg.com/firebase-admin/-/firebase-admin-11.11.0.tgz#3d6df5dfbcf85dc1c6c4302f8aee4f7c82171725"
@ -16638,6 +16703,14 @@ gcp-metadata@^6.1.0:
    gaxios "^6.0.0"
    json-bigint "^1.0.0"

+generative-bayesian-network@^2.1.52:
+  version "2.1.52"
+  resolved "https://registry.yarnpkg.com/generative-bayesian-network/-/generative-bayesian-network-2.1.52.tgz#0d8aa6dd14558bf88fb999feeff8c86d9e60322a"
+  integrity sha512-8fYemN+uiVPCjoodQX4HUH8RLDqiQeGfemlWO9yR6SqIh/6BsrW52M0YTSafsH0615BhulRy5BR2uKAqLTJ22A==
+  dependencies:
+    adm-zip "^0.5.9"
+    tslib "^2.4.0"
+
 generic-pool@3.9.0:
  version "3.9.0"
  resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4"
@ -17673,6 +17746,16 @@ header-case@^2.0.4:
    capital-case "^1.0.4"
    tslib "^2.0.3"

+header-generator@^2.1.52:
+  version "2.1.52"
+  resolved "https://registry.yarnpkg.com/header-generator/-/header-generator-2.1.52.tgz#1560fc3a2f2f65ed1d777a1660fb0c4459bf4827"
+  integrity sha512-2roqbZdd0hc7Bx+6BIQaHaCaSdnTXCnqayFbS8dpj53hmkQAXbSwiuTpfyAY1vePiaKweH6vDYhbtGOW+NmTmw==
+  dependencies:
+    browserslist "^4.21.1"
+    generative-bayesian-network "^2.1.52"
+    ow "^0.28.1"
+    tslib "^2.4.0"
+
 heap-js@^2.2.0:
  version "2.2.0"
  resolved "https://registry.yarnpkg.com/heap-js/-/heap-js-2.2.0.tgz#f4418874cd2aedc2cf3a7492d579afe23b627c5d"
@ -23446,6 +23529,11 @@ node-releases@^2.0.1, node-releases@^2.0.2:
  resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.2.tgz#7139fe71e2f4f11b47d4d2986aaf8c48699e0c01"
  integrity sha512-XxYDdcQ6eKqp/YjI+tb2C5WM2LgjnZrfYg4vgQt49EK268b6gYCHsBLrK2qvJo4FmCtqmKezb0WZFK4fkrZNsg==

+node-releases@^2.0.14:
+  version "2.0.14"
+  resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b"
+  integrity sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==
+
 nodemailer@^6.7.3:
  version "6.7.3"
  resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.3.tgz#b73f9a81b9c8fa8acb4ea14b608f5e725ea8e018"
@ -24427,6 +24515,17 @@ overlayscrollbars@^1.13.1:
  resolved "https://registry.yarnpkg.com/overlayscrollbars/-/overlayscrollbars-1.13.1.tgz#0b840a88737f43a946b9d87875a2f9e421d0338a"
  integrity sha512-gIQfzgGgu1wy80EB4/6DaJGHMEGmizq27xHIESrzXq0Y/J0Ay1P3DWk6tuVmEPIZH15zaBlxeEJOqdJKmowHCQ==

+ow@^0.28.1:
+  version "0.28.2"
+  resolved "https://registry.yarnpkg.com/ow/-/ow-0.28.2.tgz#782b28102124e665c49ec7725e2066a129acf6bf"
+  integrity sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==
+  dependencies:
+    "@sindresorhus/is" "^4.2.0"
+    callsites "^3.1.0"
+    dot-prop "^6.0.1"
+    lodash.isequal "^4.5.0"
+    vali-date "^1.0.0"
+
 p-all@^2.1.0:
  version "2.1.0"
  resolved "https://registry.yarnpkg.com/p-all/-/p-all-2.1.0.tgz#91419be56b7dee8fe4c5db875d55e0da084244a0"
@ -25212,6 +25311,11 @@ picocolors@^1.0.0:
  resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c"
  integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==

+picocolors@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1"
+  integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==
+
 picomatch@^2.0.4, picomatch@^2.2.1:
  version "2.2.2"
  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad"
@ -26089,6 +26193,17 @@ pupa@^2.1.1:
  dependencies:
    escape-goat "^2.0.0"

+puppeteer-core@^22.12.1:
+  version "22.12.1"
+  resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-22.12.1.tgz#4dacc2e9ab127ef534a4bb4793d33f9424a5b48d"
+  integrity sha512-XmqeDPVdC5/3nGJys1jbgeoZ02wP0WV1GBlPtr/ULRbGXJFuqgXMcKQ3eeNtFpBzGRbpeoCGWHge1ZWKWl0Exw==
+  dependencies:
+    "@puppeteer/browsers" "2.2.3"
+    chromium-bidi "0.5.24"
+    debug "^4.3.5"
+    devtools-protocol "0.0.1299070"
+    ws "^8.17.1"
+
 puppeteer-core@^22.8.0:
  version "22.8.0"
  resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-22.8.0.tgz#82c0e7ebf62ba5f34404394034e313b82014de5f"
@ -26110,7 +26225,17 @@ puppeteer-extra-plugin-adblocker@^2.13.5:
    node-fetch "^2.6.0"
    puppeteer-extra-plugin "^3.2.2"

-puppeteer-extra-plugin-stealth@^2.11.1:
+puppeteer-extra-plugin-adblocker@^2.13.6:
+  version "2.13.6"
+  resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.13.6.tgz#99828243579b59ed81e8b1da23d16a3a0dbae557"
+  integrity sha512-AftgnUZ1rg2RPe9RpX6rkYAxEohwp3iFeGIyjsAuTaIiw4VLZqOb1LSY8/S60vAxpeat60fbCajxoUetmLy4Dw==
+  dependencies:
+    "@cliqz/adblocker-puppeteer" "1.23.8"
+    debug "^4.1.1"
+    node-fetch "^2.6.0"
+    puppeteer-extra-plugin "^3.2.3"
+
+puppeteer-extra-plugin-stealth@^2.11.1, puppeteer-extra-plugin-stealth@^2.11.2:
  version "2.11.2"
  resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz#bd3f5a1781cac8a98c983d148086585a84fcc8f1"
  integrity sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ==
@ -26157,6 +26282,15 @@ puppeteer-extra@^3.3.4:
    debug "^4.1.1"
    deepmerge "^4.2.2"

+puppeteer-extra@^3.3.6:
+  version "3.3.6"
+  resolved "https://registry.yarnpkg.com/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz#fc16ff396aae52664842da9a557ea8fa51eaa8b7"
+  integrity sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A==
+  dependencies:
+    "@types/debug" "^4.1.0"
+    debug "^4.1.1"
+    deepmerge "^4.2.2"
+
 q@^1.5.1:
  version "1.5.1"
  resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
@ -31012,6 +31146,14 @@ upath@^1.1.1:
  resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894"
  integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==

+update-browserslist-db@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz#7ca61c0d8650766090728046e416a8cde682859e"
+  integrity sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ==
+  dependencies:
+    escalade "^3.1.2"
+    picocolors "^1.0.1"
+
 update-notifier@^5.1.0:
  version "5.1.0"
  resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9"
@ -31292,6 +31434,11 @@ v8flags@^2.0.10:
  dependencies:
    user-home "^1.1.1"

+vali-date@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/vali-date/-/vali-date-1.0.0.tgz#1b904a59609fb328ef078138420934f6b86709a6"
+  integrity sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg==
+
 valid-url@^1.0.9:
  version "1.0.9"
  resolved "https://registry.yarnpkg.com/valid-url/-/valid-url-1.0.9.tgz#1c14479b40f1397a75782f115e4086447433a200"
@ -32144,6 +32291,11 @@ ws@^7.5.6:
  resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591"
  integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==

+ws@^8.17.1:
+  version "8.18.0"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc"
+  integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==
+
 ws@^8.2.3, ws@^8.3.0, ws@^8.4.2:
  version "8.8.1"
  resolved "https://registry.yarnpkg.com/ws/-/ws-8.8.1.tgz#5dbad0feb7ade8ecc99b830c1d77c913d4955ff0"
@ -32414,6 +32566,11 @@ zod@3.22.4, zod@^3.22.3, zod@^3.22.4:
  resolved "https://registry.yarnpkg.com/zod/-/zod-3.22.4.tgz#f31c3a9386f61b1f228af56faa9255e845cf3fff"
  integrity sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==

+zod@3.23.8:
+  version "3.23.8"
+  resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d"
+  integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==
+
 zwitch@^1.0.0:
  version "1.0.5"
  resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-1.0.5.tgz#d11d7381ffed16b742f6af7b3f223d5cd9fe9920"