diff --git a/packages/content-fetch/app.js b/packages/content-fetch/app.js index 252671626..76169cd0c 100644 --- a/packages/content-fetch/app.js +++ b/packages/content-fetch/app.js @@ -14,7 +14,7 @@ if (!process.env.VERIFICATION_TOKEN) { app.get('/', async (req, res) => { if (req.query.token !== process.env.VERIFICATION_TOKEN) { console.log('query does not include valid token') - res.send(403) + res.sendStatus(403) return } await fetchContent(req, res) @@ -23,7 +23,7 @@ app.get('/', async (req, res) => { app.post('/', async (req, res) => { if (req.query.token !== process.env.VERIFICATION_TOKEN) { console.log('query does not include valid token') - res.send(403) + res.sendStatus(403) return } await fetchContent(req, res) diff --git a/packages/content-fetch/logger.js b/packages/content-fetch/logger.js new file mode 100644 index 000000000..9317e36c8 --- /dev/null +++ b/packages/content-fetch/logger.js @@ -0,0 +1,66 @@ +const { config, format, loggers, transports } = require('winston'); +const { LoggingWinston } = require('@google-cloud/logging-winston'); +const { DateTime } = require('luxon'); + +const colors = { + emerg: 'inverse underline magenta', + alert: 'underline magenta', + crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. + error: 'underline red', // Any error which is fatal to the operation, but not the service or application + warning: 'underline yellow', // Anything that can potentially cause application oddities + notice: 'underline cyan', // Normal but significant condition + info: 'underline green', // Generally useful information to log + debug: 'underline gray', +}; + +const googleConfigs = { + level: 'info', + logName: 'logger', + levels: config.syslog.levels, + resource: { + labels: { + function_name: process.env.FUNCTION_TARGET, + project_id: process.env.GCP_PROJECT, + }, + type: 'cloud_function', + }, +}; + +function localConfig(id) { + return { + level: 'debug', + format: format.combine( + format.colorize({ all: true, colors }), + format(info => + Object.assign(info, { + timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), + }), + )(), + format.printf(info => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { timestamp, message, level, ...meta } = info; + + return `[${id}@${info.timestamp}] ${info.message}${ + Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' + }`; + }), + ), + }; +} + +function buildLoggerTransport(id, options) { + return process.env.IS_LOCAL + ? new transports.Console(localConfig(id)) + : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); +} + +function buildLogger(id, options) { + return loggers.get(id, { + levels: config.syslog.levels, + transports: [buildLoggerTransport(id, options)], + }); +} + +module.exports = { + buildLogger, +} diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json index bffa43a8e..546552258 100644 --- a/packages/content-fetch/package.json +++ b/packages/content-fetch/package.json @@ -8,7 +8,8 @@ "express": "^4.17.1", "@google-cloud/functions-framework": "^3.0.0", "@omnivore/puppeteer-parse": "^1.0.0", - "@sentry/serverless": "^6.13.3" + "@sentry/serverless": "^6.13.3", + "winston": "^3.3.3" }, "devDependencies": { "chai": "^4.3.6", diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts index fd1fe5291..a05e2ac1a 100644 --- a/packages/content-handler/src/content-handler.ts +++ b/packages/content-handler/src/content-handler.ts @@ -3,6 +3,7 @@ import rfc2047 from 'rfc2047' import { v4 as uuid } from 'uuid' import { parseHTML } from 'linkedom' import axios from 'axios' +import { Browser } from 'puppeteer-core' interface Unsubscribe { mailTo?: string @@ -62,7 +63,7 @@ export abstract class ContentHandler { return false } - async preHandle(url: string): Promise { + async preHandle(url: string, browser?: Browser): Promise { return Promise.resolve({ url }) } diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index ce61c0a64..e210d025d 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -28,6 +28,7 @@ import { GhostHandler } from './newsletters/ghost-handler' import { parseHTML } from 'linkedom' import { CooperPressHandler } from './newsletters/cooper-press-handler' import { HeyWorldHandler } from './newsletters/hey-world-handler' +import { Browser } from 'puppeteer-core' const validateUrlString = (url: string) => { const u = new URL(url) @@ -80,7 +81,8 @@ const newsletterHandlers: ContentHandler[] = [ ] export const preHandleContent = async ( - url: string + url: string, + browser: Browser ): Promise => { // Before we run the regular handlers we check to see if we need tp // pre-resolve the URL. TODO: This should probably happen recursively, @@ -104,7 +106,7 @@ export const preHandleContent = async ( for (const handler of contentHandlers) { if (handler.shouldPreHandle(url)) { console.log('preHandleContent', handler.name, url) - return handler.preHandle(url) + return handler.preHandle(url, browser) } } return undefined diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 17bbdc4c2..afb8985ef 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -2,7 +2,7 @@ import { ContentHandler, PreHandleResult } from '../content-handler' import axios from 'axios' import { DateTime } from 'luxon' import _ from 'underscore' -import puppeteer from 'puppeteer-core' +import { Browser, BrowserContext } from 'puppeteer-core' interface TweetIncludes { users: { @@ -168,10 +168,11 @@ const getTweetsFromResponse = (response: Tweets): Tweet[] => { } const getOldTweets = async ( + browser: Browser, conversationId: string, username: string ): Promise => { - const tweetIds = await getTweetIds(conversationId, username) + const tweetIds = await getTweetIds(browser, conversationId, username) if (tweetIds.length === 0) { return [] } @@ -197,37 +198,21 @@ const waitFor = (ms: number) => /** * Get tweets(even older than 7 days) using puppeteer + * @param browser * @param {string} tweetId * @param {string} author */ const getTweetIds = async ( + browser: Browser, tweetId: string, author: string ): Promise => { const pageURL = `https://twitter.com/${author}/status/${tweetId}` - // Modify this variable to control the size of viewport - const factor = 0.2 - const height = Math.floor(2000 / factor) - const width = Math.floor(1700 / factor) - - const browser = await puppeteer.launch({ - executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, - defaultViewport: { - width, - height, - }, - args: [ - `--force-device-scale-factor=${factor}`, - `--window-size=${width},${height}`, - '--no-sandbox', - '--disable-setuid-sandbox', - ], - }) - + let context: BrowserContext | undefined try { - const page = await browser.newPage() + context = await browser.createIncognitoBrowserContext() + const page = await context.newPage() await page.goto(pageURL, { waitUntil: 'networkidle2', @@ -291,7 +276,9 @@ const getTweetIds = async ( console.log(error) return [] } finally { - await browser.close() + if (context) { + await context.close() + } } } @@ -305,7 +292,7 @@ export class TwitterHandler extends ContentHandler { return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) } - async preHandle(url: string): Promise { + async preHandle(url: string, browser: Browser): Promise { const tweetId = tweetIdFromStatusUrl(url) if (!tweetId) { throw new Error('could not find tweet id in url') @@ -326,7 +313,7 @@ export class TwitterHandler extends ContentHandler { const description = _.escape(tweetData.text) // use puppeteer to get all tweet replies in the thread - const tweets = await getOldTweets(conversationId, author.username) + const tweets = await getOldTweets(browser, conversationId, author.username) let tweetsContent = '' for (const tweet of tweets) { diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index cd72936af..efce2c6e0 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -9,18 +9,20 @@ const axios = require('axios'); const jwt = require('jsonwebtoken'); const { promisify } = require('util'); const signToken = promisify(jwt.sign); -const { config, format, loggers, transports } = require('winston'); -const { LoggingWinston } = require('@google-cloud/logging-winston'); -const { DateTime } = require('luxon'); const os = require('os'); const { Storage } = require('@google-cloud/storage'); const { parseHTML } = require('linkedom'); -const puppeteer = require('puppeteer-core'); const { preHandleContent } = require("@omnivore/content-handler"); +const puppeteer = require('puppeteer-extra'); + // Add stealth plugin to hide puppeteer usage -// const StealthPlugin = require('puppeteer-extra-plugin-stealth'); -// puppeteer.use(StealthPlugin()); +const StealthPlugin = require('puppeteer-extra-plugin-stealth'); +puppeteer.use(StealthPlugin()); + +// Add adblocker plugin to block all ads and trackers (saves bandwidth) +const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); +puppeteer.use(AdblockerPlugin({ blockTrackers: true })); const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; @@ -28,65 +30,6 @@ const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process. const filePath = `${os.tmpdir()}/previewImage.png`; -const colors = { - emerg: 'inverse underline magenta', - alert: 'underline magenta', - crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. - error: 'underline red', // Any error which is fatal to the operation, but not the service or application - warning: 'underline yellow', // Anything that can potentially cause application oddities - notice: 'underline cyan', // Normal but significant condition - info: 'underline green', // Generally useful information to log - debug: 'underline gray', -}; - -const googleConfigs = { - level: 'info', - logName: 'logger', - levels: config.syslog.levels, - resource: { - labels: { - function_name: process.env.FUNCTION_TARGET, - project_id: process.env.GCP_PROJECT, - }, - type: 'cloud_function', - }, -}; - -function localConfig(id) { - return { - level: 'debug', - format: format.combine( - format.colorize({ all: true, colors }), - format(info => - Object.assign(info, { - timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), - }), - )(), - format.printf(info => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { timestamp, message, level, ...meta } = info; - - return `[${id}@${info.timestamp}] ${info.message}${ - Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' - }`; - }), - ), - }; -} - -function buildLoggerTransport(id, options) { - return process.env.IS_LOCAL - ? new transports.Console(localConfig(id)) - : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); -} - -function buildLogger(id, options) { - return loggers.get(id, { - levels: config.syslog.levels, - transports: [buildLoggerTransport(id, options)], - }); -} - const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' @@ -174,8 +117,6 @@ const getBrowserPromise = (async () => { }); })(); -let logRecord, functionStartTime; - const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => { const stream = await axios.get(contentObjUrl, { responseType: 'stream' }); return await axios.put(uploadSignedUrl, stream.data, { @@ -268,22 +209,13 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId }; async function fetchContent(req, res) { - functionStartTime = Date.now(); - // Grabbing execution and trace ids to attach logs to the appropriate function call - const execution_id = req.get('function-execution-id'); - const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; - const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { - trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, - labels: { - execution_id: execution_id, - }, - }); + let functionStartTime = Date.now(); let url = getUrl(req); const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); - logRecord = { + let logRecord = { url, userId, articleSavingRequestId, @@ -292,18 +224,19 @@ async function fetchContent(req, res) { }, }; - logger.info(`Article parsing request`, logRecord); + console.info(`Article parsing request`, logRecord); if (!url) { logRecord.urlIsInvalid = true; - logger.info(`Valid URL to parse not specified`, logRecord); + console.info(`Valid URL to parse not specified`, logRecord); return res.sendStatus(400); } // pre handle url with custom handlers let title, content, contentType; try { - const result = await preHandleContent(url); + const browser = await getBrowserPromise; + const result = await preHandleContent(url, browser); if (result && result.url) { url = result.url validateUrlString(url); @@ -312,13 +245,13 @@ async function fetchContent(req, res) { if (result && result.content) { content = result.content } if (result && result.contentType) { contentType = result.contentType } } catch (e) { - logger.info('error with handler: ', e); + console.info('error with handler: ', e); } let context, page, finalUrl; try { if ((!content || !title) && contentType !== 'application/pdf') { - const result = await retrievePage(url) + const result = await retrievePage(url, logRecord, functionStartTime); if (result && result.context) { context = result.context } if (result && result.page) { page = result.page } if (result && result.finalUrl) { finalUrl = result.finalUrl } @@ -332,7 +265,7 @@ async function fetchContent(req, res) { const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId); } else { if (!content || !title) { - const result = await retrieveHtml(page); + const result = await retrieveHtml(page, logRecord); if (result.isBlocked) { const sbResult = await fetchContentWithScrapingBee(url) title = sbResult.title @@ -342,7 +275,7 @@ async function fetchContent(req, res) { content = result.domContent; } } else { - logger.info('using prefetched content and title'); + console.info('using prefetched content and title'); } logRecord.fetchContentTime = Date.now() - functionStartTime; @@ -365,7 +298,7 @@ async function fetchContent(req, res) { } } catch (e) { logRecord.error = e.message; - logger.error(`Error while retrieving page`, logRecord); + console.error(`Error while retrieving page`, logRecord); // fallback to scrapingbee const sbResult = await fetchContentWithScrapingBee(url); @@ -392,7 +325,7 @@ async function fetchContent(req, res) { if (context) { await context.close(); } - logger.info(`parse-page`, logRecord); + console.info(`parse-page`, logRecord); } return res.sendStatus(200); @@ -452,7 +385,7 @@ async function blockResources(client) { await client.send('Network.setBlockedURLs', { urls: blockedResources }); } -async function retrievePage(url) { +async function retrievePage(url, logRecord, functionStartTime) { validateUrlString(url); const browser = await getBrowserPromise; @@ -549,7 +482,7 @@ async function retrievePage(url) { }); try { - const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] }); + const response = await page.goto(url, { timeout: 30 * 1000, waitUntil: ['networkidle2'] }); const finalUrl = response.url(); const contentType = response.headers()['content-type']; @@ -566,7 +499,7 @@ async function retrievePage(url) { } } -async function retrieveHtml(page) { +async function retrieveHtml(page, logRecord) { let domContent = '', title; try { title = await page.title(); @@ -710,7 +643,7 @@ async function preview(req, res) { // Grabbing execution and trace ids to attach logs to the appropriate function call const execution_id = req.get('function-execution-id'); const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; - const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { + const console = buildconsole('cloudfunctions.googleapis.com%2Fcloud-functions', { trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, labels: { execution_id: execution_id, @@ -718,7 +651,7 @@ async function preview(req, res) { }); if (!process.env.PREVIEW_IMAGE_BUCKET) { - logger.error(`PREVIEW_IMAGE_BUCKET not set`) + console.error(`PREVIEW_IMAGE_BUCKET not set`) return res.sendStatus(500); } @@ -734,30 +667,30 @@ async function preview(req, res) { }, }; - logger.info(`Public preview image generation request`, logRecord); + console.info(`Public preview image generation request`, logRecord); if (!url) { logRecord.urlIsInvalid = true; - logger.error(`Valid URL to parse is not specified`, logRecord); + console.error(`Valid URL to parse is not specified`, logRecord); return res.sendStatus(400); } const { origin } = new URL(url); if (!ALLOWED_ORIGINS.some(o => o === origin)) { logRecord.forbiddenOrigin = true; - logger.error(`This origin is not allowed: ${origin}`, logRecord); + console.error(`This origin is not allowed: ${origin}`, logRecord); return res.sendStatus(400); } - const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH); + const browser = await getBrowserPromise; logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; const page = await browser.newPage(); const pageLoadingStart = Date.now(); const modifiedUrl = new URL(url); - modifiedUrl.searchParams.append('fontSize', 24); - modifiedUrl.searchParams.append('adjustAspectRatio', 1.91); + modifiedUrl.searchParams.append('fontSize', '24'); + modifiedUrl.searchParams.append('adjustAspectRatio', '1.91'); try { - await page.goto(modifiedUrl); + await page.goto(modifiedUrl.toString()); logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart }; } catch (error) { console.log('error going to page: ', modifiedUrl) @@ -775,7 +708,7 @@ async function preview(req, res) { ); if (!selector) { logRecord.selectorIsInvalid = true; - logger.error(`Valid element selector is not specified`, logRecord); + console.error(`Valid element selector is not specified`, logRecord); await page.close(); return res.sendStatus(400); } @@ -790,7 +723,7 @@ async function preview(req, res) { ); if (!destination) { logRecord.destinationIsInvalid = true; - logger.error(`Valid file destination is not specified`, logRecord); + console.error(`Valid file destination is not specified`, logRecord); await page.close(); return res.sendStatus(400); } @@ -801,7 +734,7 @@ async function preview(req, res) { await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load } catch (error) { logRecord.elementNotFound = true; - logger.error(`Element is not presented on the page`, logRecord); + console.error(`Element is not presented on the page`, logRecord); await page.close(); return res.sendStatus(400); } @@ -821,7 +754,7 @@ async function preview(req, res) { console.log('error uploading to bucket, this is non-fatal', e) } - logger.info(`preview-image`, logRecord); + console.info(`preview-image`, logRecord); return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); } diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index a9537f6f2..42b3a10a4 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -4,16 +4,16 @@ "description": "Accepts URL of the article and parses its content", "main": "index.js", "dependencies": { - "@google-cloud/logging-winston": "^5.1.1", "@google-cloud/storage": "^5.18.1", "@omnivore/content-handler": "1.0.0", "axios": "^0.27.2", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.9", - "luxon": "^2.3.1", "puppeteer-core": "^16.1.0", - "underscore": "^1.13.4", - "winston": "^3.3.3" + "puppeteer-extra": "^3.3.4", + "puppeteer-extra-plugin-adblocker": "^2.13.5", + "puppeteer-extra-plugin-stealth": "^2.11.1", + "underscore": "^1.13.4" }, "devDependencies": { "chai": "^4.3.6", diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index f28f688fe..50bd5d85d 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -171,20 +171,20 @@ Readability.prototype = { // Readability-readerable.js. Please keep both copies in sync. articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist/i, articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert|block-ad-.*/i, - unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon/i, + unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped/i, // okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i, get okMaybeItsACandidate() { return new RegExp(`and|(?]*>/gi, normalize: /\s{2,}/g, - videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i, + videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq|cdnapisec\.kaltura)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i, shareElements: /(\b|_)(share|sharedaddy|post-tags)(\b|_)/i, nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i, prevLink: /(prev|earl|old|new|<|«)/i, diff --git a/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json b/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json new file mode 100644 index 000000000..5e5525d51 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json @@ -0,0 +1,12 @@ +{ + "title": "Parallelizing the Naughty Dog Engine Using Fibers", + "byline": null, + "dir": null, + "excerpt": "This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall...", + "siteName": null, + "siteIcon": "http://fakehost/img/favicon.ico", + "previewImage": "https://ubm-twvideo01.s3.amazonaws.com/o1/vault/gdc2015/Images/GDC15_Vault-thumb_v1.png", + "publishedDate": null, + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/gdcvault/expected.html b/packages/readabilityjs/test/test-pages/gdcvault/expected.html new file mode 100644 index 000000000..7c253eba9 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/expected.html @@ -0,0 +1,54 @@ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ Session Name: + Parallelizing the Naughty Dog Engine Using Fibers
+ Speaker(s): + Christian Gyrling
+ Company Name(s): + Naughty Dog
+ Track / Format: + Programming
+ Overview: + This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall frame-centric engine design, the memory allocation patterns used in the title, and our strategies for dealing with locks.
+
+ + +
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/gdcvault/source.html b/packages/readabilityjs/test/test-pages/gdcvault/source.html new file mode 100644 index 000000000..1b342e352 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/source.html @@ -0,0 +1,1142 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + GDC Vault - Parallelizing the Naughty Dog Engine Using Fibers + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+
+ You've been logged out of GDC Vault since the maximum users allowed for this account has been reached. To access Members Only content on GDC Vault, please log out of GDC Vault from the computer which last accessed this account.
+
+ Click here to find out about GDC Vault Membership options for more users.
+
+ close +
+
+
+
+ + + +
+
+ + + +
+
+ +
+ + +
+      +
+ + +
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ Session Name: + + Parallelizing the Naughty Dog Engine Using Fibers +
+ Speaker(s): + + Christian Gyrling +
+ Company Name(s): + + Naughty Dog +
+ Track / Format: + + Programming +
+ Overview: + + This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall frame-centric engine design, the memory allocation patterns used in the title, and our strategies for dealing with locks. +
+
+

+ GDC 2015 +

+

+ Christian Gyrling +

+

+ Naughty Dog +

+

+ free content +

+

+ Programming +

+

+ Programming +

+
+
+
+ +

+ + +
+
+
+
+
+
+
+
+ Discover More from Informa Tech +
+
+
+
+
+ +
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ Working With Us +
+ +
+
+
+
+
+
+
+ Follow GDC on Social +
+
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+ +
+
+
+
+
+
+
+
    +
  • Copyright © 2022 Informa PLC. Informa PLC is registered in England and Wales with company number 8860726 whose registered and head office is 5 Howick Place, London, SW1P 1WG. +
    • +
+
+
+
+
+ + + + + + + diff --git a/packages/readabilityjs/test/test-pages/gdcvault/url.txt b/packages/readabilityjs/test/test-pages/gdcvault/url.txt new file mode 100644 index 000000000..64f62a296 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/url.txt @@ -0,0 +1 @@ +https://www.gdcvault.com/play/1022186/Parallelizing-the-Naughty-Dog-Engine \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index cbdd96048..7f2ee67fa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2069,6 +2069,41 @@ commander "^4.1.0" microtime "^3.0.0" +"@cliqz/adblocker-content@^1.23.8", "@cliqz/adblocker-content@^1.25.1": + version "1.25.1" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker-content/-/adblocker-content-1.25.1.tgz#da81e7838e288a6f0fdb8a97a0df8169accb74a1" + integrity sha512-7gl2VdNPBfj7aPoq34B5miwGcnda/7LCr+BqnpcSOjdLV6jjT2FrNSAKGFvcH23q0HM1IFhYDV6ydTgsdWFCnA== + dependencies: + "@cliqz/adblocker-extended-selectors" "^1.25.1" + +"@cliqz/adblocker-extended-selectors@^1.25.1": + version "1.25.1" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker-extended-selectors/-/adblocker-extended-selectors-1.25.1.tgz#cfac0080952311399805fe153cd9e7e1331b3c6d" + integrity sha512-4MdMe/YfIok5d8WYVcLR3Ak7vGrmeUV47frgmXEe945luY93vwlzk1NiLYW1JM5Gdm+VePweoS9cJ1/QUTmv+Q== + +"@cliqz/adblocker-puppeteer@1.23.8": + version "1.23.8" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker-puppeteer/-/adblocker-puppeteer-1.23.8.tgz#e74636cd200459d1734929e41504a76939504311" + integrity sha512-Ca1/DBqQXsOpKTFVAHX6OpLTSEupXmUkUWHj6iXhLLleC7RPISN5B0b801VDmaGRqoC5zKRxn0vYbIfpgCWVug== + dependencies: + "@cliqz/adblocker" "^1.23.8" + "@cliqz/adblocker-content" "^1.23.8" + tldts-experimental "^5.6.21" + +"@cliqz/adblocker@^1.23.8": + version "1.25.1" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker/-/adblocker-1.25.1.tgz#4d3e8894ce48ad0d0f8b26a4a1003f0676b7f734" + integrity sha512-1C1/ELI94/XewdUj/o1+Q4ziOigMvTZQA05UERfDoKqpJ+0cbrEF/UImrzpX7n+kYsR7xTJvmf+iNM3zS0tfsg== + dependencies: + "@cliqz/adblocker-content" "^1.25.1" + "@cliqz/adblocker-extended-selectors" "^1.25.1" + "@remusao/guess-url-type" "^1.1.2" + "@remusao/small" "^1.1.2" + "@remusao/smaz" "^1.7.1" + "@types/chrome" "^0.0.197" + "@types/firefox-webext-browser" "^94.0.0" + tldts-experimental "^5.6.21" + "@cnakazawa/watch@^1.0.3": version "1.0.4" resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a" @@ -5110,6 +5145,41 @@ resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4" integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA== +"@remusao/guess-url-type@^1.1.2": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57" + integrity sha512-rbOqre2jW8STjheOsOaQHLgYBaBZ9Owbdt8NO7WvNZftJlaG3y/K9oOkl8ZUpuFBisIhmBuMEW6c+YrQl5inRA== + +"@remusao/small@^1.1.2": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@remusao/small/-/small-1.2.1.tgz#63bfe4548832289f94ac868a0c305970c9a0e5f9" + integrity sha512-7MjoGt0TJMVw1GPKgWq6SJPws1SLsUXQRa43Umht+nkyw2jnpy3WpiLNqGdwo5rHr5Wp9B2W/Pm5RQp656UJdw== + +"@remusao/smaz-compress@^1.9.1": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@remusao/smaz-compress/-/smaz-compress-1.9.1.tgz#fc75eaf9bcac2d58bc4c3d518183a7cb9612d275" + integrity sha512-E2f48TwloQu3r6BdLOGF2aczeH7bJ/32oJGqvzT9SKur0cuUnLcZ7ZXP874E2fwmdE+cXzfC7bKzp79cDnmeyw== + dependencies: + "@remusao/trie" "^1.4.1" + +"@remusao/smaz-decompress@^1.9.1": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@remusao/smaz-decompress/-/smaz-decompress-1.9.1.tgz#8094f997e8fb591a678cda9cf08c209c825eba5b" + integrity sha512-TfjKKprYe3n47od8auhvJ/Ikj9kQTbDTe71ynKlxslrvvUhlIV3VQSuwYuMWMbdz1fIs0H/fxCN1Z8/H3km6/A== + +"@remusao/smaz@^1.7.1": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@remusao/smaz/-/smaz-1.9.1.tgz#a2b9b045385f81e1615a68d932b7cc8b04c9db8d" + integrity sha512-e6BLuP8oaXCZ9+v46Is4ilAZ/Vq6YLgmBP204Ixgk1qTjXmqvFYG7+AS7v9nsZdGOy96r9DWGFbbDVgMxwu1rA== + dependencies: + "@remusao/smaz-compress" "^1.9.1" + "@remusao/smaz-decompress" "^1.9.1" + +"@remusao/trie@^1.4.1": + version "1.4.1" + resolved "https://registry.yarnpkg.com/@remusao/trie/-/trie-1.4.1.tgz#755d09f8a007476334e611f42719b2d581f00720" + integrity sha512-yvwa+aCyYI/UjeD39BnpMypG8N06l86wIDW1/PAc6ihBRnodIfZDwccxQN3n1t74wduzaz74m4ZMHZnB06567Q== + "@rushstack/eslint-patch@^1.0.8": version "1.1.0" resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323" @@ -7611,6 +7681,14 @@ resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.2.21.tgz#9f35a5643129df132cf3b5c1ec64046ea1af0650" integrity sha512-yd+9qKmJxm496BOV9CMNaey8TWsikaZOwMRwPHQIjcOJM9oV+fi9ZMNw3JsVnbEEbo2gRTDnGEBv8pjyn67hNg== +"@types/chrome@^0.0.197": + version "0.0.197" + resolved "https://registry.yarnpkg.com/@types/chrome/-/chrome-0.0.197.tgz#c1b50cdb72ee40f9bc1411506031a9f8a925ab35" + integrity sha512-m1NfS5bOjaypyqQfaX6CxmJodZVcvj5+Mt/K94EBHkflYjPNmXHAzbxfifdLMa0YM3PDyOxohoTS5ug/e6p5jA== + dependencies: + "@types/filesystem" "*" + "@types/har-format" "*" + "@types/cls-hooked@^4.2.1": version "4.3.3" resolved "https://registry.yarnpkg.com/@types/cls-hooked/-/cls-hooked-4.3.3.tgz#c09e2f8dc62198522eaa18a5b6b873053154bd00" @@ -7667,6 +7745,13 @@ resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.12.tgz#6b2c510a7ad7039e98e7b8d3d6598f4359e5c080" integrity sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw== +"@types/debug@^4.1.0": + version "4.1.7" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82" + integrity sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg== + dependencies: + "@types/ms" "*" + "@types/diff-match-patch@^1.0.32": version "1.0.32" resolved "https://registry.yarnpkg.com/@types/diff-match-patch/-/diff-match-patch-1.0.32.tgz#d9c3b8c914aa8229485351db4865328337a3d09f" @@ -7751,11 +7836,28 @@ "@types/qs" "*" "@types/serve-static" "*" +"@types/filesystem@*": + version "0.0.32" + resolved "https://registry.yarnpkg.com/@types/filesystem/-/filesystem-0.0.32.tgz#307df7cc084a2293c3c1a31151b178063e0a8edf" + integrity sha512-Yuf4jR5YYMR2DVgwuCiP11s0xuVRyPKmz8vo6HBY3CGdeMj8af93CFZX+T82+VD1+UqHOxTq31lO7MI7lepBtQ== + dependencies: + "@types/filewriter" "*" + +"@types/filewriter@*": + version "0.0.29" + resolved "https://registry.yarnpkg.com/@types/filewriter/-/filewriter-0.0.29.tgz#a48795ecadf957f6c0d10e0c34af86c098fa5bee" + integrity sha512-BsPXH/irW0ht0Ji6iw/jJaK8Lj3FJemon2gvEqHKpCdDCeemHa+rI3WBGq5z7cDMZgoLjY40oninGxqk+8NzNQ== + "@types/fined@*": version "1.1.3" resolved "https://registry.yarnpkg.com/@types/fined/-/fined-1.1.3.tgz#83f03e8f0a8d3673dfcafb18fce3571f6250e1bc" integrity sha512-CWYnSRnun3CGbt6taXeVo2lCbuaj4mchVJ4UF/BdU5TSuIn3AmS13pGMwCsBUoehGbhZrBrpNJZSZI5EVilXww== +"@types/firefox-webext-browser@^94.0.0": + version "94.0.1" + resolved "https://registry.yarnpkg.com/@types/firefox-webext-browser/-/firefox-webext-browser-94.0.1.tgz#52afb975253dc0fd350d5d58c7fe9fd1a01f64a1" + integrity sha512-I6iHRQJSTZ+gYt2IxdH2RRAMvcUyK8v5Ig7fHQR0IwUNYP7hz9+cziBVIKxLCO6XI7fiyRsNOWObfl3/4Js2Lg== + "@types/fluent-ffmpeg@^2.1.20": version "2.1.20" resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz#3b5f42fc8263761d58284fa46ee6759a64ce54ac" @@ -7793,6 +7895,11 @@ dependencies: graphql "^15.3.0" +"@types/har-format@*": + version "1.2.9" + resolved "https://registry.yarnpkg.com/@types/har-format/-/har-format-1.2.9.tgz#b9b3a9bfc33a078e7d898a00b09662910577f4a4" + integrity sha512-rffW6MhQ9yoa75bdNi+rjZBAvu2HhehWJXlhuWXnWdENeuKe82wUgAwxYOb7KRKKmxYN+D/iRKd2NDQMLqlUmg== + "@types/hast@^2.0.0": version "2.3.4" resolved "https://registry.yarnpkg.com/@types/hast/-/hast-2.3.4.tgz#8aa5ef92c117d20d974a82bdfb6a648b08c0bafc" @@ -7995,6 +8102,11 @@ resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.2.3.tgz#bbeb55fbc73f28ea6de601fbfa4613f58d785323" integrity sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw== +"@types/ms@*": + version "0.7.31" + resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" + integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== + "@types/nanoid@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@types/nanoid/-/nanoid-3.0.0.tgz#c757b20f343f3a1dd76e80a9a431b6290fc20f35" @@ -11063,6 +11175,17 @@ cliui@^7.0.2: strip-ansi "^6.0.0" wrap-ansi "^7.0.0" +clone-deep@^0.2.4: + version "0.2.4" + resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-0.2.4.tgz#4e73dd09e9fb971cc38670c5dced9c1896481cc6" + integrity sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg== + dependencies: + for-own "^0.1.3" + is-plain-object "^2.0.1" + kind-of "^3.0.2" + lazy-cache "^1.0.3" + shallow-clone "^0.1.2" + clone-deep@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-4.0.1.tgz#c19fd9bdbbf85942b4fd979c84dcf7d5f07c2387" @@ -14103,11 +14226,23 @@ follow-redirects@^1.0.0, follow-redirects@^1.14.0, follow-redirects@^1.14.4, fol resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5" integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA== +for-in@^0.1.3: + version "0.1.8" + resolved "https://registry.yarnpkg.com/for-in/-/for-in-0.1.8.tgz#d8773908e31256109952b1fdb9b3fa867d2775e1" + integrity sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g== + for-in@^1.0.1, for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= +for-own@^0.1.3: + version "0.1.5" + resolved "https://registry.yarnpkg.com/for-own/-/for-own-0.1.5.tgz#5265c681a4f294dabbf17c9509b6763aa84510ce" + integrity sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw== + dependencies: + for-in "^1.0.1" + for-own@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/for-own/-/for-own-1.0.0.tgz#c63332f415cedc4b04dbfe70cf836494c53cb44b" @@ -14275,6 +14410,15 @@ fs-extra@^0.30.0: path-is-absolute "^1.0.0" rimraf "^2.2.8" +fs-extra@^10.0.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" + integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + fs-extra@^9.0.0, fs-extra@^9.0.1, fs-extra@^9.1.0: version "9.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" @@ -16082,7 +16226,7 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@^1.1.5, is-buffer@~1.1.6: +is-buffer@^1.0.2, is-buffer@^1.1.5, is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -16394,7 +16538,7 @@ is-plain-object@5.0.0, is-plain-object@^5.0.0: resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== -is-plain-object@^2.0.3, is-plain-object@^2.0.4: +is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677" integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og== @@ -17587,6 +17731,13 @@ keyv@^3.0.0: dependencies: json-buffer "3.0.0" +kind-of@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-2.0.1.tgz#018ec7a4ce7e3a86cb9141be519d24c8faa981b5" + integrity sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg== + dependencies: + is-buffer "^1.0.2" + kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -17687,6 +17838,16 @@ lazy-ass@^1.6.0: resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513" integrity sha1-eZllXoZGwX8In90YfRUNMyTVRRM= +lazy-cache@^0.2.3: + version "0.2.7" + resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-0.2.7.tgz#7feddf2dcb6edb77d11ef1d117ab5ffdf0ab1b65" + integrity sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ== + +lazy-cache@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-1.0.4.tgz#a1d78fc3a50474cb80845d3b3b6e1da49a446e8e" + integrity sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ== + lazy-universal-dotenv@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/lazy-universal-dotenv/-/lazy-universal-dotenv-3.0.1.tgz#a6c8938414bca426ab8c9463940da451a911db38" @@ -18608,6 +18769,15 @@ meow@^8.0.0: type-fest "^0.18.0" yargs-parser "^20.2.3" +merge-deep@^3.0.1: + version "3.0.3" + resolved "https://registry.yarnpkg.com/merge-deep/-/merge-deep-3.0.3.tgz#1a2b2ae926da8b2ae93a0ac15d90cd1922766003" + integrity sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA== + dependencies: + arr-union "^3.1.0" + clone-deep "^0.2.4" + kind-of "^3.0.2" + merge-descriptors@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" @@ -18949,6 +19119,14 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" +mixin-object@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/mixin-object/-/mixin-object-2.0.1.tgz#4fb949441dab182540f1fe035ba60e1947a5e57e" + integrity sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA== + dependencies: + for-in "^0.1.3" + is-extendable "^0.1.1" + mkdirp-classic@^0.5.2: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" @@ -19383,7 +19561,7 @@ node-fetch@2.6.1: resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== -node-fetch@2.6.7, node-fetch@^2.3.0, node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@2.6.7, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== @@ -21451,6 +21629,63 @@ puppeteer-core@^19.1.1: unbzip2-stream "1.4.3" ws "8.9.0" +puppeteer-extra-plugin-adblocker@^2.13.5: + version "2.13.5" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.13.5.tgz#c86ce94873bf6fe500555d3972eccdcca4914f6f" + integrity sha512-HMVWLA1MLrzIGr/A71PYAWZEHENqQOEaQQHtPje0uSLc6QPOQY5tbbocx4BsUiQL2V1FwgT21UU09P5lV4vrZw== + dependencies: + "@cliqz/adblocker-puppeteer" "1.23.8" + debug "^4.1.1" + node-fetch "^2.6.0" + puppeteer-extra-plugin "^3.2.2" + +puppeteer-extra-plugin-stealth@^2.11.1: + version "2.11.1" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.1.tgz#7d56a27a986cb5eb69dca3c65695ad6444f4e822" + integrity sha512-n0wdC0Ilc9tk5L6FWLyd0P2gT8b2fp+2NuB+KB0oTSw3wXaZ0D6WNakjJsayJ4waGzIJFCUHkmK9zgx5NKMoFw== + dependencies: + debug "^4.1.1" + puppeteer-extra-plugin "^3.2.2" + puppeteer-extra-plugin-user-preferences "^2.4.0" + +puppeteer-extra-plugin-user-data-dir@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.0.tgz#20e87582482b61e497abd96fec452f63bd2d9123" + integrity sha512-qrhYPTGIqzL2hpeJ5DXjf8xMy5rt1UvcqSgpGTTOUOjIMz1ROWnKHjBoE9fNBJ4+ToRZbP8MzIDXWlEk/e1zJA== + dependencies: + debug "^4.1.1" + fs-extra "^10.0.0" + puppeteer-extra-plugin "^3.2.2" + rimraf "^3.0.2" + +puppeteer-extra-plugin-user-preferences@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.0.tgz#8b75bc39c3de9913e236ae1d982a24711d84ba6f" + integrity sha512-4XxMhMkJ+qqLsPY9ULF90qS9Bj1Qrwwgp1TY9zTdp1dJuy7QSgYE7xlyamq3cKrRuzg3QUOqygJo52sVeXSg5A== + dependencies: + debug "^4.1.1" + deepmerge "^4.2.2" + puppeteer-extra-plugin "^3.2.2" + puppeteer-extra-plugin-user-data-dir "^2.4.0" + +puppeteer-extra-plugin@^3.2.2: + version "3.2.2" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.2.tgz#3c02c0a10f8eadf32e7debb7ee24105a53afc17b" + integrity sha512-0uatQxzuVn8yegbrEwSk03wvwpMB5jNs7uTTnermylLZzoT+1rmAQaJXwlS3+vADUbw6ELNgNEHC7Skm0RqHbQ== + dependencies: + "@types/debug" "^4.1.0" + debug "^4.1.1" + merge-deep "^3.0.1" + +puppeteer-extra@^3.3.4: + version "3.3.4" + resolved "https://registry.yarnpkg.com/puppeteer-extra/-/puppeteer-extra-3.3.4.tgz#e0ecf021783d1112b6b0db20546d5022e632ed55" + integrity sha512-fN5pHvSMJ8d1o7Z8wLLTQOUBpORD2BcFn+KDs7QnkGZs9SV69hcUcce67vX4L4bNSEG3A0P6Osrv+vWNhhdm8w== + dependencies: + "@types/debug" "^4.1.0" + debug "^4.1.1" + deepmerge "^4.2.2" + puppeteer@^10.1.0: version "10.4.0" resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-10.4.0.tgz#a6465ff97fda0576c4ac29601406f67e6fea3dc7" @@ -22939,6 +23174,16 @@ sha.js@^2.4.0, sha.js@^2.4.11, sha.js@^2.4.8: inherits "^2.0.1" safe-buffer "^5.0.1" +shallow-clone@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-0.1.2.tgz#5909e874ba77106d73ac414cfec1ffca87d97060" + integrity sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw== + dependencies: + is-extendable "^0.1.1" + kind-of "^2.0.1" + lazy-cache "^0.2.3" + mixin-object "^2.0.1" + shallow-clone@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3" @@ -24277,6 +24522,18 @@ title-case@^3.0.3: dependencies: tslib "^2.0.3" +tldts-core@^5.7.100: + version "5.7.100" + resolved "https://registry.yarnpkg.com/tldts-core/-/tldts-core-5.7.100.tgz#6144104277a3c4500ec395220d8e03c16fcdfaf7" + integrity sha512-56+vie1oPcJZQiPfnvIIpbyTttUketsjV7lrw/hkMMa/EACPjjDctobWwF3153gR2l+c9O+nYiHkXIL1Cmr9eQ== + +tldts-experimental@^5.6.21: + version "5.7.100" + resolved "https://registry.yarnpkg.com/tldts-experimental/-/tldts-experimental-5.7.100.tgz#fb428cf20735952c299e15e864de63ecc55fb0a7" + integrity sha512-BjdXE3YU3cXbASRXydXnzOCSc+G/bM38/5snbxwcIYaRh3AApEtD4lHVl3236x+79T/V94lKwqBXYT47EAR+TA== + dependencies: + tldts-core "^5.7.100" + tmp@^0.0.33: version "0.0.33" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"