mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
Merge pull request #1424 from omnivore-app/fix/gdcvault-parsing-issue
fix/gdcvault parsing issue
This commit is contained in:
Hongbo Wu
GitHub
commit
38ed7c9a3d
14 changed files with 1601 additions and 145 deletions
|
|
@ -14,7 +14,7 @@ if (!process.env.VERIFICATION_TOKEN) {
|
|||
app.get('/', async (req, res) => {
|
||||
if (req.query.token !== process.env.VERIFICATION_TOKEN) {
|
||||
console.log('query does not include valid token')
|
||||
res.send(403)
|
||||
res.sendStatus(403)
|
||||
return
|
||||
}
|
||||
await fetchContent(req, res)
|
||||
|
|
@ -23,7 +23,7 @@ app.get('/', async (req, res) => {
|
|||
app.post('/', async (req, res) => {
|
||||
if (req.query.token !== process.env.VERIFICATION_TOKEN) {
|
||||
console.log('query does not include valid token')
|
||||
res.send(403)
|
||||
res.sendStatus(403)
|
||||
return
|
||||
}
|
||||
await fetchContent(req, res)
|
||||
|
|
|
|||
66
packages/content-fetch/logger.js
Normal file
66
packages/content-fetch/logger.js
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
const { config, format, loggers, transports } = require('winston');
|
||||
const { LoggingWinston } = require('@google-cloud/logging-winston');
|
||||
const { DateTime } = require('luxon');
|
||||
|
||||
const colors = {
|
||||
emerg: 'inverse underline magenta',
|
||||
alert: 'underline magenta',
|
||||
crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss.
|
||||
error: 'underline red', // Any error which is fatal to the operation, but not the service or application
|
||||
warning: 'underline yellow', // Anything that can potentially cause application oddities
|
||||
notice: 'underline cyan', // Normal but significant condition
|
||||
info: 'underline green', // Generally useful information to log
|
||||
debug: 'underline gray',
|
||||
};
|
||||
|
||||
const googleConfigs = {
|
||||
level: 'info',
|
||||
logName: 'logger',
|
||||
levels: config.syslog.levels,
|
||||
resource: {
|
||||
labels: {
|
||||
function_name: process.env.FUNCTION_TARGET,
|
||||
project_id: process.env.GCP_PROJECT,
|
||||
},
|
||||
type: 'cloud_function',
|
||||
},
|
||||
};
|
||||
|
||||
function localConfig(id) {
|
||||
return {
|
||||
level: 'debug',
|
||||
format: format.combine(
|
||||
format.colorize({ all: true, colors }),
|
||||
format(info =>
|
||||
Object.assign(info, {
|
||||
timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS),
|
||||
}),
|
||||
)(),
|
||||
format.printf(info => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { timestamp, message, level, ...meta } = info;
|
||||
|
||||
return `[${id}@${info.timestamp}] ${info.message}${
|
||||
Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : ''
|
||||
}`;
|
||||
}),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function buildLoggerTransport(id, options) {
|
||||
return process.env.IS_LOCAL
|
||||
? new transports.Console(localConfig(id))
|
||||
: new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options });
|
||||
}
|
||||
|
||||
function buildLogger(id, options) {
|
||||
return loggers.get(id, {
|
||||
levels: config.syslog.levels,
|
||||
transports: [buildLoggerTransport(id, options)],
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
buildLogger,
|
||||
}
|
||||
|
|
@ -8,7 +8,8 @@
|
|||
"express": "^4.17.1",
|
||||
"@google-cloud/functions-framework": "^3.0.0",
|
||||
"@omnivore/puppeteer-parse": "^1.0.0",
|
||||
"@sentry/serverless": "^6.13.3"
|
||||
"@sentry/serverless": "^6.13.3",
|
||||
"winston": "^3.3.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"chai": "^4.3.6",
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import rfc2047 from 'rfc2047'
|
|||
import { v4 as uuid } from 'uuid'
|
||||
import { parseHTML } from 'linkedom'
|
||||
import axios from 'axios'
|
||||
import { Browser } from 'puppeteer-core'
|
||||
|
||||
interface Unsubscribe {
|
||||
mailTo?: string
|
||||
|
|
@ -62,7 +63,7 @@ export abstract class ContentHandler {
|
|||
return false
|
||||
}
|
||||
|
||||
async preHandle(url: string): Promise<PreHandleResult> {
|
||||
async preHandle(url: string, browser?: Browser): Promise<PreHandleResult> {
|
||||
return Promise.resolve({ url })
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import { GhostHandler } from './newsletters/ghost-handler'
|
|||
import { parseHTML } from 'linkedom'
|
||||
import { CooperPressHandler } from './newsletters/cooper-press-handler'
|
||||
import { HeyWorldHandler } from './newsletters/hey-world-handler'
|
||||
import { Browser } from 'puppeteer-core'
|
||||
|
||||
const validateUrlString = (url: string) => {
|
||||
const u = new URL(url)
|
||||
|
|
@ -80,7 +81,8 @@ const newsletterHandlers: ContentHandler[] = [
|
|||
]
|
||||
|
||||
export const preHandleContent = async (
|
||||
url: string
|
||||
url: string,
|
||||
browser: Browser
|
||||
): Promise<PreHandleResult | undefined> => {
|
||||
// Before we run the regular handlers we check to see if we need tp
|
||||
// pre-resolve the URL. TODO: This should probably happen recursively,
|
||||
|
|
@ -104,7 +106,7 @@ export const preHandleContent = async (
|
|||
for (const handler of contentHandlers) {
|
||||
if (handler.shouldPreHandle(url)) {
|
||||
console.log('preHandleContent', handler.name, url)
|
||||
return handler.preHandle(url)
|
||||
return handler.preHandle(url, browser)
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { ContentHandler, PreHandleResult } from '../content-handler'
|
|||
import axios from 'axios'
|
||||
import { DateTime } from 'luxon'
|
||||
import _ from 'underscore'
|
||||
import puppeteer from 'puppeteer-core'
|
||||
import { Browser, BrowserContext } from 'puppeteer-core'
|
||||
|
||||
interface TweetIncludes {
|
||||
users: {
|
||||
|
|
@ -168,10 +168,11 @@ const getTweetsFromResponse = (response: Tweets): Tweet[] => {
|
|||
}
|
||||
|
||||
const getOldTweets = async (
|
||||
browser: Browser,
|
||||
conversationId: string,
|
||||
username: string
|
||||
): Promise<Tweet[]> => {
|
||||
const tweetIds = await getTweetIds(conversationId, username)
|
||||
const tweetIds = await getTweetIds(browser, conversationId, username)
|
||||
if (tweetIds.length === 0) {
|
||||
return []
|
||||
}
|
||||
|
|
@ -197,37 +198,21 @@ const waitFor = (ms: number) =>
|
|||
|
||||
/**
|
||||
* Get tweets(even older than 7 days) using puppeteer
|
||||
* @param browser
|
||||
* @param {string} tweetId
|
||||
* @param {string} author
|
||||
*/
|
||||
const getTweetIds = async (
|
||||
browser: Browser,
|
||||
tweetId: string,
|
||||
author: string
|
||||
): Promise<string[]> => {
|
||||
const pageURL = `https://twitter.com/${author}/status/${tweetId}`
|
||||
|
||||
// Modify this variable to control the size of viewport
|
||||
const factor = 0.2
|
||||
const height = Math.floor(2000 / factor)
|
||||
const width = Math.floor(1700 / factor)
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath: process.env.CHROMIUM_PATH,
|
||||
headless: !!process.env.LAUNCH_HEADLESS,
|
||||
defaultViewport: {
|
||||
width,
|
||||
height,
|
||||
},
|
||||
args: [
|
||||
`--force-device-scale-factor=${factor}`,
|
||||
`--window-size=${width},${height}`,
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
],
|
||||
})
|
||||
|
||||
let context: BrowserContext | undefined
|
||||
try {
|
||||
const page = await browser.newPage()
|
||||
context = await browser.createIncognitoBrowserContext()
|
||||
const page = await context.newPage()
|
||||
|
||||
await page.goto(pageURL, {
|
||||
waitUntil: 'networkidle2',
|
||||
|
|
@ -291,7 +276,9 @@ const getTweetIds = async (
|
|||
console.log(error)
|
||||
return []
|
||||
} finally {
|
||||
await browser.close()
|
||||
if (context) {
|
||||
await context.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -305,7 +292,7 @@ export class TwitterHandler extends ContentHandler {
|
|||
return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
|
||||
}
|
||||
|
||||
async preHandle(url: string): Promise<PreHandleResult> {
|
||||
async preHandle(url: string, browser: Browser): Promise<PreHandleResult> {
|
||||
const tweetId = tweetIdFromStatusUrl(url)
|
||||
if (!tweetId) {
|
||||
throw new Error('could not find tweet id in url')
|
||||
|
|
@ -326,7 +313,7 @@ export class TwitterHandler extends ContentHandler {
|
|||
const description = _.escape(tweetData.text)
|
||||
|
||||
// use puppeteer to get all tweet replies in the thread
|
||||
const tweets = await getOldTweets(conversationId, author.username)
|
||||
const tweets = await getOldTweets(browser, conversationId, author.username)
|
||||
|
||||
let tweetsContent = ''
|
||||
for (const tweet of tweets) {
|
||||
|
|
|
|||
|
|
@ -9,18 +9,20 @@ const axios = require('axios');
|
|||
const jwt = require('jsonwebtoken');
|
||||
const { promisify } = require('util');
|
||||
const signToken = promisify(jwt.sign);
|
||||
const { config, format, loggers, transports } = require('winston');
|
||||
const { LoggingWinston } = require('@google-cloud/logging-winston');
|
||||
const { DateTime } = require('luxon');
|
||||
const os = require('os');
|
||||
const { Storage } = require('@google-cloud/storage');
|
||||
const { parseHTML } = require('linkedom');
|
||||
const puppeteer = require('puppeteer-core');
|
||||
const { preHandleContent } = require("@omnivore/content-handler");
|
||||
|
||||
const puppeteer = require('puppeteer-extra');
|
||||
|
||||
// Add stealth plugin to hide puppeteer usage
|
||||
// const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
||||
// puppeteer.use(StealthPlugin());
|
||||
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
||||
puppeteer.use(StealthPlugin());
|
||||
|
||||
// Add adblocker plugin to block all ads and trackers (saves bandwidth)
|
||||
const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker');
|
||||
puppeteer.use(AdblockerPlugin({ blockTrackers: true }));
|
||||
|
||||
const storage = new Storage();
|
||||
const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : [];
|
||||
|
|
@ -28,65 +30,6 @@ const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.
|
|||
|
||||
const filePath = `${os.tmpdir()}/previewImage.png`;
|
||||
|
||||
const colors = {
|
||||
emerg: 'inverse underline magenta',
|
||||
alert: 'underline magenta',
|
||||
crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss.
|
||||
error: 'underline red', // Any error which is fatal to the operation, but not the service or application
|
||||
warning: 'underline yellow', // Anything that can potentially cause application oddities
|
||||
notice: 'underline cyan', // Normal but significant condition
|
||||
info: 'underline green', // Generally useful information to log
|
||||
debug: 'underline gray',
|
||||
};
|
||||
|
||||
const googleConfigs = {
|
||||
level: 'info',
|
||||
logName: 'logger',
|
||||
levels: config.syslog.levels,
|
||||
resource: {
|
||||
labels: {
|
||||
function_name: process.env.FUNCTION_TARGET,
|
||||
project_id: process.env.GCP_PROJECT,
|
||||
},
|
||||
type: 'cloud_function',
|
||||
},
|
||||
};
|
||||
|
||||
function localConfig(id) {
|
||||
return {
|
||||
level: 'debug',
|
||||
format: format.combine(
|
||||
format.colorize({ all: true, colors }),
|
||||
format(info =>
|
||||
Object.assign(info, {
|
||||
timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS),
|
||||
}),
|
||||
)(),
|
||||
format.printf(info => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { timestamp, message, level, ...meta } = info;
|
||||
|
||||
return `[${id}@${info.timestamp}] ${info.message}${
|
||||
Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : ''
|
||||
}`;
|
||||
}),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function buildLoggerTransport(id, options) {
|
||||
return process.env.IS_LOCAL
|
||||
? new transports.Console(localConfig(id))
|
||||
: new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options });
|
||||
}
|
||||
|
||||
function buildLogger(id, options) {
|
||||
return loggers.get(id, {
|
||||
levels: config.syslog.levels,
|
||||
transports: [buildLoggerTransport(id, options)],
|
||||
});
|
||||
}
|
||||
|
||||
const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
|
||||
const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
|
||||
const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
|
||||
|
|
@ -174,8 +117,6 @@ const getBrowserPromise = (async () => {
|
|||
});
|
||||
})();
|
||||
|
||||
let logRecord, functionStartTime;
|
||||
|
||||
const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => {
|
||||
const stream = await axios.get(contentObjUrl, { responseType: 'stream' });
|
||||
return await axios.put(uploadSignedUrl, stream.data, {
|
||||
|
|
@ -268,22 +209,13 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId
|
|||
};
|
||||
|
||||
async function fetchContent(req, res) {
|
||||
functionStartTime = Date.now();
|
||||
// Grabbing execution and trace ids to attach logs to the appropriate function call
|
||||
const execution_id = req.get('function-execution-id');
|
||||
const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0];
|
||||
const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', {
|
||||
trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`,
|
||||
labels: {
|
||||
execution_id: execution_id,
|
||||
},
|
||||
});
|
||||
let functionStartTime = Date.now();
|
||||
|
||||
let url = getUrl(req);
|
||||
const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined);
|
||||
const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined);
|
||||
|
||||
logRecord = {
|
||||
let logRecord = {
|
||||
url,
|
||||
userId,
|
||||
articleSavingRequestId,
|
||||
|
|
@ -292,18 +224,19 @@ async function fetchContent(req, res) {
|
|||
},
|
||||
};
|
||||
|
||||
logger.info(`Article parsing request`, logRecord);
|
||||
console.info(`Article parsing request`, logRecord);
|
||||
|
||||
if (!url) {
|
||||
logRecord.urlIsInvalid = true;
|
||||
logger.info(`Valid URL to parse not specified`, logRecord);
|
||||
console.info(`Valid URL to parse not specified`, logRecord);
|
||||
return res.sendStatus(400);
|
||||
}
|
||||
|
||||
// pre handle url with custom handlers
|
||||
let title, content, contentType;
|
||||
try {
|
||||
const result = await preHandleContent(url);
|
||||
const browser = await getBrowserPromise;
|
||||
const result = await preHandleContent(url, browser);
|
||||
if (result && result.url) {
|
||||
url = result.url
|
||||
validateUrlString(url);
|
||||
|
|
@ -312,13 +245,13 @@ async function fetchContent(req, res) {
|
|||
if (result && result.content) { content = result.content }
|
||||
if (result && result.contentType) { contentType = result.contentType }
|
||||
} catch (e) {
|
||||
logger.info('error with handler: ', e);
|
||||
console.info('error with handler: ', e);
|
||||
}
|
||||
|
||||
let context, page, finalUrl;
|
||||
try {
|
||||
if ((!content || !title) && contentType !== 'application/pdf') {
|
||||
const result = await retrievePage(url)
|
||||
const result = await retrievePage(url, logRecord, functionStartTime);
|
||||
if (result && result.context) { context = result.context }
|
||||
if (result && result.page) { page = result.page }
|
||||
if (result && result.finalUrl) { finalUrl = result.finalUrl }
|
||||
|
|
@ -332,7 +265,7 @@ async function fetchContent(req, res) {
|
|||
const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId);
|
||||
} else {
|
||||
if (!content || !title) {
|
||||
const result = await retrieveHtml(page);
|
||||
const result = await retrieveHtml(page, logRecord);
|
||||
if (result.isBlocked) {
|
||||
const sbResult = await fetchContentWithScrapingBee(url)
|
||||
title = sbResult.title
|
||||
|
|
@ -342,7 +275,7 @@ async function fetchContent(req, res) {
|
|||
content = result.domContent;
|
||||
}
|
||||
} else {
|
||||
logger.info('using prefetched content and title');
|
||||
console.info('using prefetched content and title');
|
||||
}
|
||||
|
||||
logRecord.fetchContentTime = Date.now() - functionStartTime;
|
||||
|
|
@ -365,7 +298,7 @@ async function fetchContent(req, res) {
|
|||
}
|
||||
} catch (e) {
|
||||
logRecord.error = e.message;
|
||||
logger.error(`Error while retrieving page`, logRecord);
|
||||
console.error(`Error while retrieving page`, logRecord);
|
||||
|
||||
// fallback to scrapingbee
|
||||
const sbResult = await fetchContentWithScrapingBee(url);
|
||||
|
|
@ -392,7 +325,7 @@ async function fetchContent(req, res) {
|
|||
if (context) {
|
||||
await context.close();
|
||||
}
|
||||
logger.info(`parse-page`, logRecord);
|
||||
console.info(`parse-page`, logRecord);
|
||||
}
|
||||
|
||||
return res.sendStatus(200);
|
||||
|
|
@ -452,7 +385,7 @@ async function blockResources(client) {
|
|||
await client.send('Network.setBlockedURLs', { urls: blockedResources });
|
||||
}
|
||||
|
||||
async function retrievePage(url) {
|
||||
async function retrievePage(url, logRecord, functionStartTime) {
|
||||
validateUrlString(url);
|
||||
|
||||
const browser = await getBrowserPromise;
|
||||
|
|
@ -549,7 +482,7 @@ async function retrievePage(url) {
|
|||
});
|
||||
|
||||
try {
|
||||
const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] });
|
||||
const response = await page.goto(url, { timeout: 30 * 1000, waitUntil: ['networkidle2'] });
|
||||
const finalUrl = response.url();
|
||||
const contentType = response.headers()['content-type'];
|
||||
|
||||
|
|
@ -566,7 +499,7 @@ async function retrievePage(url) {
|
|||
}
|
||||
}
|
||||
|
||||
async function retrieveHtml(page) {
|
||||
async function retrieveHtml(page, logRecord) {
|
||||
let domContent = '', title;
|
||||
try {
|
||||
title = await page.title();
|
||||
|
|
@ -710,7 +643,7 @@ async function preview(req, res) {
|
|||
// Grabbing execution and trace ids to attach logs to the appropriate function call
|
||||
const execution_id = req.get('function-execution-id');
|
||||
const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0];
|
||||
const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', {
|
||||
const console = buildconsole('cloudfunctions.googleapis.com%2Fcloud-functions', {
|
||||
trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`,
|
||||
labels: {
|
||||
execution_id: execution_id,
|
||||
|
|
@ -718,7 +651,7 @@ async function preview(req, res) {
|
|||
});
|
||||
|
||||
if (!process.env.PREVIEW_IMAGE_BUCKET) {
|
||||
logger.error(`PREVIEW_IMAGE_BUCKET not set`)
|
||||
console.error(`PREVIEW_IMAGE_BUCKET not set`)
|
||||
return res.sendStatus(500);
|
||||
}
|
||||
|
||||
|
|
@ -734,30 +667,30 @@ async function preview(req, res) {
|
|||
},
|
||||
};
|
||||
|
||||
logger.info(`Public preview image generation request`, logRecord);
|
||||
console.info(`Public preview image generation request`, logRecord);
|
||||
|
||||
if (!url) {
|
||||
logRecord.urlIsInvalid = true;
|
||||
logger.error(`Valid URL to parse is not specified`, logRecord);
|
||||
console.error(`Valid URL to parse is not specified`, logRecord);
|
||||
return res.sendStatus(400);
|
||||
}
|
||||
const { origin } = new URL(url);
|
||||
if (!ALLOWED_ORIGINS.some(o => o === origin)) {
|
||||
logRecord.forbiddenOrigin = true;
|
||||
logger.error(`This origin is not allowed: ${origin}`, logRecord);
|
||||
console.error(`This origin is not allowed: ${origin}`, logRecord);
|
||||
return res.sendStatus(400);
|
||||
}
|
||||
|
||||
const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH);
|
||||
const browser = await getBrowserPromise;
|
||||
logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime };
|
||||
|
||||
const page = await browser.newPage();
|
||||
const pageLoadingStart = Date.now();
|
||||
const modifiedUrl = new URL(url);
|
||||
modifiedUrl.searchParams.append('fontSize', 24);
|
||||
modifiedUrl.searchParams.append('adjustAspectRatio', 1.91);
|
||||
modifiedUrl.searchParams.append('fontSize', '24');
|
||||
modifiedUrl.searchParams.append('adjustAspectRatio', '1.91');
|
||||
try {
|
||||
await page.goto(modifiedUrl);
|
||||
await page.goto(modifiedUrl.toString());
|
||||
logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart };
|
||||
} catch (error) {
|
||||
console.log('error going to page: ', modifiedUrl)
|
||||
|
|
@ -775,7 +708,7 @@ async function preview(req, res) {
|
|||
);
|
||||
if (!selector) {
|
||||
logRecord.selectorIsInvalid = true;
|
||||
logger.error(`Valid element selector is not specified`, logRecord);
|
||||
console.error(`Valid element selector is not specified`, logRecord);
|
||||
await page.close();
|
||||
return res.sendStatus(400);
|
||||
}
|
||||
|
|
@ -790,7 +723,7 @@ async function preview(req, res) {
|
|||
);
|
||||
if (!destination) {
|
||||
logRecord.destinationIsInvalid = true;
|
||||
logger.error(`Valid file destination is not specified`, logRecord);
|
||||
console.error(`Valid file destination is not specified`, logRecord);
|
||||
await page.close();
|
||||
return res.sendStatus(400);
|
||||
}
|
||||
|
|
@ -801,7 +734,7 @@ async function preview(req, res) {
|
|||
await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load
|
||||
} catch (error) {
|
||||
logRecord.elementNotFound = true;
|
||||
logger.error(`Element is not presented on the page`, logRecord);
|
||||
console.error(`Element is not presented on the page`, logRecord);
|
||||
await page.close();
|
||||
return res.sendStatus(400);
|
||||
}
|
||||
|
|
@ -821,7 +754,7 @@ async function preview(req, res) {
|
|||
console.log('error uploading to bucket, this is non-fatal', e)
|
||||
}
|
||||
|
||||
logger.info(`preview-image`, logRecord);
|
||||
console.info(`preview-image`, logRecord);
|
||||
return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,16 +4,16 @@
|
|||
"description": "Accepts URL of the article and parses its content",
|
||||
"main": "index.js",
|
||||
"dependencies": {
|
||||
"@google-cloud/logging-winston": "^5.1.1",
|
||||
"@google-cloud/storage": "^5.18.1",
|
||||
"@omnivore/content-handler": "1.0.0",
|
||||
"axios": "^0.27.2",
|
||||
"jsonwebtoken": "^8.5.1",
|
||||
"linkedom": "^0.14.9",
|
||||
"luxon": "^2.3.1",
|
||||
"puppeteer-core": "^16.1.0",
|
||||
"underscore": "^1.13.4",
|
||||
"winston": "^3.3.3"
|
||||
"puppeteer-extra": "^3.3.4",
|
||||
"puppeteer-extra-plugin-adblocker": "^2.13.5",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.1",
|
||||
"underscore": "^1.13.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"chai": "^4.3.6",
|
||||
|
|
|
|||
|
|
@ -171,20 +171,20 @@ Readability.prototype = {
|
|||
// Readability-readerable.js. Please keep both copies in sync.
|
||||
articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist/i,
|
||||
articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert|block-ad-.*/i,
|
||||
unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon/i,
|
||||
unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped/i,
|
||||
// okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i,
|
||||
get okMaybeItsACandidate() {
|
||||
return new RegExp(`and|(?<!${this.articleNegativeLookAheadCandidates.source})article(?!-(${this.articleNegativeLookBehindCandidates.source}))|body|column|content|^(?!main-navigation|main-header)main|shadow|post-header|hfeed site|blog-posts hfeed|container-banners|menu-opacity|header-with-anchor-widget`, 'i')
|
||||
},
|
||||
|
||||
positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story|tweet(-\w+)?|instagram|image|container-banners/i,
|
||||
positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story|tweet(-\w+)?|instagram|image|container-banners|player/i,
|
||||
negative: /\bad\b|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|controls|video-controls/i,
|
||||
extraneous: /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,
|
||||
byline: /byline|author|dateline|writtenby|p-author/i,
|
||||
publishedDate: /published|modified|created|updated/i,
|
||||
replaceFonts: /<(\/?)font[^>]*>/gi,
|
||||
normalize: /\s{2,}/g,
|
||||
videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,
|
||||
videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq|cdnapisec\.kaltura)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,
|
||||
shareElements: /(\b|_)(share|sharedaddy|post-tags)(\b|_)/i,
|
||||
nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,
|
||||
prevLink: /(prev|earl|old|new|<|«)/i,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"title": "Parallelizing the Naughty Dog Engine Using Fibers",
|
||||
"byline": null,
|
||||
"dir": null,
|
||||
"excerpt": "This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall...",
|
||||
"siteName": null,
|
||||
"siteIcon": "http://fakehost/img/favicon.ico",
|
||||
"previewImage": "https://ubm-twvideo01.s3.amazonaws.com/o1/vault/gdc2015/Images/GDC15_Vault-thumb_v1.png",
|
||||
"publishedDate": null,
|
||||
"language": "English",
|
||||
"readerable": true
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
<DIV class="page" id="readability-page-1">
|
||||
<div id="player">
|
||||
<iframe src="https://cdnapisec.kaltura.com/p/1670711/sp/167071100/embedIframeJs/uiconf_id/43558772/partner_id/1670711?iframeembed=true&playerId=kaltura_player_1547062087&cache_st=1547072087&width=1000&height=570&entry_id=0_6kvuqtmi&videoid=6aafd73d83118f0e6c301022186" width="1000" height="570" scrolling="no" frameborder="0" marginheight="0" marginwidth="0" allowfullscreen="allowfullscreen"></iframe>
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<strong>Session Name:</strong>
|
||||
</td>
|
||||
<td> Parallelizing the Naughty Dog Engine Using Fibers </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<strong>Speaker(s):</strong>
|
||||
</td>
|
||||
<td> Christian Gyrling </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<strong>Company Name(s):</strong>
|
||||
</td>
|
||||
<td> Naughty Dog </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<strong>Track / Format:</strong>
|
||||
</td>
|
||||
<td> Programming </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<strong>Overview:</strong>
|
||||
</td>
|
||||
<td> This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall frame-centric engine design, the memory allocation patterns used in the title, and our strategies for dealing with locks. </td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<br>
|
||||
<!-- <div id="ad_lower">
|
||||
<div id="div-gpt-ad-rec1">
|
||||
<script type='text/javascript'>
|
||||
googletag.display('div-gpt-ad-rec1');
|
||||
</script>
|
||||
</div>
|
||||
</div> -->
|
||||
<section id="recommended">
|
||||
<img src="http://fakehost/img/icon_members_only_text.png">
|
||||
<div id="recommended_body">
|
||||
<p><button id="load_recommended_videos">LOAD MORE RECOMMENDED VIDEOS</button>
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</DIV>
|
||||
1142
packages/readabilityjs/test/test-pages/gdcvault/source.html
Normal file
1142
packages/readabilityjs/test/test-pages/gdcvault/source.html
Normal file
File diff suppressed because one or more lines are too long
1
packages/readabilityjs/test/test-pages/gdcvault/url.txt
Normal file
1
packages/readabilityjs/test/test-pages/gdcvault/url.txt
Normal file
|
|
@ -0,0 +1 @@
|
|||
https://www.gdcvault.com/play/1022186/Parallelizing-the-Naughty-Dog-Engine
|
||||
263
yarn.lock
263
yarn.lock
|
|
@ -2069,6 +2069,41 @@
|
|||
commander "^4.1.0"
|
||||
microtime "^3.0.0"
|
||||
|
||||
"@cliqz/adblocker-content@^1.23.8", "@cliqz/adblocker-content@^1.25.1":
|
||||
version "1.25.1"
|
||||
resolved "https://registry.yarnpkg.com/@cliqz/adblocker-content/-/adblocker-content-1.25.1.tgz#da81e7838e288a6f0fdb8a97a0df8169accb74a1"
|
||||
integrity sha512-7gl2VdNPBfj7aPoq34B5miwGcnda/7LCr+BqnpcSOjdLV6jjT2FrNSAKGFvcH23q0HM1IFhYDV6ydTgsdWFCnA==
|
||||
dependencies:
|
||||
"@cliqz/adblocker-extended-selectors" "^1.25.1"
|
||||
|
||||
"@cliqz/adblocker-extended-selectors@^1.25.1":
|
||||
version "1.25.1"
|
||||
resolved "https://registry.yarnpkg.com/@cliqz/adblocker-extended-selectors/-/adblocker-extended-selectors-1.25.1.tgz#cfac0080952311399805fe153cd9e7e1331b3c6d"
|
||||
integrity sha512-4MdMe/YfIok5d8WYVcLR3Ak7vGrmeUV47frgmXEe945luY93vwlzk1NiLYW1JM5Gdm+VePweoS9cJ1/QUTmv+Q==
|
||||
|
||||
"@cliqz/adblocker-puppeteer@1.23.8":
|
||||
version "1.23.8"
|
||||
resolved "https://registry.yarnpkg.com/@cliqz/adblocker-puppeteer/-/adblocker-puppeteer-1.23.8.tgz#e74636cd200459d1734929e41504a76939504311"
|
||||
integrity sha512-Ca1/DBqQXsOpKTFVAHX6OpLTSEupXmUkUWHj6iXhLLleC7RPISN5B0b801VDmaGRqoC5zKRxn0vYbIfpgCWVug==
|
||||
dependencies:
|
||||
"@cliqz/adblocker" "^1.23.8"
|
||||
"@cliqz/adblocker-content" "^1.23.8"
|
||||
tldts-experimental "^5.6.21"
|
||||
|
||||
"@cliqz/adblocker@^1.23.8":
|
||||
version "1.25.1"
|
||||
resolved "https://registry.yarnpkg.com/@cliqz/adblocker/-/adblocker-1.25.1.tgz#4d3e8894ce48ad0d0f8b26a4a1003f0676b7f734"
|
||||
integrity sha512-1C1/ELI94/XewdUj/o1+Q4ziOigMvTZQA05UERfDoKqpJ+0cbrEF/UImrzpX7n+kYsR7xTJvmf+iNM3zS0tfsg==
|
||||
dependencies:
|
||||
"@cliqz/adblocker-content" "^1.25.1"
|
||||
"@cliqz/adblocker-extended-selectors" "^1.25.1"
|
||||
"@remusao/guess-url-type" "^1.1.2"
|
||||
"@remusao/small" "^1.1.2"
|
||||
"@remusao/smaz" "^1.7.1"
|
||||
"@types/chrome" "^0.0.197"
|
||||
"@types/firefox-webext-browser" "^94.0.0"
|
||||
tldts-experimental "^5.6.21"
|
||||
|
||||
"@cnakazawa/watch@^1.0.3":
|
||||
version "1.0.4"
|
||||
resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a"
|
||||
|
|
@ -5110,6 +5145,41 @@
|
|||
resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4"
|
||||
integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA==
|
||||
|
||||
"@remusao/guess-url-type@^1.1.2":
|
||||
version "1.2.1"
|
||||
resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57"
|
||||
integrity sha512-rbOqre2jW8STjheOsOaQHLgYBaBZ9Owbdt8NO7WvNZftJlaG3y/K9oOkl8ZUpuFBisIhmBuMEW6c+YrQl5inRA==
|
||||
|
||||
"@remusao/small@^1.1.2":
|
||||
version "1.2.1"
|
||||
resolved "https://registry.yarnpkg.com/@remusao/small/-/small-1.2.1.tgz#63bfe4548832289f94ac868a0c305970c9a0e5f9"
|
||||
integrity sha512-7MjoGt0TJMVw1GPKgWq6SJPws1SLsUXQRa43Umht+nkyw2jnpy3WpiLNqGdwo5rHr5Wp9B2W/Pm5RQp656UJdw==
|
||||
|
||||
"@remusao/smaz-compress@^1.9.1":
|
||||
version "1.9.1"
|
||||
resolved "https://registry.yarnpkg.com/@remusao/smaz-compress/-/smaz-compress-1.9.1.tgz#fc75eaf9bcac2d58bc4c3d518183a7cb9612d275"
|
||||
integrity sha512-E2f48TwloQu3r6BdLOGF2aczeH7bJ/32oJGqvzT9SKur0cuUnLcZ7ZXP874E2fwmdE+cXzfC7bKzp79cDnmeyw==
|
||||
dependencies:
|
||||
"@remusao/trie" "^1.4.1"
|
||||
|
||||
"@remusao/smaz-decompress@^1.9.1":
|
||||
version "1.9.1"
|
||||
resolved "https://registry.yarnpkg.com/@remusao/smaz-decompress/-/smaz-decompress-1.9.1.tgz#8094f997e8fb591a678cda9cf08c209c825eba5b"
|
||||
integrity sha512-TfjKKprYe3n47od8auhvJ/Ikj9kQTbDTe71ynKlxslrvvUhlIV3VQSuwYuMWMbdz1fIs0H/fxCN1Z8/H3km6/A==
|
||||
|
||||
"@remusao/smaz@^1.7.1":
|
||||
version "1.9.1"
|
||||
resolved "https://registry.yarnpkg.com/@remusao/smaz/-/smaz-1.9.1.tgz#a2b9b045385f81e1615a68d932b7cc8b04c9db8d"
|
||||
integrity sha512-e6BLuP8oaXCZ9+v46Is4ilAZ/Vq6YLgmBP204Ixgk1qTjXmqvFYG7+AS7v9nsZdGOy96r9DWGFbbDVgMxwu1rA==
|
||||
dependencies:
|
||||
"@remusao/smaz-compress" "^1.9.1"
|
||||
"@remusao/smaz-decompress" "^1.9.1"
|
||||
|
||||
"@remusao/trie@^1.4.1":
|
||||
version "1.4.1"
|
||||
resolved "https://registry.yarnpkg.com/@remusao/trie/-/trie-1.4.1.tgz#755d09f8a007476334e611f42719b2d581f00720"
|
||||
integrity sha512-yvwa+aCyYI/UjeD39BnpMypG8N06l86wIDW1/PAc6ihBRnodIfZDwccxQN3n1t74wduzaz74m4ZMHZnB06567Q==
|
||||
|
||||
"@rushstack/eslint-patch@^1.0.8":
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323"
|
||||
|
|
@ -7611,6 +7681,14 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.2.21.tgz#9f35a5643129df132cf3b5c1ec64046ea1af0650"
|
||||
integrity sha512-yd+9qKmJxm496BOV9CMNaey8TWsikaZOwMRwPHQIjcOJM9oV+fi9ZMNw3JsVnbEEbo2gRTDnGEBv8pjyn67hNg==
|
||||
|
||||
"@types/chrome@^0.0.197":
|
||||
version "0.0.197"
|
||||
resolved "https://registry.yarnpkg.com/@types/chrome/-/chrome-0.0.197.tgz#c1b50cdb72ee40f9bc1411506031a9f8a925ab35"
|
||||
integrity sha512-m1NfS5bOjaypyqQfaX6CxmJodZVcvj5+Mt/K94EBHkflYjPNmXHAzbxfifdLMa0YM3PDyOxohoTS5ug/e6p5jA==
|
||||
dependencies:
|
||||
"@types/filesystem" "*"
|
||||
"@types/har-format" "*"
|
||||
|
||||
"@types/cls-hooked@^4.2.1":
|
||||
version "4.3.3"
|
||||
resolved "https://registry.yarnpkg.com/@types/cls-hooked/-/cls-hooked-4.3.3.tgz#c09e2f8dc62198522eaa18a5b6b873053154bd00"
|
||||
|
|
@ -7667,6 +7745,13 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.12.tgz#6b2c510a7ad7039e98e7b8d3d6598f4359e5c080"
|
||||
integrity sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==
|
||||
|
||||
"@types/debug@^4.1.0":
|
||||
version "4.1.7"
|
||||
resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82"
|
||||
integrity sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==
|
||||
dependencies:
|
||||
"@types/ms" "*"
|
||||
|
||||
"@types/diff-match-patch@^1.0.32":
|
||||
version "1.0.32"
|
||||
resolved "https://registry.yarnpkg.com/@types/diff-match-patch/-/diff-match-patch-1.0.32.tgz#d9c3b8c914aa8229485351db4865328337a3d09f"
|
||||
|
|
@ -7751,11 +7836,28 @@
|
|||
"@types/qs" "*"
|
||||
"@types/serve-static" "*"
|
||||
|
||||
"@types/filesystem@*":
|
||||
version "0.0.32"
|
||||
resolved "https://registry.yarnpkg.com/@types/filesystem/-/filesystem-0.0.32.tgz#307df7cc084a2293c3c1a31151b178063e0a8edf"
|
||||
integrity sha512-Yuf4jR5YYMR2DVgwuCiP11s0xuVRyPKmz8vo6HBY3CGdeMj8af93CFZX+T82+VD1+UqHOxTq31lO7MI7lepBtQ==
|
||||
dependencies:
|
||||
"@types/filewriter" "*"
|
||||
|
||||
"@types/filewriter@*":
|
||||
version "0.0.29"
|
||||
resolved "https://registry.yarnpkg.com/@types/filewriter/-/filewriter-0.0.29.tgz#a48795ecadf957f6c0d10e0c34af86c098fa5bee"
|
||||
integrity sha512-BsPXH/irW0ht0Ji6iw/jJaK8Lj3FJemon2gvEqHKpCdDCeemHa+rI3WBGq5z7cDMZgoLjY40oninGxqk+8NzNQ==
|
||||
|
||||
"@types/fined@*":
|
||||
version "1.1.3"
|
||||
resolved "https://registry.yarnpkg.com/@types/fined/-/fined-1.1.3.tgz#83f03e8f0a8d3673dfcafb18fce3571f6250e1bc"
|
||||
integrity sha512-CWYnSRnun3CGbt6taXeVo2lCbuaj4mchVJ4UF/BdU5TSuIn3AmS13pGMwCsBUoehGbhZrBrpNJZSZI5EVilXww==
|
||||
|
||||
"@types/firefox-webext-browser@^94.0.0":
|
||||
version "94.0.1"
|
||||
resolved "https://registry.yarnpkg.com/@types/firefox-webext-browser/-/firefox-webext-browser-94.0.1.tgz#52afb975253dc0fd350d5d58c7fe9fd1a01f64a1"
|
||||
integrity sha512-I6iHRQJSTZ+gYt2IxdH2RRAMvcUyK8v5Ig7fHQR0IwUNYP7hz9+cziBVIKxLCO6XI7fiyRsNOWObfl3/4Js2Lg==
|
||||
|
||||
"@types/fluent-ffmpeg@^2.1.20":
|
||||
version "2.1.20"
|
||||
resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz#3b5f42fc8263761d58284fa46ee6759a64ce54ac"
|
||||
|
|
@ -7793,6 +7895,11 @@
|
|||
dependencies:
|
||||
graphql "^15.3.0"
|
||||
|
||||
"@types/har-format@*":
|
||||
version "1.2.9"
|
||||
resolved "https://registry.yarnpkg.com/@types/har-format/-/har-format-1.2.9.tgz#b9b3a9bfc33a078e7d898a00b09662910577f4a4"
|
||||
integrity sha512-rffW6MhQ9yoa75bdNi+rjZBAvu2HhehWJXlhuWXnWdENeuKe82wUgAwxYOb7KRKKmxYN+D/iRKd2NDQMLqlUmg==
|
||||
|
||||
"@types/hast@^2.0.0":
|
||||
version "2.3.4"
|
||||
resolved "https://registry.yarnpkg.com/@types/hast/-/hast-2.3.4.tgz#8aa5ef92c117d20d974a82bdfb6a648b08c0bafc"
|
||||
|
|
@ -7995,6 +8102,11 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.2.3.tgz#bbeb55fbc73f28ea6de601fbfa4613f58d785323"
|
||||
integrity sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw==
|
||||
|
||||
"@types/ms@*":
|
||||
version "0.7.31"
|
||||
resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197"
|
||||
integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==
|
||||
|
||||
"@types/nanoid@^3.0.0":
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/@types/nanoid/-/nanoid-3.0.0.tgz#c757b20f343f3a1dd76e80a9a431b6290fc20f35"
|
||||
|
|
@ -11063,6 +11175,17 @@ cliui@^7.0.2:
|
|||
strip-ansi "^6.0.0"
|
||||
wrap-ansi "^7.0.0"
|
||||
|
||||
clone-deep@^0.2.4:
|
||||
version "0.2.4"
|
||||
resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-0.2.4.tgz#4e73dd09e9fb971cc38670c5dced9c1896481cc6"
|
||||
integrity sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==
|
||||
dependencies:
|
||||
for-own "^0.1.3"
|
||||
is-plain-object "^2.0.1"
|
||||
kind-of "^3.0.2"
|
||||
lazy-cache "^1.0.3"
|
||||
shallow-clone "^0.1.2"
|
||||
|
||||
clone-deep@^4.0.1:
|
||||
version "4.0.1"
|
||||
resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-4.0.1.tgz#c19fd9bdbbf85942b4fd979c84dcf7d5f07c2387"
|
||||
|
|
@ -14103,11 +14226,23 @@ follow-redirects@^1.0.0, follow-redirects@^1.14.0, follow-redirects@^1.14.4, fol
|
|||
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5"
|
||||
integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==
|
||||
|
||||
for-in@^0.1.3:
|
||||
version "0.1.8"
|
||||
resolved "https://registry.yarnpkg.com/for-in/-/for-in-0.1.8.tgz#d8773908e31256109952b1fdb9b3fa867d2775e1"
|
||||
integrity sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==
|
||||
|
||||
for-in@^1.0.1, for-in@^1.0.2:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
|
||||
integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=
|
||||
|
||||
for-own@^0.1.3:
|
||||
version "0.1.5"
|
||||
resolved "https://registry.yarnpkg.com/for-own/-/for-own-0.1.5.tgz#5265c681a4f294dabbf17c9509b6763aa84510ce"
|
||||
integrity sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==
|
||||
dependencies:
|
||||
for-in "^1.0.1"
|
||||
|
||||
for-own@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/for-own/-/for-own-1.0.0.tgz#c63332f415cedc4b04dbfe70cf836494c53cb44b"
|
||||
|
|
@ -14275,6 +14410,15 @@ fs-extra@^0.30.0:
|
|||
path-is-absolute "^1.0.0"
|
||||
rimraf "^2.2.8"
|
||||
|
||||
fs-extra@^10.0.0:
|
||||
version "10.1.0"
|
||||
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf"
|
||||
integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==
|
||||
dependencies:
|
||||
graceful-fs "^4.2.0"
|
||||
jsonfile "^6.0.1"
|
||||
universalify "^2.0.0"
|
||||
|
||||
fs-extra@^9.0.0, fs-extra@^9.0.1, fs-extra@^9.1.0:
|
||||
version "9.1.0"
|
||||
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d"
|
||||
|
|
@ -16082,7 +16226,7 @@ is-boolean-object@^1.1.0:
|
|||
call-bind "^1.0.2"
|
||||
has-tostringtag "^1.0.0"
|
||||
|
||||
is-buffer@^1.1.5, is-buffer@~1.1.6:
|
||||
is-buffer@^1.0.2, is-buffer@^1.1.5, is-buffer@~1.1.6:
|
||||
version "1.1.6"
|
||||
resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
|
||||
integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==
|
||||
|
|
@ -16394,7 +16538,7 @@ is-plain-object@5.0.0, is-plain-object@^5.0.0:
|
|||
resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344"
|
||||
integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==
|
||||
|
||||
is-plain-object@^2.0.3, is-plain-object@^2.0.4:
|
||||
is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4:
|
||||
version "2.0.4"
|
||||
resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677"
|
||||
integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==
|
||||
|
|
@ -17587,6 +17731,13 @@ keyv@^3.0.0:
|
|||
dependencies:
|
||||
json-buffer "3.0.0"
|
||||
|
||||
kind-of@^2.0.1:
|
||||
version "2.0.1"
|
||||
resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-2.0.1.tgz#018ec7a4ce7e3a86cb9141be519d24c8faa981b5"
|
||||
integrity sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==
|
||||
dependencies:
|
||||
is-buffer "^1.0.2"
|
||||
|
||||
kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0:
|
||||
version "3.2.2"
|
||||
resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
|
||||
|
|
@ -17687,6 +17838,16 @@ lazy-ass@^1.6.0:
|
|||
resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513"
|
||||
integrity sha1-eZllXoZGwX8In90YfRUNMyTVRRM=
|
||||
|
||||
lazy-cache@^0.2.3:
|
||||
version "0.2.7"
|
||||
resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-0.2.7.tgz#7feddf2dcb6edb77d11ef1d117ab5ffdf0ab1b65"
|
||||
integrity sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==
|
||||
|
||||
lazy-cache@^1.0.3:
|
||||
version "1.0.4"
|
||||
resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-1.0.4.tgz#a1d78fc3a50474cb80845d3b3b6e1da49a446e8e"
|
||||
integrity sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==
|
||||
|
||||
lazy-universal-dotenv@^3.0.1:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/lazy-universal-dotenv/-/lazy-universal-dotenv-3.0.1.tgz#a6c8938414bca426ab8c9463940da451a911db38"
|
||||
|
|
@ -18608,6 +18769,15 @@ meow@^8.0.0:
|
|||
type-fest "^0.18.0"
|
||||
yargs-parser "^20.2.3"
|
||||
|
||||
merge-deep@^3.0.1:
|
||||
version "3.0.3"
|
||||
resolved "https://registry.yarnpkg.com/merge-deep/-/merge-deep-3.0.3.tgz#1a2b2ae926da8b2ae93a0ac15d90cd1922766003"
|
||||
integrity sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==
|
||||
dependencies:
|
||||
arr-union "^3.1.0"
|
||||
clone-deep "^0.2.4"
|
||||
kind-of "^3.0.2"
|
||||
|
||||
merge-descriptors@1.0.1:
|
||||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61"
|
||||
|
|
@ -18949,6 +19119,14 @@ mixin-deep@^1.2.0:
|
|||
for-in "^1.0.2"
|
||||
is-extendable "^1.0.1"
|
||||
|
||||
mixin-object@^2.0.1:
|
||||
version "2.0.1"
|
||||
resolved "https://registry.yarnpkg.com/mixin-object/-/mixin-object-2.0.1.tgz#4fb949441dab182540f1fe035ba60e1947a5e57e"
|
||||
integrity sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==
|
||||
dependencies:
|
||||
for-in "^0.1.3"
|
||||
is-extendable "^0.1.1"
|
||||
|
||||
mkdirp-classic@^0.5.2:
|
||||
version "0.5.3"
|
||||
resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
|
||||
|
|
@ -19383,7 +19561,7 @@ node-fetch@2.6.1:
|
|||
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
|
||||
integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
|
||||
|
||||
node-fetch@2.6.7, node-fetch@^2.3.0, node-fetch@^2.6.1, node-fetch@^2.6.7:
|
||||
node-fetch@2.6.7, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7:
|
||||
version "2.6.7"
|
||||
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
|
||||
integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
|
||||
|
|
@ -21451,6 +21629,63 @@ puppeteer-core@^19.1.1:
|
|||
unbzip2-stream "1.4.3"
|
||||
ws "8.9.0"
|
||||
|
||||
puppeteer-extra-plugin-adblocker@^2.13.5:
|
||||
version "2.13.5"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.13.5.tgz#c86ce94873bf6fe500555d3972eccdcca4914f6f"
|
||||
integrity sha512-HMVWLA1MLrzIGr/A71PYAWZEHENqQOEaQQHtPje0uSLc6QPOQY5tbbocx4BsUiQL2V1FwgT21UU09P5lV4vrZw==
|
||||
dependencies:
|
||||
"@cliqz/adblocker-puppeteer" "1.23.8"
|
||||
debug "^4.1.1"
|
||||
node-fetch "^2.6.0"
|
||||
puppeteer-extra-plugin "^3.2.2"
|
||||
|
||||
puppeteer-extra-plugin-stealth@^2.11.1:
|
||||
version "2.11.1"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.1.tgz#7d56a27a986cb5eb69dca3c65695ad6444f4e822"
|
||||
integrity sha512-n0wdC0Ilc9tk5L6FWLyd0P2gT8b2fp+2NuB+KB0oTSw3wXaZ0D6WNakjJsayJ4waGzIJFCUHkmK9zgx5NKMoFw==
|
||||
dependencies:
|
||||
debug "^4.1.1"
|
||||
puppeteer-extra-plugin "^3.2.2"
|
||||
puppeteer-extra-plugin-user-preferences "^2.4.0"
|
||||
|
||||
puppeteer-extra-plugin-user-data-dir@^2.4.0:
|
||||
version "2.4.0"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.0.tgz#20e87582482b61e497abd96fec452f63bd2d9123"
|
||||
integrity sha512-qrhYPTGIqzL2hpeJ5DXjf8xMy5rt1UvcqSgpGTTOUOjIMz1ROWnKHjBoE9fNBJ4+ToRZbP8MzIDXWlEk/e1zJA==
|
||||
dependencies:
|
||||
debug "^4.1.1"
|
||||
fs-extra "^10.0.0"
|
||||
puppeteer-extra-plugin "^3.2.2"
|
||||
rimraf "^3.0.2"
|
||||
|
||||
puppeteer-extra-plugin-user-preferences@^2.4.0:
|
||||
version "2.4.0"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.0.tgz#8b75bc39c3de9913e236ae1d982a24711d84ba6f"
|
||||
integrity sha512-4XxMhMkJ+qqLsPY9ULF90qS9Bj1Qrwwgp1TY9zTdp1dJuy7QSgYE7xlyamq3cKrRuzg3QUOqygJo52sVeXSg5A==
|
||||
dependencies:
|
||||
debug "^4.1.1"
|
||||
deepmerge "^4.2.2"
|
||||
puppeteer-extra-plugin "^3.2.2"
|
||||
puppeteer-extra-plugin-user-data-dir "^2.4.0"
|
||||
|
||||
puppeteer-extra-plugin@^3.2.2:
|
||||
version "3.2.2"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.2.tgz#3c02c0a10f8eadf32e7debb7ee24105a53afc17b"
|
||||
integrity sha512-0uatQxzuVn8yegbrEwSk03wvwpMB5jNs7uTTnermylLZzoT+1rmAQaJXwlS3+vADUbw6ELNgNEHC7Skm0RqHbQ==
|
||||
dependencies:
|
||||
"@types/debug" "^4.1.0"
|
||||
debug "^4.1.1"
|
||||
merge-deep "^3.0.1"
|
||||
|
||||
puppeteer-extra@^3.3.4:
|
||||
version "3.3.4"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer-extra/-/puppeteer-extra-3.3.4.tgz#e0ecf021783d1112b6b0db20546d5022e632ed55"
|
||||
integrity sha512-fN5pHvSMJ8d1o7Z8wLLTQOUBpORD2BcFn+KDs7QnkGZs9SV69hcUcce67vX4L4bNSEG3A0P6Osrv+vWNhhdm8w==
|
||||
dependencies:
|
||||
"@types/debug" "^4.1.0"
|
||||
debug "^4.1.1"
|
||||
deepmerge "^4.2.2"
|
||||
|
||||
puppeteer@^10.1.0:
|
||||
version "10.4.0"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-10.4.0.tgz#a6465ff97fda0576c4ac29601406f67e6fea3dc7"
|
||||
|
|
@ -22939,6 +23174,16 @@ sha.js@^2.4.0, sha.js@^2.4.11, sha.js@^2.4.8:
|
|||
inherits "^2.0.1"
|
||||
safe-buffer "^5.0.1"
|
||||
|
||||
shallow-clone@^0.1.2:
|
||||
version "0.1.2"
|
||||
resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-0.1.2.tgz#5909e874ba77106d73ac414cfec1ffca87d97060"
|
||||
integrity sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==
|
||||
dependencies:
|
||||
is-extendable "^0.1.1"
|
||||
kind-of "^2.0.1"
|
||||
lazy-cache "^0.2.3"
|
||||
mixin-object "^2.0.1"
|
||||
|
||||
shallow-clone@^3.0.0:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3"
|
||||
|
|
@ -24277,6 +24522,18 @@ title-case@^3.0.3:
|
|||
dependencies:
|
||||
tslib "^2.0.3"
|
||||
|
||||
tldts-core@^5.7.100:
|
||||
version "5.7.100"
|
||||
resolved "https://registry.yarnpkg.com/tldts-core/-/tldts-core-5.7.100.tgz#6144104277a3c4500ec395220d8e03c16fcdfaf7"
|
||||
integrity sha512-56+vie1oPcJZQiPfnvIIpbyTttUketsjV7lrw/hkMMa/EACPjjDctobWwF3153gR2l+c9O+nYiHkXIL1Cmr9eQ==
|
||||
|
||||
tldts-experimental@^5.6.21:
|
||||
version "5.7.100"
|
||||
resolved "https://registry.yarnpkg.com/tldts-experimental/-/tldts-experimental-5.7.100.tgz#fb428cf20735952c299e15e864de63ecc55fb0a7"
|
||||
integrity sha512-BjdXE3YU3cXbASRXydXnzOCSc+G/bM38/5snbxwcIYaRh3AApEtD4lHVl3236x+79T/V94lKwqBXYT47EAR+TA==
|
||||
dependencies:
|
||||
tldts-core "^5.7.100"
|
||||
|
||||
tmp@^0.0.33:
|
||||
version "0.0.33"
|
||||
resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"
|
||||
|
|
|
|||
Loading…
Reference in a new issue