From a4ad78652a8e5c8d83793f70f12084ca5352e3b0 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 14 Jun 2022 13:30:32 -0700 Subject: [PATCH 1/5] Allow specifying a proxy url when launching puppeteer --- packages/content-fetch/fetch-content.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js index f90c753ce..be6b65e68 100644 --- a/packages/content-fetch/fetch-content.js +++ b/packages/content-fetch/fetch-content.js @@ -86,8 +86,9 @@ const getBrowserPromise = (async () => { '--no-sandbox', '--no-zygote', '--use-gl=swiftshader', - '--window-size=1920,1080', // https://source.chromium.org/search?q=lang:cpp+symbol:kWindowSize&ss=chromium - ], + process.env.PROXY_URL ? '--proxy-server=' + process.env.PROXY_URL : undefined, + '--window-size=1920,1080', + ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, executablePath: process.env.CHROMIUM_PATH , headless: true, @@ -177,7 +178,6 @@ const sendCreateArticleMutation = async (userId, input) => { 'Content-Type': 'application/json', }, }); - console.log('response', response); return response.data.data.createArticle; }; From 814f6098a3b6c33cf68c7118764898a830e1b04e Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 14 Jun 2022 14:27:19 -0700 Subject: [PATCH 2/5] Log proxy url --- packages/content-fetch/fetch-content.js | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js index be6b65e68..cf862d558 100644 --- a/packages/content-fetch/fetch-content.js +++ b/packages/content-fetch/fetch-content.js @@ -63,6 +63,7 @@ const enableJavascriptForUrl = async (url) => { // launch Puppeteer const getBrowserPromise = (async () => { + console.log("starting with proxy url", process.env.PROXY_URL) return puppeteer.launch({ args: [ '--allow-running-insecure-content', @@ -328,20 +329,6 @@ async function fetchContent(req, res) { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; console.log(`parse-page`, logRecord); - - // return res.send({ - // url: finalUrl, - // articleSavingRequestId, - // preparedDocument: { - // document: content, - // pageInfo: { - // title, - // canonicalUrl: finalUrl, - // }, - // }, - // skipParsing: !content, - // timeTaken: Date.now() - functionStartTime, - // }) } } catch (e) { console.log('error', e) From 159a7f89502c5a898fd27abdee98a441c11e5246 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 14 Jun 2022 16:06:01 -0700 Subject: [PATCH 3/5] Fallback to scrapingbee if a page cant fetch content --- packages/content-fetch/fetch-content.js | 41 ++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js index cf862d558..8087f69bc 100644 --- a/packages/content-fetch/fetch-content.js +++ b/packages/content-fetch/fetch-content.js @@ -28,6 +28,8 @@ const NON_SCRIPT_HOSTS= ['medium.com'] const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; +const { parseHTML } = require('linkedom'); + // Add stealth plugin to hide puppeteer usage const StealthPlugin = require('puppeteer-extra-plugin-stealth'); puppeteer.use(StealthPlugin()); @@ -47,6 +49,21 @@ const userAgentForUrl = (url) => { return DESKTOP_USER_AGENT }; +const fetchContentWithScrapingBee = async (url) => { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + 'api_key': process.env.SCRAPINGBEE_API_KEY, + 'url': url, + 'render_js': 'false', + 'premium_proxy': 'true', + 'country_code':'us' + } + }) + + const dom = parseHTML(response.data).document; + return { title: dom.title, domContent: dom.documentElement.outerHTML, url: url } +} + const enableJavascriptForUrl = async (url) => { try { const u = new URL(url); @@ -304,8 +321,15 @@ async function fetchContent(req, res) { } else { if (!content || !title) { const result = await retrieveHtml(page); - title = result.title; - content = result.domContent; + if (result.isBlocked) { + console.log("PAGE IS BLOCKED", url) + const sbResult = await fetchContentWithScrapingBee(url) + title = sbResult.title + content = sbResult.domContent + } else { + title = result.title; + content = result.domContent; + } } else { console.log('using prefetched content and title'); console.log(content); @@ -407,7 +431,8 @@ async function retrievePage(url) { logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; const context = await browser.createIncognitoBrowserContext(); - const page = await context.newPage(); + const page = await context.newPage() + if (!enableJavascriptForUrl(url)) { page.setJavaScriptEnabled(false) } @@ -547,7 +572,7 @@ async function retrieveHtml(page) { } })(); }), - page.waitForTimeout(1000), //5 second timeout + page.waitForTimeout(1000), ]); logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; @@ -615,6 +640,11 @@ async function retrieveHtml(page) { } } }); + + if (document.querySelector('[data-translate="managed_checking_msg"]')) { + return 'IS_BLOCKED' + } + return document.documentElement.outerHTML; }, iframes); logRecord.puppeteerSuccess = true; @@ -635,6 +665,9 @@ async function retrieveHtml(page) { }; } } + if (domContent == 'IS_BLOCKED') { + return { isBlocked: true }; + } return { domContent, title }; } From be2801477b4dc39b44f0ace384222778cee6343f Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 14 Jun 2022 16:13:43 -0700 Subject: [PATCH 4/5] Add some extra debugging --- packages/content-fetch/fetch-content.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js index 8087f69bc..60ec985ec 100644 --- a/packages/content-fetch/fetch-content.js +++ b/packages/content-fetch/fetch-content.js @@ -665,6 +665,9 @@ async function retrieveHtml(page) { }; } } + console.log('DOM CONTENT') + console.log(domContent) + if (domContent == 'IS_BLOCKED') { return { isBlocked: true }; } From ec5bbb8350fb3db08760c027b149d2f17a6fe76e Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 14 Jun 2022 16:23:07 -0700 Subject: [PATCH 5/5] Return URL as a string --- packages/content-fetch/medium-handler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/content-fetch/medium-handler.js b/packages/content-fetch/medium-handler.js index 8d0443447..c119486f1 100644 --- a/packages/content-fetch/medium-handler.js +++ b/packages/content-fetch/medium-handler.js @@ -22,7 +22,7 @@ exports.mediumHandler = { try { const res = new URL(url); res.searchParams.delete('source'); - return { url: res } + return { url: res.toString() } } catch (error) { console.error('error prehandling medium url', error) throw error