diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js index f90c753ce..60ec985ec 100644 --- a/packages/content-fetch/fetch-content.js +++ b/packages/content-fetch/fetch-content.js @@ -28,6 +28,8 @@ const NON_SCRIPT_HOSTS= ['medium.com'] const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; +const { parseHTML } = require('linkedom'); + // Add stealth plugin to hide puppeteer usage const StealthPlugin = require('puppeteer-extra-plugin-stealth'); puppeteer.use(StealthPlugin()); @@ -47,6 +49,21 @@ const userAgentForUrl = (url) => { return DESKTOP_USER_AGENT }; +const fetchContentWithScrapingBee = async (url) => { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + 'api_key': process.env.SCRAPINGBEE_API_KEY, + 'url': url, + 'render_js': 'false', + 'premium_proxy': 'true', + 'country_code':'us' + } + }) + + const dom = parseHTML(response.data).document; + return { title: dom.title, domContent: dom.documentElement.outerHTML, url: url } +} + const enableJavascriptForUrl = async (url) => { try { const u = new URL(url); @@ -63,6 +80,7 @@ const enableJavascriptForUrl = async (url) => { // launch Puppeteer const getBrowserPromise = (async () => { + console.log("starting with proxy url", process.env.PROXY_URL) return puppeteer.launch({ args: [ '--allow-running-insecure-content', @@ -86,8 +104,9 @@ const getBrowserPromise = (async () => { '--no-sandbox', '--no-zygote', '--use-gl=swiftshader', - '--window-size=1920,1080', // https://source.chromium.org/search?q=lang:cpp+symbol:kWindowSize&ss=chromium - ], + process.env.PROXY_URL ? '--proxy-server=' + process.env.PROXY_URL : undefined, + '--window-size=1920,1080', + ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, executablePath: process.env.CHROMIUM_PATH , headless: true, @@ -177,7 +196,6 @@ const sendCreateArticleMutation = async (userId, input) => { 'Content-Type': 'application/json', }, }); - console.log('response', response); return response.data.data.createArticle; }; @@ -303,8 +321,15 @@ async function fetchContent(req, res) { } else { if (!content || !title) { const result = await retrieveHtml(page); - title = result.title; - content = result.domContent; + if (result.isBlocked) { + console.log("PAGE IS BLOCKED", url) + const sbResult = await fetchContentWithScrapingBee(url) + title = sbResult.title + content = sbResult.domContent + } else { + title = result.title; + content = result.domContent; + } } else { console.log('using prefetched content and title'); console.log(content); @@ -328,20 +353,6 @@ async function fetchContent(req, res) { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; console.log(`parse-page`, logRecord); - - // return res.send({ - // url: finalUrl, - // articleSavingRequestId, - // preparedDocument: { - // document: content, - // pageInfo: { - // title, - // canonicalUrl: finalUrl, - // }, - // }, - // skipParsing: !content, - // timeTaken: Date.now() - functionStartTime, - // }) } } catch (e) { console.log('error', e) @@ -420,7 +431,8 @@ async function retrievePage(url) { logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; const context = await browser.createIncognitoBrowserContext(); - const page = await context.newPage(); + const page = await context.newPage() + if (!enableJavascriptForUrl(url)) { page.setJavaScriptEnabled(false) } @@ -560,7 +572,7 @@ async function retrieveHtml(page) { } })(); }), - page.waitForTimeout(1000), //5 second timeout + page.waitForTimeout(1000), ]); logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; @@ -628,6 +640,11 @@ async function retrieveHtml(page) { } } }); + + if (document.querySelector('[data-translate="managed_checking_msg"]')) { + return 'IS_BLOCKED' + } + return document.documentElement.outerHTML; }, iframes); logRecord.puppeteerSuccess = true; @@ -648,6 +665,12 @@ async function retrieveHtml(page) { }; } } + console.log('DOM CONTENT') + console.log(domContent) + + if (domContent == 'IS_BLOCKED') { + return { isBlocked: true }; + } return { domContent, title }; } diff --git a/packages/content-fetch/medium-handler.js b/packages/content-fetch/medium-handler.js index 8d0443447..c119486f1 100644 --- a/packages/content-fetch/medium-handler.js +++ b/packages/content-fetch/medium-handler.js @@ -22,7 +22,7 @@ exports.mediumHandler = { try { const res = new URL(url); res.searchParams.delete('source'); - return { url: res } + return { url: res.toString() } } catch (error) { console.error('error prehandling medium url', error) throw error