Merge pull request #798 from omnivore-app/fix/proxy-urls

Allow specifying a proxy url when launching puppeteer

packages/content-fetch/fetch-content.js

@@ -28,6 +28,8 @@ const NON_SCRIPT_HOSTS= ['medium.com']
 
 const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf'];
 
+const { parseHTML } = require('linkedom');
+
 // Add stealth plugin to hide puppeteer usage
 const StealthPlugin = require('puppeteer-extra-plugin-stealth');
 puppeteer.use(StealthPlugin());
@@ -47,6 +49,21 @@ const userAgentForUrl = (url) => {
   return DESKTOP_USER_AGENT
 };
 
+const fetchContentWithScrapingBee = async (url) => {
+  const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+    params: {
+      'api_key':  process.env.SCRAPINGBEE_API_KEY,
+      'url': url,
+      'render_js': 'false',
+      'premium_proxy': 'true',
+      'country_code':'us'
+    }
+  })
+
+  const dom = parseHTML(response.data).document;
+  return { title: dom.title, domContent: dom.documentElement.outerHTML, url: url }
+}
+
 const enableJavascriptForUrl = async (url) => {
   try {
     const u = new URL(url);
@@ -63,6 +80,7 @@ const enableJavascriptForUrl = async (url) => {
 
 // launch Puppeteer
 const getBrowserPromise = (async () => {
+  console.log("starting with proxy url", process.env.PROXY_URL)
   return puppeteer.launch({
     args: [
       '--allow-running-insecure-content',
@@ -86,8 +104,9 @@ const getBrowserPromise = (async () => {
       '--no-sandbox',
       '--no-zygote',
       '--use-gl=swiftshader',
-      '--window-size=1920,1080', // https://source.chromium.org/search?q=lang:cpp+symbol:kWindowSize&ss=chromium
-    ],
+      process.env.PROXY_URL ? '--proxy-server=' + process.env.PROXY_URL : undefined,
+      '--window-size=1920,1080',
+    ].filter((item) => !!item),
     defaultViewport: { height: 1080, width: 1920 },
     executablePath: process.env.CHROMIUM_PATH ,
     headless: true,
@@ -177,7 +196,6 @@ const sendCreateArticleMutation = async (userId, input) => {
       'Content-Type': 'application/json',
     },
   });
-  console.log('response', response);
   return response.data.data.createArticle;
 };
 
@@ -303,8 +321,15 @@ async function fetchContent(req, res) {
     } else {
       if (!content || !title) {
         const result = await retrieveHtml(page);
-        title = result.title;
-        content = result.domContent;
+        if (result.isBlocked) {
+          console.log("PAGE IS BLOCKED", url)
+          const sbResult = await fetchContentWithScrapingBee(url)
+          title = sbResult.title
+          content = sbResult.domContent
+        } else {
+          title = result.title;
+          content = result.domContent;
+        }
       } else {
         console.log('using prefetched content and title');
         console.log(content);
@@ -328,20 +353,6 @@ async function fetchContent(req, res) {
       logRecord.totalTime = Date.now() - functionStartTime;
       logRecord.result = apiResponse.createArticle;
       console.log(`parse-page`, logRecord);
-
-      // return res.send({
-      //     url: finalUrl,
-      //     articleSavingRequestId,
-      //     preparedDocument: {
-      //       document: content,
-      //       pageInfo: {
-      //         title,
-      //         canonicalUrl: finalUrl,
-      //       },
-      //     },
-      //     skipParsing: !content,
-      //     timeTaken: Date.now() - functionStartTime,
-      //   })
     }
   } catch (e) {
     console.log('error', e)
@@ -420,7 +431,8 @@ async function retrievePage(url) {
   logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime };
 
   const context = await browser.createIncognitoBrowserContext();
-  const page = await context.newPage();
+  const page = await context.newPage()
+
   if (!enableJavascriptForUrl(url)) {
     page.setJavaScriptEnabled(false)
   }
@@ -560,7 +572,7 @@ async function retrieveHtml(page) {
           }
         })();
       }),
-      page.waitForTimeout(1000), //5 second timeout
+      page.waitForTimeout(1000),
     ]);
     logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart };
 
@@ -628,6 +640,11 @@ async function retrieveHtml(page) {
           }
         }
       });
+
+      if (document.querySelector('[data-translate="managed_checking_msg"]')) {
+        return 'IS_BLOCKED'
+      }
+
       return document.documentElement.outerHTML;
     }, iframes);
     logRecord.puppeteerSuccess = true;
@@ -648,6 +665,12 @@ async function retrieveHtml(page) {
       };
     }
   }
+  console.log('DOM CONTENT')
+  console.log(domContent)
+
+  if (domContent == 'IS_BLOCKED') {
+    return { isBlocked: true };
+  }
   return { domContent, title };
 }
 

packages/content-fetch/medium-handler.js

@@ -22,7 +22,7 @@ exports.mediumHandler = {
     try {
       const res = new URL(url);
       res.searchParams.delete('source');
-      return { url: res }
+      return { url: res.toString() }
     } catch (error) {
       console.error('error prehandling medium url', error)
       throw error