Merge pull request #978 from omnivore-app/fix/content-fetch-hosts

Use ScrapingBee for some hosts

packages/content-fetch/fetch-content.js

@@ -18,6 +18,7 @@ const { pdfHandler } = require('./pdf-handler');
 const { mediumHandler } = require('./medium-handler');
 const { derstandardHandler } = require('./derstandard-handler');
 const { imageHandler } = require('./image-handler');
+const { scrapingBeeHandler } = require('./scrapingBee-handler')
 
 const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
 const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
@@ -215,6 +216,7 @@ const handlers = {
   'medium': mediumHandler,
   'derstandard': derstandardHandler,
   'image': imageHandler,
+  'scrapingBee': scrapingBeeHandler,
 };
 
 

packages/content-fetch/scrapingBee-handler.js

@@ -0,0 +1,44 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const axios = require('axios');
+const { parseHTML } = require('linkedom');
+
+const os = require('os');
+
+exports.scrapingBeeHandler = {
+
+  shouldPrehandle: (url, env) => {
+    const u = new URL(url);
+    const hostnames = [
+      'nytimes.com',
+      'news.google.com',
+    ]
+
+    return hostnames.some((h) => u.hostname.endsWith(h))
+  },
+
+  prehandle: async (url, env) => {
+    console.log('prehandling url with scrapingbee', url)
+
+    try {
+      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+        params: {
+          'api_key': process.env.SCRAPINGBEE_API_KEY,
+          'url': url,
+          'return_page_source': true,
+          'block_ads': true,
+          'block_resources': false,
+        }
+      })
+      const dom = parseHTML(response.data).document;
+      return { title: dom.title, content: response.data, url: url }
+    } catch (error) {
+      console.error('error prehandling url w/scrapingbee', error)
+      throw error
+    }
+  }
+}

packages/content-fetch/t-dot-co-handler.js

@@ -12,7 +12,6 @@ exports.tDotCoHandler = {
 
   shouldResolve: function (url, env) {
     const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/;
-    console.log('should preresolve?', T_DOT_CO_URL_MATCH.test(url), url)
     return T_DOT_CO_URL_MATCH.test(url);
   },
 

packages/puppeteer-parse/index.js

@@ -23,6 +23,7 @@ const { pdfHandler } = require('./pdf-handler');
 const { mediumHandler } = require('./medium-handler');
 const { derstandardHandler } = require('./derstandard-handler');
 const { imageHandler } = require('./image-handler');
+const { scrappingBeeHandler } = require('./scrapingBee-handler');
 
 const chromium = require('chrome-aws-lambda');
 const puppeteer = require('puppeteer-core');
@@ -265,6 +266,7 @@ const handlers = {
   'medium': mediumHandler,
   'derstandard': derstandardHandler,
   'image': imageHandler,
+  'scrappingBee': scrappingBeeHandler,
 };
 
 /**

packages/puppeteer-parse/scrapingBee-handler.js

@@ -0,0 +1,44 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const axios = require('axios');
+const { parseHTML } = require('linkedom');
+
+const os = require('os');
+
+exports.scrapingBeeHandler = {
+
+  shouldPrehandle: (url, env) => {
+    const u = new URL(url);
+    const hostnames = [
+      'nytimes.com',
+      'news.google.com',
+    ]
+
+    return hostnames.some((h) => u.hostname.endsWith(h))
+  },
+
+  prehandle: async (url, env) => {
+    console.log('prehandling url with scrapingbee', url)
+
+    try {
+      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+        params: {
+          'api_key': process.env.SCRAPINGBEE_API_KEY,
+          'url': url,
+          'return_page_source': true,
+          'block_ads': true,
+          'block_resources': false,
+        }
+      })
+      const dom = parseHTML(response.data).document;
+      return { title: dom.title, content: response.data, url: url }
+    } catch (error) {
+      console.error('error prehandling url w/scrapingbee', error)
+      throw error
+    }
+  }
+}