From f17ee64676359da9b4315576c1e349a73c1b78e7 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Sat, 16 Jul 2022 14:08:01 -0700
Subject: [PATCH] Use ScrapingBee for some hosts

---
 packages/content-fetch/fetch-content.js       |  2 +
 packages/content-fetch/scrapingBee-handler.js | 44 +++++++++++++++++++
 packages/content-fetch/t-dot-co-handler.js    |  1 -
 packages/puppeteer-parse/index.js             |  2 +
 .../puppeteer-parse/scrapingBee-handler.js    | 44 +++++++++++++++++++
 5 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 packages/content-fetch/scrapingBee-handler.js
 create mode 100644 packages/puppeteer-parse/scrapingBee-handler.js

diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js
index e70e2acc3..6a87f912a 100644
--- a/packages/content-fetch/fetch-content.js
+++ b/packages/content-fetch/fetch-content.js
@@ -18,6 +18,7 @@ const { pdfHandler } = require('./pdf-handler');
 const { mediumHandler } = require('./medium-handler');
 const { derstandardHandler } = require('./derstandard-handler');
 const { imageHandler } = require('./image-handler');
+const { scrapingBeeHandler } = require('./scrapingBee-handler')
 
 const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
 const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
@@ -215,6 +216,7 @@ const handlers = {
   'medium': mediumHandler,
   'derstandard': derstandardHandler,
   'image': imageHandler,
+  'scrapingBee': scrapingBeeHandler,
 };
 
 
diff --git a/packages/content-fetch/scrapingBee-handler.js b/packages/content-fetch/scrapingBee-handler.js
new file mode 100644
index 000000000..6563fca44
--- /dev/null
+++ b/packages/content-fetch/scrapingBee-handler.js
@@ -0,0 +1,44 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const axios = require('axios');
+const { parseHTML } = require('linkedom');
+
+const os = require('os');
+
+exports.scrapingBeeHandler = {
+
+  shouldPrehandle: (url, env) => {
+    const u = new URL(url);
+    const hostnames = [
+      'nytimes.com',
+      'news.google.com',
+    ]
+
+    return hostnames.some((h) => u.hostname.endsWith(h))
+  },
+
+  prehandle: async (url, env) => {
+    console.log('prehandling url with scrapingbee', url)
+
+    try {
+      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+        params: {
+          'api_key': process.env.SCRAPINGBEE_API_KEY,
+          'url': url,
+          'return_page_source': true,
+          'block_ads': true,
+          'block_resources': false,
+        }
+      })
+      const dom = parseHTML(response.data).document;
+      return { title: dom.title, content: response.data, url: url }
+    } catch (error) {
+      console.error('error prehandling url w/scrapingbee', error)
+      throw error
+    }
+  }
+}
diff --git a/packages/content-fetch/t-dot-co-handler.js b/packages/content-fetch/t-dot-co-handler.js
index cbbfb304a..170f97fb7 100644
--- a/packages/content-fetch/t-dot-co-handler.js
+++ b/packages/content-fetch/t-dot-co-handler.js
@@ -12,7 +12,6 @@ exports.tDotCoHandler = {
 
   shouldResolve: function (url, env) {
     const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/;
-    console.log('should preresolve?', T_DOT_CO_URL_MATCH.test(url), url)
     return T_DOT_CO_URL_MATCH.test(url);
   },
 
diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js
index 695b5a1aa..23e743f57 100644
--- a/packages/puppeteer-parse/index.js
+++ b/packages/puppeteer-parse/index.js
@@ -23,6 +23,7 @@ const { pdfHandler } = require('./pdf-handler');
 const { mediumHandler } = require('./medium-handler');
 const { derstandardHandler } = require('./derstandard-handler');
 const { imageHandler } = require('./image-handler');
+const { scrappingBeeHandler } = require('./scrapingBee-handler');
 
 const chromium = require('chrome-aws-lambda');
 const puppeteer = require('puppeteer-core');
@@ -265,6 +266,7 @@ const handlers = {
   'medium': mediumHandler,
   'derstandard': derstandardHandler,
   'image': imageHandler,
+  'scrappingBee': scrappingBeeHandler,
 };
 
 /**
diff --git a/packages/puppeteer-parse/scrapingBee-handler.js b/packages/puppeteer-parse/scrapingBee-handler.js
new file mode 100644
index 000000000..6563fca44
--- /dev/null
+++ b/packages/puppeteer-parse/scrapingBee-handler.js
@@ -0,0 +1,44 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const axios = require('axios');
+const { parseHTML } = require('linkedom');
+
+const os = require('os');
+
+exports.scrapingBeeHandler = {
+
+  shouldPrehandle: (url, env) => {
+    const u = new URL(url);
+    const hostnames = [
+      'nytimes.com',
+      'news.google.com',
+    ]
+
+    return hostnames.some((h) => u.hostname.endsWith(h))
+  },
+
+  prehandle: async (url, env) => {
+    console.log('prehandling url with scrapingbee', url)
+
+    try {
+      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+        params: {
+          'api_key': process.env.SCRAPINGBEE_API_KEY,
+          'url': url,
+          'return_page_source': true,
+          'block_ads': true,
+          'block_resources': false,
+        }
+      })
+      const dom = parseHTML(response.data).document;
+      return { title: dom.title, content: response.data, url: url }
+    } catch (error) {
+      console.error('error prehandling url w/scrapingbee', error)
+      throw error
+    }
+  }
+}