Add createImageProxyUrl callback to the readability in puppeteer-parse service

packages/puppeteer-parse/index.js

@@ -3,6 +3,9 @@
 /* eslint-disable @typescript-eslint/explicit-function-return-type */
 /* eslint-disable @typescript-eslint/no-var-requires */
 /* eslint-disable @typescript-eslint/no-require-imports */
+const { encode } = require("urlsafe-base64");
+const crypto = require("crypto");
+
 const Url = require('url');
 // const puppeteer = require('puppeteer-extra');
 const axios = require('axios');
@@ -23,6 +26,7 @@ puppeteer.use(StealthPlugin());
 
 // Add adblocker plugin to block all ads and trackers (saves bandwidth)
 const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker');
+const createDOMPurify = require("dompurify");
 puppeteer.use(AdblockerPlugin({ blockTrackers: true }));
 
 const storage = new Storage();
@@ -782,10 +786,97 @@ async function preview(req, res) {
   return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`);
 }
 
-async function getReadabilityResult(url, domContent) {
-  const document = parseHTML(domContent).document;
-  const readability = new Readability(document, { url });
-  return readability.parse();
+const DOM_PURIFY_CONFIG = {
+  ADD_TAGS: ['iframe'],
+  ADD_ATTR: ['allow', 'allowfullscreen', 'frameborder', 'scrolling'],
+  FORBID_ATTR: [
+    'data-ml-dynamic',
+    'data-ml-dynamic-type',
+    'data-orig-url',
+    'data-ml-id',
+    'data-ml',
+    'data-xid',
+    'data-feature',
+  ],
+}
+
+function domPurifySanitizeHook(node, data) {
+  if (data.tagName === 'iframe') {
+    const urlRegex = /^(https?:)?\/\/www\.youtube(-nocookie)?\.com\/embed\//i
+    const src = node.getAttribute('src') || ''
+    const dataSrc = node.getAttribute('data-src') || ''
+
+    if (src && urlRegex.test(src)) {
+      return
+    }
+
+    if (dataSrc && urlRegex.test(dataSrc)) {
+      node.setAttribute('src', dataSrc)
+      return
+    }
+
+    node.parentNode?.removeChild(node)
+  }
+}
+
+function getPurifiedContent(html) {
+  const newWindow = parseHTML('')
+  const DOMPurify = createDOMPurify(newWindow)
+  DOMPurify.addHook('uponSanitizeElement', domPurifySanitizeHook)
+  const clean = DOMPurify.sanitize(html, DOM_PURIFY_CONFIG)
+  return parseHTML(clean).document
+}
+
+function signImageProxyUrl(url) {
+  return encode(
+    crypto.createHmac('sha256', process.env.IMAGE_PROXY_SECRET).update(url).digest()
+  )
+}
+
+function createImageProxyUrl(url, width = 0, height = 0) {
+  if (!process.env.IMAGE_PROXY_URL || !process.env.IMAGE_PROXY_SECRET) {
+    return url
+  }
+
+  const urlWithOptions = `${url}#${width}x${height}`
+  const signature = signImageProxyUrl(urlWithOptions)
+
+  return `${process.env.IMAGE_PROXY_URL}/${width}x${height},s${signature}/${url}`
+}
+
+async function getReadabilityResult(url, document) {
+  // First attempt to read the article as is.
+  // if that fails attempt to purify then read
+  const sources = [
+    () => {
+      return document
+    },
+    () => {
+      return getPurifiedContent(document)
+    },
+  ]
+
+  for (const source of sources) {
+    const document = source()
+    if (!document) {
+      continue
+    }
+
+    try {
+      const article = await new Readability(document, {
+        createImageProxyUrl,
+        url,
+      }).parse()
+
+      if (article) {
+        return article
+      }
+    } catch (error) {
+      console.log('parsing error for url', url, error)
+    }
+  }
+
+  return null
 }
 
 module.exports = {

packages/puppeteer-parse/package.json

@@ -8,13 +8,16 @@
     "@omnivore/content-handler": "1.0.0",
     "@omnivore/readability": "1.0.0",
     "axios": "^0.27.2",
+    "crypto": "^1.0.1",
+    "dompurify": "^2.4.1",
     "jsonwebtoken": "^8.5.1",
     "linkedom": "^0.14.9",
     "puppeteer-core": "^16.1.0",
     "puppeteer-extra": "^3.3.4",
     "puppeteer-extra-plugin-adblocker": "^2.13.5",
     "puppeteer-extra-plugin-stealth": "^2.11.1",
-    "underscore": "^1.13.4"
+    "underscore": "^1.13.4",
+    "urlsafe-base64": "^1.0.0"
   },
   "devDependencies": {
     "chai": "^4.3.6",

yarn.lock

@@ -12251,6 +12251,11 @@ crypto-random-string@^2.0.0:
   resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5"
   integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==
 
+crypto@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/crypto/-/crypto-1.0.1.tgz#2af1b7cad8175d24c8a1b0778255794a21803037"
+  integrity sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==
+
 css-loader@^3.6.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-3.6.0.tgz#2e4b2c7e6e2d27f8c8f28f61bffcd2e6c91ef645"
@@ -13055,6 +13060,11 @@ dompurify@^2.0.17:
   resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.3.8.tgz#224fe9ae57d7ebd9a1ae1ac18c1c1ca3f532226f"
   integrity sha512-eVhaWoVibIzqdGYjwsBWodIQIaXFSB+cKDf4cfxLMsK0xiud6SE+/WCVx/Xw/UwQsa4cS3T2eITcdtmTg2UKcw==
 
+dompurify@^2.4.1:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.4.1.tgz#f9cb1a275fde9af6f2d0a2644ef648dd6847b631"
+  integrity sha512-ewwFzHzrrneRjxzmK6oVz/rZn9VWspGFRDb4/rRtIsM1n36t9AKma/ye8syCpcw+XJ25kOK/hOG7t1j2I2yBqA==
+
 domutils@^2.0.0, domutils@^2.5.2:
   version "2.7.0"
   resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.7.0.tgz#8ebaf0c41ebafcf55b0b72ec31c56323712c5442"
@@ -26280,7 +26290,7 @@ url@^0.11.0:
 urlsafe-base64@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/urlsafe-base64/-/urlsafe-base64-1.0.0.tgz#23f89069a6c62f46cf3a1d3b00169cefb90be0c6"
-  integrity sha1-I/iQaabGL0bPOh07ABac77kL4MY=
+  integrity sha512-RtuPeMy7c1UrHwproMZN9gN6kiZ0SvJwRaEzwZY0j9MypEkFqyBaKv176jvlPtg58Zh36bOkS0NFABXMHvvGCA==
 
 use-callback-ref@^1.2.3:
   version "1.2.5"