diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index d03c5f3fb..f2ec56f42 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -3,6 +3,9 @@ /* eslint-disable @typescript-eslint/explicit-function-return-type */ /* eslint-disable @typescript-eslint/no-var-requires */ /* eslint-disable @typescript-eslint/no-require-imports */ +const { encode } = require("urlsafe-base64"); +const crypto = require("crypto"); + const Url = require('url'); // const puppeteer = require('puppeteer-extra'); const axios = require('axios'); @@ -23,6 +26,7 @@ puppeteer.use(StealthPlugin()); // Add adblocker plugin to block all ads and trackers (saves bandwidth) const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); +const createDOMPurify = require("dompurify"); puppeteer.use(AdblockerPlugin({ blockTrackers: true })); const storage = new Storage(); @@ -782,10 +786,97 @@ async function preview(req, res) { return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); } -async function getReadabilityResult(url, domContent) { - const document = parseHTML(domContent).document; - const readability = new Readability(document, { url }); - return readability.parse(); +const DOM_PURIFY_CONFIG = { + ADD_TAGS: ['iframe'], + ADD_ATTR: ['allow', 'allowfullscreen', 'frameborder', 'scrolling'], + FORBID_ATTR: [ + 'data-ml-dynamic', + 'data-ml-dynamic-type', + 'data-orig-url', + 'data-ml-id', + 'data-ml', + 'data-xid', + 'data-feature', + ], +} + +function domPurifySanitizeHook(node, data) { + if (data.tagName === 'iframe') { + const urlRegex = /^(https?:)?\/\/www\.youtube(-nocookie)?\.com\/embed\//i + const src = node.getAttribute('src') || '' + const dataSrc = node.getAttribute('data-src') || '' + + if (src && urlRegex.test(src)) { + return + } + + if (dataSrc && urlRegex.test(dataSrc)) { + node.setAttribute('src', dataSrc) + return + } + + node.parentNode?.removeChild(node) + } +} + +function getPurifiedContent(html) { + const newWindow = parseHTML('') + const DOMPurify = createDOMPurify(newWindow) + DOMPurify.addHook('uponSanitizeElement', domPurifySanitizeHook) + const clean = DOMPurify.sanitize(html, DOM_PURIFY_CONFIG) + return parseHTML(clean).document +} + +function signImageProxyUrl(url) { + return encode( + crypto.createHmac('sha256', process.env.IMAGE_PROXY_SECRET).update(url).digest() + ) +} + +function createImageProxyUrl(url, width = 0, height = 0) { + if (!process.env.IMAGE_PROXY_URL || !process.env.IMAGE_PROXY_SECRET) { + return url + } + + const urlWithOptions = `${url}#${width}x${height}` + const signature = signImageProxyUrl(urlWithOptions) + + return `${process.env.IMAGE_PROXY_URL}/${width}x${height},s${signature}/${url}` +} + +async function getReadabilityResult(url, document) { + // First attempt to read the article as is. + // if that fails attempt to purify then read + const sources = [ + () => { + return document + }, + () => { + return getPurifiedContent(document) + }, + ] + + for (const source of sources) { + const document = source() + if (!document) { + continue + } + + try { + const article = await new Readability(document, { + createImageProxyUrl, + url, + }).parse() + + if (article) { + return article + } + } catch (error) { + console.log('parsing error for url', url, error) + } + } + + return null } module.exports = { diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 8798377a9..540070b50 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -8,13 +8,16 @@ "@omnivore/content-handler": "1.0.0", "@omnivore/readability": "1.0.0", "axios": "^0.27.2", + "crypto": "^1.0.1", + "dompurify": "^2.4.1", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.9", "puppeteer-core": "^16.1.0", "puppeteer-extra": "^3.3.4", "puppeteer-extra-plugin-adblocker": "^2.13.5", "puppeteer-extra-plugin-stealth": "^2.11.1", - "underscore": "^1.13.4" + "underscore": "^1.13.4", + "urlsafe-base64": "^1.0.0" }, "devDependencies": { "chai": "^4.3.6", diff --git a/yarn.lock b/yarn.lock index 2395d7803..c4990abaa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12251,6 +12251,11 @@ crypto-random-string@^2.0.0: resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA== +crypto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/crypto/-/crypto-1.0.1.tgz#2af1b7cad8175d24c8a1b0778255794a21803037" + integrity sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig== + css-loader@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-3.6.0.tgz#2e4b2c7e6e2d27f8c8f28f61bffcd2e6c91ef645" @@ -13055,6 +13060,11 @@ dompurify@^2.0.17: resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.3.8.tgz#224fe9ae57d7ebd9a1ae1ac18c1c1ca3f532226f" integrity sha512-eVhaWoVibIzqdGYjwsBWodIQIaXFSB+cKDf4cfxLMsK0xiud6SE+/WCVx/Xw/UwQsa4cS3T2eITcdtmTg2UKcw== +dompurify@^2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.4.1.tgz#f9cb1a275fde9af6f2d0a2644ef648dd6847b631" + integrity sha512-ewwFzHzrrneRjxzmK6oVz/rZn9VWspGFRDb4/rRtIsM1n36t9AKma/ye8syCpcw+XJ25kOK/hOG7t1j2I2yBqA== + domutils@^2.0.0, domutils@^2.5.2: version "2.7.0" resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.7.0.tgz#8ebaf0c41ebafcf55b0b72ec31c56323712c5442" @@ -26280,7 +26290,7 @@ url@^0.11.0: urlsafe-base64@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/urlsafe-base64/-/urlsafe-base64-1.0.0.tgz#23f89069a6c62f46cf3a1d3b00169cefb90be0c6" - integrity sha1-I/iQaabGL0bPOh07ABac77kL4MY= + integrity sha512-RtuPeMy7c1UrHwproMZN9gN6kiZ0SvJwRaEzwZY0j9MypEkFqyBaKv176jvlPtg58Zh36bOkS0NFABXMHvvGCA== use-callback-ref@^1.2.3: version "1.2.5"