From 8bdbd86e972b167b6bc9ab925df5f494d881ad6d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 11:38:35 +0800 Subject: [PATCH 01/10] Make headless = false in local testing --- packages/content-handler/src/websites/twitter-handler.ts | 2 +- packages/puppeteer-parse/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 17bbdc4c2..e4b23394c 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -213,7 +213,7 @@ const getTweetIds = async ( const browser = await puppeteer.launch({ executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, + headless: process.env.LAUNCH_HEADLESS === 'true', defaultViewport: { width, height, diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index cd72936af..13623fe90 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -169,7 +169,7 @@ const getBrowserPromise = (async () => { ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, + headless: process.env.LAUNCH_HEADLESS === 'true', timeout: 120000, // 2 minutes }); })(); From b3e73cd84be0cc5eb2447bd3cfab20e7a11861c0 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 11:42:13 +0800 Subject: [PATCH 02/10] Add puppeteer-extra plugins --- packages/puppeteer-parse/index.js | 11 +- packages/puppeteer-parse/package.json | 3 + yarn.lock | 263 +++++++++++++++++++++++++- 3 files changed, 271 insertions(+), 6 deletions(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 13623fe90..148dfb7d6 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -15,12 +15,17 @@ const { DateTime } = require('luxon'); const os = require('os'); const { Storage } = require('@google-cloud/storage'); const { parseHTML } = require('linkedom'); -const puppeteer = require('puppeteer-core'); const { preHandleContent } = require("@omnivore/content-handler"); +const puppeteer = require('puppeteer-extra'); + // Add stealth plugin to hide puppeteer usage -// const StealthPlugin = require('puppeteer-extra-plugin-stealth'); -// puppeteer.use(StealthPlugin()); +const StealthPlugin = require('puppeteer-extra-plugin-stealth'); +puppeteer.use(StealthPlugin()); + +// Add adblocker plugin to block all ads and trackers (saves bandwidth) +const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); +puppeteer.use(AdblockerPlugin({ blockTrackers: true })); const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index a9537f6f2..41378e1bf 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -12,6 +12,9 @@ "linkedom": "^0.14.9", "luxon": "^2.3.1", "puppeteer-core": "^16.1.0", + "puppeteer-extra": "^3.3.4", + "puppeteer-extra-plugin-adblocker": "^2.13.5", + "puppeteer-extra-plugin-stealth": "^2.11.1", "underscore": "^1.13.4", "winston": "^3.3.3" }, diff --git a/yarn.lock b/yarn.lock index cbdd96048..7f2ee67fa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2069,6 +2069,41 @@ commander "^4.1.0" microtime "^3.0.0" +"@cliqz/adblocker-content@^1.23.8", "@cliqz/adblocker-content@^1.25.1": + version "1.25.1" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker-content/-/adblocker-content-1.25.1.tgz#da81e7838e288a6f0fdb8a97a0df8169accb74a1" + integrity sha512-7gl2VdNPBfj7aPoq34B5miwGcnda/7LCr+BqnpcSOjdLV6jjT2FrNSAKGFvcH23q0HM1IFhYDV6ydTgsdWFCnA== + dependencies: + "@cliqz/adblocker-extended-selectors" "^1.25.1" + +"@cliqz/adblocker-extended-selectors@^1.25.1": + version "1.25.1" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker-extended-selectors/-/adblocker-extended-selectors-1.25.1.tgz#cfac0080952311399805fe153cd9e7e1331b3c6d" + integrity sha512-4MdMe/YfIok5d8WYVcLR3Ak7vGrmeUV47frgmXEe945luY93vwlzk1NiLYW1JM5Gdm+VePweoS9cJ1/QUTmv+Q== + +"@cliqz/adblocker-puppeteer@1.23.8": + version "1.23.8" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker-puppeteer/-/adblocker-puppeteer-1.23.8.tgz#e74636cd200459d1734929e41504a76939504311" + integrity sha512-Ca1/DBqQXsOpKTFVAHX6OpLTSEupXmUkUWHj6iXhLLleC7RPISN5B0b801VDmaGRqoC5zKRxn0vYbIfpgCWVug== + dependencies: + "@cliqz/adblocker" "^1.23.8" + "@cliqz/adblocker-content" "^1.23.8" + tldts-experimental "^5.6.21" + +"@cliqz/adblocker@^1.23.8": + version "1.25.1" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker/-/adblocker-1.25.1.tgz#4d3e8894ce48ad0d0f8b26a4a1003f0676b7f734" + integrity sha512-1C1/ELI94/XewdUj/o1+Q4ziOigMvTZQA05UERfDoKqpJ+0cbrEF/UImrzpX7n+kYsR7xTJvmf+iNM3zS0tfsg== + dependencies: + "@cliqz/adblocker-content" "^1.25.1" + "@cliqz/adblocker-extended-selectors" "^1.25.1" + "@remusao/guess-url-type" "^1.1.2" + "@remusao/small" "^1.1.2" + "@remusao/smaz" "^1.7.1" + "@types/chrome" "^0.0.197" + "@types/firefox-webext-browser" "^94.0.0" + tldts-experimental "^5.6.21" + "@cnakazawa/watch@^1.0.3": version "1.0.4" resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a" @@ -5110,6 +5145,41 @@ resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4" integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA== +"@remusao/guess-url-type@^1.1.2": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57" + integrity sha512-rbOqre2jW8STjheOsOaQHLgYBaBZ9Owbdt8NO7WvNZftJlaG3y/K9oOkl8ZUpuFBisIhmBuMEW6c+YrQl5inRA== + +"@remusao/small@^1.1.2": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@remusao/small/-/small-1.2.1.tgz#63bfe4548832289f94ac868a0c305970c9a0e5f9" + integrity sha512-7MjoGt0TJMVw1GPKgWq6SJPws1SLsUXQRa43Umht+nkyw2jnpy3WpiLNqGdwo5rHr5Wp9B2W/Pm5RQp656UJdw== + +"@remusao/smaz-compress@^1.9.1": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@remusao/smaz-compress/-/smaz-compress-1.9.1.tgz#fc75eaf9bcac2d58bc4c3d518183a7cb9612d275" + integrity sha512-E2f48TwloQu3r6BdLOGF2aczeH7bJ/32oJGqvzT9SKur0cuUnLcZ7ZXP874E2fwmdE+cXzfC7bKzp79cDnmeyw== + dependencies: + "@remusao/trie" "^1.4.1" + +"@remusao/smaz-decompress@^1.9.1": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@remusao/smaz-decompress/-/smaz-decompress-1.9.1.tgz#8094f997e8fb591a678cda9cf08c209c825eba5b" + integrity sha512-TfjKKprYe3n47od8auhvJ/Ikj9kQTbDTe71ynKlxslrvvUhlIV3VQSuwYuMWMbdz1fIs0H/fxCN1Z8/H3km6/A== + +"@remusao/smaz@^1.7.1": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@remusao/smaz/-/smaz-1.9.1.tgz#a2b9b045385f81e1615a68d932b7cc8b04c9db8d" + integrity sha512-e6BLuP8oaXCZ9+v46Is4ilAZ/Vq6YLgmBP204Ixgk1qTjXmqvFYG7+AS7v9nsZdGOy96r9DWGFbbDVgMxwu1rA== + dependencies: + "@remusao/smaz-compress" "^1.9.1" + "@remusao/smaz-decompress" "^1.9.1" + +"@remusao/trie@^1.4.1": + version "1.4.1" + resolved "https://registry.yarnpkg.com/@remusao/trie/-/trie-1.4.1.tgz#755d09f8a007476334e611f42719b2d581f00720" + integrity sha512-yvwa+aCyYI/UjeD39BnpMypG8N06l86wIDW1/PAc6ihBRnodIfZDwccxQN3n1t74wduzaz74m4ZMHZnB06567Q== + "@rushstack/eslint-patch@^1.0.8": version "1.1.0" resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323" @@ -7611,6 +7681,14 @@ resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.2.21.tgz#9f35a5643129df132cf3b5c1ec64046ea1af0650" integrity sha512-yd+9qKmJxm496BOV9CMNaey8TWsikaZOwMRwPHQIjcOJM9oV+fi9ZMNw3JsVnbEEbo2gRTDnGEBv8pjyn67hNg== +"@types/chrome@^0.0.197": + version "0.0.197" + resolved "https://registry.yarnpkg.com/@types/chrome/-/chrome-0.0.197.tgz#c1b50cdb72ee40f9bc1411506031a9f8a925ab35" + integrity sha512-m1NfS5bOjaypyqQfaX6CxmJodZVcvj5+Mt/K94EBHkflYjPNmXHAzbxfifdLMa0YM3PDyOxohoTS5ug/e6p5jA== + dependencies: + "@types/filesystem" "*" + "@types/har-format" "*" + "@types/cls-hooked@^4.2.1": version "4.3.3" resolved "https://registry.yarnpkg.com/@types/cls-hooked/-/cls-hooked-4.3.3.tgz#c09e2f8dc62198522eaa18a5b6b873053154bd00" @@ -7667,6 +7745,13 @@ resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.12.tgz#6b2c510a7ad7039e98e7b8d3d6598f4359e5c080" integrity sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw== +"@types/debug@^4.1.0": + version "4.1.7" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82" + integrity sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg== + dependencies: + "@types/ms" "*" + "@types/diff-match-patch@^1.0.32": version "1.0.32" resolved "https://registry.yarnpkg.com/@types/diff-match-patch/-/diff-match-patch-1.0.32.tgz#d9c3b8c914aa8229485351db4865328337a3d09f" @@ -7751,11 +7836,28 @@ "@types/qs" "*" "@types/serve-static" "*" +"@types/filesystem@*": + version "0.0.32" + resolved "https://registry.yarnpkg.com/@types/filesystem/-/filesystem-0.0.32.tgz#307df7cc084a2293c3c1a31151b178063e0a8edf" + integrity sha512-Yuf4jR5YYMR2DVgwuCiP11s0xuVRyPKmz8vo6HBY3CGdeMj8af93CFZX+T82+VD1+UqHOxTq31lO7MI7lepBtQ== + dependencies: + "@types/filewriter" "*" + +"@types/filewriter@*": + version "0.0.29" + resolved "https://registry.yarnpkg.com/@types/filewriter/-/filewriter-0.0.29.tgz#a48795ecadf957f6c0d10e0c34af86c098fa5bee" + integrity sha512-BsPXH/irW0ht0Ji6iw/jJaK8Lj3FJemon2gvEqHKpCdDCeemHa+rI3WBGq5z7cDMZgoLjY40oninGxqk+8NzNQ== + "@types/fined@*": version "1.1.3" resolved "https://registry.yarnpkg.com/@types/fined/-/fined-1.1.3.tgz#83f03e8f0a8d3673dfcafb18fce3571f6250e1bc" integrity sha512-CWYnSRnun3CGbt6taXeVo2lCbuaj4mchVJ4UF/BdU5TSuIn3AmS13pGMwCsBUoehGbhZrBrpNJZSZI5EVilXww== +"@types/firefox-webext-browser@^94.0.0": + version "94.0.1" + resolved "https://registry.yarnpkg.com/@types/firefox-webext-browser/-/firefox-webext-browser-94.0.1.tgz#52afb975253dc0fd350d5d58c7fe9fd1a01f64a1" + integrity sha512-I6iHRQJSTZ+gYt2IxdH2RRAMvcUyK8v5Ig7fHQR0IwUNYP7hz9+cziBVIKxLCO6XI7fiyRsNOWObfl3/4Js2Lg== + "@types/fluent-ffmpeg@^2.1.20": version "2.1.20" resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz#3b5f42fc8263761d58284fa46ee6759a64ce54ac" @@ -7793,6 +7895,11 @@ dependencies: graphql "^15.3.0" +"@types/har-format@*": + version "1.2.9" + resolved "https://registry.yarnpkg.com/@types/har-format/-/har-format-1.2.9.tgz#b9b3a9bfc33a078e7d898a00b09662910577f4a4" + integrity sha512-rffW6MhQ9yoa75bdNi+rjZBAvu2HhehWJXlhuWXnWdENeuKe82wUgAwxYOb7KRKKmxYN+D/iRKd2NDQMLqlUmg== + "@types/hast@^2.0.0": version "2.3.4" resolved "https://registry.yarnpkg.com/@types/hast/-/hast-2.3.4.tgz#8aa5ef92c117d20d974a82bdfb6a648b08c0bafc" @@ -7995,6 +8102,11 @@ resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.2.3.tgz#bbeb55fbc73f28ea6de601fbfa4613f58d785323" integrity sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw== +"@types/ms@*": + version "0.7.31" + resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" + integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== + "@types/nanoid@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@types/nanoid/-/nanoid-3.0.0.tgz#c757b20f343f3a1dd76e80a9a431b6290fc20f35" @@ -11063,6 +11175,17 @@ cliui@^7.0.2: strip-ansi "^6.0.0" wrap-ansi "^7.0.0" +clone-deep@^0.2.4: + version "0.2.4" + resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-0.2.4.tgz#4e73dd09e9fb971cc38670c5dced9c1896481cc6" + integrity sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg== + dependencies: + for-own "^0.1.3" + is-plain-object "^2.0.1" + kind-of "^3.0.2" + lazy-cache "^1.0.3" + shallow-clone "^0.1.2" + clone-deep@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-4.0.1.tgz#c19fd9bdbbf85942b4fd979c84dcf7d5f07c2387" @@ -14103,11 +14226,23 @@ follow-redirects@^1.0.0, follow-redirects@^1.14.0, follow-redirects@^1.14.4, fol resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5" integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA== +for-in@^0.1.3: + version "0.1.8" + resolved "https://registry.yarnpkg.com/for-in/-/for-in-0.1.8.tgz#d8773908e31256109952b1fdb9b3fa867d2775e1" + integrity sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g== + for-in@^1.0.1, for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= +for-own@^0.1.3: + version "0.1.5" + resolved "https://registry.yarnpkg.com/for-own/-/for-own-0.1.5.tgz#5265c681a4f294dabbf17c9509b6763aa84510ce" + integrity sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw== + dependencies: + for-in "^1.0.1" + for-own@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/for-own/-/for-own-1.0.0.tgz#c63332f415cedc4b04dbfe70cf836494c53cb44b" @@ -14275,6 +14410,15 @@ fs-extra@^0.30.0: path-is-absolute "^1.0.0" rimraf "^2.2.8" +fs-extra@^10.0.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" + integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + fs-extra@^9.0.0, fs-extra@^9.0.1, fs-extra@^9.1.0: version "9.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" @@ -16082,7 +16226,7 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@^1.1.5, is-buffer@~1.1.6: +is-buffer@^1.0.2, is-buffer@^1.1.5, is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -16394,7 +16538,7 @@ is-plain-object@5.0.0, is-plain-object@^5.0.0: resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== -is-plain-object@^2.0.3, is-plain-object@^2.0.4: +is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677" integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og== @@ -17587,6 +17731,13 @@ keyv@^3.0.0: dependencies: json-buffer "3.0.0" +kind-of@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-2.0.1.tgz#018ec7a4ce7e3a86cb9141be519d24c8faa981b5" + integrity sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg== + dependencies: + is-buffer "^1.0.2" + kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -17687,6 +17838,16 @@ lazy-ass@^1.6.0: resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513" integrity sha1-eZllXoZGwX8In90YfRUNMyTVRRM= +lazy-cache@^0.2.3: + version "0.2.7" + resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-0.2.7.tgz#7feddf2dcb6edb77d11ef1d117ab5ffdf0ab1b65" + integrity sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ== + +lazy-cache@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-1.0.4.tgz#a1d78fc3a50474cb80845d3b3b6e1da49a446e8e" + integrity sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ== + lazy-universal-dotenv@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/lazy-universal-dotenv/-/lazy-universal-dotenv-3.0.1.tgz#a6c8938414bca426ab8c9463940da451a911db38" @@ -18608,6 +18769,15 @@ meow@^8.0.0: type-fest "^0.18.0" yargs-parser "^20.2.3" +merge-deep@^3.0.1: + version "3.0.3" + resolved "https://registry.yarnpkg.com/merge-deep/-/merge-deep-3.0.3.tgz#1a2b2ae926da8b2ae93a0ac15d90cd1922766003" + integrity sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA== + dependencies: + arr-union "^3.1.0" + clone-deep "^0.2.4" + kind-of "^3.0.2" + merge-descriptors@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" @@ -18949,6 +19119,14 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" +mixin-object@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/mixin-object/-/mixin-object-2.0.1.tgz#4fb949441dab182540f1fe035ba60e1947a5e57e" + integrity sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA== + dependencies: + for-in "^0.1.3" + is-extendable "^0.1.1" + mkdirp-classic@^0.5.2: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" @@ -19383,7 +19561,7 @@ node-fetch@2.6.1: resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== -node-fetch@2.6.7, node-fetch@^2.3.0, node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@2.6.7, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== @@ -21451,6 +21629,63 @@ puppeteer-core@^19.1.1: unbzip2-stream "1.4.3" ws "8.9.0" +puppeteer-extra-plugin-adblocker@^2.13.5: + version "2.13.5" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.13.5.tgz#c86ce94873bf6fe500555d3972eccdcca4914f6f" + integrity sha512-HMVWLA1MLrzIGr/A71PYAWZEHENqQOEaQQHtPje0uSLc6QPOQY5tbbocx4BsUiQL2V1FwgT21UU09P5lV4vrZw== + dependencies: + "@cliqz/adblocker-puppeteer" "1.23.8" + debug "^4.1.1" + node-fetch "^2.6.0" + puppeteer-extra-plugin "^3.2.2" + +puppeteer-extra-plugin-stealth@^2.11.1: + version "2.11.1" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.1.tgz#7d56a27a986cb5eb69dca3c65695ad6444f4e822" + integrity sha512-n0wdC0Ilc9tk5L6FWLyd0P2gT8b2fp+2NuB+KB0oTSw3wXaZ0D6WNakjJsayJ4waGzIJFCUHkmK9zgx5NKMoFw== + dependencies: + debug "^4.1.1" + puppeteer-extra-plugin "^3.2.2" + puppeteer-extra-plugin-user-preferences "^2.4.0" + +puppeteer-extra-plugin-user-data-dir@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.0.tgz#20e87582482b61e497abd96fec452f63bd2d9123" + integrity sha512-qrhYPTGIqzL2hpeJ5DXjf8xMy5rt1UvcqSgpGTTOUOjIMz1ROWnKHjBoE9fNBJ4+ToRZbP8MzIDXWlEk/e1zJA== + dependencies: + debug "^4.1.1" + fs-extra "^10.0.0" + puppeteer-extra-plugin "^3.2.2" + rimraf "^3.0.2" + +puppeteer-extra-plugin-user-preferences@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.0.tgz#8b75bc39c3de9913e236ae1d982a24711d84ba6f" + integrity sha512-4XxMhMkJ+qqLsPY9ULF90qS9Bj1Qrwwgp1TY9zTdp1dJuy7QSgYE7xlyamq3cKrRuzg3QUOqygJo52sVeXSg5A== + dependencies: + debug "^4.1.1" + deepmerge "^4.2.2" + puppeteer-extra-plugin "^3.2.2" + puppeteer-extra-plugin-user-data-dir "^2.4.0" + +puppeteer-extra-plugin@^3.2.2: + version "3.2.2" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.2.tgz#3c02c0a10f8eadf32e7debb7ee24105a53afc17b" + integrity sha512-0uatQxzuVn8yegbrEwSk03wvwpMB5jNs7uTTnermylLZzoT+1rmAQaJXwlS3+vADUbw6ELNgNEHC7Skm0RqHbQ== + dependencies: + "@types/debug" "^4.1.0" + debug "^4.1.1" + merge-deep "^3.0.1" + +puppeteer-extra@^3.3.4: + version "3.3.4" + resolved "https://registry.yarnpkg.com/puppeteer-extra/-/puppeteer-extra-3.3.4.tgz#e0ecf021783d1112b6b0db20546d5022e632ed55" + integrity sha512-fN5pHvSMJ8d1o7Z8wLLTQOUBpORD2BcFn+KDs7QnkGZs9SV69hcUcce67vX4L4bNSEG3A0P6Osrv+vWNhhdm8w== + dependencies: + "@types/debug" "^4.1.0" + debug "^4.1.1" + deepmerge "^4.2.2" + puppeteer@^10.1.0: version "10.4.0" resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-10.4.0.tgz#a6465ff97fda0576c4ac29601406f67e6fea3dc7" @@ -22939,6 +23174,16 @@ sha.js@^2.4.0, sha.js@^2.4.11, sha.js@^2.4.8: inherits "^2.0.1" safe-buffer "^5.0.1" +shallow-clone@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-0.1.2.tgz#5909e874ba77106d73ac414cfec1ffca87d97060" + integrity sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw== + dependencies: + is-extendable "^0.1.1" + kind-of "^2.0.1" + lazy-cache "^0.2.3" + mixin-object "^2.0.1" + shallow-clone@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3" @@ -24277,6 +24522,18 @@ title-case@^3.0.3: dependencies: tslib "^2.0.3" +tldts-core@^5.7.100: + version "5.7.100" + resolved "https://registry.yarnpkg.com/tldts-core/-/tldts-core-5.7.100.tgz#6144104277a3c4500ec395220d8e03c16fcdfaf7" + integrity sha512-56+vie1oPcJZQiPfnvIIpbyTttUketsjV7lrw/hkMMa/EACPjjDctobWwF3153gR2l+c9O+nYiHkXIL1Cmr9eQ== + +tldts-experimental@^5.6.21: + version "5.7.100" + resolved "https://registry.yarnpkg.com/tldts-experimental/-/tldts-experimental-5.7.100.tgz#fb428cf20735952c299e15e864de63ecc55fb0a7" + integrity sha512-BjdXE3YU3cXbASRXydXnzOCSc+G/bM38/5snbxwcIYaRh3AApEtD4lHVl3236x+79T/V94lKwqBXYT47EAR+TA== + dependencies: + tldts-core "^5.7.100" + tmp@^0.0.33: version "0.0.33" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" From 4ee5675e23930bc360df7c75ee5b065e19348ca2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 12:03:35 +0800 Subject: [PATCH 03/10] Add gdcvault parsing test-case --- .../gdcvault/expected-metadata.json | 12 + .../test/test-pages/gdcvault/expected.html | 9 + .../test/test-pages/gdcvault/source.html | 1142 +++++++++++++++++ .../test/test-pages/gdcvault/url.txt | 1 + 4 files changed, 1164 insertions(+) create mode 100644 packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json create mode 100644 packages/readabilityjs/test/test-pages/gdcvault/expected.html create mode 100644 packages/readabilityjs/test/test-pages/gdcvault/source.html create mode 100644 packages/readabilityjs/test/test-pages/gdcvault/url.txt diff --git a/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json b/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json new file mode 100644 index 000000000..5e5525d51 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json @@ -0,0 +1,12 @@ +{ + "title": "Parallelizing the Naughty Dog Engine Using Fibers", + "byline": null, + "dir": null, + "excerpt": "This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall...", + "siteName": null, + "siteIcon": "http://fakehost/img/favicon.ico", + "previewImage": "https://ubm-twvideo01.s3.amazonaws.com/o1/vault/gdc2015/Images/GDC15_Vault-thumb_v1.png", + "publishedDate": null, + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/gdcvault/expected.html b/packages/readabilityjs/test/test-pages/gdcvault/expected.html new file mode 100644 index 000000000..64124891e --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/expected.html @@ -0,0 +1,9 @@ +
+
+

You've been logged out of GDC Vault since the maximum users allowed for this account has been reached. To access Members Only content on GDC Vault, please log out of GDC Vault from the computer which last accessed this account.

+

Click here to find out about GDC Vault Membership options for more users.

+

+ close +

+
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/gdcvault/source.html b/packages/readabilityjs/test/test-pages/gdcvault/source.html new file mode 100644 index 000000000..1b342e352 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/source.html @@ -0,0 +1,1142 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + GDC Vault - Parallelizing the Naughty Dog Engine Using Fibers + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+
+ You've been logged out of GDC Vault since the maximum users allowed for this account has been reached. To access Members Only content on GDC Vault, please log out of GDC Vault from the computer which last accessed this account.
+
+ Click here to find out about GDC Vault Membership options for more users.
+
+ close +
+
+
+
+ + + +
+
+ + + +
+
+ +
+ + +
+      +
+ + +
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ Session Name: + + Parallelizing the Naughty Dog Engine Using Fibers +
+ Speaker(s): + + Christian Gyrling +
+ Company Name(s): + + Naughty Dog +
+ Track / Format: + + Programming +
+ Overview: + + This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall frame-centric engine design, the memory allocation patterns used in the title, and our strategies for dealing with locks. +
+
+

+ GDC 2015 +

+

+ Christian Gyrling +

+

+ Naughty Dog +

+

+ free content +

+

+ Programming +

+

+ Programming +

+
+
+
+ +

+ + +
+
+
+
+
+
+
+
+ Discover More from Informa Tech +
+
+
+
+
+ +
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ Working With Us +
+ +
+
+
+
+
+
+
+ Follow GDC on Social +
+
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+ +
+
+
+
+
+
+
+
    +
  • Copyright © 2022 Informa PLC. Informa PLC is registered in England and Wales with company number 8860726 whose registered and head office is 5 Howick Place, London, SW1P 1WG. +
    • +
+
+
+
+
+ + + + + + + diff --git a/packages/readabilityjs/test/test-pages/gdcvault/url.txt b/packages/readabilityjs/test/test-pages/gdcvault/url.txt new file mode 100644 index 000000000..64f62a296 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/url.txt @@ -0,0 +1 @@ +https://www.gdcvault.com/play/1022186/Parallelizing-the-Naughty-Dog-Engine \ No newline at end of file From 13e925c503de61b9fac74e526889d37481aa39fe Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 12:17:37 +0800 Subject: [PATCH 04/10] Allow kaltura videos through in readablity --- packages/readabilityjs/Readability.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index f28f688fe..01b6999f5 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -184,7 +184,7 @@ Readability.prototype = { publishedDate: /published|modified|created|updated/i, replaceFonts: /<(\/?)font[^>]*>/gi, normalize: /\s{2,}/g, - videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i, + videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq|cdnapisec\.kaltura)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i, shareElements: /(\b|_)(share|sharedaddy|post-tags)(\b|_)/i, nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i, prevLink: /(prev|earl|old|new|<|«)/i, From 1b62ada73ea53cb7044fd97c76051a3a249e178d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 12:45:24 +0800 Subject: [PATCH 05/10] Make overlay element an unlikelyCandidates and give player element positive score --- packages/readabilityjs/Readability.js | 4 +- .../test/test-pages/gdcvault/expected.html | 57 +++++++++++++++++-- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 01b6999f5..50bd5d85d 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -171,13 +171,13 @@ Readability.prototype = { // Readability-readerable.js. Please keep both copies in sync. articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist/i, articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert|block-ad-.*/i, - unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon/i, + unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped/i, // okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i, get okMaybeItsACandidate() { return new RegExp(`and|(? -
-

You've been logged out of GDC Vault since the maximum users allowed for this account has been reached. To access Members Only content on GDC Vault, please log out of GDC Vault from the computer which last accessed this account.

-

Click here to find out about GDC Vault Membership options for more users.

-

- close -

+
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ Session Name: + Parallelizing the Naughty Dog Engine Using Fibers
+ Speaker(s): + Christian Gyrling
+ Company Name(s): + Naughty Dog
+ Track / Format: + Programming
+ Overview: + This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall frame-centric engine design, the memory allocation patterns used in the title, and our strategies for dealing with locks.
+
+ +
\ No newline at end of file From e75e49a7b44795ee10d135291757a1d4d20daf70 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 17:23:49 +0800 Subject: [PATCH 06/10] Remove logging dependecies in puppeteer-parse --- packages/content-fetch/logger.js | 66 +++++++++++++++++ packages/content-fetch/package.json | 3 +- packages/puppeteer-parse/index.js | 101 ++++---------------------- packages/puppeteer-parse/package.json | 5 +- 4 files changed, 84 insertions(+), 91 deletions(-) create mode 100644 packages/content-fetch/logger.js diff --git a/packages/content-fetch/logger.js b/packages/content-fetch/logger.js new file mode 100644 index 000000000..9317e36c8 --- /dev/null +++ b/packages/content-fetch/logger.js @@ -0,0 +1,66 @@ +const { config, format, loggers, transports } = require('winston'); +const { LoggingWinston } = require('@google-cloud/logging-winston'); +const { DateTime } = require('luxon'); + +const colors = { + emerg: 'inverse underline magenta', + alert: 'underline magenta', + crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. + error: 'underline red', // Any error which is fatal to the operation, but not the service or application + warning: 'underline yellow', // Anything that can potentially cause application oddities + notice: 'underline cyan', // Normal but significant condition + info: 'underline green', // Generally useful information to log + debug: 'underline gray', +}; + +const googleConfigs = { + level: 'info', + logName: 'logger', + levels: config.syslog.levels, + resource: { + labels: { + function_name: process.env.FUNCTION_TARGET, + project_id: process.env.GCP_PROJECT, + }, + type: 'cloud_function', + }, +}; + +function localConfig(id) { + return { + level: 'debug', + format: format.combine( + format.colorize({ all: true, colors }), + format(info => + Object.assign(info, { + timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), + }), + )(), + format.printf(info => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { timestamp, message, level, ...meta } = info; + + return `[${id}@${info.timestamp}] ${info.message}${ + Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' + }`; + }), + ), + }; +} + +function buildLoggerTransport(id, options) { + return process.env.IS_LOCAL + ? new transports.Console(localConfig(id)) + : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); +} + +function buildLogger(id, options) { + return loggers.get(id, { + levels: config.syslog.levels, + transports: [buildLoggerTransport(id, options)], + }); +} + +module.exports = { + buildLogger, +} diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json index bffa43a8e..546552258 100644 --- a/packages/content-fetch/package.json +++ b/packages/content-fetch/package.json @@ -8,7 +8,8 @@ "express": "^4.17.1", "@google-cloud/functions-framework": "^3.0.0", "@omnivore/puppeteer-parse": "^1.0.0", - "@sentry/serverless": "^6.13.3" + "@sentry/serverless": "^6.13.3", + "winston": "^3.3.3" }, "devDependencies": { "chai": "^4.3.6", diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 148dfb7d6..8cb311ac1 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -9,9 +9,6 @@ const axios = require('axios'); const jwt = require('jsonwebtoken'); const { promisify } = require('util'); const signToken = promisify(jwt.sign); -const { config, format, loggers, transports } = require('winston'); -const { LoggingWinston } = require('@google-cloud/logging-winston'); -const { DateTime } = require('luxon'); const os = require('os'); const { Storage } = require('@google-cloud/storage'); const { parseHTML } = require('linkedom'); @@ -33,65 +30,6 @@ const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process. const filePath = `${os.tmpdir()}/previewImage.png`; -const colors = { - emerg: 'inverse underline magenta', - alert: 'underline magenta', - crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. - error: 'underline red', // Any error which is fatal to the operation, but not the service or application - warning: 'underline yellow', // Anything that can potentially cause application oddities - notice: 'underline cyan', // Normal but significant condition - info: 'underline green', // Generally useful information to log - debug: 'underline gray', -}; - -const googleConfigs = { - level: 'info', - logName: 'logger', - levels: config.syslog.levels, - resource: { - labels: { - function_name: process.env.FUNCTION_TARGET, - project_id: process.env.GCP_PROJECT, - }, - type: 'cloud_function', - }, -}; - -function localConfig(id) { - return { - level: 'debug', - format: format.combine( - format.colorize({ all: true, colors }), - format(info => - Object.assign(info, { - timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), - }), - )(), - format.printf(info => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { timestamp, message, level, ...meta } = info; - - return `[${id}@${info.timestamp}] ${info.message}${ - Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' - }`; - }), - ), - }; -} - -function buildLoggerTransport(id, options) { - return process.env.IS_LOCAL - ? new transports.Console(localConfig(id)) - : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); -} - -function buildLogger(id, options) { - return loggers.get(id, { - levels: config.syslog.levels, - transports: [buildLoggerTransport(id, options)], - }); -} - const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' @@ -274,15 +212,6 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId async function fetchContent(req, res) { functionStartTime = Date.now(); - // Grabbing execution and trace ids to attach logs to the appropriate function call - const execution_id = req.get('function-execution-id'); - const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; - const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { - trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, - labels: { - execution_id: execution_id, - }, - }); let url = getUrl(req); const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); @@ -297,11 +226,11 @@ async function fetchContent(req, res) { }, }; - logger.info(`Article parsing request`, logRecord); + console.info(`Article parsing request`, logRecord); if (!url) { logRecord.urlIsInvalid = true; - logger.info(`Valid URL to parse not specified`, logRecord); + console.info(`Valid URL to parse not specified`, logRecord); return res.sendStatus(400); } @@ -317,7 +246,7 @@ async function fetchContent(req, res) { if (result && result.content) { content = result.content } if (result && result.contentType) { contentType = result.contentType } } catch (e) { - logger.info('error with handler: ', e); + console.info('error with handler: ', e); } let context, page, finalUrl; @@ -347,7 +276,7 @@ async function fetchContent(req, res) { content = result.domContent; } } else { - logger.info('using prefetched content and title'); + console.info('using prefetched content and title'); } logRecord.fetchContentTime = Date.now() - functionStartTime; @@ -370,7 +299,7 @@ async function fetchContent(req, res) { } } catch (e) { logRecord.error = e.message; - logger.error(`Error while retrieving page`, logRecord); + console.error(`Error while retrieving page`, logRecord); // fallback to scrapingbee const sbResult = await fetchContentWithScrapingBee(url); @@ -397,7 +326,7 @@ async function fetchContent(req, res) { if (context) { await context.close(); } - logger.info(`parse-page`, logRecord); + console.info(`parse-page`, logRecord); } return res.sendStatus(200); @@ -715,7 +644,7 @@ async function preview(req, res) { // Grabbing execution and trace ids to attach logs to the appropriate function call const execution_id = req.get('function-execution-id'); const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; - const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { + const console = buildconsole('cloudfunctions.googleapis.com%2Fcloud-functions', { trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, labels: { execution_id: execution_id, @@ -723,7 +652,7 @@ async function preview(req, res) { }); if (!process.env.PREVIEW_IMAGE_BUCKET) { - logger.error(`PREVIEW_IMAGE_BUCKET not set`) + console.error(`PREVIEW_IMAGE_BUCKET not set`) return res.sendStatus(500); } @@ -739,17 +668,17 @@ async function preview(req, res) { }, }; - logger.info(`Public preview image generation request`, logRecord); + console.info(`Public preview image generation request`, logRecord); if (!url) { logRecord.urlIsInvalid = true; - logger.error(`Valid URL to parse is not specified`, logRecord); + console.error(`Valid URL to parse is not specified`, logRecord); return res.sendStatus(400); } const { origin } = new URL(url); if (!ALLOWED_ORIGINS.some(o => o === origin)) { logRecord.forbiddenOrigin = true; - logger.error(`This origin is not allowed: ${origin}`, logRecord); + console.error(`This origin is not allowed: ${origin}`, logRecord); return res.sendStatus(400); } @@ -780,7 +709,7 @@ async function preview(req, res) { ); if (!selector) { logRecord.selectorIsInvalid = true; - logger.error(`Valid element selector is not specified`, logRecord); + console.error(`Valid element selector is not specified`, logRecord); await page.close(); return res.sendStatus(400); } @@ -795,7 +724,7 @@ async function preview(req, res) { ); if (!destination) { logRecord.destinationIsInvalid = true; - logger.error(`Valid file destination is not specified`, logRecord); + console.error(`Valid file destination is not specified`, logRecord); await page.close(); return res.sendStatus(400); } @@ -806,7 +735,7 @@ async function preview(req, res) { await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load } catch (error) { logRecord.elementNotFound = true; - logger.error(`Element is not presented on the page`, logRecord); + console.error(`Element is not presented on the page`, logRecord); await page.close(); return res.sendStatus(400); } @@ -826,7 +755,7 @@ async function preview(req, res) { console.log('error uploading to bucket, this is non-fatal', e) } - logger.info(`preview-image`, logRecord); + console.info(`preview-image`, logRecord); return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); } diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 41378e1bf..42b3a10a4 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -4,19 +4,16 @@ "description": "Accepts URL of the article and parses its content", "main": "index.js", "dependencies": { - "@google-cloud/logging-winston": "^5.1.1", "@google-cloud/storage": "^5.18.1", "@omnivore/content-handler": "1.0.0", "axios": "^0.27.2", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.9", - "luxon": "^2.3.1", "puppeteer-core": "^16.1.0", "puppeteer-extra": "^3.3.4", "puppeteer-extra-plugin-adblocker": "^2.13.5", "puppeteer-extra-plugin-stealth": "^2.11.1", - "underscore": "^1.13.4", - "winston": "^3.3.3" + "underscore": "^1.13.4" }, "devDependencies": { "chai": "^4.3.6", From e866541ae1d09498c0db4b852973b6d9dc460204 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 17 Nov 2022 10:01:21 +0800 Subject: [PATCH 07/10] Fix puppeteer launch in head mode --- packages/content-fetch/app.js | 4 ++-- packages/content-handler/src/websites/twitter-handler.ts | 2 +- packages/puppeteer-parse/index.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/content-fetch/app.js b/packages/content-fetch/app.js index 252671626..76169cd0c 100644 --- a/packages/content-fetch/app.js +++ b/packages/content-fetch/app.js @@ -14,7 +14,7 @@ if (!process.env.VERIFICATION_TOKEN) { app.get('/', async (req, res) => { if (req.query.token !== process.env.VERIFICATION_TOKEN) { console.log('query does not include valid token') - res.send(403) + res.sendStatus(403) return } await fetchContent(req, res) @@ -23,7 +23,7 @@ app.get('/', async (req, res) => { app.post('/', async (req, res) => { if (req.query.token !== process.env.VERIFICATION_TOKEN) { console.log('query does not include valid token') - res.send(403) + res.sendStatus(403) return } await fetchContent(req, res) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index e4b23394c..17bbdc4c2 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -213,7 +213,7 @@ const getTweetIds = async ( const browser = await puppeteer.launch({ executablePath: process.env.CHROMIUM_PATH, - headless: process.env.LAUNCH_HEADLESS === 'true', + headless: !!process.env.LAUNCH_HEADLESS, defaultViewport: { width, height, diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 8cb311ac1..ccf74d39e 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -112,7 +112,7 @@ const getBrowserPromise = (async () => { ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, executablePath: process.env.CHROMIUM_PATH, - headless: process.env.LAUNCH_HEADLESS === 'true', + headless: !!process.env.LAUNCH_HEADLESS, timeout: 120000, // 2 minutes }); })(); From d9237723c57d181ae0bfc977b2cfe5b1f77ad8a9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 17 Nov 2022 10:43:30 +0800 Subject: [PATCH 08/10] Share puppeteer browser instance between content handlers --- .../content-handler/src/content-handler.ts | 3 +- packages/content-handler/src/index.ts | 6 ++- .../src/websites/twitter-handler.ts | 54 ++++++++++--------- packages/puppeteer-parse/index.js | 11 ++-- 4 files changed, 41 insertions(+), 33 deletions(-) diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts index fd1fe5291..a05e2ac1a 100644 --- a/packages/content-handler/src/content-handler.ts +++ b/packages/content-handler/src/content-handler.ts @@ -3,6 +3,7 @@ import rfc2047 from 'rfc2047' import { v4 as uuid } from 'uuid' import { parseHTML } from 'linkedom' import axios from 'axios' +import { Browser } from 'puppeteer-core' interface Unsubscribe { mailTo?: string @@ -62,7 +63,7 @@ export abstract class ContentHandler { return false } - async preHandle(url: string): Promise { + async preHandle(url: string, browser?: Browser): Promise { return Promise.resolve({ url }) } diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index ce61c0a64..e210d025d 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -28,6 +28,7 @@ import { GhostHandler } from './newsletters/ghost-handler' import { parseHTML } from 'linkedom' import { CooperPressHandler } from './newsletters/cooper-press-handler' import { HeyWorldHandler } from './newsletters/hey-world-handler' +import { Browser } from 'puppeteer-core' const validateUrlString = (url: string) => { const u = new URL(url) @@ -80,7 +81,8 @@ const newsletterHandlers: ContentHandler[] = [ ] export const preHandleContent = async ( - url: string + url: string, + browser: Browser ): Promise => { // Before we run the regular handlers we check to see if we need tp // pre-resolve the URL. TODO: This should probably happen recursively, @@ -104,7 +106,7 @@ export const preHandleContent = async ( for (const handler of contentHandlers) { if (handler.shouldPreHandle(url)) { console.log('preHandleContent', handler.name, url) - return handler.preHandle(url) + return handler.preHandle(url, browser) } } return undefined diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 17bbdc4c2..788301b8e 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -2,7 +2,7 @@ import { ContentHandler, PreHandleResult } from '../content-handler' import axios from 'axios' import { DateTime } from 'luxon' import _ from 'underscore' -import puppeteer from 'puppeteer-core' +import { Browser } from 'puppeteer-core' interface TweetIncludes { users: { @@ -168,10 +168,11 @@ const getTweetsFromResponse = (response: Tweets): Tweet[] => { } const getOldTweets = async ( + browser: Browser, conversationId: string, username: string ): Promise => { - const tweetIds = await getTweetIds(conversationId, username) + const tweetIds = await getTweetIds(browser, conversationId, username) if (tweetIds.length === 0) { return [] } @@ -197,37 +198,40 @@ const waitFor = (ms: number) => /** * Get tweets(even older than 7 days) using puppeteer + * @param browser * @param {string} tweetId * @param {string} author */ const getTweetIds = async ( + browser: Browser, tweetId: string, author: string ): Promise => { const pageURL = `https://twitter.com/${author}/status/${tweetId}` - // Modify this variable to control the size of viewport - const factor = 0.2 - const height = Math.floor(2000 / factor) - const width = Math.floor(1700 / factor) - - const browser = await puppeteer.launch({ - executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, - defaultViewport: { - width, - height, - }, - args: [ - `--force-device-scale-factor=${factor}`, - `--window-size=${width},${height}`, - '--no-sandbox', - '--disable-setuid-sandbox', - ], - }) + // // Modify this variable to control the size of viewport + // const factor = 0.2 + // const height = Math.floor(2000 / factor) + // const width = Math.floor(1700 / factor) + // + // const browser = await puppeteer.launch({ + // executablePath: process.env.CHROMIUM_PATH, + // headless: !!process.env.LAUNCH_HEADLESS, + // defaultViewport: { + // width, + // height, + // }, + // args: [ + // `--force-device-scale-factor=${factor}`, + // `--window-size=${width},${height}`, + // '--no-sandbox', + // '--disable-setuid-sandbox', + // ], + // }) + const context = await browser.createIncognitoBrowserContext() try { - const page = await browser.newPage() + const page = await context.newPage() await page.goto(pageURL, { waitUntil: 'networkidle2', @@ -291,7 +295,7 @@ const getTweetIds = async ( console.log(error) return [] } finally { - await browser.close() + await context.close() } } @@ -305,7 +309,7 @@ export class TwitterHandler extends ContentHandler { return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) } - async preHandle(url: string): Promise { + async preHandle(url: string, browser: Browser): Promise { const tweetId = tweetIdFromStatusUrl(url) if (!tweetId) { throw new Error('could not find tweet id in url') @@ -326,7 +330,7 @@ export class TwitterHandler extends ContentHandler { const description = _.escape(tweetData.text) // use puppeteer to get all tweet replies in the thread - const tweets = await getOldTweets(conversationId, author.username) + const tweets = await getOldTweets(browser, conversationId, author.username) let tweetsContent = '' for (const tweet of tweets) { diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index ccf74d39e..fca2e292b 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -237,7 +237,8 @@ async function fetchContent(req, res) { // pre handle url with custom handlers let title, content, contentType; try { - const result = await preHandleContent(url); + const browser = await getBrowserPromise; + const result = await preHandleContent(url, browser); if (result && result.url) { url = result.url validateUrlString(url); @@ -682,16 +683,16 @@ async function preview(req, res) { return res.sendStatus(400); } - const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH); + const browser = await getBrowserPromise; logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; const page = await browser.newPage(); const pageLoadingStart = Date.now(); const modifiedUrl = new URL(url); - modifiedUrl.searchParams.append('fontSize', 24); - modifiedUrl.searchParams.append('adjustAspectRatio', 1.91); + modifiedUrl.searchParams.append('fontSize', '24'); + modifiedUrl.searchParams.append('adjustAspectRatio', '1.91'); try { - await page.goto(modifiedUrl); + await page.goto(modifiedUrl.toString()); logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart }; } catch (error) { console.log('error going to page: ', modifiedUrl) From c18217e6974c3ae5c7dd1fa0a4d10c14988f32d3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 17 Nov 2022 11:06:56 +0800 Subject: [PATCH 09/10] Fix logRecord being shared by requests --- .../src/websites/twitter-handler.ts | 29 ++++--------------- packages/puppeteer-parse/index.js | 14 ++++----- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 788301b8e..afb8985ef 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -2,7 +2,7 @@ import { ContentHandler, PreHandleResult } from '../content-handler' import axios from 'axios' import { DateTime } from 'luxon' import _ from 'underscore' -import { Browser } from 'puppeteer-core' +import { Browser, BrowserContext } from 'puppeteer-core' interface TweetIncludes { users: { @@ -209,28 +209,9 @@ const getTweetIds = async ( ): Promise => { const pageURL = `https://twitter.com/${author}/status/${tweetId}` - // // Modify this variable to control the size of viewport - // const factor = 0.2 - // const height = Math.floor(2000 / factor) - // const width = Math.floor(1700 / factor) - // - // const browser = await puppeteer.launch({ - // executablePath: process.env.CHROMIUM_PATH, - // headless: !!process.env.LAUNCH_HEADLESS, - // defaultViewport: { - // width, - // height, - // }, - // args: [ - // `--force-device-scale-factor=${factor}`, - // `--window-size=${width},${height}`, - // '--no-sandbox', - // '--disable-setuid-sandbox', - // ], - // }) - - const context = await browser.createIncognitoBrowserContext() + let context: BrowserContext | undefined try { + context = await browser.createIncognitoBrowserContext() const page = await context.newPage() await page.goto(pageURL, { @@ -295,7 +276,9 @@ const getTweetIds = async ( console.log(error) return [] } finally { - await context.close() + if (context) { + await context.close() + } } } diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index fca2e292b..8c54c18d7 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -117,8 +117,6 @@ const getBrowserPromise = (async () => { }); })(); -let logRecord, functionStartTime; - const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => { const stream = await axios.get(contentObjUrl, { responseType: 'stream' }); return await axios.put(uploadSignedUrl, stream.data, { @@ -211,13 +209,13 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId }; async function fetchContent(req, res) { - functionStartTime = Date.now(); + let functionStartTime = Date.now(); let url = getUrl(req); const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); - logRecord = { + let logRecord = { url, userId, articleSavingRequestId, @@ -253,7 +251,7 @@ async function fetchContent(req, res) { let context, page, finalUrl; try { if ((!content || !title) && contentType !== 'application/pdf') { - const result = await retrievePage(url) + const result = await retrievePage(url, logRecord, functionStartTime); if (result && result.context) { context = result.context } if (result && result.page) { page = result.page } if (result && result.finalUrl) { finalUrl = result.finalUrl } @@ -267,7 +265,7 @@ async function fetchContent(req, res) { const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId); } else { if (!content || !title) { - const result = await retrieveHtml(page); + const result = await retrieveHtml(page, logRecord); if (result.isBlocked) { const sbResult = await fetchContentWithScrapingBee(url) title = sbResult.title @@ -387,7 +385,7 @@ async function blockResources(client) { await client.send('Network.setBlockedURLs', { urls: blockedResources }); } -async function retrievePage(url) { +async function retrievePage(url, logRecord, functionStartTime) { validateUrlString(url); const browser = await getBrowserPromise; @@ -501,7 +499,7 @@ async function retrievePage(url) { } } -async function retrieveHtml(page) { +async function retrieveHtml(page, logRecord) { let domContent = '', title; try { title = await page.title(); From 44e4e080dd3af52c230ed9ec2917c6df90c01c63 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 17 Nov 2022 11:23:32 +0800 Subject: [PATCH 10/10] increase navigation timeout to 30 seconds --- packages/puppeteer-parse/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 8c54c18d7..efce2c6e0 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -482,7 +482,7 @@ async function retrievePage(url, logRecord, functionStartTime) { }); try { - const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] }); + const response = await page.goto(url, { timeout: 30 * 1000, waitUntil: ['networkidle2'] }); const finalUrl = response.url(); const contentType = response.headers()['content-type'];