From 76ff6906f34ec18e13f12f69f891f47efb43bda6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:23:40 +0000 Subject: [PATCH 01/15] Bump apollo-server-core from 3.10.0 to 3.11.0 Bumps [apollo-server-core](https://github.com/apollographql/apollo-server/tree/HEAD/packages/apollo-server-core) from 3.10.0 to 3.11.0. - [Release notes](https://github.com/apollographql/apollo-server/releases) - [Changelog](https://github.com/apollographql/apollo-server/blob/apollo-server-core@3.11.0/CHANGELOG.md) - [Commits](https://github.com/apollographql/apollo-server/commits/apollo-server-core@3.11.0/packages/apollo-server-core) --- updated-dependencies: - dependency-name: apollo-server-core dependency-type: indirect ... Signed-off-by: dependabot[bot] --- yarn.lock | 161 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 102 insertions(+), 59 deletions(-) diff --git a/yarn.lock b/yarn.lock index 27452d1b1..fe597b75b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16,10 +16,10 @@ dependencies: "@jridgewell/trace-mapping" "^0.3.0" -"@apollo/protobufjs@1.2.4": - version "1.2.4" - resolved "https://registry.yarnpkg.com/@apollo/protobufjs/-/protobufjs-1.2.4.tgz#d913e7627210ec5efd758ceeb751c776c68ba133" - integrity sha512-npVJ9NVU/pynj+SCU+fambvTneJDyCnif738DnZ7pCxdDtzeEz7WkpSIq5wNUmWm5Td55N+S2xfqZ+WP4hDLng== +"@apollo/protobufjs@1.2.6": + version "1.2.6" + resolved "https://registry.yarnpkg.com/@apollo/protobufjs/-/protobufjs-1.2.6.tgz#d601e65211e06ae1432bf5993a1a0105f2862f27" + integrity sha512-Wqo1oSHNUj/jxmsVp4iR3I480p6qdqHikn38lKrFhfzcDJ7lwd7Ck7cHRl4JE81tWNArl77xhnG/OkZhxKBYOw== dependencies: "@protobufjs/aspromise" "^1.1.2" "@protobufjs/base64" "^1.1.2" @@ -49,9 +49,9 @@ lru-cache "^7.10.1" "@apollo/utils.logger@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@apollo/utils.logger/-/utils.logger-1.0.0.tgz#6e3460a2250c2ef7c2c3b0be6b5e148a1596f12b" - integrity sha512-dx9XrjyisD2pOa+KsB5RcDbWIAdgC91gJfeyLCgy0ctJMjQe7yZK5kdWaWlaOoCeX0z6YI9iYlg7vMPyMpQF3Q== + version "1.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.logger/-/utils.logger-1.0.1.tgz#aea0d1bb7ceb237f506c6bbf38f10a555b99a695" + integrity sha512-XdlzoY7fYNK4OIcvMD2G94RoFZbzTQaNP0jozmqqMudmaGo2I/2Jx71xlDJ801mWA/mbYRihyaw6KJii7k5RVA== "@apollo/utils.printwithreducedwhitespace@^1.1.0": version "1.1.0" @@ -2828,6 +2828,14 @@ "@graphql-tools/utils" "8.9.0" tslib "^2.4.0" +"@graphql-tools/merge@8.3.10": + version "8.3.10" + resolved "https://registry.yarnpkg.com/@graphql-tools/merge/-/merge-8.3.10.tgz#81f374bc1e8c81d45cb1003d8ed05f181b7e6bd5" + integrity sha512-/hSg69JwqEA+t01wQmMGKPuaJ9VJBSz6uAXhbNNrTBJu8bmXljw305NVXM49pCwDKFVUGtbTqYrBeLcfT3RoYw== + dependencies: + "@graphql-tools/utils" "9.0.1" + tslib "^2.4.0" + "@graphql-tools/merge@^8.2.1": version "8.2.2" resolved "https://registry.yarnpkg.com/@graphql-tools/merge/-/merge-8.2.2.tgz#433566c662a33f5a9c3cc5f3ce3753fb0019477a" @@ -2837,14 +2845,14 @@ tslib "~2.3.0" "@graphql-tools/mock@^8.1.2": - version "8.5.1" - resolved "https://registry.yarnpkg.com/@graphql-tools/mock/-/mock-8.5.1.tgz#379d18eafdcb65486beb8f9247b33b7b693c53aa" - integrity sha512-cwwqGs9Rofev1JdMheAseqM/rw1uw4CYb35vv3Kcv2bbyiPF+490xdlHqFeIazceotMFxC60LlQztwb64rsEnw== + version "8.7.10" + resolved "https://registry.yarnpkg.com/@graphql-tools/mock/-/mock-8.7.10.tgz#1a277f29ba96b8111c063eb6f5899df441be786d" + integrity sha512-PuRGfk6TQger7EfE08yO3+QCAcZ6nYo3kyoEmTPc27w4yiqKCwZIyD8vegzl/EQphEourjaOhO149te6qNEUeQ== dependencies: - "@graphql-tools/schema" "^8.3.1" - "@graphql-tools/utils" "^8.6.0" + "@graphql-tools/schema" "9.0.8" + "@graphql-tools/utils" "9.0.1" fast-json-stable-stringify "^2.1.0" - tslib "~2.3.0" + tslib "^2.4.0" "@graphql-tools/optimize@^1.0.1": version "1.2.0" @@ -2908,6 +2916,16 @@ tslib "^2.4.0" value-or-promise "1.0.11" +"@graphql-tools/schema@9.0.8": + version "9.0.8" + resolved "https://registry.yarnpkg.com/@graphql-tools/schema/-/schema-9.0.8.tgz#df3119c8543e6dacf425998f83aa714e2ee86eb0" + integrity sha512-PnES7sNkhQ/FdPQhP7cup0OIzwzQh+nfjklilU7YJzE209ACIyEQtxoNCfvPW5eV6hc9bWsBQeI3Jm4mMtwxNA== + dependencies: + "@graphql-tools/merge" "8.3.10" + "@graphql-tools/utils" "9.0.1" + tslib "^2.4.0" + value-or-promise "1.0.11" + "@graphql-tools/url-loader@^7.0.11", "@graphql-tools/url-loader@^7.4.2": version "7.7.1" resolved "https://registry.yarnpkg.com/@graphql-tools/url-loader/-/url-loader-7.7.1.tgz#2faabdc1d2c47edc8edc9cc938eee2767189869f" @@ -2947,6 +2965,13 @@ dependencies: tslib "^2.4.0" +"@graphql-tools/utils@9.0.1": + version "9.0.1" + resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-9.0.1.tgz#04933b34c3435ef9add4f8bdfdf452040376f9d0" + integrity sha512-z6FimVa5E44bHKmqK0/uMp9hHvHo2Tkt9A5rlLb40ReD/8IFKehSXLzM4b2N1vcP7mSsbXIdDK9Aoc8jT/he1Q== + dependencies: + tslib "^2.4.0" + "@graphql-tools/utils@^8.1.1": version "8.1.2" resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-8.1.2.tgz#a376259fafbca7532fda657e3abeec23b545e5d3" @@ -4563,7 +4588,7 @@ "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" - integrity sha1-m4sMxmPWaafY9vXQiToU00jzD78= + integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== "@protobufjs/base64@^1.1.2": version "1.1.2" @@ -4578,12 +4603,12 @@ "@protobufjs/eventemitter@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" - integrity sha1-NVy8mLr61ZePntCV85diHx0Ga3A= + integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== "@protobufjs/fetch@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" - integrity sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU= + integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== dependencies: "@protobufjs/aspromise" "^1.1.1" "@protobufjs/inquire" "^1.1.0" @@ -4591,27 +4616,27 @@ "@protobufjs/float@^1.0.2": version "1.0.2" resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" - integrity sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E= + integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== "@protobufjs/inquire@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" - integrity sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik= + integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== "@protobufjs/path@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" - integrity sha1-bMKyDFya1q0NzP0hynZz2Nf79o0= + integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== "@protobufjs/pool@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" - integrity sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q= + integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== "@protobufjs/utf8@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" - integrity sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA= + integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== "@radix-ui/popper@0.1.0": version "0.1.0" @@ -7892,7 +7917,12 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.180.tgz#4ab7c9ddfc92ec4a887886483bc14c79fb380670" integrity sha512-XOKXa1KIxtNXgASAnwj7cnttJxS4fksBRywK/9LzRV5YxrF80BXZIGeQSuoESQ/VkUj30Ae0+YcuHc15wJCB2g== -"@types/long@^4.0.0", "@types/long@^4.0.1": +"@types/long@^4.0.0": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" + integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA== + +"@types/long@^4.0.1": version "4.0.1" resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9" integrity sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w== @@ -9146,17 +9176,17 @@ apollo-datasource@^3.3.1, apollo-datasource@^3.3.2: "@apollo/utils.keyvaluecache" "^1.0.1" apollo-server-env "^4.2.1" -apollo-reporting-protobuf@^3.3.1, apollo-reporting-protobuf@^3.3.2: - version "3.3.2" - resolved "https://registry.yarnpkg.com/apollo-reporting-protobuf/-/apollo-reporting-protobuf-3.3.2.tgz#2078c53d3140bc6221c6040c5326623e0c21c8d4" - integrity sha512-j1tx9tmkVdsLt1UPzBrvz90PdjAeKW157WxGn+aXlnnGfVjZLIRXX3x5t1NWtXvB7rVaAsLLILLtDHW382TSoQ== +apollo-reporting-protobuf@^3.3.1, apollo-reporting-protobuf@^3.3.3: + version "3.3.3" + resolved "https://registry.yarnpkg.com/apollo-reporting-protobuf/-/apollo-reporting-protobuf-3.3.3.tgz#df2b7ff73422cd682af3f1805d32301aefdd9e89" + integrity sha512-L3+DdClhLMaRZWVmMbBcwl4Ic77CnEBPXLW53F7hkYhkaZD88ivbCVB1w/x5gunO6ZHrdzhjq0FHmTsBvPo7aQ== dependencies: - "@apollo/protobufjs" "1.2.4" + "@apollo/protobufjs" "1.2.6" apollo-server-core@^3.10.0: - version "3.10.0" - resolved "https://registry.yarnpkg.com/apollo-server-core/-/apollo-server-core-3.10.0.tgz#6680b4eb4699829ed50d8a592721ee5e5e11e041" - integrity sha512-ln5drIk3oW/ycYhcYL9TvM7vRf7OZwJrgHWlnjnMakozBQIBSumdMi4pN001DhU9mVBWTfnmBv3CdcxJdGXIvA== + version "3.11.0" + resolved "https://registry.yarnpkg.com/apollo-server-core/-/apollo-server-core-3.11.0.tgz#dbbf4c03ac0fdd8774e03c1f4f0d1ea1448b743c" + integrity sha512-5iRlkbilXpQeY66/F2/t2oNO0YSqb+kFb5lyMUIqK9VLuBfI/hILQDa5H71ar7hhexKwoDzIDfSJRg5ASNmnQw== dependencies: "@apollo/utils.keyvaluecache" "^1.0.1" "@apollo/utils.logger" "^1.0.0" @@ -9167,18 +9197,19 @@ apollo-server-core@^3.10.0: "@graphql-tools/schema" "^8.0.0" "@josephg/resolvable" "^1.0.0" apollo-datasource "^3.3.2" - apollo-reporting-protobuf "^3.3.2" + apollo-reporting-protobuf "^3.3.3" apollo-server-env "^4.2.1" apollo-server-errors "^3.3.1" - apollo-server-plugin-base "^3.6.2" - apollo-server-types "^3.6.2" + apollo-server-plugin-base "^3.7.0" + apollo-server-types "^3.7.0" async-retry "^1.2.1" fast-json-stable-stringify "^2.1.0" graphql-tag "^2.11.0" loglevel "^1.6.8" lru-cache "^6.0.0" + node-abort-controller "^3.0.1" sha.js "^2.4.11" - uuid "^8.0.0" + uuid "^9.0.0" whatwg-mimetype "^3.0.0" apollo-server-env@^4.2.1: @@ -9210,21 +9241,21 @@ apollo-server-express@^3.6.3: cors "^2.8.5" parseurl "^1.3.3" -apollo-server-plugin-base@^3.6.2: - version "3.6.2" - resolved "https://registry.yarnpkg.com/apollo-server-plugin-base/-/apollo-server-plugin-base-3.6.2.tgz#f256e1f274c8fee0d7267b6944f402da71788fb3" - integrity sha512-erWXjLOO1u7fxQkbxJ2cwSO7p0tYzNied91I1SJ9tikXZ/2eZUyDyvrpI+4g70kOdEi+AmJ5Fo8ahEXKJ75zdg== +apollo-server-plugin-base@^3.7.0: + version "3.7.0" + resolved "https://registry.yarnpkg.com/apollo-server-plugin-base/-/apollo-server-plugin-base-3.7.0.tgz#b7170c2be0344d5f4382fea951f6d1dd274d6635" + integrity sha512-YRPjqFHvWK9eM4gN3D4ArrAtPY7Mb1FL+YoXXwq2GxdrsZSolnDYQkqZ6BhK11J8lUmAQpnpunK91IPZshWluA== dependencies: - apollo-server-types "^3.6.2" + apollo-server-types "^3.7.0" -apollo-server-types@^3.6.2: - version "3.6.2" - resolved "https://registry.yarnpkg.com/apollo-server-types/-/apollo-server-types-3.6.2.tgz#34bb0c335fcce3057cbdf72b3b63da182de6fc84" - integrity sha512-9Z54S7NB+qW1VV+kmiqwU2Q6jxWfX89HlSGCGOo3zrkrperh85LrzABgN9S92+qyeHYd72noMDg2aI039sF3dg== +apollo-server-types@^3.6.2, apollo-server-types@^3.7.0: + version "3.7.0" + resolved "https://registry.yarnpkg.com/apollo-server-types/-/apollo-server-types-3.7.0.tgz#5a6f6f05a3c2ed937ad339b91665248dad957733" + integrity sha512-Y2wx7eH/dqqYDdzt0KBJRbVKR10bLiup2aT8huoBbp/u3nbCN88jo1yW+FvlETeV+iKuoY3RiZDlHIvcDQ5/lA== dependencies: "@apollo/utils.keyvaluecache" "^1.0.1" "@apollo/utils.logger" "^1.0.0" - apollo-reporting-protobuf "^3.3.2" + apollo-reporting-protobuf "^3.3.3" apollo-server-env "^4.2.1" app-root-dir@^1.0.2: @@ -11869,7 +11900,7 @@ cssesc@^3.0.0: cssfilter@0.0.10: version "0.0.10" resolved "https://registry.yarnpkg.com/cssfilter/-/cssfilter-0.0.10.tgz#c6d2672632a2e5c83e013e6864a42ce8defd20ae" - integrity sha1-xtJnJjKi5cg+AT5oZKQs6N79IK4= + integrity sha512-FAaLDaplstoRsDR8XGYH51znUN0UY7nMc6Z9/fvE8EXGwvJE9hu7W2vHwx1+bd6gCYnln9nLbzxFTrcO9YQDZw== cssom@^0.4.4: version "0.4.4" @@ -14952,9 +14983,11 @@ graphql-sse@^1.0.1: integrity sha512-y2mVBN2KwNrzxX2KBncQ6kzc6JWvecxuBernrl0j65hsr6MAS3+Yn8PTFSOgRmtolxugepxveyZVQEuaNEbw3w== graphql-tag@^2.11.0: - version "2.11.0" - resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.11.0.tgz#1deb53a01c46a7eb401d6cb59dec86fa1cccbffd" - integrity sha512-VmsD5pJqWJnQZMUeRwrDhfgoyqcfwEkvtpANqcoUG8/tOLkwNgU9mzub/Mc78OJMhHjx7gfAMTxzdG43VGg3bA== + version "2.12.6" + resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.12.6.tgz#d441a569c1d2537ef10ca3d1633b48725329b5f1" + integrity sha512-FdSNcu2QQcWnM2VNvSCCDCVS5PpPqpzgFT8+GXzqJuoDd0CBncxCY278u4mhRO7tMgo2JjgJA5aZ+nWSQ/Z+xg== + dependencies: + tslib "^2.1.0" graphql-ws@^5.4.1: version "5.5.5" @@ -18060,7 +18093,7 @@ lodash.snakecase@^4.1.1: lodash.sortby@^4.7.0: version "4.7.0" resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" - integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg= + integrity sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA== lodash.template@^4.5.0: version "4.5.0" @@ -18254,9 +18287,9 @@ lru-cache@^6.0.0: yallist "^4.0.0" lru-cache@^7.10.1: - version "7.12.0" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.12.0.tgz#be2649a992c8a9116efda5c487538dcf715f3476" - integrity sha512-OIP3DwzRZDfLg9B9VP/huWBlpvbkmbfiBy8xmsXp4RPmE4A3MhwNozc5ZJ3fWnSg8fDcdlE/neRTPG2ycEKliw== + version "7.14.1" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.14.1.tgz#8da8d2f5f59827edb388e63e459ac23d6d408fea" + integrity sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA== lru-cache@~4.0.0: version "4.0.2" @@ -19303,6 +19336,11 @@ nock@^13.2.9: lodash "^4.17.21" propagate "^2.0.0" +node-abort-controller@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e" + integrity sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw== + node-addon-api@^1.2.0: version "1.7.2" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-1.7.2.tgz#3df30b95720b53c24e59948b49532b662444f54d" @@ -24354,7 +24392,7 @@ tr46@^2.1.0: tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" - integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== tree-kill@^1.2.2: version "1.2.2" @@ -24498,11 +24536,16 @@ tslib@^1.0.0, tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== -tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.3.1, tslib@^2.4.0, tslib@~2.4.0: +tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.3.0, tslib@^2.3.1, tslib@~2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== +tslib@^2.1.0, tslib@^2.4.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.1.tgz#0d0bfbaac2880b91e22df0768e55be9753a5b17e" + integrity sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA== + tslib@~2.0.1: version "2.0.3" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.3.tgz#8e0741ac45fc0c226e58a17bfc3e64b9bc6ca61c" @@ -25361,7 +25404,7 @@ web-streams-polyfill@^3.2.0: webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" - integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== webidl-conversions@^5.0.0: version "5.0.0" @@ -25660,7 +25703,7 @@ whatwg-mimetype@^3.0.0: whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" - integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== dependencies: tr46 "~0.0.3" webidl-conversions "^3.0.0" @@ -25966,9 +26009,9 @@ xorshift@^0.2.0: integrity sha1-/NgiZ+k1HBPw+5xzMH8lMx0pxjo= xss@^1.0.8: - version "1.0.9" - resolved "https://registry.yarnpkg.com/xss/-/xss-1.0.9.tgz#3ffd565571ff60d2e40db7f3b80b4677bec770d2" - integrity sha512-2t7FahYnGJys6DpHLhajusId7R0Pm2yTmuL0GV9+mV0ZlaLSnb2toBmppATfg5sWIhZQGlsTLoecSzya+l4EAQ== + version "1.0.14" + resolved "https://registry.yarnpkg.com/xss/-/xss-1.0.14.tgz#4f3efbde75ad0d82e9921cc3c95e6590dd336694" + integrity sha512-og7TEJhXvn1a7kzZGQ7ETjdQVS2UfZyTlsEdDOqvQF7GoxNfY+0YLCzBy1kPdsDDx4QuNAonQPddpsn6Xl/7sw== dependencies: commander "^2.20.3" cssfilter "0.0.10" From 0d83ecb06875089b8287302c5ed1ec1d58db7955 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 16 Nov 2022 13:52:39 +0800 Subject: [PATCH 02/15] Use greedy matching on the GitHub title regex This fixes issues with titles that have slashes (/) in their title as the regex match would look for the last possible matching slash. --- .../src/websites/github-handler.ts | 2 +- .../test/github-handler.test.ts | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 packages/content-handler/test/github-handler.test.ts diff --git a/packages/content-handler/src/websites/github-handler.ts b/packages/content-handler/src/websites/github-handler.ts index f086f72d3..7ac64c3fb 100644 --- a/packages/content-handler/src/websites/github-handler.ts +++ b/packages/content-handler/src/websites/github-handler.ts @@ -35,7 +35,7 @@ export class GitHubHandler extends ContentHandler { if (twitterTitle && twitterTitleContent) { twitterTitle.setAttribute( 'content', - twitterTitleContent.replace(/GitHub - .*\//, '') + twitterTitleContent.replace(/GitHub - (.*?)\//, '') ) } diff --git a/packages/content-handler/test/github-handler.test.ts b/packages/content-handler/test/github-handler.test.ts new file mode 100644 index 000000000..16cbb12b3 --- /dev/null +++ b/packages/content-handler/test/github-handler.test.ts @@ -0,0 +1,32 @@ +import { GitHubHandler } from '../src/websites/github-handler' +import 'mocha' +import { expect } from 'chai' +import { parseHTML } from 'linkedom' + +describe('preParse', () => { + it('should update the title on the page', async () => { + const dom = parseHTML( + ` + + + + + +
this is the content of the article
+ + + ` + ) + const result = await new GitHubHandler().preParse( + 'https://github.com/siyuan-note/siyuan', + dom.document + ) + const title = result + .querySelector(`meta[name='twitter:title']`) + ?.getAttribute('content') + + expect(title).to.eq(`repo: This is a title with a / char`) + }) +}) From 8bdbd86e972b167b6bc9ab925df5f494d881ad6d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 11:38:35 +0800 Subject: [PATCH 03/15] Make headless = false in local testing --- packages/content-handler/src/websites/twitter-handler.ts | 2 +- packages/puppeteer-parse/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 17bbdc4c2..e4b23394c 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -213,7 +213,7 @@ const getTweetIds = async ( const browser = await puppeteer.launch({ executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, + headless: process.env.LAUNCH_HEADLESS === 'true', defaultViewport: { width, height, diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index cd72936af..13623fe90 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -169,7 +169,7 @@ const getBrowserPromise = (async () => { ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, + headless: process.env.LAUNCH_HEADLESS === 'true', timeout: 120000, // 2 minutes }); })(); From b3e73cd84be0cc5eb2447bd3cfab20e7a11861c0 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 11:42:13 +0800 Subject: [PATCH 04/15] Add puppeteer-extra plugins --- packages/puppeteer-parse/index.js | 11 +- packages/puppeteer-parse/package.json | 3 + yarn.lock | 263 +++++++++++++++++++++++++- 3 files changed, 271 insertions(+), 6 deletions(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 13623fe90..148dfb7d6 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -15,12 +15,17 @@ const { DateTime } = require('luxon'); const os = require('os'); const { Storage } = require('@google-cloud/storage'); const { parseHTML } = require('linkedom'); -const puppeteer = require('puppeteer-core'); const { preHandleContent } = require("@omnivore/content-handler"); +const puppeteer = require('puppeteer-extra'); + // Add stealth plugin to hide puppeteer usage -// const StealthPlugin = require('puppeteer-extra-plugin-stealth'); -// puppeteer.use(StealthPlugin()); +const StealthPlugin = require('puppeteer-extra-plugin-stealth'); +puppeteer.use(StealthPlugin()); + +// Add adblocker plugin to block all ads and trackers (saves bandwidth) +const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); +puppeteer.use(AdblockerPlugin({ blockTrackers: true })); const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index a9537f6f2..41378e1bf 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -12,6 +12,9 @@ "linkedom": "^0.14.9", "luxon": "^2.3.1", "puppeteer-core": "^16.1.0", + "puppeteer-extra": "^3.3.4", + "puppeteer-extra-plugin-adblocker": "^2.13.5", + "puppeteer-extra-plugin-stealth": "^2.11.1", "underscore": "^1.13.4", "winston": "^3.3.3" }, diff --git a/yarn.lock b/yarn.lock index cbdd96048..7f2ee67fa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2069,6 +2069,41 @@ commander "^4.1.0" microtime "^3.0.0" +"@cliqz/adblocker-content@^1.23.8", "@cliqz/adblocker-content@^1.25.1": + version "1.25.1" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker-content/-/adblocker-content-1.25.1.tgz#da81e7838e288a6f0fdb8a97a0df8169accb74a1" + integrity sha512-7gl2VdNPBfj7aPoq34B5miwGcnda/7LCr+BqnpcSOjdLV6jjT2FrNSAKGFvcH23q0HM1IFhYDV6ydTgsdWFCnA== + dependencies: + "@cliqz/adblocker-extended-selectors" "^1.25.1" + +"@cliqz/adblocker-extended-selectors@^1.25.1": + version "1.25.1" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker-extended-selectors/-/adblocker-extended-selectors-1.25.1.tgz#cfac0080952311399805fe153cd9e7e1331b3c6d" + integrity sha512-4MdMe/YfIok5d8WYVcLR3Ak7vGrmeUV47frgmXEe945luY93vwlzk1NiLYW1JM5Gdm+VePweoS9cJ1/QUTmv+Q== + +"@cliqz/adblocker-puppeteer@1.23.8": + version "1.23.8" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker-puppeteer/-/adblocker-puppeteer-1.23.8.tgz#e74636cd200459d1734929e41504a76939504311" + integrity sha512-Ca1/DBqQXsOpKTFVAHX6OpLTSEupXmUkUWHj6iXhLLleC7RPISN5B0b801VDmaGRqoC5zKRxn0vYbIfpgCWVug== + dependencies: + "@cliqz/adblocker" "^1.23.8" + "@cliqz/adblocker-content" "^1.23.8" + tldts-experimental "^5.6.21" + +"@cliqz/adblocker@^1.23.8": + version "1.25.1" + resolved "https://registry.yarnpkg.com/@cliqz/adblocker/-/adblocker-1.25.1.tgz#4d3e8894ce48ad0d0f8b26a4a1003f0676b7f734" + integrity sha512-1C1/ELI94/XewdUj/o1+Q4ziOigMvTZQA05UERfDoKqpJ+0cbrEF/UImrzpX7n+kYsR7xTJvmf+iNM3zS0tfsg== + dependencies: + "@cliqz/adblocker-content" "^1.25.1" + "@cliqz/adblocker-extended-selectors" "^1.25.1" + "@remusao/guess-url-type" "^1.1.2" + "@remusao/small" "^1.1.2" + "@remusao/smaz" "^1.7.1" + "@types/chrome" "^0.0.197" + "@types/firefox-webext-browser" "^94.0.0" + tldts-experimental "^5.6.21" + "@cnakazawa/watch@^1.0.3": version "1.0.4" resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a" @@ -5110,6 +5145,41 @@ resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4" integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA== +"@remusao/guess-url-type@^1.1.2": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57" + integrity sha512-rbOqre2jW8STjheOsOaQHLgYBaBZ9Owbdt8NO7WvNZftJlaG3y/K9oOkl8ZUpuFBisIhmBuMEW6c+YrQl5inRA== + +"@remusao/small@^1.1.2": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@remusao/small/-/small-1.2.1.tgz#63bfe4548832289f94ac868a0c305970c9a0e5f9" + integrity sha512-7MjoGt0TJMVw1GPKgWq6SJPws1SLsUXQRa43Umht+nkyw2jnpy3WpiLNqGdwo5rHr5Wp9B2W/Pm5RQp656UJdw== + +"@remusao/smaz-compress@^1.9.1": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@remusao/smaz-compress/-/smaz-compress-1.9.1.tgz#fc75eaf9bcac2d58bc4c3d518183a7cb9612d275" + integrity sha512-E2f48TwloQu3r6BdLOGF2aczeH7bJ/32oJGqvzT9SKur0cuUnLcZ7ZXP874E2fwmdE+cXzfC7bKzp79cDnmeyw== + dependencies: + "@remusao/trie" "^1.4.1" + +"@remusao/smaz-decompress@^1.9.1": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@remusao/smaz-decompress/-/smaz-decompress-1.9.1.tgz#8094f997e8fb591a678cda9cf08c209c825eba5b" + integrity sha512-TfjKKprYe3n47od8auhvJ/Ikj9kQTbDTe71ynKlxslrvvUhlIV3VQSuwYuMWMbdz1fIs0H/fxCN1Z8/H3km6/A== + +"@remusao/smaz@^1.7.1": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@remusao/smaz/-/smaz-1.9.1.tgz#a2b9b045385f81e1615a68d932b7cc8b04c9db8d" + integrity sha512-e6BLuP8oaXCZ9+v46Is4ilAZ/Vq6YLgmBP204Ixgk1qTjXmqvFYG7+AS7v9nsZdGOy96r9DWGFbbDVgMxwu1rA== + dependencies: + "@remusao/smaz-compress" "^1.9.1" + "@remusao/smaz-decompress" "^1.9.1" + +"@remusao/trie@^1.4.1": + version "1.4.1" + resolved "https://registry.yarnpkg.com/@remusao/trie/-/trie-1.4.1.tgz#755d09f8a007476334e611f42719b2d581f00720" + integrity sha512-yvwa+aCyYI/UjeD39BnpMypG8N06l86wIDW1/PAc6ihBRnodIfZDwccxQN3n1t74wduzaz74m4ZMHZnB06567Q== + "@rushstack/eslint-patch@^1.0.8": version "1.1.0" resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323" @@ -7611,6 +7681,14 @@ resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.2.21.tgz#9f35a5643129df132cf3b5c1ec64046ea1af0650" integrity sha512-yd+9qKmJxm496BOV9CMNaey8TWsikaZOwMRwPHQIjcOJM9oV+fi9ZMNw3JsVnbEEbo2gRTDnGEBv8pjyn67hNg== +"@types/chrome@^0.0.197": + version "0.0.197" + resolved "https://registry.yarnpkg.com/@types/chrome/-/chrome-0.0.197.tgz#c1b50cdb72ee40f9bc1411506031a9f8a925ab35" + integrity sha512-m1NfS5bOjaypyqQfaX6CxmJodZVcvj5+Mt/K94EBHkflYjPNmXHAzbxfifdLMa0YM3PDyOxohoTS5ug/e6p5jA== + dependencies: + "@types/filesystem" "*" + "@types/har-format" "*" + "@types/cls-hooked@^4.2.1": version "4.3.3" resolved "https://registry.yarnpkg.com/@types/cls-hooked/-/cls-hooked-4.3.3.tgz#c09e2f8dc62198522eaa18a5b6b873053154bd00" @@ -7667,6 +7745,13 @@ resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.12.tgz#6b2c510a7ad7039e98e7b8d3d6598f4359e5c080" integrity sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw== +"@types/debug@^4.1.0": + version "4.1.7" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82" + integrity sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg== + dependencies: + "@types/ms" "*" + "@types/diff-match-patch@^1.0.32": version "1.0.32" resolved "https://registry.yarnpkg.com/@types/diff-match-patch/-/diff-match-patch-1.0.32.tgz#d9c3b8c914aa8229485351db4865328337a3d09f" @@ -7751,11 +7836,28 @@ "@types/qs" "*" "@types/serve-static" "*" +"@types/filesystem@*": + version "0.0.32" + resolved "https://registry.yarnpkg.com/@types/filesystem/-/filesystem-0.0.32.tgz#307df7cc084a2293c3c1a31151b178063e0a8edf" + integrity sha512-Yuf4jR5YYMR2DVgwuCiP11s0xuVRyPKmz8vo6HBY3CGdeMj8af93CFZX+T82+VD1+UqHOxTq31lO7MI7lepBtQ== + dependencies: + "@types/filewriter" "*" + +"@types/filewriter@*": + version "0.0.29" + resolved "https://registry.yarnpkg.com/@types/filewriter/-/filewriter-0.0.29.tgz#a48795ecadf957f6c0d10e0c34af86c098fa5bee" + integrity sha512-BsPXH/irW0ht0Ji6iw/jJaK8Lj3FJemon2gvEqHKpCdDCeemHa+rI3WBGq5z7cDMZgoLjY40oninGxqk+8NzNQ== + "@types/fined@*": version "1.1.3" resolved "https://registry.yarnpkg.com/@types/fined/-/fined-1.1.3.tgz#83f03e8f0a8d3673dfcafb18fce3571f6250e1bc" integrity sha512-CWYnSRnun3CGbt6taXeVo2lCbuaj4mchVJ4UF/BdU5TSuIn3AmS13pGMwCsBUoehGbhZrBrpNJZSZI5EVilXww== +"@types/firefox-webext-browser@^94.0.0": + version "94.0.1" + resolved "https://registry.yarnpkg.com/@types/firefox-webext-browser/-/firefox-webext-browser-94.0.1.tgz#52afb975253dc0fd350d5d58c7fe9fd1a01f64a1" + integrity sha512-I6iHRQJSTZ+gYt2IxdH2RRAMvcUyK8v5Ig7fHQR0IwUNYP7hz9+cziBVIKxLCO6XI7fiyRsNOWObfl3/4Js2Lg== + "@types/fluent-ffmpeg@^2.1.20": version "2.1.20" resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz#3b5f42fc8263761d58284fa46ee6759a64ce54ac" @@ -7793,6 +7895,11 @@ dependencies: graphql "^15.3.0" +"@types/har-format@*": + version "1.2.9" + resolved "https://registry.yarnpkg.com/@types/har-format/-/har-format-1.2.9.tgz#b9b3a9bfc33a078e7d898a00b09662910577f4a4" + integrity sha512-rffW6MhQ9yoa75bdNi+rjZBAvu2HhehWJXlhuWXnWdENeuKe82wUgAwxYOb7KRKKmxYN+D/iRKd2NDQMLqlUmg== + "@types/hast@^2.0.0": version "2.3.4" resolved "https://registry.yarnpkg.com/@types/hast/-/hast-2.3.4.tgz#8aa5ef92c117d20d974a82bdfb6a648b08c0bafc" @@ -7995,6 +8102,11 @@ resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.2.3.tgz#bbeb55fbc73f28ea6de601fbfa4613f58d785323" integrity sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw== +"@types/ms@*": + version "0.7.31" + resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" + integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== + "@types/nanoid@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@types/nanoid/-/nanoid-3.0.0.tgz#c757b20f343f3a1dd76e80a9a431b6290fc20f35" @@ -11063,6 +11175,17 @@ cliui@^7.0.2: strip-ansi "^6.0.0" wrap-ansi "^7.0.0" +clone-deep@^0.2.4: + version "0.2.4" + resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-0.2.4.tgz#4e73dd09e9fb971cc38670c5dced9c1896481cc6" + integrity sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg== + dependencies: + for-own "^0.1.3" + is-plain-object "^2.0.1" + kind-of "^3.0.2" + lazy-cache "^1.0.3" + shallow-clone "^0.1.2" + clone-deep@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-4.0.1.tgz#c19fd9bdbbf85942b4fd979c84dcf7d5f07c2387" @@ -14103,11 +14226,23 @@ follow-redirects@^1.0.0, follow-redirects@^1.14.0, follow-redirects@^1.14.4, fol resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5" integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA== +for-in@^0.1.3: + version "0.1.8" + resolved "https://registry.yarnpkg.com/for-in/-/for-in-0.1.8.tgz#d8773908e31256109952b1fdb9b3fa867d2775e1" + integrity sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g== + for-in@^1.0.1, for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= +for-own@^0.1.3: + version "0.1.5" + resolved "https://registry.yarnpkg.com/for-own/-/for-own-0.1.5.tgz#5265c681a4f294dabbf17c9509b6763aa84510ce" + integrity sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw== + dependencies: + for-in "^1.0.1" + for-own@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/for-own/-/for-own-1.0.0.tgz#c63332f415cedc4b04dbfe70cf836494c53cb44b" @@ -14275,6 +14410,15 @@ fs-extra@^0.30.0: path-is-absolute "^1.0.0" rimraf "^2.2.8" +fs-extra@^10.0.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" + integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + fs-extra@^9.0.0, fs-extra@^9.0.1, fs-extra@^9.1.0: version "9.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" @@ -16082,7 +16226,7 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@^1.1.5, is-buffer@~1.1.6: +is-buffer@^1.0.2, is-buffer@^1.1.5, is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -16394,7 +16538,7 @@ is-plain-object@5.0.0, is-plain-object@^5.0.0: resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== -is-plain-object@^2.0.3, is-plain-object@^2.0.4: +is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677" integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og== @@ -17587,6 +17731,13 @@ keyv@^3.0.0: dependencies: json-buffer "3.0.0" +kind-of@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-2.0.1.tgz#018ec7a4ce7e3a86cb9141be519d24c8faa981b5" + integrity sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg== + dependencies: + is-buffer "^1.0.2" + kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -17687,6 +17838,16 @@ lazy-ass@^1.6.0: resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513" integrity sha1-eZllXoZGwX8In90YfRUNMyTVRRM= +lazy-cache@^0.2.3: + version "0.2.7" + resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-0.2.7.tgz#7feddf2dcb6edb77d11ef1d117ab5ffdf0ab1b65" + integrity sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ== + +lazy-cache@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-1.0.4.tgz#a1d78fc3a50474cb80845d3b3b6e1da49a446e8e" + integrity sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ== + lazy-universal-dotenv@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/lazy-universal-dotenv/-/lazy-universal-dotenv-3.0.1.tgz#a6c8938414bca426ab8c9463940da451a911db38" @@ -18608,6 +18769,15 @@ meow@^8.0.0: type-fest "^0.18.0" yargs-parser "^20.2.3" +merge-deep@^3.0.1: + version "3.0.3" + resolved "https://registry.yarnpkg.com/merge-deep/-/merge-deep-3.0.3.tgz#1a2b2ae926da8b2ae93a0ac15d90cd1922766003" + integrity sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA== + dependencies: + arr-union "^3.1.0" + clone-deep "^0.2.4" + kind-of "^3.0.2" + merge-descriptors@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" @@ -18949,6 +19119,14 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" +mixin-object@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/mixin-object/-/mixin-object-2.0.1.tgz#4fb949441dab182540f1fe035ba60e1947a5e57e" + integrity sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA== + dependencies: + for-in "^0.1.3" + is-extendable "^0.1.1" + mkdirp-classic@^0.5.2: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" @@ -19383,7 +19561,7 @@ node-fetch@2.6.1: resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== -node-fetch@2.6.7, node-fetch@^2.3.0, node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@2.6.7, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== @@ -21451,6 +21629,63 @@ puppeteer-core@^19.1.1: unbzip2-stream "1.4.3" ws "8.9.0" +puppeteer-extra-plugin-adblocker@^2.13.5: + version "2.13.5" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.13.5.tgz#c86ce94873bf6fe500555d3972eccdcca4914f6f" + integrity sha512-HMVWLA1MLrzIGr/A71PYAWZEHENqQOEaQQHtPje0uSLc6QPOQY5tbbocx4BsUiQL2V1FwgT21UU09P5lV4vrZw== + dependencies: + "@cliqz/adblocker-puppeteer" "1.23.8" + debug "^4.1.1" + node-fetch "^2.6.0" + puppeteer-extra-plugin "^3.2.2" + +puppeteer-extra-plugin-stealth@^2.11.1: + version "2.11.1" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.1.tgz#7d56a27a986cb5eb69dca3c65695ad6444f4e822" + integrity sha512-n0wdC0Ilc9tk5L6FWLyd0P2gT8b2fp+2NuB+KB0oTSw3wXaZ0D6WNakjJsayJ4waGzIJFCUHkmK9zgx5NKMoFw== + dependencies: + debug "^4.1.1" + puppeteer-extra-plugin "^3.2.2" + puppeteer-extra-plugin-user-preferences "^2.4.0" + +puppeteer-extra-plugin-user-data-dir@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.0.tgz#20e87582482b61e497abd96fec452f63bd2d9123" + integrity sha512-qrhYPTGIqzL2hpeJ5DXjf8xMy5rt1UvcqSgpGTTOUOjIMz1ROWnKHjBoE9fNBJ4+ToRZbP8MzIDXWlEk/e1zJA== + dependencies: + debug "^4.1.1" + fs-extra "^10.0.0" + puppeteer-extra-plugin "^3.2.2" + rimraf "^3.0.2" + +puppeteer-extra-plugin-user-preferences@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.0.tgz#8b75bc39c3de9913e236ae1d982a24711d84ba6f" + integrity sha512-4XxMhMkJ+qqLsPY9ULF90qS9Bj1Qrwwgp1TY9zTdp1dJuy7QSgYE7xlyamq3cKrRuzg3QUOqygJo52sVeXSg5A== + dependencies: + debug "^4.1.1" + deepmerge "^4.2.2" + puppeteer-extra-plugin "^3.2.2" + puppeteer-extra-plugin-user-data-dir "^2.4.0" + +puppeteer-extra-plugin@^3.2.2: + version "3.2.2" + resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.2.tgz#3c02c0a10f8eadf32e7debb7ee24105a53afc17b" + integrity sha512-0uatQxzuVn8yegbrEwSk03wvwpMB5jNs7uTTnermylLZzoT+1rmAQaJXwlS3+vADUbw6ELNgNEHC7Skm0RqHbQ== + dependencies: + "@types/debug" "^4.1.0" + debug "^4.1.1" + merge-deep "^3.0.1" + +puppeteer-extra@^3.3.4: + version "3.3.4" + resolved "https://registry.yarnpkg.com/puppeteer-extra/-/puppeteer-extra-3.3.4.tgz#e0ecf021783d1112b6b0db20546d5022e632ed55" + integrity sha512-fN5pHvSMJ8d1o7Z8wLLTQOUBpORD2BcFn+KDs7QnkGZs9SV69hcUcce67vX4L4bNSEG3A0P6Osrv+vWNhhdm8w== + dependencies: + "@types/debug" "^4.1.0" + debug "^4.1.1" + deepmerge "^4.2.2" + puppeteer@^10.1.0: version "10.4.0" resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-10.4.0.tgz#a6465ff97fda0576c4ac29601406f67e6fea3dc7" @@ -22939,6 +23174,16 @@ sha.js@^2.4.0, sha.js@^2.4.11, sha.js@^2.4.8: inherits "^2.0.1" safe-buffer "^5.0.1" +shallow-clone@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-0.1.2.tgz#5909e874ba77106d73ac414cfec1ffca87d97060" + integrity sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw== + dependencies: + is-extendable "^0.1.1" + kind-of "^2.0.1" + lazy-cache "^0.2.3" + mixin-object "^2.0.1" + shallow-clone@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3" @@ -24277,6 +24522,18 @@ title-case@^3.0.3: dependencies: tslib "^2.0.3" +tldts-core@^5.7.100: + version "5.7.100" + resolved "https://registry.yarnpkg.com/tldts-core/-/tldts-core-5.7.100.tgz#6144104277a3c4500ec395220d8e03c16fcdfaf7" + integrity sha512-56+vie1oPcJZQiPfnvIIpbyTttUketsjV7lrw/hkMMa/EACPjjDctobWwF3153gR2l+c9O+nYiHkXIL1Cmr9eQ== + +tldts-experimental@^5.6.21: + version "5.7.100" + resolved "https://registry.yarnpkg.com/tldts-experimental/-/tldts-experimental-5.7.100.tgz#fb428cf20735952c299e15e864de63ecc55fb0a7" + integrity sha512-BjdXE3YU3cXbASRXydXnzOCSc+G/bM38/5snbxwcIYaRh3AApEtD4lHVl3236x+79T/V94lKwqBXYT47EAR+TA== + dependencies: + tldts-core "^5.7.100" + tmp@^0.0.33: version "0.0.33" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" From 4ee5675e23930bc360df7c75ee5b065e19348ca2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 12:03:35 +0800 Subject: [PATCH 05/15] Add gdcvault parsing test-case --- .../gdcvault/expected-metadata.json | 12 + .../test/test-pages/gdcvault/expected.html | 9 + .../test/test-pages/gdcvault/source.html | 1142 +++++++++++++++++ .../test/test-pages/gdcvault/url.txt | 1 + 4 files changed, 1164 insertions(+) create mode 100644 packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json create mode 100644 packages/readabilityjs/test/test-pages/gdcvault/expected.html create mode 100644 packages/readabilityjs/test/test-pages/gdcvault/source.html create mode 100644 packages/readabilityjs/test/test-pages/gdcvault/url.txt diff --git a/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json b/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json new file mode 100644 index 000000000..5e5525d51 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json @@ -0,0 +1,12 @@ +{ + "title": "Parallelizing the Naughty Dog Engine Using Fibers", + "byline": null, + "dir": null, + "excerpt": "This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall...", + "siteName": null, + "siteIcon": "http://fakehost/img/favicon.ico", + "previewImage": "https://ubm-twvideo01.s3.amazonaws.com/o1/vault/gdc2015/Images/GDC15_Vault-thumb_v1.png", + "publishedDate": null, + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/gdcvault/expected.html b/packages/readabilityjs/test/test-pages/gdcvault/expected.html new file mode 100644 index 000000000..64124891e --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/expected.html @@ -0,0 +1,9 @@ +
+
+

You've been logged out of GDC Vault since the maximum users allowed for this account has been reached. To access Members Only content on GDC Vault, please log out of GDC Vault from the computer which last accessed this account.

+

Click here to find out about GDC Vault Membership options for more users.

+

+ close +

+
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/gdcvault/source.html b/packages/readabilityjs/test/test-pages/gdcvault/source.html new file mode 100644 index 000000000..1b342e352 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/source.html @@ -0,0 +1,1142 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + GDC Vault - Parallelizing the Naughty Dog Engine Using Fibers + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+
+ You've been logged out of GDC Vault since the maximum users allowed for this account has been reached. To access Members Only content on GDC Vault, please log out of GDC Vault from the computer which last accessed this account.
+
+ Click here to find out about GDC Vault Membership options for more users.
+
+ close +
+
+
+
+ + + +
+
+ + + +
+
+ +
+ + +
+      +
+ + +
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ Session Name: + + Parallelizing the Naughty Dog Engine Using Fibers +
+ Speaker(s): + + Christian Gyrling +
+ Company Name(s): + + Naughty Dog +
+ Track / Format: + + Programming +
+ Overview: + + This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall frame-centric engine design, the memory allocation patterns used in the title, and our strategies for dealing with locks. +
+
+

+ GDC 2015 +

+

+ Christian Gyrling +

+

+ Naughty Dog +

+

+ free content +

+

+ Programming +

+

+ Programming +

+
+
+
+ +

+ + +
+
+
+
+
+
+
+
+ Discover More from Informa Tech +
+
+
+
+
+ +
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ Working With Us +
+ +
+
+
+
+
+
+
+ Follow GDC on Social +
+
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+ +
+
+
+
+
+
+
+
    +
  • Copyright © 2022 Informa PLC. Informa PLC is registered in England and Wales with company number 8860726 whose registered and head office is 5 Howick Place, London, SW1P 1WG. +
    • +
+
+
+
+
+ + + + + + + diff --git a/packages/readabilityjs/test/test-pages/gdcvault/url.txt b/packages/readabilityjs/test/test-pages/gdcvault/url.txt new file mode 100644 index 000000000..64f62a296 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/url.txt @@ -0,0 +1 @@ +https://www.gdcvault.com/play/1022186/Parallelizing-the-Naughty-Dog-Engine \ No newline at end of file From 13e925c503de61b9fac74e526889d37481aa39fe Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 12:17:37 +0800 Subject: [PATCH 06/15] Allow kaltura videos through in readablity --- packages/readabilityjs/Readability.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index f28f688fe..01b6999f5 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -184,7 +184,7 @@ Readability.prototype = { publishedDate: /published|modified|created|updated/i, replaceFonts: /<(\/?)font[^>]*>/gi, normalize: /\s{2,}/g, - videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i, + videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq|cdnapisec\.kaltura)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i, shareElements: /(\b|_)(share|sharedaddy|post-tags)(\b|_)/i, nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i, prevLink: /(prev|earl|old|new|<|«)/i, From 1b62ada73ea53cb7044fd97c76051a3a249e178d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 12:45:24 +0800 Subject: [PATCH 07/15] Make overlay element an unlikelyCandidates and give player element positive score --- packages/readabilityjs/Readability.js | 4 +- .../test/test-pages/gdcvault/expected.html | 57 +++++++++++++++++-- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 01b6999f5..50bd5d85d 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -171,13 +171,13 @@ Readability.prototype = { // Readability-readerable.js. Please keep both copies in sync. articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist/i, articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert|block-ad-.*/i, - unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon/i, + unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped/i, // okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i, get okMaybeItsACandidate() { return new RegExp(`and|(? -
-

You've been logged out of GDC Vault since the maximum users allowed for this account has been reached. To access Members Only content on GDC Vault, please log out of GDC Vault from the computer which last accessed this account.

-

Click here to find out about GDC Vault Membership options for more users.

-

- close -

+
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ Session Name: + Parallelizing the Naughty Dog Engine Using Fibers
+ Speaker(s): + Christian Gyrling
+ Company Name(s): + Naughty Dog
+ Track / Format: + Programming
+ Overview: + This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall frame-centric engine design, the memory allocation patterns used in the title, and our strategies for dealing with locks.
+
+ +
\ No newline at end of file From e75e49a7b44795ee10d135291757a1d4d20daf70 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 16 Nov 2022 17:23:49 +0800 Subject: [PATCH 08/15] Remove logging dependecies in puppeteer-parse --- packages/content-fetch/logger.js | 66 +++++++++++++++++ packages/content-fetch/package.json | 3 +- packages/puppeteer-parse/index.js | 101 ++++---------------------- packages/puppeteer-parse/package.json | 5 +- 4 files changed, 84 insertions(+), 91 deletions(-) create mode 100644 packages/content-fetch/logger.js diff --git a/packages/content-fetch/logger.js b/packages/content-fetch/logger.js new file mode 100644 index 000000000..9317e36c8 --- /dev/null +++ b/packages/content-fetch/logger.js @@ -0,0 +1,66 @@ +const { config, format, loggers, transports } = require('winston'); +const { LoggingWinston } = require('@google-cloud/logging-winston'); +const { DateTime } = require('luxon'); + +const colors = { + emerg: 'inverse underline magenta', + alert: 'underline magenta', + crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. + error: 'underline red', // Any error which is fatal to the operation, but not the service or application + warning: 'underline yellow', // Anything that can potentially cause application oddities + notice: 'underline cyan', // Normal but significant condition + info: 'underline green', // Generally useful information to log + debug: 'underline gray', +}; + +const googleConfigs = { + level: 'info', + logName: 'logger', + levels: config.syslog.levels, + resource: { + labels: { + function_name: process.env.FUNCTION_TARGET, + project_id: process.env.GCP_PROJECT, + }, + type: 'cloud_function', + }, +}; + +function localConfig(id) { + return { + level: 'debug', + format: format.combine( + format.colorize({ all: true, colors }), + format(info => + Object.assign(info, { + timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), + }), + )(), + format.printf(info => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { timestamp, message, level, ...meta } = info; + + return `[${id}@${info.timestamp}] ${info.message}${ + Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' + }`; + }), + ), + }; +} + +function buildLoggerTransport(id, options) { + return process.env.IS_LOCAL + ? new transports.Console(localConfig(id)) + : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); +} + +function buildLogger(id, options) { + return loggers.get(id, { + levels: config.syslog.levels, + transports: [buildLoggerTransport(id, options)], + }); +} + +module.exports = { + buildLogger, +} diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json index bffa43a8e..546552258 100644 --- a/packages/content-fetch/package.json +++ b/packages/content-fetch/package.json @@ -8,7 +8,8 @@ "express": "^4.17.1", "@google-cloud/functions-framework": "^3.0.0", "@omnivore/puppeteer-parse": "^1.0.0", - "@sentry/serverless": "^6.13.3" + "@sentry/serverless": "^6.13.3", + "winston": "^3.3.3" }, "devDependencies": { "chai": "^4.3.6", diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 148dfb7d6..8cb311ac1 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -9,9 +9,6 @@ const axios = require('axios'); const jwt = require('jsonwebtoken'); const { promisify } = require('util'); const signToken = promisify(jwt.sign); -const { config, format, loggers, transports } = require('winston'); -const { LoggingWinston } = require('@google-cloud/logging-winston'); -const { DateTime } = require('luxon'); const os = require('os'); const { Storage } = require('@google-cloud/storage'); const { parseHTML } = require('linkedom'); @@ -33,65 +30,6 @@ const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process. const filePath = `${os.tmpdir()}/previewImage.png`; -const colors = { - emerg: 'inverse underline magenta', - alert: 'underline magenta', - crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. - error: 'underline red', // Any error which is fatal to the operation, but not the service or application - warning: 'underline yellow', // Anything that can potentially cause application oddities - notice: 'underline cyan', // Normal but significant condition - info: 'underline green', // Generally useful information to log - debug: 'underline gray', -}; - -const googleConfigs = { - level: 'info', - logName: 'logger', - levels: config.syslog.levels, - resource: { - labels: { - function_name: process.env.FUNCTION_TARGET, - project_id: process.env.GCP_PROJECT, - }, - type: 'cloud_function', - }, -}; - -function localConfig(id) { - return { - level: 'debug', - format: format.combine( - format.colorize({ all: true, colors }), - format(info => - Object.assign(info, { - timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), - }), - )(), - format.printf(info => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { timestamp, message, level, ...meta } = info; - - return `[${id}@${info.timestamp}] ${info.message}${ - Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' - }`; - }), - ), - }; -} - -function buildLoggerTransport(id, options) { - return process.env.IS_LOCAL - ? new transports.Console(localConfig(id)) - : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); -} - -function buildLogger(id, options) { - return loggers.get(id, { - levels: config.syslog.levels, - transports: [buildLoggerTransport(id, options)], - }); -} - const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' @@ -274,15 +212,6 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId async function fetchContent(req, res) { functionStartTime = Date.now(); - // Grabbing execution and trace ids to attach logs to the appropriate function call - const execution_id = req.get('function-execution-id'); - const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; - const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { - trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, - labels: { - execution_id: execution_id, - }, - }); let url = getUrl(req); const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); @@ -297,11 +226,11 @@ async function fetchContent(req, res) { }, }; - logger.info(`Article parsing request`, logRecord); + console.info(`Article parsing request`, logRecord); if (!url) { logRecord.urlIsInvalid = true; - logger.info(`Valid URL to parse not specified`, logRecord); + console.info(`Valid URL to parse not specified`, logRecord); return res.sendStatus(400); } @@ -317,7 +246,7 @@ async function fetchContent(req, res) { if (result && result.content) { content = result.content } if (result && result.contentType) { contentType = result.contentType } } catch (e) { - logger.info('error with handler: ', e); + console.info('error with handler: ', e); } let context, page, finalUrl; @@ -347,7 +276,7 @@ async function fetchContent(req, res) { content = result.domContent; } } else { - logger.info('using prefetched content and title'); + console.info('using prefetched content and title'); } logRecord.fetchContentTime = Date.now() - functionStartTime; @@ -370,7 +299,7 @@ async function fetchContent(req, res) { } } catch (e) { logRecord.error = e.message; - logger.error(`Error while retrieving page`, logRecord); + console.error(`Error while retrieving page`, logRecord); // fallback to scrapingbee const sbResult = await fetchContentWithScrapingBee(url); @@ -397,7 +326,7 @@ async function fetchContent(req, res) { if (context) { await context.close(); } - logger.info(`parse-page`, logRecord); + console.info(`parse-page`, logRecord); } return res.sendStatus(200); @@ -715,7 +644,7 @@ async function preview(req, res) { // Grabbing execution and trace ids to attach logs to the appropriate function call const execution_id = req.get('function-execution-id'); const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; - const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { + const console = buildconsole('cloudfunctions.googleapis.com%2Fcloud-functions', { trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, labels: { execution_id: execution_id, @@ -723,7 +652,7 @@ async function preview(req, res) { }); if (!process.env.PREVIEW_IMAGE_BUCKET) { - logger.error(`PREVIEW_IMAGE_BUCKET not set`) + console.error(`PREVIEW_IMAGE_BUCKET not set`) return res.sendStatus(500); } @@ -739,17 +668,17 @@ async function preview(req, res) { }, }; - logger.info(`Public preview image generation request`, logRecord); + console.info(`Public preview image generation request`, logRecord); if (!url) { logRecord.urlIsInvalid = true; - logger.error(`Valid URL to parse is not specified`, logRecord); + console.error(`Valid URL to parse is not specified`, logRecord); return res.sendStatus(400); } const { origin } = new URL(url); if (!ALLOWED_ORIGINS.some(o => o === origin)) { logRecord.forbiddenOrigin = true; - logger.error(`This origin is not allowed: ${origin}`, logRecord); + console.error(`This origin is not allowed: ${origin}`, logRecord); return res.sendStatus(400); } @@ -780,7 +709,7 @@ async function preview(req, res) { ); if (!selector) { logRecord.selectorIsInvalid = true; - logger.error(`Valid element selector is not specified`, logRecord); + console.error(`Valid element selector is not specified`, logRecord); await page.close(); return res.sendStatus(400); } @@ -795,7 +724,7 @@ async function preview(req, res) { ); if (!destination) { logRecord.destinationIsInvalid = true; - logger.error(`Valid file destination is not specified`, logRecord); + console.error(`Valid file destination is not specified`, logRecord); await page.close(); return res.sendStatus(400); } @@ -806,7 +735,7 @@ async function preview(req, res) { await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load } catch (error) { logRecord.elementNotFound = true; - logger.error(`Element is not presented on the page`, logRecord); + console.error(`Element is not presented on the page`, logRecord); await page.close(); return res.sendStatus(400); } @@ -826,7 +755,7 @@ async function preview(req, res) { console.log('error uploading to bucket, this is non-fatal', e) } - logger.info(`preview-image`, logRecord); + console.info(`preview-image`, logRecord); return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); } diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 41378e1bf..42b3a10a4 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -4,19 +4,16 @@ "description": "Accepts URL of the article and parses its content", "main": "index.js", "dependencies": { - "@google-cloud/logging-winston": "^5.1.1", "@google-cloud/storage": "^5.18.1", "@omnivore/content-handler": "1.0.0", "axios": "^0.27.2", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.9", - "luxon": "^2.3.1", "puppeteer-core": "^16.1.0", "puppeteer-extra": "^3.3.4", "puppeteer-extra-plugin-adblocker": "^2.13.5", "puppeteer-extra-plugin-stealth": "^2.11.1", - "underscore": "^1.13.4", - "winston": "^3.3.3" + "underscore": "^1.13.4" }, "devDependencies": { "chai": "^4.3.6", From e866541ae1d09498c0db4b852973b6d9dc460204 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 17 Nov 2022 10:01:21 +0800 Subject: [PATCH 09/15] Fix puppeteer launch in head mode --- packages/content-fetch/app.js | 4 ++-- packages/content-handler/src/websites/twitter-handler.ts | 2 +- packages/puppeteer-parse/index.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/content-fetch/app.js b/packages/content-fetch/app.js index 252671626..76169cd0c 100644 --- a/packages/content-fetch/app.js +++ b/packages/content-fetch/app.js @@ -14,7 +14,7 @@ if (!process.env.VERIFICATION_TOKEN) { app.get('/', async (req, res) => { if (req.query.token !== process.env.VERIFICATION_TOKEN) { console.log('query does not include valid token') - res.send(403) + res.sendStatus(403) return } await fetchContent(req, res) @@ -23,7 +23,7 @@ app.get('/', async (req, res) => { app.post('/', async (req, res) => { if (req.query.token !== process.env.VERIFICATION_TOKEN) { console.log('query does not include valid token') - res.send(403) + res.sendStatus(403) return } await fetchContent(req, res) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index e4b23394c..17bbdc4c2 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -213,7 +213,7 @@ const getTweetIds = async ( const browser = await puppeteer.launch({ executablePath: process.env.CHROMIUM_PATH, - headless: process.env.LAUNCH_HEADLESS === 'true', + headless: !!process.env.LAUNCH_HEADLESS, defaultViewport: { width, height, diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 8cb311ac1..ccf74d39e 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -112,7 +112,7 @@ const getBrowserPromise = (async () => { ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, executablePath: process.env.CHROMIUM_PATH, - headless: process.env.LAUNCH_HEADLESS === 'true', + headless: !!process.env.LAUNCH_HEADLESS, timeout: 120000, // 2 minutes }); })(); From d9237723c57d181ae0bfc977b2cfe5b1f77ad8a9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 17 Nov 2022 10:43:30 +0800 Subject: [PATCH 10/15] Share puppeteer browser instance between content handlers --- .../content-handler/src/content-handler.ts | 3 +- packages/content-handler/src/index.ts | 6 ++- .../src/websites/twitter-handler.ts | 54 ++++++++++--------- packages/puppeteer-parse/index.js | 11 ++-- 4 files changed, 41 insertions(+), 33 deletions(-) diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts index fd1fe5291..a05e2ac1a 100644 --- a/packages/content-handler/src/content-handler.ts +++ b/packages/content-handler/src/content-handler.ts @@ -3,6 +3,7 @@ import rfc2047 from 'rfc2047' import { v4 as uuid } from 'uuid' import { parseHTML } from 'linkedom' import axios from 'axios' +import { Browser } from 'puppeteer-core' interface Unsubscribe { mailTo?: string @@ -62,7 +63,7 @@ export abstract class ContentHandler { return false } - async preHandle(url: string): Promise { + async preHandle(url: string, browser?: Browser): Promise { return Promise.resolve({ url }) } diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index ce61c0a64..e210d025d 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -28,6 +28,7 @@ import { GhostHandler } from './newsletters/ghost-handler' import { parseHTML } from 'linkedom' import { CooperPressHandler } from './newsletters/cooper-press-handler' import { HeyWorldHandler } from './newsletters/hey-world-handler' +import { Browser } from 'puppeteer-core' const validateUrlString = (url: string) => { const u = new URL(url) @@ -80,7 +81,8 @@ const newsletterHandlers: ContentHandler[] = [ ] export const preHandleContent = async ( - url: string + url: string, + browser: Browser ): Promise => { // Before we run the regular handlers we check to see if we need tp // pre-resolve the URL. TODO: This should probably happen recursively, @@ -104,7 +106,7 @@ export const preHandleContent = async ( for (const handler of contentHandlers) { if (handler.shouldPreHandle(url)) { console.log('preHandleContent', handler.name, url) - return handler.preHandle(url) + return handler.preHandle(url, browser) } } return undefined diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 17bbdc4c2..788301b8e 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -2,7 +2,7 @@ import { ContentHandler, PreHandleResult } from '../content-handler' import axios from 'axios' import { DateTime } from 'luxon' import _ from 'underscore' -import puppeteer from 'puppeteer-core' +import { Browser } from 'puppeteer-core' interface TweetIncludes { users: { @@ -168,10 +168,11 @@ const getTweetsFromResponse = (response: Tweets): Tweet[] => { } const getOldTweets = async ( + browser: Browser, conversationId: string, username: string ): Promise => { - const tweetIds = await getTweetIds(conversationId, username) + const tweetIds = await getTweetIds(browser, conversationId, username) if (tweetIds.length === 0) { return [] } @@ -197,37 +198,40 @@ const waitFor = (ms: number) => /** * Get tweets(even older than 7 days) using puppeteer + * @param browser * @param {string} tweetId * @param {string} author */ const getTweetIds = async ( + browser: Browser, tweetId: string, author: string ): Promise => { const pageURL = `https://twitter.com/${author}/status/${tweetId}` - // Modify this variable to control the size of viewport - const factor = 0.2 - const height = Math.floor(2000 / factor) - const width = Math.floor(1700 / factor) - - const browser = await puppeteer.launch({ - executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, - defaultViewport: { - width, - height, - }, - args: [ - `--force-device-scale-factor=${factor}`, - `--window-size=${width},${height}`, - '--no-sandbox', - '--disable-setuid-sandbox', - ], - }) + // // Modify this variable to control the size of viewport + // const factor = 0.2 + // const height = Math.floor(2000 / factor) + // const width = Math.floor(1700 / factor) + // + // const browser = await puppeteer.launch({ + // executablePath: process.env.CHROMIUM_PATH, + // headless: !!process.env.LAUNCH_HEADLESS, + // defaultViewport: { + // width, + // height, + // }, + // args: [ + // `--force-device-scale-factor=${factor}`, + // `--window-size=${width},${height}`, + // '--no-sandbox', + // '--disable-setuid-sandbox', + // ], + // }) + const context = await browser.createIncognitoBrowserContext() try { - const page = await browser.newPage() + const page = await context.newPage() await page.goto(pageURL, { waitUntil: 'networkidle2', @@ -291,7 +295,7 @@ const getTweetIds = async ( console.log(error) return [] } finally { - await browser.close() + await context.close() } } @@ -305,7 +309,7 @@ export class TwitterHandler extends ContentHandler { return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) } - async preHandle(url: string): Promise { + async preHandle(url: string, browser: Browser): Promise { const tweetId = tweetIdFromStatusUrl(url) if (!tweetId) { throw new Error('could not find tweet id in url') @@ -326,7 +330,7 @@ export class TwitterHandler extends ContentHandler { const description = _.escape(tweetData.text) // use puppeteer to get all tweet replies in the thread - const tweets = await getOldTweets(conversationId, author.username) + const tweets = await getOldTweets(browser, conversationId, author.username) let tweetsContent = '' for (const tweet of tweets) { diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index ccf74d39e..fca2e292b 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -237,7 +237,8 @@ async function fetchContent(req, res) { // pre handle url with custom handlers let title, content, contentType; try { - const result = await preHandleContent(url); + const browser = await getBrowserPromise; + const result = await preHandleContent(url, browser); if (result && result.url) { url = result.url validateUrlString(url); @@ -682,16 +683,16 @@ async function preview(req, res) { return res.sendStatus(400); } - const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH); + const browser = await getBrowserPromise; logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; const page = await browser.newPage(); const pageLoadingStart = Date.now(); const modifiedUrl = new URL(url); - modifiedUrl.searchParams.append('fontSize', 24); - modifiedUrl.searchParams.append('adjustAspectRatio', 1.91); + modifiedUrl.searchParams.append('fontSize', '24'); + modifiedUrl.searchParams.append('adjustAspectRatio', '1.91'); try { - await page.goto(modifiedUrl); + await page.goto(modifiedUrl.toString()); logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart }; } catch (error) { console.log('error going to page: ', modifiedUrl) From c18217e6974c3ae5c7dd1fa0a4d10c14988f32d3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 17 Nov 2022 11:06:56 +0800 Subject: [PATCH 11/15] Fix logRecord being shared by requests --- .../src/websites/twitter-handler.ts | 29 ++++--------------- packages/puppeteer-parse/index.js | 14 ++++----- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 788301b8e..afb8985ef 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -2,7 +2,7 @@ import { ContentHandler, PreHandleResult } from '../content-handler' import axios from 'axios' import { DateTime } from 'luxon' import _ from 'underscore' -import { Browser } from 'puppeteer-core' +import { Browser, BrowserContext } from 'puppeteer-core' interface TweetIncludes { users: { @@ -209,28 +209,9 @@ const getTweetIds = async ( ): Promise => { const pageURL = `https://twitter.com/${author}/status/${tweetId}` - // // Modify this variable to control the size of viewport - // const factor = 0.2 - // const height = Math.floor(2000 / factor) - // const width = Math.floor(1700 / factor) - // - // const browser = await puppeteer.launch({ - // executablePath: process.env.CHROMIUM_PATH, - // headless: !!process.env.LAUNCH_HEADLESS, - // defaultViewport: { - // width, - // height, - // }, - // args: [ - // `--force-device-scale-factor=${factor}`, - // `--window-size=${width},${height}`, - // '--no-sandbox', - // '--disable-setuid-sandbox', - // ], - // }) - - const context = await browser.createIncognitoBrowserContext() + let context: BrowserContext | undefined try { + context = await browser.createIncognitoBrowserContext() const page = await context.newPage() await page.goto(pageURL, { @@ -295,7 +276,9 @@ const getTweetIds = async ( console.log(error) return [] } finally { - await context.close() + if (context) { + await context.close() + } } } diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index fca2e292b..8c54c18d7 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -117,8 +117,6 @@ const getBrowserPromise = (async () => { }); })(); -let logRecord, functionStartTime; - const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => { const stream = await axios.get(contentObjUrl, { responseType: 'stream' }); return await axios.put(uploadSignedUrl, stream.data, { @@ -211,13 +209,13 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId }; async function fetchContent(req, res) { - functionStartTime = Date.now(); + let functionStartTime = Date.now(); let url = getUrl(req); const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); - logRecord = { + let logRecord = { url, userId, articleSavingRequestId, @@ -253,7 +251,7 @@ async function fetchContent(req, res) { let context, page, finalUrl; try { if ((!content || !title) && contentType !== 'application/pdf') { - const result = await retrievePage(url) + const result = await retrievePage(url, logRecord, functionStartTime); if (result && result.context) { context = result.context } if (result && result.page) { page = result.page } if (result && result.finalUrl) { finalUrl = result.finalUrl } @@ -267,7 +265,7 @@ async function fetchContent(req, res) { const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId); } else { if (!content || !title) { - const result = await retrieveHtml(page); + const result = await retrieveHtml(page, logRecord); if (result.isBlocked) { const sbResult = await fetchContentWithScrapingBee(url) title = sbResult.title @@ -387,7 +385,7 @@ async function blockResources(client) { await client.send('Network.setBlockedURLs', { urls: blockedResources }); } -async function retrievePage(url) { +async function retrievePage(url, logRecord, functionStartTime) { validateUrlString(url); const browser = await getBrowserPromise; @@ -501,7 +499,7 @@ async function retrievePage(url) { } } -async function retrieveHtml(page) { +async function retrieveHtml(page, logRecord) { let domContent = '', title; try { title = await page.title(); From 44e4e080dd3af52c230ed9ec2917c6df90c01c63 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 17 Nov 2022 11:23:32 +0800 Subject: [PATCH 12/15] increase navigation timeout to 30 seconds --- packages/puppeteer-parse/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 8c54c18d7..efce2c6e0 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -482,7 +482,7 @@ async function retrievePage(url, logRecord, functionStartTime) { }); try { - const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] }); + const response = await page.goto(url, { timeout: 30 * 1000, waitUntil: ['networkidle2'] }); const finalUrl = response.url(); const contentType = response.headers()['content-type']; From 5d9b705be38585ecef640ab3742312d69a58f276 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 17 Nov 2022 15:49:32 +0800 Subject: [PATCH 13/15] Fix issue where stripping classNames could cause a crash --- packages/readabilityjs/Readability.js | 6 +- .../techcrunch/expected-metadata.json | 12 + .../test/test-pages/techcrunch/expected.html | 25 ++ .../test/test-pages/techcrunch/source.html | 252 ++++++++++++++++++ .../test/test-pages/techcrunch/url.txt | 1 + 5 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 packages/readabilityjs/test/test-pages/techcrunch/expected-metadata.json create mode 100644 packages/readabilityjs/test/test-pages/techcrunch/expected.html create mode 100644 packages/readabilityjs/test/test-pages/techcrunch/source.html create mode 100644 packages/readabilityjs/test/test-pages/techcrunch/url.txt diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 50bd5d85d..7f787bbb5 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -414,7 +414,11 @@ Readability.prototype = { * @return void */ _cleanClasses: function (node) { - if (node.className.startsWith("_omnivore")) { + if (node.className && node.className.startsWith && node.className.startsWith('_omnivore')) { + return; + } + + if (node.className && node.className.hasOwnProperty && node.className.hasOwnProperty('_omnivore')) { return; } diff --git a/packages/readabilityjs/test/test-pages/techcrunch/expected-metadata.json b/packages/readabilityjs/test/test-pages/techcrunch/expected-metadata.json new file mode 100644 index 000000000..120703136 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/techcrunch/expected-metadata.json @@ -0,0 +1,12 @@ +{ + "title": "SBF regrets declaring FTX bankrupt", + "byline": "Alex Wilhelm, Natasha Mascarenhas", + "dir": null, + "excerpt": "The saga of FTX, formerly one of the world’s largest crypto exchanges that fell rapidly into bankruptcy, took a new turn today after Vox published a series of messages with its former CEO Sam Bankman-Fried. The erstwhile executive, known in the crypto world as SBF, discussed regulators, ethics and bankruptcy regrets, amongst other issues that […]", + "siteName": "TechCrunch", + "siteIcon": "https://techcrunch.com/wp-content/uploads/2015/02/cropped-cropped-favicon-gradient.png?w=32", + "previewImage": "https://techcrunch.com/wp-content/uploads/2022/11/GettyImages-1238326461.jpg?w=680", + "publishedDate": "2022-11-16T21:58:29.000Z", + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/techcrunch/expected.html b/packages/readabilityjs/test/test-pages/techcrunch/expected.html new file mode 100644 index 000000000..d5a33f3d8 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/techcrunch/expected.html @@ -0,0 +1,25 @@ +
+
+
+
+

+

+
+
+

The saga of FTX, formerly one of the world’s largest crypto exchanges that fell rapidly into bankruptcy, took a new turn today after Vox published a series of messages with its former CEO Sam Bankman-Fried. The erstwhile executive, known in the crypto world as SBF, discussed regulators, ethics and bankruptcy regrets, amongst other issues that have become the de jure conversation in tech since FTX itself immolated.

+

“Everyone goes around pretending that perception reflects reality, it doesn’t,” SBF said in a Twitter conversation with reporter Kelsey Piper. “Some of this decade’s greatest heroes will never be known, and some of its most beloved people are basically shams.”

+

In the notes, shared in screenshot form by the publication, SBF spoke harshly of regulators, saying that they “make everything worse” and that “they don’t protect customers at all.” Given that SBF’s former company will soon face at least the American Congress, the approach and tone are notable.

+

His take on regulators is predicated, later messages make clear, on his view that their methods of control are too simplified — “just ‘do more business’ vs ‘do less business’ and ‘put up more moats’ vs ‘put up fewer moats’” — which doesn’t distinguish “between good and bad” in his estimation.

+

The Vox interview spent a good chunk of its time discussing ethics and philanthropy, an unsurprising choice given that SBF was a well-known person in the “effective altruism” movement, a method of helping others that focuses on what is practical. SBF was also an active political donor until recently, further keeping him in the media limelight.

+

Back on the matters most pertinent to TechCrunch, while discussing his own activities, SBF wrote that he “didn’t want to do sketchy stuff [as] there are huge negative effects from it,” adding in a following message that he “didn’t mean to.” Last week, SBF officially stepped down as chief executive of FTX while Enron wind-down veteran John J. Ray III was appointed as the new CEO.

+

In response to SBF’s public statements, although we’re not exactly sure which ones as there are many, Ray published a statement saying that “Mr. Bankman-Fried has no ongoing role at FTX…and does not speak on their behalf.”

+

Later in the conversation with Vox, SBF brought up CZ, the well-known leader of Binance, the largest crypto exchange in the world. CZ and SBF’s dueling Twitter accounts up until, and after, the FTX meltdown centered the attention of the world on their different business approaches, and leverage.

+

“A month ago CZ was a walking example of ‘don’t do unethical shit or your money is worthless,’” SBF Wrote, “now he’s a hero,” later asking if the shift in his view of market perception of CZ was due to his being virtuous, or simply having had the “bigger balance sheet,” leading to CZ winning and not SBF. CZ’s comments about FTX’s native token FTT are viewed by some as a precipitating event in the collapse of the latter exchange; precisely where blame lies is not yet entirely clear, so grains of salt, please.

+

Interestingly enough, Bankman-Fried tells Vox that his “biggest single fuckup [was] the one thing everyone told” him to do: file for Chapter 11 bankruptcy. He thinks if he hadn’t filed for bankruptcy, “withdrawals would be opening up in a month with customers fully whole.”

+

He adds: “But instead I filed, and the people in charge of it are trying to burn it all to the ground out of shame.” So Vox inquired whether he was suggesting he should’ve just kept trying to raise the $8 billion lifeline. SBF added that he might still get there, but with way more “collateral damage.”

+

Damage is correct. The impact is still being felt; at the other firms in the crypto trading and investing business or the smaller individuals and businesses that had assets on the platform (pre-bankruptcy). The fall-out even hurts early-stage entrepreneurs, with MIT Media Lab canceling its fellowship that was originally backed by FTX Future Fund.

+

There are entire chapters, if not volumes to come. And thankfully for those of us observing, and reporting, SBF continues to talk.

+
+
+
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/techcrunch/source.html b/packages/readabilityjs/test/test-pages/techcrunch/source.html new file mode 100644 index 000000000..7d2068718 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/techcrunch/source.html @@ -0,0 +1,252 @@ + + + + + + + + + + + + + + + + SBF regrets declaring FTX bankrupt • TechCrunch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+

SBF regrets declaring FTX bankrupt

+
+ +
+ +
+ + Alex Wilhelm + + Natasha Mascarenhas + + 18 hours +
+
+ +
+ +
+
+ +
+

The saga of FTX, formerly one of the world’s largest crypto exchanges that fell rapidly into bankruptcy, took a new turn today after Vox published a series of messages with its former CEO Sam Bankman-Fried. The erstwhile executive, known in the crypto world as SBF, discussed regulators, ethics and bankruptcy regrets, amongst other issues that have become the de jure conversation in tech since FTX itself immolated.

+

“Everyone goes around pretending that perception reflects reality, it doesn’t,” SBF said in a Twitter conversation with reporter Kelsey Piper. “Some of this decade’s greatest heroes will never be known, and some of its most beloved people are basically shams.”

+

In the notes, shared in screenshot form by the publication, SBF spoke harshly of regulators, saying that they “make everything worse” and that “they don’t protect customers at all.” Given that SBF’s former company will soon face at least the American Congress, the approach and tone are notable.

+

His take on regulators is predicated, later messages make clear, on his view that their methods of control are too simplified — “just ‘do more business’ vs ‘do less business’ and ‘put up more moats’ vs ‘put up fewer moats’” — which doesn’t distinguish “between good and bad” in his estimation.

+

The Vox interview spent a good chunk of its time discussing ethics and philanthropy, an unsurprising choice given that SBF was a well-known person in the “effective altruism” movement, a method of helping others that focuses on what is practical. SBF was also an active political donor until recently, further keeping him in the media limelight.

+

Back on the matters most pertinent to TechCrunch, while discussing his own activities, SBF wrote that he “didn’t want to do sketchy stuff [as] there are huge negative effects from it,” adding in a following message that he “didn’t mean to.” Last week, SBF officially stepped down as chief executive of FTX while Enron wind-down veteran John J. Ray III was appointed as the new CEO.

+

In response to SBF’s public statements, although we’re not exactly sure which ones as there are many, Ray published a statement saying that “Mr. Bankman-Fried has no ongoing role at FTX…and does not speak on their behalf.”

+

Later in the conversation with Vox, SBF brought up CZ, the well-known leader of Binance, the largest crypto exchange in the world. CZ and SBF’s dueling Twitter accounts up until, and after, the FTX meltdown centered the attention of the world on their different business approaches, and leverage.

+

“A month ago CZ was a walking example of ‘don’t do unethical shit or your money is worthless,’” SBF Wrote, “now he’s a hero,” later asking if the shift in his view of market perception of CZ was due to his being virtuous, or simply having had the “bigger balance sheet,” leading to CZ winning and not SBF. CZ’s comments about FTX’s native token FTT are viewed by some as a precipitating event in the collapse of the latter exchange; precisely where blame lies is not yet entirely clear, so grains of salt, please.

+

Interestingly enough, Bankman-Fried tells Vox that his “biggest single fuckup [was] the one thing everyone told” him to do: file for Chapter 11 bankruptcy. He thinks if he hadn’t filed for bankruptcy, “withdrawals would be opening up in a month with customers fully whole.”

+

He adds: “But instead I filed, and the people in charge of it are trying to burn it all to the ground out of shame.” So Vox inquired whether he was suggesting he should’ve just kept trying to raise the $8 billion lifeline. SBF added that he might still get there, but with way more “collateral damage.”

+

Damage is correct. The impact is still being felt; at the other firms in the crypto trading and investing business or the smaller individuals and businesses that had assets on the platform (pre-bankruptcy). The fall-out even hurts early-stage entrepreneurs, with MIT Media Lab canceling its fellowship that was originally backed by FTX Future Fund.

+

There are entire chapters, if not volumes to come. And thankfully for those of us observing, and reporting, SBF continues to talk.

+

 

+
+ +
+
+
+
+
+ +
+ +
+
+
+
+
+ + + + + + + + + + + diff --git a/packages/readabilityjs/test/test-pages/techcrunch/url.txt b/packages/readabilityjs/test/test-pages/techcrunch/url.txt new file mode 100644 index 000000000..91947cb76 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/techcrunch/url.txt @@ -0,0 +1 @@ +https://techcrunch.com/2022/11/16/sbf-regrets-declaring-ftx-bankrupt-per-his-dms-to-vox/ \ No newline at end of file From dabada68ff54b2404a46307942757a867f818f2d Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 17 Nov 2022 18:40:56 +0800 Subject: [PATCH 14/15] Add missing package --- packages/api/package.json | 1 + yarn.lock | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/packages/api/package.json b/packages/api/package.json index ffd240c8f..2e5d0ab24 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -19,6 +19,7 @@ "@google-cloud/pubsub": "^2.16.0", "@google-cloud/storage": "^5.18.1", "@google-cloud/tasks": "^2.3.0", + "@graphql-tools/utils": "^9.1.1", "@omnivore/content-handler": "1.0.0", "@omnivore/readability": "1.0.0", "@omnivore/text-to-speech-handler": "1.0.0", diff --git a/yarn.lock b/yarn.lock index b0dabc55a..935668ba3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3068,6 +3068,13 @@ dependencies: tslib "~2.3.0" +"@graphql-tools/utils@^9.1.1": + version "9.1.1" + resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-9.1.1.tgz#b47ea8f0d18c038c5c1c429e72caa5c25039fbab" + integrity sha512-DXKLIEDbihK24fktR2hwp/BNIVwULIHaSTNTNhXS+19vgT50eX9wndx1bPxGwHnVBOONcwjXy0roQac49vdt/w== + dependencies: + tslib "^2.4.0" + "@graphql-tools/wrap@^8.3.1": version "8.3.3" resolved "https://registry.yarnpkg.com/@graphql-tools/wrap/-/wrap-8.3.3.tgz#014aa04a6cf671ffe477516255d1134777da056a" From 981457038081a126cfd6c7f18007499f3f925932 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 17 Nov 2022 18:45:36 +0800 Subject: [PATCH 15/15] Always show the loading headers now that filtering is faster --- .../OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift index 448271929..cc97741df 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift @@ -345,9 +345,8 @@ import Views } List { - if viewModel.items.count > 0 || viewModel.searchTerm.count > 0 { - filtersHeader - } + filtersHeader + ForEach(viewModel.items) { item in FeedCardNavigationLink( item: item,