diff --git a/packages/import-handler/Dockerfile b/packages/import-handler/Dockerfile index 84730b9d2..f4a167620 100644 --- a/packages/import-handler/Dockerfile +++ b/packages/import-handler/Dockerfile @@ -1,30 +1,10 @@ FROM node:14.18-alpine -# Installs latest Chromium (92) package. -RUN apk add --no-cache \ - chromium \ - nss \ - freetype \ - harfbuzz \ - ca-certificates \ - ttf-freefont \ - nodejs \ - yarn \ - g++ \ - make \ - python3 - -# Add user so we don't need --no-sandbox. -RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ - && mkdir -p /home/pptruser/Downloads /app \ - && chown -R pptruser:pptruser /home/pptruser \ - && chown -R pptruser:pptruser /app - -# Run everything after as non-privileged user. WORKDIR /app -ENV CHROMIUM_PATH /usr/bin/chromium-browser -ENV LAUNCH_HEADLESS=true +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +RUN apk add g++ make python3 + ENV PORT 9090 COPY package.json . @@ -34,13 +14,10 @@ COPY .prettierrc . COPY .eslintrc . COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json -COPY /packages/content-handler/package.json ./packages/content-handler/package.json COPY /packages/import-handler/package.json ./packages/import-handler/package.json - RUN yarn install --pure-lockfile -ADD /packages/content-handler ./packages/content-handler ADD /packages/import-handler ./packages/import-handler ADD /packages/readabilityjs ./packages/readabilityjs RUN yarn workspace @omnivore/import-handler build @@ -52,5 +29,4 @@ RUN yarn install --pure-lockfile --production EXPOSE 9090 -# USER pptruser ENTRYPOINT ["yarn", "workspace", "@omnivore/import-handler", "start_gcf"] diff --git a/packages/import-handler/package.json b/packages/import-handler/package.json index 609cd51b9..75b4b85bd 100644 --- a/packages/import-handler/package.json +++ b/packages/import-handler/package.json @@ -20,10 +20,16 @@ "deploy": "yarn build && yarn gcloud-deploy" }, "devDependencies": { + "@types/chai": "^4.3.4", + "@types/chai-string": "^1.4.2", + "@types/dompurify": "^2.4.0", "@types/fs-extra": "^11.0.1", "@types/jsonwebtoken": "^8.5.0", + "@types/mocha": "^10.0.1", "@types/node": "^14.11.2", "@types/unzip-stream": "^0.3.1", + "@types/urlsafe-base64": "^1.0.28", + "@types/uuid": "^9.0.0", "eslint-plugin-prettier": "^4.0.0" }, "dependencies": { @@ -31,7 +37,6 @@ "@google-cloud/functions-framework": "3.1.2", "@google-cloud/storage": "^5.18.1", "@google-cloud/tasks": "^3.0.5", - "@omnivore/content-handler": "1.0.0", "@omnivore/readability": "1.0.0", "@types/express": "^4.17.13", "csv-parser": "^3.0.0", @@ -39,6 +44,7 @@ "fs-extra": "^11.1.0", "jsonwebtoken": "^8.5.1", "nodemon": "^2.0.15", - "unzip-stream": "^0.3.1" + "unzip-stream": "^0.3.1", + "urlsafe-base64": "^1.0.0" } } diff --git a/yarn.lock b/yarn.lock index d469f24e8..8fccfb80d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7865,6 +7865,11 @@ resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.2.21.tgz#9f35a5643129df132cf3b5c1ec64046ea1af0650" integrity sha512-yd+9qKmJxm496BOV9CMNaey8TWsikaZOwMRwPHQIjcOJM9oV+fi9ZMNw3JsVnbEEbo2gRTDnGEBv8pjyn67hNg== +"@types/chai@^4.3.4": + version "4.3.4" + resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.3.4.tgz#e913e8175db8307d78b4e8fa690408ba6b65dee4" + integrity sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw== + "@types/chrome@^0.0.197": version "0.0.197" resolved "https://registry.yarnpkg.com/@types/chrome/-/chrome-0.0.197.tgz#c1b50cdb72ee40f9bc1411506031a9f8a925ab35" @@ -7948,6 +7953,13 @@ dependencies: "@types/trusted-types" "*" +"@types/dompurify@^2.4.0": + version "2.4.0" + resolved "https://registry.yarnpkg.com/@types/dompurify/-/dompurify-2.4.0.tgz#fd9706392a88e0e0e6d367f3588482d817df0ab9" + integrity sha512-IDBwO5IZhrKvHFUl+clZxgf3hn2b/lU6H1KaBShPkQyGJUQ0xwebezIPSuiyGwfz1UzJWQl4M7BDxtHtCCPlTg== + dependencies: + "@types/trusted-types" "*" + "@types/duplexify@^3.6.0": version "3.6.0" resolved "https://registry.yarnpkg.com/@types/duplexify/-/duplexify-3.6.0.tgz#dfc82b64bd3a2168f5bd26444af165bf0237dcd8" @@ -8058,6 +8070,14 @@ dependencies: "@types/node" "*" +"@types/fs-extra@^11.0.1": + version "11.0.1" + resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-11.0.1.tgz#f542ec47810532a8a252127e6e105f487e0a6ea5" + integrity sha512-MxObHvNl4A69ofaTRU8DFqvgzzv8s9yRtaPPm5gud9HDNvpB3GPQFvNuTWAI59B9huVGV5jXYJwbCsmBsOGYWA== + dependencies: + "@types/jsonfile" "*" + "@types/node" "*" + "@types/glob@*": version "7.2.0" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb" @@ -8225,6 +8245,13 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= +"@types/jsonfile@*": + version "6.1.1" + resolved "https://registry.yarnpkg.com/@types/jsonfile/-/jsonfile-6.1.1.tgz#ac84e9aefa74a2425a0fb3012bdea44f58970f1b" + integrity sha512-GSgiRCVeapDN+3pqA35IkQwasaCh/0YFH5dEF6S88iDvEn901DjOeH3/QPY+XYP1DFzDZPvIvfeEgk+7br5png== + dependencies: + "@types/node" "*" + "@types/jsonwebtoken@^8.5.0": version "8.5.0" resolved "https://registry.yarnpkg.com/@types/jsonwebtoken/-/jsonwebtoken-8.5.0.tgz#2531d5e300803aa63279b232c014acf780c981c5" @@ -8313,6 +8340,11 @@ resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c" integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ== +"@types/mocha@^10.0.1": + version "10.0.1" + resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.1.tgz#2f4f65bb08bc368ac39c96da7b2f09140b26851b" + integrity sha512-/fvYntiO1GeICvqbQ3doGDIP97vWmvFt83GKguJ6prmQM2iXZfFcq6YE8KteFyRtX2/h5Hf91BYvPodJKFYv5Q== + "@types/mocha@^8.2.2": version "8.2.3" resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.2.3.tgz#bbeb55fbc73f28ea6de601fbfa4613f58d785323" @@ -8653,10 +8685,17 @@ resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.6.tgz#250a7b16c3b91f672a24552ec64678eeb1d3a08d" integrity sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ== +"@types/unzip-stream@^0.3.1": + version "0.3.1" + resolved "https://registry.yarnpkg.com/@types/unzip-stream/-/unzip-stream-0.3.1.tgz#a57b8365be3d7f89960c755c8cc37c83223dc7d0" + integrity sha512-RlE3qaqvu4XaMwxkG/zR1gIunCbqXvNrmZ4BCG7OiQ8QUactFUPxm0TTrOCRJZQfPW3T6XBH7PcHQiiqkdcijw== + dependencies: + "@types/node" "*" + "@types/urlsafe-base64@^1.0.28": version "1.0.28" resolved "https://registry.yarnpkg.com/@types/urlsafe-base64/-/urlsafe-base64-1.0.28.tgz#2cf2098518e98c730a7e00de79a455e0269a3f4d" - integrity sha1-LPIJhRjpjHMKfgDeeaRV4CaaP00= + integrity sha512-TG5dKbqx75FUTXfiARIPvLvMCImVYJbKM+Fvy9HgpxkunHnMHNAn78xpvcZxIbPITyRzf0b2Gl8fnd1Ja3p1eQ== dependencies: "@types/node" "*" @@ -8665,6 +8704,11 @@ resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.4.tgz#bd86a43617df0594787d38b735f55c805becf1bc" integrity sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw== +"@types/uuid@^9.0.0": + version "9.0.0" + resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.0.tgz#53ef263e5239728b56096b0a869595135b7952d2" + integrity sha512-kr90f+ERiQtKWMz5rP32ltJ/BtULDI5RVO0uavn1HQUOwjx0R1h0rnDYNL0CepF1zL5bSY6FISAfd9tOdDhU5Q== + "@types/voca@^1.4.0": version "1.4.2" resolved "https://registry.yarnpkg.com/@types/voca/-/voca-1.4.2.tgz#4d2ceef3582c2a1f6a1574f71f0897400d910583" @@ -10494,6 +10538,14 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== +binary@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/binary/-/binary-0.3.0.tgz#9f60553bc5ce8c3386f3b553cff47462adecaa79" + integrity sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg== + dependencies: + buffers "~0.1.1" + chainsaw "~0.1.0" + binaryextensions@^2.1.2: version "2.3.0" resolved "https://registry.yarnpkg.com/binaryextensions/-/binaryextensions-2.3.0.tgz#1d269cbf7e6243ea886aa41453c3651ccbe13c22" @@ -10793,6 +10845,11 @@ buffer@^6.0.3: base64-js "^1.3.1" ieee754 "^1.2.1" +buffers@~0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/buffers/-/buffers-0.1.1.tgz#b24579c3bed4d6d396aeee6d9a8ae7f5482ab7bb" + integrity sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ== + bufrw@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/bufrw/-/bufrw-1.3.0.tgz#28d6cfdaf34300376836310f5c31d57eeb40c8fa" @@ -11089,6 +11146,13 @@ chai@^4.3.6: pathval "^1.1.1" type-detect "^4.0.5" +chainsaw@~0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/chainsaw/-/chainsaw-0.1.0.tgz#5eab50b28afe58074d0d58291388828b5e5fbc98" + integrity sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ== + dependencies: + traverse ">=0.3.0 <0.4" + chalk@^1.0.0, chalk@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" @@ -13062,6 +13126,11 @@ dompurify@^2.4.1: resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.4.1.tgz#f9cb1a275fde9af6f2d0a2644ef648dd6847b631" integrity sha512-ewwFzHzrrneRjxzmK6oVz/rZn9VWspGFRDb4/rRtIsM1n36t9AKma/ye8syCpcw+XJ25kOK/hOG7t1j2I2yBqA== +dompurify@^2.4.3: + version "2.4.3" + resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.4.3.tgz#f4133af0e6a50297fc8874e2eaedc13a3c308c03" + integrity sha512-q6QaLcakcRjebxjg8/+NP+h0rPfatOgOzc46Fst9VAA3jF2ApfKBNKMzdP4DYTqtUMXSCd5pRS/8Po/OmoCHZQ== + domutils@^2.0.0, domutils@^2.5.2: version "2.7.0" resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.7.0.tgz#8ebaf0c41ebafcf55b0b72ec31c56323712c5442" @@ -14783,6 +14852,15 @@ fs-extra@^10.0.0: jsonfile "^6.0.1" universalify "^2.0.0" +fs-extra@^11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.1.0.tgz#5784b102104433bb0e090f48bfc4a30742c357ed" + integrity sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + fs-extra@^9.0.0, fs-extra@^9.0.1, fs-extra@^9.1.0: version "9.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" @@ -25566,6 +25644,11 @@ tr46@~0.0.3: resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== +"traverse@>=0.3.0 <0.4": + version "0.3.9" + resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9" + integrity sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ== + tree-kill@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.2.tgz#4ca09a9092c88b73a7cdc5e8a01b507b0790a0cc" @@ -26151,6 +26234,14 @@ untildify@^4.0.0: resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== +unzip-stream@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/unzip-stream/-/unzip-stream-0.3.1.tgz#2333b5cd035d29db86fb701ca212cf8517400083" + integrity sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw== + dependencies: + binary "^0.3.0" + mkdirp "^0.5.1" + upath@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894"