From 63423751c6f598e9ff177e9c83a4c038b24b47dd Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 29 Jun 2022 11:18:13 +0800 Subject: [PATCH] Create a new service for gcf --- packages/content-fetch-gcf/.env.example | 8 + packages/content-fetch-gcf/.gcloudignore | 21 +++ packages/content-fetch-gcf/.gitignore | 1 + packages/content-fetch-gcf/Dockerfile | 112 ++++++++++++ packages/content-fetch-gcf/README.md | 24 +++ packages/content-fetch-gcf/index.js | 221 +++++++++++++++++++++++ packages/content-fetch-gcf/package.json | 25 +++ 7 files changed, 412 insertions(+) create mode 100644 packages/content-fetch-gcf/.env.example create mode 100644 packages/content-fetch-gcf/.gcloudignore create mode 100644 packages/content-fetch-gcf/.gitignore create mode 100644 packages/content-fetch-gcf/Dockerfile create mode 100644 packages/content-fetch-gcf/README.md create mode 100644 packages/content-fetch-gcf/index.js create mode 100644 packages/content-fetch-gcf/package.json diff --git a/packages/content-fetch-gcf/.env.example b/packages/content-fetch-gcf/.env.example new file mode 100644 index 000000000..64242a22d --- /dev/null +++ b/packages/content-fetch-gcf/.env.example @@ -0,0 +1,8 @@ +# Should match with the JWT_SECRET that the api uses +JWT_SECRET=some_secret + +# Address of the backend that is running locally +REST_BACKEND_ENDPOINT=http://localhost:4000/api + +# set for local development +IS_LOCAL=true diff --git a/packages/content-fetch-gcf/.gcloudignore b/packages/content-fetch-gcf/.gcloudignore new file mode 100644 index 000000000..fc644d8d3 --- /dev/null +++ b/packages/content-fetch-gcf/.gcloudignore @@ -0,0 +1,21 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +node_modules +.env* +.secrets* +Dockerfile* +previewImage.* +*.sa.json diff --git a/packages/content-fetch-gcf/.gitignore b/packages/content-fetch-gcf/.gitignore new file mode 100644 index 000000000..9bfbc5e8b --- /dev/null +++ b/packages/content-fetch-gcf/.gitignore @@ -0,0 +1 @@ +previewImage.* \ No newline at end of file diff --git a/packages/content-fetch-gcf/Dockerfile b/packages/content-fetch-gcf/Dockerfile new file mode 100644 index 000000000..c0df04111 --- /dev/null +++ b/packages/content-fetch-gcf/Dockerfile @@ -0,0 +1,112 @@ +# FROM node:14-slim + +# # Taken from pu + +# # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) +# # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer +# # installs, work. +# RUN apt-get update \ +# && apt-get install -y wget gnupg \ +# && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ +# && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ +# && apt-get update \ +# && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \ +# --no-install-recommends \ +# && rm -rf /var/lib/apt/lists/* + +# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +# ENV CHROMIUM_PATH "/usr/bin/google-chrome-stable" + +# ------------------------ + +# FROM --platform=linux/arm64 node:14.18 + +# RUN apt-get update \ +# && apt-get install -y chromium \ +# && apt-get install -y ca-certificates \ +# fonts-liberation \ +# libappindicator3-1 \ +# libasound2 \ +# libatk-bridge2.0-0 \ +# libatk1.0-0 \ +# libc6 \ +# libcairo2 \ +# libcups2 \ +# libdbus-1-3 \ +# libexpat1 \ +# libfontconfig1 \ +# libgbm1 \ +# libgcc1 \ +# libglib2.0-0 \ +# libgtk-3-0 \ +# libnspr4 \ +# libnss3 \ +# libpango-1.0-0 \ +# libpangocairo-1.0-0 \ +# libstdc++6 \ +# libx11-6 \ +# libx11-xcb1 \ +# libxcb1 \ +# libxcomposite1 \ +# libxcursor1 \ +# libxdamage1 \ +# libxext6 \ +# libxfixes3 \ +# libxi6 \ +# libxrandr2 \ +# libxrender1 \ +# libxss1 \ +# libxtst6 \ +# lsb-release \ +# wget \ +# xdg-utils + +FROM node:14.18-alpine + +# Installs latest Chromium (92) package. +RUN apk add --no-cache \ + chromium \ + nss \ + freetype \ + harfbuzz \ + ca-certificates \ + ttf-freefont \ + nodejs \ + yarn + +# Tell Puppeteer to skip installing Chrome. We'll be using the installed package. +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ + PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + +# Puppeteer v10.0.0 works with Chromium 92. +RUN yarn add puppeteer@10.0.0 + +# Add user so we don't need --no-sandbox. +RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ + && mkdir -p /home/pptruser/Downloads /app \ + && chown -R pptruser:pptruser /home/pptruser \ + && chown -R pptruser:pptruser /app + +# Run everything after as non-privileged user. +WORKDIR /app + +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +ENV CHROMIUM_PATH /usr/bin/chromium-browser +ENV LAUNCH_HEADLESS=true + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .prettierrc . +COPY .eslintrc . + +COPY /packages/content-fetch-gcf/package.json ./packages/content-fetch-gcf/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/content-fetch-gcf ./packages/content-fetch-gcf + +EXPOSE 8080 + +# USER pptruser +ENTRYPOINT ["yarn", "workspace", "@omnivore/content-fetch-gcf", "start"] diff --git a/packages/content-fetch-gcf/README.md b/packages/content-fetch-gcf/README.md new file mode 100644 index 000000000..501c004ee --- /dev/null +++ b/packages/content-fetch-gcf/README.md @@ -0,0 +1,24 @@ +# Puppeteer parsing function handler + +This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer. + +## Using locally + +Copy .env.example file to .env file: `cp .env.example .env` + +Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading). + +After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) + +## Deployment + +To deploy the function use the following command: + +`gcloud functions deploy puppeteer --runtime nodejs12 --trigger-http --memory 1GB --set-env-vars REST_BACKEND_ENDPOINT=,JWT_SECRET=` + + +where: + +`` - address of the backend server (e.g "http://localhost:4000") + +`` - JWT secret that the backend server is using (e.g "some_secret") diff --git a/packages/content-fetch-gcf/index.js b/packages/content-fetch-gcf/index.js new file mode 100644 index 000000000..2c18d99f8 --- /dev/null +++ b/packages/content-fetch-gcf/index.js @@ -0,0 +1,221 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const { config, format, loggers, transports } = require('winston'); +const { LoggingWinston } = require('@google-cloud/logging-winston'); +const { DateTime } = require('luxon'); +const os = require('os'); +const Sentry = require('@sentry/serverless'); +const { Storage } = require('@google-cloud/storage'); +const { fetchContent, getBrowserPromise, getUrl } = require("@omnivore/puppeteer-parse"); + +const storage = new Storage(); +const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; +const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.env.PREVIEW_IMAGE_BUCKET) : undefined; + +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}); + +const filePath = `${os.tmpdir()}/previewImage.png`; + +const colors = { + emerg: 'inverse underline magenta', + alert: 'underline magenta', + crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. + error: 'underline red', // Any error which is fatal to the operation, but not the service or application + warning: 'underline yellow', // Anything that can potentially cause application oddities + notice: 'underline cyan', // Normal but significant condition + info: 'underline green', // Generally useful information to log + debug: 'underline gray', +}; + +const googleConfigs = { + level: 'info', + logName: 'logger', + levels: config.syslog.levels, + resource: { + labels: { + function_name: process.env.FUNCTION_TARGET, + project_id: process.env.GCP_PROJECT, + }, + type: 'cloud_function', + }, +}; + +function localConfig(id) { + return { + level: 'debug', + format: format.combine( + format.colorize({ all: true, colors }), + format(info => + Object.assign(info, { + timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), + }), + )(), + format.printf(info => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { timestamp, message, level, ...meta } = info; + + return `[${id}@${info.timestamp}] ${info.message}${ + Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' + }`; + }), + ), + }; +} + +function buildLoggerTransport(id, options) { + return process.env.IS_LOCAL + ? new transports.Console(localConfig(id)) + : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); +} + +function buildLogger(id, options) { + return loggers.get(id, { + levels: config.syslog.levels, + transports: [buildLoggerTransport(id, options)], + }); +} + +/** + * Cloud Function entry point, HTTP trigger. + * Loads the requested URL via Puppeteer, captures page content and sends it to backend + * + * @param {Object} req Cloud Function request context. + * @param {Object} res Cloud Function response context. + */ +exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(fetchContent); + +/** + * Cloud Function entry point, HTTP trigger. + * Loads the requested URL via Puppeteer and captures a screenshot of the provided element + * + * @param {Object} req Cloud Function request context. + * Inlcudes: + * * url - URL address of the page to open + * @param {Object} res Cloud Function response context. + */ +exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { + const functionStartTime = Date.now(); + // Grabbing execution and trace ids to attach logs to the appropriate function call + const execution_id = req.get('function-execution-id'); + const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; + const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { + trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, + labels: { + execution_id: execution_id, + }, + }); + + if (!process.env.PREVIEW_IMAGE_BUCKET) { + logger.error(`PREVIEW_IMAGE_BUCKET not set`) + return res.sendStatus(500); + } + + const url = getUrl(req); + console.log('preview request url', url); + + const logRecord = { + url, + query: req.query, + origin: req.get('Origin'), + labels: { + source: 'publicImagePreview', + }, + }; + + logger.info(`Public preview image generation request`, logRecord); + + if (!url) { + logRecord.urlIsInvalid = true; + logger.error(`Valid URL to parse is not specified`, logRecord); + return res.sendStatus(400); + } + const { origin } = new URL(url); + if (!ALLOWED_ORIGINS.some(o => o === origin)) { + logRecord.forbiddenOrigin = true; + logger.error(`This origin is not allowed: ${origin}`, logRecord); + return res.sendStatus(400); + } + + const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH); + logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; + + const page = await browser.newPage(); + const pageLoadingStart = Date.now(); + const modifiedUrl = new URL(url); + modifiedUrl.searchParams.append('fontSize', 24); + modifiedUrl.searchParams.append('adjustAspectRatio', 1.91); + try { + await page.goto(modifiedUrl); + logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart }; + } catch (error) { + console.log('error going to page: ', modifiedUrl) + console.log(error) + throw error + } + + // We lookup the destination path from our own page content and avoid trusting any passed query params + // selector - CSS selector of the element to get screenshot of + const selector = decodeURIComponent( + await page.$eval( + "head > meta[name='omnivore:preview_image_selector']", + element => element.content, + ), + ); + if (!selector) { + logRecord.selectorIsInvalid = true; + logger.error(`Valid element selector is not specified`, logRecord); + await page.close(); + return res.sendStatus(400); + } + logRecord.selector = selector; + + // destination - destination pathname for the image to save with + const destination = decodeURIComponent( + await page.$eval( + "head > meta[name='omnivore:preview_image_destination']", + element => element.content, + ), + ); + if (!destination) { + logRecord.destinationIsInvalid = true; + logger.error(`Valid file destination is not specified`, logRecord); + await page.close(); + return res.sendStatus(400); + } + logRecord.destination = destination; + + const screenshotTakingStart = Date.now(); + try { + await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load + } catch (error) { + logRecord.elementNotFound = true; + logger.error(`Element is not presented on the page`, logRecord); + await page.close(); + return res.sendStatus(400); + } + const element = await page.$(selector); + await element.screenshot({ path: filePath }); // take screenshot of the element in puppeteer + logRecord.timing = { ...logRecord.timing, screenshotTaken: Date.now() - screenshotTakingStart }; + + await page.close(); + + try { + const [file] = await previewBucket.upload(filePath, { + destination, + metadata: logRecord, + }); + logRecord.file = file.metadata; + } catch (e) { + console.log('error uploading to bucket, this is non-fatal', e) + } + + logger.info(`preview-image`, logRecord); + return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); +}); diff --git a/packages/content-fetch-gcf/package.json b/packages/content-fetch-gcf/package.json new file mode 100644 index 000000000..efec3ca5e --- /dev/null +++ b/packages/content-fetch-gcf/package.json @@ -0,0 +1,25 @@ +{ + "name": "@omnivore/content-fetch-gcf", + "version": "1.0.0", + "description": "Google Cloud Function that accepts URL of the article and parses its content", + "main": "index.js", + "dependencies": { + "@google-cloud/logging-winston": "^4.1.2", + "@google-cloud/storage": "^5.18.1", + "@sentry/serverless": "^6.13.3", + "axios": "^0.26.0", + "dotenv": "^8.2.0", + "jsonwebtoken": "^8.5.1", + "linkedom": "^0.14.9", + "luxon": "^2.3.1", + "winston": "^3.3.3", + "@omnivore/puppeteer-parse": "^1.0.0" + }, + "devDependencies": { + "@google-cloud/functions-framework": "^3.0.0" + }, + "scripts": { + "start": "npx functions-framework --port=9090 --target=puppeteer", + "start_preview": "npx functions-framework --target=preview" + } +}