Create a new service for gcf

packages/content-fetch-gcf/.env.example

@@ -0,0 +1,8 @@
+# Should match with the JWT_SECRET that the api uses
+JWT_SECRET=some_secret
+
+# Address of the backend that is running locally
+REST_BACKEND_ENDPOINT=http://localhost:4000/api
+
+# set for local development
+IS_LOCAL=true

packages/content-fetch-gcf/.gcloudignore

@@ -0,0 +1,21 @@
+# This file specifies files that are *not* uploaded to Google Cloud Platform
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+node_modules
+.env*
+.secrets*
+Dockerfile*
+previewImage.*
+*.sa.json

packages/content-fetch-gcf/.gitignore

@@ -0,0 +1 @@
+previewImage.*

packages/content-fetch-gcf/Dockerfile

@@ -0,0 +1,112 @@
+# FROM node:14-slim
+
+# # Taken from pu
+
+# # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others)
+# # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer
+# # installs, work.
+# RUN apt-get update \
+#     && apt-get install -y wget gnupg \
+#     && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+#     && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+#     && apt-get update \
+#     && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \
+#       --no-install-recommends \
+#     && rm -rf /var/lib/apt/lists/*
+
+# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+# ENV CHROMIUM_PATH "/usr/bin/google-chrome-stable"
+
+# ------------------------
+
+# FROM --platform=linux/arm64 node:14.18
+
+# RUN apt-get update \
+#   && apt-get install -y chromium \
+#   && apt-get install -y ca-certificates \
+#   fonts-liberation \
+#   libappindicator3-1 \
+#   libasound2 \
+#   libatk-bridge2.0-0 \
+#   libatk1.0-0 \
+#   libc6 \
+#   libcairo2 \
+#   libcups2 \
+#   libdbus-1-3 \
+#   libexpat1 \
+#   libfontconfig1 \
+#   libgbm1 \
+#   libgcc1 \
+#   libglib2.0-0 \
+#   libgtk-3-0 \
+#   libnspr4 \
+#   libnss3 \
+#   libpango-1.0-0 \
+#   libpangocairo-1.0-0 \
+#   libstdc++6 \
+#   libx11-6 \
+#   libx11-xcb1 \
+#   libxcb1 \
+#   libxcomposite1 \
+#   libxcursor1 \
+#   libxdamage1 \
+#   libxext6 \
+#   libxfixes3 \
+#   libxi6 \
+#   libxrandr2 \
+#   libxrender1 \
+#   libxss1 \
+#   libxtst6 \
+#   lsb-release \
+#   wget \
+#   xdg-utils
+
+FROM node:14.18-alpine
+
+# Installs latest Chromium (92) package.
+RUN apk add --no-cache \
+      chromium \
+      nss \
+      freetype \
+      harfbuzz \
+      ca-certificates \
+      ttf-freefont \
+      nodejs \
+      yarn
+
+# Tell Puppeteer to skip installing Chrome. We'll be using the installed package.
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
+
+# Puppeteer v10.0.0 works with Chromium 92.
+RUN yarn add puppeteer@10.0.0
+
+# Add user so we don't need --no-sandbox.
+RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \
+    && mkdir -p /home/pptruser/Downloads /app \
+    && chown -R pptruser:pptruser /home/pptruser \
+    && chown -R pptruser:pptruser /app
+
+# Run everything after as non-privileged user.
+WORKDIR /app
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+ENV CHROMIUM_PATH /usr/bin/chromium-browser
+ENV LAUNCH_HEADLESS=true
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/content-fetch-gcf/package.json ./packages/content-fetch-gcf/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/content-fetch-gcf ./packages/content-fetch-gcf
+
+EXPOSE 8080
+
+# USER pptruser
+ENTRYPOINT ["yarn", "workspace", "@omnivore/content-fetch-gcf", "start"]

packages/content-fetch-gcf/README.md

@@ -0,0 +1,24 @@
+# Puppeteer parsing function handler
+
+This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer.
+     
+## Using locally
+
+Copy .env.example file to .env file: `cp .env.example .env`
+
+Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading).
+
+After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer)
+
+## Deployment
+
+To deploy the function use the following command:
+
+`gcloud functions deploy puppeteer --runtime nodejs12 --trigger-http --memory 1GB --set-env-vars REST_BACKEND_ENDPOINT=<backend-address>,JWT_SECRET=<jwt-secret>`
+
+
+where:
+
+`<backend-address>` - address of the backend server (e.g "http://localhost:4000")
+
+`<jwt-secret>` - JWT secret that the backend server is using (e.g "some_secret")

packages/content-fetch-gcf/index.js

@@ -0,0 +1,221 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const { config, format, loggers, transports } = require('winston');
+const { LoggingWinston } = require('@google-cloud/logging-winston');
+const { DateTime } = require('luxon');
+const os = require('os');
+const Sentry = require('@sentry/serverless');
+const { Storage } = require('@google-cloud/storage');
+const { fetchContent, getBrowserPromise, getUrl } = require("@omnivore/puppeteer-parse");
+
+const storage = new Storage();
+const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : [];
+const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.env.PREVIEW_IMAGE_BUCKET) : undefined;
+
+Sentry.GCPFunction.init({
+  dsn: process.env.SENTRY_DSN,
+  tracesSampleRate: 0,
+});
+
+const filePath = `${os.tmpdir()}/previewImage.png`;
+
+const colors = {
+  emerg: 'inverse underline magenta',
+  alert: 'underline magenta',
+  crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss.
+  error: 'underline red', // Any error which is fatal to the operation, but not the service or application
+  warning: 'underline yellow', // Anything that can potentially cause application oddities
+  notice: 'underline cyan', // Normal but significant condition
+  info: 'underline green', // Generally useful information to log
+  debug: 'underline gray',
+};
+
+const googleConfigs = {
+  level: 'info',
+  logName: 'logger',
+  levels: config.syslog.levels,
+  resource: {
+    labels: {
+      function_name: process.env.FUNCTION_TARGET,
+      project_id: process.env.GCP_PROJECT,
+    },
+    type: 'cloud_function',
+  },
+};
+
+function localConfig(id) {
+  return {
+    level: 'debug',
+    format: format.combine(
+      format.colorize({ all: true, colors }),
+      format(info =>
+        Object.assign(info, {
+          timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS),
+        }),
+      )(),
+      format.printf(info => {
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const { timestamp, message, level, ...meta } = info;
+
+        return `[${id}@${info.timestamp}] ${info.message}${
+          Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : ''
+        }`;
+      }),
+    ),
+  };
+}
+
+function buildLoggerTransport(id, options) {
+  return process.env.IS_LOCAL
+    ? new transports.Console(localConfig(id))
+    : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options });
+}
+
+function buildLogger(id, options) {
+  return loggers.get(id, {
+    levels: config.syslog.levels,
+    transports: [buildLoggerTransport(id, options)],
+  });
+}
+
+/**
+ * Cloud Function entry point, HTTP trigger.
+ * Loads the requested URL via Puppeteer, captures page content and sends it to backend
+ *
+ * @param {Object} req Cloud Function request context.
+ * @param {Object} res Cloud Function response context.
+ */
+exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(fetchContent);
+
+/**
+ * Cloud Function entry point, HTTP trigger.
+ * Loads the requested URL via Puppeteer and captures a screenshot of the provided element
+ *
+ * @param {Object} req Cloud Function request context.
+ * Inlcudes:
+ *  * url - URL address of the page to open
+ * @param {Object} res Cloud Function response context.
+ */
+exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
+  const functionStartTime = Date.now();
+  // Grabbing execution and trace ids to attach logs to the appropriate function call
+  const execution_id = req.get('function-execution-id');
+  const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0];
+  const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', {
+    trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`,
+    labels: {
+      execution_id: execution_id,
+    },
+  });
+
+  if (!process.env.PREVIEW_IMAGE_BUCKET) {
+    logger.error(`PREVIEW_IMAGE_BUCKET not set`)
+    return res.sendStatus(500);
+  }
+
+  const url = getUrl(req);
+  console.log('preview request url', url);
+
+  const logRecord = {
+    url,
+    query: req.query,
+    origin: req.get('Origin'),
+    labels: {
+      source: 'publicImagePreview',
+    },
+  };
+
+  logger.info(`Public preview image generation request`, logRecord);
+
+  if (!url) {
+    logRecord.urlIsInvalid = true;
+    logger.error(`Valid URL to parse is not specified`, logRecord);
+    return res.sendStatus(400);
+  }
+  const { origin } = new URL(url);
+  if (!ALLOWED_ORIGINS.some(o => o === origin)) {
+    logRecord.forbiddenOrigin = true;
+    logger.error(`This origin is not allowed: ${origin}`, logRecord);
+    return res.sendStatus(400);
+  }
+
+  const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH);
+  logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime };
+
+  const page = await browser.newPage();
+  const pageLoadingStart = Date.now();
+  const modifiedUrl = new URL(url);
+  modifiedUrl.searchParams.append('fontSize', 24);
+  modifiedUrl.searchParams.append('adjustAspectRatio', 1.91);
+  try {
+    await page.goto(modifiedUrl);
+    logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart };
+  } catch (error) {
+    console.log('error going to page: ', modifiedUrl)
+    console.log(error)
+    throw error
+  }
+
+  // We lookup the destination path from our own page content and avoid trusting any passed query params
+  // selector - CSS selector of the element to get screenshot of
+  const selector = decodeURIComponent(
+    await page.$eval(
+      "head > meta[name='omnivore:preview_image_selector']",
+      element => element.content,
+    ),
+  );
+  if (!selector) {
+    logRecord.selectorIsInvalid = true;
+    logger.error(`Valid element selector is not specified`, logRecord);
+    await page.close();
+    return res.sendStatus(400);
+  }
+  logRecord.selector = selector;
+
+  // destination - destination pathname for the image to save with
+  const destination = decodeURIComponent(
+    await page.$eval(
+      "head > meta[name='omnivore:preview_image_destination']",
+      element => element.content,
+    ),
+  );
+  if (!destination) {
+    logRecord.destinationIsInvalid = true;
+    logger.error(`Valid file destination is not specified`, logRecord);
+    await page.close();
+    return res.sendStatus(400);
+  }
+  logRecord.destination = destination;
+
+  const screenshotTakingStart = Date.now();
+  try {
+    await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load
+  } catch (error) {
+    logRecord.elementNotFound = true;
+    logger.error(`Element is not presented on the page`, logRecord);
+    await page.close();
+    return res.sendStatus(400);
+  }
+  const element = await page.$(selector);
+  await element.screenshot({ path: filePath }); // take screenshot of the element in puppeteer
+  logRecord.timing = { ...logRecord.timing, screenshotTaken: Date.now() - screenshotTakingStart };
+
+  await page.close();
+
+  try {
+    const [file] = await previewBucket.upload(filePath, {
+      destination,
+      metadata: logRecord,
+    });
+    logRecord.file = file.metadata;
+  } catch (e) {
+    console.log('error uploading to bucket, this is non-fatal', e)
+  }
+
+  logger.info(`preview-image`, logRecord);
+  return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`);
+});

packages/content-fetch-gcf/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@omnivore/content-fetch-gcf",
+  "version": "1.0.0",
+  "description": "Google Cloud Function that accepts URL of the article and parses its content",
+  "main": "index.js",
+  "dependencies": {
+    "@google-cloud/logging-winston": "^4.1.2",
+    "@google-cloud/storage": "^5.18.1",
+    "@sentry/serverless": "^6.13.3",
+    "axios": "^0.26.0",
+    "dotenv": "^8.2.0",
+    "jsonwebtoken": "^8.5.1",
+    "linkedom": "^0.14.9",
+    "luxon": "^2.3.1",
+    "winston": "^3.3.3",
+    "@omnivore/puppeteer-parse": "^1.0.0"
+  },
+  "devDependencies": {
+    "@google-cloud/functions-framework": "^3.0.0"
+  },
+  "scripts": {
+    "start": "npx functions-framework --port=9090 --target=puppeteer",
+    "start_preview": "npx functions-framework --target=preview"
+  }
+}