From cb858484c69cda861a6c29900661c077c783e087 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 29 Jun 2022 11:17:43 +0800 Subject: [PATCH 01/11] Make puppeteer-parse a module --- packages/content-fetch/fetch-content.js | 635 ------------------------ packages/puppeteer-parse/.env.example | 8 - packages/puppeteer-parse/.gcloudignore | 21 - packages/puppeteer-parse/.gitignore | 1 - packages/puppeteer-parse/Dockerfile | 113 ----- packages/puppeteer-parse/README.md | 23 +- packages/puppeteer-parse/index.js | 393 +++++---------- packages/puppeteer-parse/package.json | 4 +- 8 files changed, 125 insertions(+), 1073 deletions(-) delete mode 100644 packages/content-fetch/fetch-content.js delete mode 100644 packages/puppeteer-parse/.env.example delete mode 100644 packages/puppeteer-parse/.gcloudignore delete mode 100644 packages/puppeteer-parse/.gitignore delete mode 100644 packages/puppeteer-parse/Dockerfile diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js deleted file mode 100644 index 7a35e1eeb..000000000 --- a/packages/content-fetch/fetch-content.js +++ /dev/null @@ -1,635 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const Url = require('url'); -const puppeteer = require('puppeteer-core'); -const axios = require('axios'); -const jwt = require('jsonwebtoken'); -const { promisify } = require('util'); -const { parseHTML } = require('linkedom'); -const { preHandleContent } = require('@omnivore/content-handler'); - -const signToken = promisify(jwt.sign); - -const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' -const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] -const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; - -const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; - -// Add stealth plugin to hide puppeteer usage -// const StealthPlugin = require('puppeteer-extra-plugin-stealth'); -// puppeteer.use(StealthPlugin()); - - -const userAgentForUrl = (url) => { - try { - const u = new URL(url); - for (const host of NON_BOT_HOSTS) { - if (u.hostname.endsWith(host)) { - return NON_BOT_DESKTOP_USER_AGENT; - } - } - } catch (e) { - console.log('error getting user agent for url', url, e) - } - return DESKTOP_USER_AGENT -}; - -const fetchContentWithScrapingBee = async (url) => { - const response = await axios.get('https://app.scrapingbee.com/api/v1', { - params: { - 'api_key': process.env.SCRAPINGBEE_API_KEY, - 'url': url, - 'render_js': 'false', - 'premium_proxy': 'true', - 'country_code':'us' - } - }) - - const dom = parseHTML(response.data).document; - return { title: dom.title, domContent: dom.documentElement.outerHTML, url: url } -} - -const enableJavascriptForUrl = (url) => { - try { - const u = new URL(url); - for (const host of NON_SCRIPT_HOSTS) { - if (u.hostname.endsWith(host)) { - return false; - } - } - } catch (e) { - console.log('error getting hostname for url', url, e) - } - return true -}; - -// launch Puppeteer -const getBrowserPromise = (async () => { - console.log("starting with proxy url", process.env.PROXY_URL) - return puppeteer.launch({ - args: [ - '--allow-running-insecure-content', - '--autoplay-policy=user-gesture-required', - '--disable-component-update', - '--disable-domain-reliability', - '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process', - '--disable-print-preview', - '--disable-setuid-sandbox', - '--disable-site-isolation-trials', - '--disable-speech-api', - '--disable-web-security', - '--disk-cache-size=33554432', - '--enable-features=SharedArrayBuffer', - '--hide-scrollbars', - '--ignore-gpu-blocklist', - '--in-process-gpu', - '--mute-audio', - '--no-default-browser-check', - '--no-pings', - '--no-sandbox', - '--no-zygote', - '--use-gl=swiftshader', - '--window-size=1920,1080', - ].filter((item) => !!item), - defaultViewport: { height: 1080, width: 1920 }, - executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, - timeout: 120000, // 2 minutes - }); -})(); - -let logRecord, functionStartTime; - -const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => { - const stream = await axios.get(contentObjUrl, { responseType: 'stream' }); - return await axios.put(uploadSignedUrl, stream.data, { - headers: { - 'Content-Type': contentType, - }, - maxBodyLength: 1000000000, - maxContentLength: 100000000, - }) -}; - -const getUploadIdAndSignedUrl = async (userId, url, articleSavingRequestId) => { - const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); - const data = JSON.stringify({ - query: `mutation UploadFileRequest($input: UploadFileRequestInput!) { - uploadFileRequest(input:$input) { - ... on UploadFileRequestError { - errorCodes - } - ... on UploadFileRequestSuccess { - id - uploadSignedUrl - } - } - }`, - variables: { - input: { - url, - contentType: 'application/pdf', - clientRequestId: articleSavingRequestId, - } - } - }); - - const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); - return response.data.data.uploadFileRequest; -}; - -const uploadPdf = async (url, userId, articleSavingRequestId) => { - validateUrlString(url); - - const uploadResult = await getUploadIdAndSignedUrl(userId, url, articleSavingRequestId); - await uploadToSignedUrl(uploadResult, 'application/pdf', url); - return uploadResult.id; -}; - -const sendCreateArticleMutation = async (userId, input) => { - const data = JSON.stringify({ - query: `mutation CreateArticle ($input: CreateArticleInput!){ - createArticle(input:$input){ - ... on CreateArticleSuccess{ - createdArticle{ - id - } - } - ... on CreateArticleError{ - errorCodes - } - } - }`, - variables: { - input: Object.assign({}, input , { source: 'puppeteer-parse' }), - }, - }); - - const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); - const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); - return response.data.data.createArticle; -}; - -const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId) => { - return sendCreateArticleMutation(userId, { - url: encodeURI(url), - articleSavingRequestId, - uploadFileId: uploadFileId, - }, - ); -}; - -async function fetchContent(req, res) { - functionStartTime = Date.now(); - - let url = getUrl(req); - const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); - const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); - - console.log('user id', userId, 'url', url) - - logRecord = { - url, - userId, - articleSavingRequestId, - labels: { - source: 'parseContent', - }, - }; - - console.log(`Article parsing request`, logRecord); - - if (!url) { - logRecord.urlIsInvalid = true; - console.log(`Valid URL to parse not specified`, logRecord); - return res.sendStatus(400); - } - - // pre handle url with custom handlers - let title, content, contentType; - try { - const result = await preHandleContent(url); - if (result && result.url) { - url = result.url - validateUrlString(url); - } - if (result && result.title) { title = result.title } - if (result && result.content) { content = result.content } - if (result && result.contentType) { contentType = result.contentType } - } catch (e) { - console.log('error with handler: ', e); - } - - let context, page, finalUrl; - try { - if ((!content || !title) && contentType !== 'application/pdf') { - const result = await retrievePage(url) - if (result && result.context) { context = result.context } - if (result && result.page) { page = result.page } - if (result && result.finalUrl) { finalUrl = result.finalUrl } - if (result && result.contentType) { contentType = result.contentType } - } else { - finalUrl = url - } - - if (contentType === 'application/pdf') { - const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId); - const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId); - } else { - if (!content || !title) { - const result = await retrieveHtml(page); - if (result.isBlocked) { - const sbResult = await fetchContentWithScrapingBee(url) - title = sbResult.title - content = sbResult.domContent - } else { - title = result.title; - content = result.domContent; - } - } else { - console.log('using prefetched content and title'); - } - - logRecord.fetchContentTime = Date.now() - functionStartTime; - - const apiResponse = await sendCreateArticleMutation(userId, { - url: finalUrl, - articleSavingRequestId, - preparedDocument: { - document: content, - pageInfo: { - title, - canonicalUrl: finalUrl, - }, - }, - skipParsing: !content, - }); - - logRecord.totalTime = Date.now() - functionStartTime; - logRecord.result = apiResponse.createArticle; - } - } catch (e) { - logRecord.error = e.message; - console.log(`Error while retrieving page`, logRecord); - - // fallback to scrapingbee - const sbResult = await fetchContentWithScrapingBee(url); - const sbUrl = finalUrl || sbResult.url; - const content = sbResult.domContent; - logRecord.fetchContentTime = Date.now() - functionStartTime; - - const apiResponse = await sendCreateArticleMutation(userId, { - url: sbUrl, - articleSavingRequestId, - preparedDocument: { - document: content, - pageInfo: { - title: sbResult.title, - canonicalUrl: sbUrl, - }, - }, - skipParsing: !content, - }); - - logRecord.totalTime = Date.now() - functionStartTime; - logRecord.result = apiResponse.createArticle; - } finally { - if (context) { - await context.close(); - } - console.log(`parse-page`, logRecord); - } - - return res.sendStatus(200); -} - -function validateUrlString(url) { - const u = new URL(url); - // Make sure the URL is http or https - if (u.protocol !== 'http:' && u.protocol !== 'https:') { - throw new Error('Invalid URL protocol check failed') - } - // Make sure the domain is not localhost - if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') { - throw new Error('Invalid URL is localhost') - } - // Make sure the domain is not a private IP - if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) { - throw new Error('Invalid URL is private ip') - } -} - -function getUrl(req) { - console.log('body', req.body) - const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined); - if (!urlStr) { - throw new Error('No URL specified'); - } - - validateUrlString(urlStr); - - const parsed = Url.parse(urlStr); - return parsed.href; -} - - -async function blockResources(client) { - const blockedResources = [ - // Assets - // '*/favicon.ico', - // '.css', - // '.jpg', - // '.jpeg', - // '.png', - // '.svg', - // '.woff', - - // Analytics and other fluff - '*.optimizely.com', - 'everesttech.net', - 'userzoom.com', - 'doubleclick.net', - 'googleadservices.com', - 'adservice.google.com/*', - 'connect.facebook.com', - 'connect.facebook.net', - 'sp.analytics.yahoo.com', - ] - - await client.send('Network.setBlockedURLs', { urls: blockedResources }); -} - -async function retrievePage(url) { - validateUrlString(url); - - const browser = await getBrowserPromise; - logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; - - const context = await browser.createIncognitoBrowserContext(); - const page = await context.newPage() - - if (!enableJavascriptForUrl(url)) { - await page.setJavaScriptEnabled(false); - } - await page.setUserAgent(userAgentForUrl(url)); - - const client = await page.target().createCDPSession(); - - // intercept request when response headers was received - await client.send('Network.setRequestInterception', { - patterns: [ - { - urlPattern: '*', - resourceType: 'Document', - interceptionStage: 'HeadersReceived', - }, - ], - }); - - const path = require('path'); - const download_path = path.resolve('./download_dir/'); - - await client.send('Page.setDownloadBehavior', { - behavior: 'allow', - userDataDir: './', - downloadPath: download_path, - }) - - client.on('Network.requestIntercepted', async e => { - const headers = e.responseHeaders || {}; - - const [contentType] = (headers['content-type'] || headers['Content-Type'] || '') - .toLowerCase() - .split(';'); - const obj = { interceptionId: e.interceptionId }; - - if (e.responseStatusCode >= 200 && e.responseStatusCode < 300) { - // We only check content-type on success responses - // as it doesn't matter what the content type is for things - // like redirects - if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) { - obj['errorReason'] = 'BlockedByClient'; - } - } - - try { - await client.send('Network.continueInterceptedRequest', obj); - // eslint-disable-next-line no-empty - } catch {} - }); - - await blockResources(client); - - /* - * Disallow MathJax from running in Puppeteer and modifying the document, - * we shall instead run it in our frontend application to transform any - * mathjax content when present. - */ - await page.setRequestInterception(true); - let requestCount = 0; - page.on('request', request => { - if (['font', 'image', 'media'].includes(request.resourceType())) { - request.abort(); - return; - } - if (requestCount++ > 100) { - request.abort(); - return; - } - if ( - request.resourceType() === 'script' && - request.url().toLowerCase().indexOf('mathjax') > -1 - ) { - request.abort(); - return - } - request.continue(); - }); - - // Puppeteer fails during download of PDf files, - // so record the failure and use those items - let lastPdfUrl = undefined; - page.on('response', response => { - if (response.headers()['content-type'] === 'application/pdf') { - lastPdfUrl = response.url(); - } - }); - - try { - const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] }); - const finalUrl = response.url(); - const contentType = response.headers()['content-type']; - - logRecord.finalUrl = response.url(); - logRecord.contentType = response.headers()['content-type']; - - return { context, page, response, finalUrl, contentType }; - } catch (error) { - if (lastPdfUrl) { - return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' }; - } - await context.close(); - throw error; - } -} - -async function retrieveHtml(page) { - let domContent = '', title; - try { - title = await page.title(); - logRecord.title = title; - - const pageScrollingStart = Date.now(); - /* scroll with a 5 second timeout */ - await Promise.race([ - new Promise(resolve => { - (async function () { - try { - await page.evaluate(`(async () => { - /* credit: https://github.com/puppeteer/puppeteer/issues/305 */ - return new Promise((resolve, reject) => { - let scrollHeight = document.body.scrollHeight; - let totalHeight = 0; - let distance = 500; - let timer = setInterval(() => { - window.scrollBy(0, distance); - totalHeight += distance; - if(totalHeight >= scrollHeight){ - clearInterval(timer); - resolve(true); - } - }, 10); - }); - })()`); - } catch (e) { - logRecord.scrollError = true; - } finally { - resolve(true); - } - })(); - }), - await page.waitForTimeout(1000), - ]); - logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; - - const iframes = {}; - const urls = []; - const framesPromises = []; - const allowedUrls = /instagram\.com/gi; - - for (const frame of page.mainFrame().childFrames()) { - if (frame.url() && allowedUrls.test(frame.url())) { - urls.push(frame.url()); - framesPromises.push(frame.evaluate(el => el.innerHTML, await frame.$('body'))); - } - } - - (await Promise.all(framesPromises)).forEach((frame, index) => (iframes[urls[index]] = frame)); - - const domContentCapturingStart = Date.now(); - // get document body with all hidden elements removed - domContent = await page.evaluate(iframes => { - const BI_SRC_REGEXP = /url\("(.+?)"\)/gi; - - Array.from(document.body.getElementsByTagName('*')).forEach(el => { - const style = window.getComputedStyle(el); - - try { - // Removing blurred images since they are mostly the copies of lazy loaded ones - if (['img', 'image'].includes(el.tagName.toLowerCase())) { - const filter = style.getPropertyValue('filter'); - if (filter && filter.startsWith('blur')) { - el.parentNode && el.parentNode.removeChild(el); - } - } - } catch (err) { - // throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*')))) - } - - // convert all nodes with background image to img nodes - if (!['', 'none'].includes(style.getPropertyValue('background-image'))) { - const filter = style.getPropertyValue('filter'); - // avoiding image nodes with a blur effect creation - if (filter && filter.startsWith('blur')) { - el && el.parentNode && el.parentNode.removeChild(el); - } else { - const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image')); - // Using "g" flag with a regex we have to manually break down lastIndex to zero after every usage - // More details here: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results - BI_SRC_REGEXP.lastIndex = 0; - - if (matchedSRC && matchedSRC[1] && !el.src) { - // Replacing element only of there are no content inside, b/c might remove important div with content. - // Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html - // DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image. - if (el.innerHTML.length < 25) { - const img = document.createElement('img'); - img.src = matchedSRC[1]; - el && el.parentNode && el.parentNode.removeChild(el); - } - } - } - } - - if (el.tagName === 'IFRAME') { - if (iframes[el.src]) { - const newNode = document.createElement('div'); - newNode.className = 'omnivore-instagram-embed'; - newNode.innerHTML = iframes[el.src]; - el && el.parentNode && el.parentNode.replaceChild(newNode, el); - } - } - }); - - if (document.querySelector('[data-translate="managed_checking_msg"]') || - document.getElementById('px-block-form-wrapper')) { - return 'IS_BLOCKED' - } - - return document.documentElement.outerHTML; - }, iframes); - logRecord.puppeteerSuccess = true; - logRecord.timing = { - ...logRecord.timing, - contenCaptured: Date.now() - domContentCapturingStart, - }; - - // [END puppeteer-block] - } catch (e) { - if (e.message.startsWith('net::ERR_BLOCKED_BY_CLIENT at ')) { - logRecord.blockedByClient = true; - } else { - logRecord.puppeteerSuccess = false; - logRecord.puppeteerError = { - message: e.message, - stack: e.stack, - }; - } - } - if (domContent === 'IS_BLOCKED') { - return { isBlocked: true }; - } - return { domContent, title }; -} - -module.exports = fetchContent; diff --git a/packages/puppeteer-parse/.env.example b/packages/puppeteer-parse/.env.example deleted file mode 100644 index 64242a22d..000000000 --- a/packages/puppeteer-parse/.env.example +++ /dev/null @@ -1,8 +0,0 @@ -# Should match with the JWT_SECRET that the api uses -JWT_SECRET=some_secret - -# Address of the backend that is running locally -REST_BACKEND_ENDPOINT=http://localhost:4000/api - -# set for local development -IS_LOCAL=true diff --git a/packages/puppeteer-parse/.gcloudignore b/packages/puppeteer-parse/.gcloudignore deleted file mode 100644 index fc644d8d3..000000000 --- a/packages/puppeteer-parse/.gcloudignore +++ /dev/null @@ -1,21 +0,0 @@ -# This file specifies files that are *not* uploaded to Google Cloud Platform -# using gcloud. It follows the same syntax as .gitignore, with the addition of -# "#!include" directives (which insert the entries of the given .gitignore-style -# file at that point). -# -# For more information, run: -# $ gcloud topic gcloudignore -# -.gcloudignore -# If you would like to upload your .git directory, .gitignore file or files -# from your .gitignore file, remove the corresponding line -# below: -.git -.gitignore - -node_modules -.env* -.secrets* -Dockerfile* -previewImage.* -*.sa.json diff --git a/packages/puppeteer-parse/.gitignore b/packages/puppeteer-parse/.gitignore deleted file mode 100644 index 9bfbc5e8b..000000000 --- a/packages/puppeteer-parse/.gitignore +++ /dev/null @@ -1 +0,0 @@ -previewImage.* \ No newline at end of file diff --git a/packages/puppeteer-parse/Dockerfile b/packages/puppeteer-parse/Dockerfile deleted file mode 100644 index a52f55122..000000000 --- a/packages/puppeteer-parse/Dockerfile +++ /dev/null @@ -1,113 +0,0 @@ -# FROM node:14-slim - -# # Taken from pu - -# # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) -# # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer -# # installs, work. -# RUN apt-get update \ -# && apt-get install -y wget gnupg \ -# && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ -# && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ -# && apt-get update \ -# && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \ -# --no-install-recommends \ -# && rm -rf /var/lib/apt/lists/* - -# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true -# ENV CHROMIUM_PATH "/usr/bin/google-chrome-stable" - -# ------------------------ - -# FROM --platform=linux/arm64 node:14.18 - -# RUN apt-get update \ -# && apt-get install -y chromium \ -# && apt-get install -y ca-certificates \ -# fonts-liberation \ -# libappindicator3-1 \ -# libasound2 \ -# libatk-bridge2.0-0 \ -# libatk1.0-0 \ -# libc6 \ -# libcairo2 \ -# libcups2 \ -# libdbus-1-3 \ -# libexpat1 \ -# libfontconfig1 \ -# libgbm1 \ -# libgcc1 \ -# libglib2.0-0 \ -# libgtk-3-0 \ -# libnspr4 \ -# libnss3 \ -# libpango-1.0-0 \ -# libpangocairo-1.0-0 \ -# libstdc++6 \ -# libx11-6 \ -# libx11-xcb1 \ -# libxcb1 \ -# libxcomposite1 \ -# libxcursor1 \ -# libxdamage1 \ -# libxext6 \ -# libxfixes3 \ -# libxi6 \ -# libxrandr2 \ -# libxrender1 \ -# libxss1 \ -# libxtst6 \ -# lsb-release \ -# wget \ -# xdg-utils - -FROM node:14.18-alpine - -# Installs latest Chromium (92) package. -RUN apk add --no-cache \ - chromium \ - nss \ - freetype \ - harfbuzz \ - ca-certificates \ - ttf-freefont \ - nodejs \ - yarn - -# Add user so we don't need --no-sandbox. -RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ - && mkdir -p /home/pptruser/Downloads /app \ - && chown -R pptruser:pptruser /home/pptruser \ - && chown -R pptruser:pptruser /app - -# Run everything after as non-privileged user. -WORKDIR /app - -ENV CHROMIUM_PATH /usr/bin/chromium-browser -ENV LAUNCH_HEADLESS=true -ENV PORT 9090 - -COPY package.json . -COPY yarn.lock . -COPY tsconfig.json . -COPY .prettierrc . -COPY .eslintrc . - -COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json -COPY /packages/content-handler/package.json ./packages/content-handler/package.json - -RUN yarn install --pure-lockfile - -ADD /packages/puppeteer-parse ./packages/puppeteer-parse -ADD /packages/content-handler ./packages/content-handler -RUN yarn workspace @omnivore/content-handler build - -# After building, fetch the production dependencies -RUN rm -rf /app/packages/puppeteer-parse/node_modules -RUN rm -rf /app/node_modules -RUN yarn install --pure-lockfile --production - -EXPOSE 9090 - -# USER pptruser -ENTRYPOINT ["yarn", "workspace", "@omnivore/puppeteer-parse", "start"] diff --git a/packages/puppeteer-parse/README.md b/packages/puppeteer-parse/README.md index 501c004ee..6155c45b0 100644 --- a/packages/puppeteer-parse/README.md +++ b/packages/puppeteer-parse/README.md @@ -1,24 +1,3 @@ # Puppeteer parsing function handler -This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer. - -## Using locally - -Copy .env.example file to .env file: `cp .env.example .env` - -Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading). - -After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) - -## Deployment - -To deploy the function use the following command: - -`gcloud functions deploy puppeteer --runtime nodejs12 --trigger-http --memory 1GB --set-env-vars REST_BACKEND_ENDPOINT=,JWT_SECRET=` - - -where: - -`` - address of the backend server (e.g "http://localhost:4000") - -`` - JWT secret that the backend server is using (e.g "some_secret") +This workspace is used to provide the module for the app to hande requests for the article parsing via Puppeteer. diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index b602ff1ef..f2f4cf02f 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -5,6 +5,7 @@ /* eslint-disable @typescript-eslint/no-require-imports */ require('dotenv').config(); const Url = require('url'); +const puppeteer = require('puppeteer-extra'); const axios = require('axios'); const jwt = require('jsonwebtoken'); const { promisify } = require('util'); @@ -38,69 +39,16 @@ const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) Apple const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] +const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; -const filePath = `${os.tmpdir()}/previewImage.png`; +const path = require("path"); const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; +const { parseHTML } = require('linkedom'); -const colors = { - emerg: 'inverse underline magenta', - alert: 'underline magenta', - crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. - error: 'underline red', // Any error which is fatal to the operation, but not the service or application - warning: 'underline yellow', // Anything that can potentially cause application oddities - notice: 'underline cyan', // Normal but significant condition - info: 'underline green', // Generally useful information to log - debug: 'underline gray', -}; - -const googleConfigs = { - level: 'info', - logName: 'logger', - levels: config.syslog.levels, - resource: { - labels: { - function_name: process.env.FUNCTION_TARGET, - project_id: process.env.GCP_PROJECT, - }, - type: 'cloud_function', - }, -}; - -function localConfig(id) { - return { - level: 'debug', - format: format.combine( - format.colorize({ all: true, colors }), - format(info => - Object.assign(info, { - timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), - }), - )(), - format.printf(info => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { timestamp, message, level, ...meta } = info; - - return `[${id}@${info.timestamp}] ${info.message}${ - Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' - }`; - }), - ), - }; -} - -function buildLoggerTransport(id, options) { - return process.env.IS_LOCAL - ? new transports.Console(localConfig(id)) - : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); -} - -function buildLogger(id, options) { - return loggers.get(id, { - levels: config.syslog.levels, - transports: [buildLoggerTransport(id, options)], - }); -} +// Add stealth plugin to hide puppeteer usage +const StealthPlugin = require('puppeteer-extra-plugin-stealth'); +puppeteer.use(StealthPlugin()); const userAgentForUrl = (url) => { try { @@ -116,15 +64,38 @@ const userAgentForUrl = (url) => { return DESKTOP_USER_AGENT }; +const fetchContentWithScrapingBee = async (url) => { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + 'api_key': process.env.SCRAPINGBEE_API_KEY, + 'url': url, + 'render_js': 'false', + 'premium_proxy': 'true', + 'country_code':'us' + } + }) + + const dom = parseHTML(response.data).document; + return { title: dom.title, domContent: dom.documentElement.outerHTML, url: url } +} + +const enableJavascriptForUrl = (url) => { + try { + const u = new URL(url); + for (const host of NON_SCRIPT_HOSTS) { + if (u.hostname.endsWith(host)) { + return false; + } + } + } catch (e) { + console.log('error getting hostname for url', url, e) + } + return true +}; + // launch Puppeteer -const getBrowserPromise = (async () => { - // return puppeteer.launch({ - // args: chromium.args, - // defaultViewport: chromium.defaultViewport, - // executablePath: process.env.CHROMIUM_PATH, - // headless: chromium.headless, - // ignoreHTTPSErrors: true, - // }); +const getBrowserPromise = (async (proxyUrl, chromiumPath) => { + console.log("starting with proxy url", proxyUrl) return puppeteer.launch({ args: [ '--allow-running-insecure-content', @@ -151,8 +122,8 @@ const getBrowserPromise = (async () => { '--window-size=1920,1080', ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, - executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, + executablePath: chromiumPath, + headless: true, timeout: 120000, // 2 minutes }); })(); @@ -170,7 +141,7 @@ const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentOb }) }; -const getUploadIdAndSignedUrl = async (userId, url) => { +const getUploadIdAndSignedUrl = async (userId, url, articleSavingRequestId) => { const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); const data = JSON.stringify({ query: `mutation UploadFileRequest($input: UploadFileRequestInput!) { @@ -188,17 +159,18 @@ const getUploadIdAndSignedUrl = async (userId, url) => { input: { url, contentType: 'application/pdf', + clientRequestId: articleSavingRequestId, } } }); const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + }); return response.data.data.uploadFileRequest; }; @@ -231,12 +203,12 @@ const sendCreateArticleMutation = async (userId, input) => { const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + }); return response.data.data.createArticle; }; @@ -249,24 +221,8 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId ); }; -/** - * Cloud Function entry point, HTTP trigger. - * Loads the requested URL via Puppeteer, captures page content and sends it to backend - * - * @param {Object} req Cloud Function request context. - * @param {Object} res Cloud Function response context. - */ -exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { +async function fetchContent(req, res) { functionStartTime = Date.now(); - // Grabbing execution and trace ids to attach logs to the appropriate function call - const execution_id = req.get('function-execution-id'); - const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; - const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { - trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, - labels: { - execution_id: execution_id, - }, - }); let url = getUrl(req); const userId = req.body.userId || req.query.userId; @@ -281,7 +237,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { }, }; - logger.info(`Article parsing request`, logRecord); + console.log(`Article parsing request`, logRecord); if (!url) { logRecord.urlIsInvalid = true; @@ -306,16 +262,15 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { let context, page, finalUrl; try { - if ((!content || !title) && contentType !== 'application/pdf') { - const result = await retrievePage(url) - if (result && result.context) { context = result.context } - if (result && result.page) { page = result.page } - if (result && result.finalUrl) { finalUrl = result.finalUrl } - if (result && result.contentType) { contentType = result.contentType } - console.log('context, page, finalUrl, contentType', context, page, finalUrl, contentType); - } else { - finalUrl = url - } + if ((!content || !title) && contentType !== 'application/pdf') { + const result = await retrievePage(url) + if (result && result.context) { context = result.context } + if (result && result.page) { page = result.page } + if (result && result.finalUrl) { finalUrl = result.finalUrl } + if (result && result.contentType) { contentType = result.contentType } + } else { + finalUrl = url + } if (contentType === 'application/pdf') { const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId); @@ -323,14 +278,20 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { } else { if (!content || !title) { const result = await retrieveHtml(page); - title = result.title; - content = result.domContent; + if (result.isBlocked) { + const sbResult = await fetchContentWithScrapingBee(url) + title = sbResult.title + content = sbResult.domContent + } else { + title = result.title; + content = result.domContent; + } } else { console.log('using prefetched content and title'); console.log(content); } - logRecord.contentFetchTime = Date.now() - functionStartTime; + logRecord.fetchContentTime = Date.now() - functionStartTime; const apiResponse = await sendCreateArticleMutation(userId, { url: finalUrl, @@ -347,12 +308,12 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; - logger.info(`parse-page`, logRecord); + console.log(`parse-page`, logRecord); } } catch (e) { console.log('error', e) logRecord.error = e.message; - logger.error(`Error while retrieving page`, logRecord); + console.log(`Error while retrieving page`, logRecord); return res.sendStatus(503); } finally { if (context) { @@ -361,136 +322,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { } return res.sendStatus(200); -}); - -/** - * Cloud Function entry point, HTTP trigger. - * Loads the requested URL via Puppeteer and captures a screenshot of the provided element - * - * @param {Object} req Cloud Function request context. - * Inlcudes: - * * url - URL address of the page to open - * @param {Object} res Cloud Function response context. - */ -exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { - functionStartTime = Date.now(); - // Grabbing execution and trace ids to attach logs to the appropriate function call - const execution_id = req.get('function-execution-id'); - const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; - const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { - trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, - labels: { - execution_id: execution_id, - }, - }); - - if (!process.env.PREVIEW_IMAGE_BUCKET) { - logger.error(`PREVIEW_IMAGE_BUCKET not set`) - return res.sendStatus(500); - } - - const url = getUrl(req); - console.log('preview request url', url); - - logRecord = { - url, - query: req.query, - origin: req.get('Origin'), - labels: { - source: 'publicImagePreview', - }, - }; - - logger.info(`Public preview image generation request`, logRecord); - - if (!url) { - logRecord.urlIsInvalid = true; - logger.error(`Valid URL to parse is not specified`, logRecord); - return res.sendStatus(400); - } - const { origin } = new URL(url); - if (!ALLOWED_ORIGINS.some(o => o === origin)) { - logRecord.forbiddenOrigin = true; - logger.error(`This origin is not allowed: ${origin}`, logRecord); - return res.sendStatus(400); - } - - const browser = await getBrowserPromise; - logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; - - const page = await browser.newPage(); - const pageLoadingStart = Date.now(); - const modifiedUrl = new URL(url); - modifiedUrl.searchParams.append('fontSize', 24); - modifiedUrl.searchParams.append('adjustAspectRatio', 1.91); - try { - await page.goto(modifiedUrl); - logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart }; - } catch (error) { - console.log('error going to page: ', modifiedUrl) - console.log(error) - throw error - } - - // We lookup the destination path from our own page content and avoid trusting any passed query params - // selector - CSS selector of the element to get screenshot of - const selector = decodeURIComponent( - await page.$eval( - "head > meta[name='omnivore:preview_image_selector']", - element => element.content, - ), - ); - if (!selector) { - logRecord.selectorIsInvalid = true; - logger.error(`Valid element selector is not specified`, logRecord); - await page.close(); - return res.sendStatus(400); - } - logRecord.selector = selector; - - // destination - destination pathname for the image to save with - const destination = decodeURIComponent( - await page.$eval( - "head > meta[name='omnivore:preview_image_destination']", - element => element.content, - ), - ); - if (!destination) { - logRecord.destinationIsInvalid = true; - logger.error(`Valid file destination is not specified`, logRecord); - await page.close(); - return res.sendStatus(400); - } - logRecord.destination = destination; - - const screenshotTakingStart = Date.now(); - try { - await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load - } catch (error) { - logRecord.elementNotFound = true; - logger.error(`Element is not presented on the page`, logRecord); - await page.close(); - return res.sendStatus(400); - } - const element = await page.$(selector); - await element.screenshot({ path: filePath }); // take screenshot of the element in puppeteer - logRecord.timing = { ...logRecord.timing, screenshotTaken: Date.now() - screenshotTakingStart }; - - await page.close(); - - try { - const [file] = await previewBucket.upload(filePath, { - destination, - metadata: logRecord, - }); - logRecord.file = file.metadata; - } catch (e) { - console.log('error uploading to bucket, this is non-fatal', e) - } - - logger.info(`preview-image`, logRecord); - return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); -}); +} function validateUrlString(url) { const u = new URL(url); @@ -509,16 +341,15 @@ function validateUrlString(url) { } function getUrl(req) { - if (req.query.url || req.body.url) { - const urlStr = req.query.url || req.body.url; - validateUrlString(urlStr); - - const url = Url.parse(urlStr); - return url.href; + const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined); + if (!urlStr) { + throw new Error('No URL specified'); } - try { - return Url.parse(JSON.parse(req.body).url).href; - } catch (e) {} + + validateUrlString(urlStr); + + const parsed = Url.parse(urlStr); + return parsed.href; } async function blockResources(client) { @@ -554,7 +385,11 @@ async function retrievePage(url) { logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; const context = await browser.createIncognitoBrowserContext(); - const page = await context.newPage(); + const page = await context.newPage() + + if (!enableJavascriptForUrl(url)) { + await page.setJavaScriptEnabled(false); + } await page.setUserAgent(userAgentForUrl(url)); const client = await page.target().createCDPSession(); @@ -570,16 +405,15 @@ async function retrievePage(url) { ], }); - const path = require('path'); const download_path = path.resolve('./download_dir/'); await client.send('Page.setDownloadBehavior', { - behavior: 'allow', - userDataDir: './', - downloadPath: download_path, + behavior: 'allow', + userDataDir: './', + downloadPath: download_path, }) - client.on('Network.requestIntercepted', async e => { + client.on('Network.requestIntercepted', async (e) => { const headers = e.responseHeaders || {}; const [contentType] = (headers['content-type'] || headers['Content-Type'] || '') @@ -605,10 +439,10 @@ async function retrievePage(url) { await blockResources(client); /* - * Disallow MathJax from running in Puppeteer and modifying the document, - * we shall instead run it in our frontend application to transform any - * mathjax content when present. - */ + * Disallow MathJax from running in Puppeteer and modifying the document, + * we shall instead run it in our frontend application to transform any + * mathjax content when present. + */ await page.setRequestInterception(true); let requestCount = 0; page.on('request', request => { @@ -630,7 +464,6 @@ async function retrievePage(url) { request.continue(); }); - // Puppeteer fails during download of PDf files, // so record the failure and use those items let lastPdfUrl = undefined; @@ -719,12 +552,16 @@ async function retrieveHtml(page) { Array.from(document.body.getElementsByTagName('*')).forEach(el => { const style = window.getComputedStyle(el); - // Removing blurred images since they are mostly the copies of lazy loaded ones - if (['img', 'image'].includes(el.tagName.toLowerCase())) { - const filter = style.getPropertyValue('filter'); - if (filter && filter.startsWith('blur')) { - el.parentNode && el.parentNode.removeChild(el); + try { + // Removing blurred images since they are mostly the copies of lazy loaded ones + if (['img', 'image'].includes(el.tagName.toLowerCase())) { + const filter = style.getPropertyValue('filter'); + if (filter && filter.startsWith('blur')) { + el.parentNode && el.parentNode.removeChild(el); + } } + } catch (err) { + // throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*')))) } // convert all nodes with background image to img nodes @@ -761,6 +598,12 @@ async function retrieveHtml(page) { } } }); + + if (document.querySelector('[data-translate="managed_checking_msg"]') || + document.getElementById('px-block-form-wrapper')) { + return 'IS_BLOCKED' + } + return document.documentElement.outerHTML; }, iframes); logRecord.puppeteerSuccess = true; @@ -781,5 +624,15 @@ async function retrieveHtml(page) { }; } } + if (domContent === 'IS_BLOCKED') { + return { isBlocked: true }; + } return { domContent, title }; } + +module.exports = { + fetchContent, + getBrowserPromise, + getUrl, +}; + diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 4ef6bb4d9..e73914fa4 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -1,7 +1,7 @@ { "name": "@omnivore/puppeteer-parse", "version": "1.0.0", - "description": "Google Cloud Function that accepts URL of the article and parses its content", + "description": "Accepts URL of the article and parses its content", "main": "index.js", "dependencies": { "@google-cloud/functions-framework": "^3.1.2", @@ -20,8 +20,6 @@ "winston": "^3.3.3" }, "scripts": { - "start": "npx functions-framework --port=9090 --target=puppeteer", - "start_preview": "npx functions-framework --target=preview", "test": "yarn mocha" } } From 63423751c6f598e9ff177e9c83a4c038b24b47dd Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 29 Jun 2022 11:18:13 +0800 Subject: [PATCH 02/11] Create a new service for gcf --- packages/content-fetch-gcf/.env.example | 8 + packages/content-fetch-gcf/.gcloudignore | 21 +++ packages/content-fetch-gcf/.gitignore | 1 + packages/content-fetch-gcf/Dockerfile | 112 ++++++++++++ packages/content-fetch-gcf/README.md | 24 +++ packages/content-fetch-gcf/index.js | 221 +++++++++++++++++++++++ packages/content-fetch-gcf/package.json | 25 +++ 7 files changed, 412 insertions(+) create mode 100644 packages/content-fetch-gcf/.env.example create mode 100644 packages/content-fetch-gcf/.gcloudignore create mode 100644 packages/content-fetch-gcf/.gitignore create mode 100644 packages/content-fetch-gcf/Dockerfile create mode 100644 packages/content-fetch-gcf/README.md create mode 100644 packages/content-fetch-gcf/index.js create mode 100644 packages/content-fetch-gcf/package.json diff --git a/packages/content-fetch-gcf/.env.example b/packages/content-fetch-gcf/.env.example new file mode 100644 index 000000000..64242a22d --- /dev/null +++ b/packages/content-fetch-gcf/.env.example @@ -0,0 +1,8 @@ +# Should match with the JWT_SECRET that the api uses +JWT_SECRET=some_secret + +# Address of the backend that is running locally +REST_BACKEND_ENDPOINT=http://localhost:4000/api + +# set for local development +IS_LOCAL=true diff --git a/packages/content-fetch-gcf/.gcloudignore b/packages/content-fetch-gcf/.gcloudignore new file mode 100644 index 000000000..fc644d8d3 --- /dev/null +++ b/packages/content-fetch-gcf/.gcloudignore @@ -0,0 +1,21 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +node_modules +.env* +.secrets* +Dockerfile* +previewImage.* +*.sa.json diff --git a/packages/content-fetch-gcf/.gitignore b/packages/content-fetch-gcf/.gitignore new file mode 100644 index 000000000..9bfbc5e8b --- /dev/null +++ b/packages/content-fetch-gcf/.gitignore @@ -0,0 +1 @@ +previewImage.* \ No newline at end of file diff --git a/packages/content-fetch-gcf/Dockerfile b/packages/content-fetch-gcf/Dockerfile new file mode 100644 index 000000000..c0df04111 --- /dev/null +++ b/packages/content-fetch-gcf/Dockerfile @@ -0,0 +1,112 @@ +# FROM node:14-slim + +# # Taken from pu + +# # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) +# # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer +# # installs, work. +# RUN apt-get update \ +# && apt-get install -y wget gnupg \ +# && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ +# && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ +# && apt-get update \ +# && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \ +# --no-install-recommends \ +# && rm -rf /var/lib/apt/lists/* + +# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +# ENV CHROMIUM_PATH "/usr/bin/google-chrome-stable" + +# ------------------------ + +# FROM --platform=linux/arm64 node:14.18 + +# RUN apt-get update \ +# && apt-get install -y chromium \ +# && apt-get install -y ca-certificates \ +# fonts-liberation \ +# libappindicator3-1 \ +# libasound2 \ +# libatk-bridge2.0-0 \ +# libatk1.0-0 \ +# libc6 \ +# libcairo2 \ +# libcups2 \ +# libdbus-1-3 \ +# libexpat1 \ +# libfontconfig1 \ +# libgbm1 \ +# libgcc1 \ +# libglib2.0-0 \ +# libgtk-3-0 \ +# libnspr4 \ +# libnss3 \ +# libpango-1.0-0 \ +# libpangocairo-1.0-0 \ +# libstdc++6 \ +# libx11-6 \ +# libx11-xcb1 \ +# libxcb1 \ +# libxcomposite1 \ +# libxcursor1 \ +# libxdamage1 \ +# libxext6 \ +# libxfixes3 \ +# libxi6 \ +# libxrandr2 \ +# libxrender1 \ +# libxss1 \ +# libxtst6 \ +# lsb-release \ +# wget \ +# xdg-utils + +FROM node:14.18-alpine + +# Installs latest Chromium (92) package. +RUN apk add --no-cache \ + chromium \ + nss \ + freetype \ + harfbuzz \ + ca-certificates \ + ttf-freefont \ + nodejs \ + yarn + +# Tell Puppeteer to skip installing Chrome. We'll be using the installed package. +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ + PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + +# Puppeteer v10.0.0 works with Chromium 92. +RUN yarn add puppeteer@10.0.0 + +# Add user so we don't need --no-sandbox. +RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ + && mkdir -p /home/pptruser/Downloads /app \ + && chown -R pptruser:pptruser /home/pptruser \ + && chown -R pptruser:pptruser /app + +# Run everything after as non-privileged user. +WORKDIR /app + +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +ENV CHROMIUM_PATH /usr/bin/chromium-browser +ENV LAUNCH_HEADLESS=true + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .prettierrc . +COPY .eslintrc . + +COPY /packages/content-fetch-gcf/package.json ./packages/content-fetch-gcf/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/content-fetch-gcf ./packages/content-fetch-gcf + +EXPOSE 8080 + +# USER pptruser +ENTRYPOINT ["yarn", "workspace", "@omnivore/content-fetch-gcf", "start"] diff --git a/packages/content-fetch-gcf/README.md b/packages/content-fetch-gcf/README.md new file mode 100644 index 000000000..501c004ee --- /dev/null +++ b/packages/content-fetch-gcf/README.md @@ -0,0 +1,24 @@ +# Puppeteer parsing function handler + +This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer. + +## Using locally + +Copy .env.example file to .env file: `cp .env.example .env` + +Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading). + +After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) + +## Deployment + +To deploy the function use the following command: + +`gcloud functions deploy puppeteer --runtime nodejs12 --trigger-http --memory 1GB --set-env-vars REST_BACKEND_ENDPOINT=,JWT_SECRET=` + + +where: + +`` - address of the backend server (e.g "http://localhost:4000") + +`` - JWT secret that the backend server is using (e.g "some_secret") diff --git a/packages/content-fetch-gcf/index.js b/packages/content-fetch-gcf/index.js new file mode 100644 index 000000000..2c18d99f8 --- /dev/null +++ b/packages/content-fetch-gcf/index.js @@ -0,0 +1,221 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const { config, format, loggers, transports } = require('winston'); +const { LoggingWinston } = require('@google-cloud/logging-winston'); +const { DateTime } = require('luxon'); +const os = require('os'); +const Sentry = require('@sentry/serverless'); +const { Storage } = require('@google-cloud/storage'); +const { fetchContent, getBrowserPromise, getUrl } = require("@omnivore/puppeteer-parse"); + +const storage = new Storage(); +const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; +const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.env.PREVIEW_IMAGE_BUCKET) : undefined; + +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}); + +const filePath = `${os.tmpdir()}/previewImage.png`; + +const colors = { + emerg: 'inverse underline magenta', + alert: 'underline magenta', + crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. + error: 'underline red', // Any error which is fatal to the operation, but not the service or application + warning: 'underline yellow', // Anything that can potentially cause application oddities + notice: 'underline cyan', // Normal but significant condition + info: 'underline green', // Generally useful information to log + debug: 'underline gray', +}; + +const googleConfigs = { + level: 'info', + logName: 'logger', + levels: config.syslog.levels, + resource: { + labels: { + function_name: process.env.FUNCTION_TARGET, + project_id: process.env.GCP_PROJECT, + }, + type: 'cloud_function', + }, +}; + +function localConfig(id) { + return { + level: 'debug', + format: format.combine( + format.colorize({ all: true, colors }), + format(info => + Object.assign(info, { + timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), + }), + )(), + format.printf(info => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { timestamp, message, level, ...meta } = info; + + return `[${id}@${info.timestamp}] ${info.message}${ + Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' + }`; + }), + ), + }; +} + +function buildLoggerTransport(id, options) { + return process.env.IS_LOCAL + ? new transports.Console(localConfig(id)) + : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); +} + +function buildLogger(id, options) { + return loggers.get(id, { + levels: config.syslog.levels, + transports: [buildLoggerTransport(id, options)], + }); +} + +/** + * Cloud Function entry point, HTTP trigger. + * Loads the requested URL via Puppeteer, captures page content and sends it to backend + * + * @param {Object} req Cloud Function request context. + * @param {Object} res Cloud Function response context. + */ +exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(fetchContent); + +/** + * Cloud Function entry point, HTTP trigger. + * Loads the requested URL via Puppeteer and captures a screenshot of the provided element + * + * @param {Object} req Cloud Function request context. + * Inlcudes: + * * url - URL address of the page to open + * @param {Object} res Cloud Function response context. + */ +exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { + const functionStartTime = Date.now(); + // Grabbing execution and trace ids to attach logs to the appropriate function call + const execution_id = req.get('function-execution-id'); + const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; + const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { + trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, + labels: { + execution_id: execution_id, + }, + }); + + if (!process.env.PREVIEW_IMAGE_BUCKET) { + logger.error(`PREVIEW_IMAGE_BUCKET not set`) + return res.sendStatus(500); + } + + const url = getUrl(req); + console.log('preview request url', url); + + const logRecord = { + url, + query: req.query, + origin: req.get('Origin'), + labels: { + source: 'publicImagePreview', + }, + }; + + logger.info(`Public preview image generation request`, logRecord); + + if (!url) { + logRecord.urlIsInvalid = true; + logger.error(`Valid URL to parse is not specified`, logRecord); + return res.sendStatus(400); + } + const { origin } = new URL(url); + if (!ALLOWED_ORIGINS.some(o => o === origin)) { + logRecord.forbiddenOrigin = true; + logger.error(`This origin is not allowed: ${origin}`, logRecord); + return res.sendStatus(400); + } + + const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH); + logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; + + const page = await browser.newPage(); + const pageLoadingStart = Date.now(); + const modifiedUrl = new URL(url); + modifiedUrl.searchParams.append('fontSize', 24); + modifiedUrl.searchParams.append('adjustAspectRatio', 1.91); + try { + await page.goto(modifiedUrl); + logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart }; + } catch (error) { + console.log('error going to page: ', modifiedUrl) + console.log(error) + throw error + } + + // We lookup the destination path from our own page content and avoid trusting any passed query params + // selector - CSS selector of the element to get screenshot of + const selector = decodeURIComponent( + await page.$eval( + "head > meta[name='omnivore:preview_image_selector']", + element => element.content, + ), + ); + if (!selector) { + logRecord.selectorIsInvalid = true; + logger.error(`Valid element selector is not specified`, logRecord); + await page.close(); + return res.sendStatus(400); + } + logRecord.selector = selector; + + // destination - destination pathname for the image to save with + const destination = decodeURIComponent( + await page.$eval( + "head > meta[name='omnivore:preview_image_destination']", + element => element.content, + ), + ); + if (!destination) { + logRecord.destinationIsInvalid = true; + logger.error(`Valid file destination is not specified`, logRecord); + await page.close(); + return res.sendStatus(400); + } + logRecord.destination = destination; + + const screenshotTakingStart = Date.now(); + try { + await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load + } catch (error) { + logRecord.elementNotFound = true; + logger.error(`Element is not presented on the page`, logRecord); + await page.close(); + return res.sendStatus(400); + } + const element = await page.$(selector); + await element.screenshot({ path: filePath }); // take screenshot of the element in puppeteer + logRecord.timing = { ...logRecord.timing, screenshotTaken: Date.now() - screenshotTakingStart }; + + await page.close(); + + try { + const [file] = await previewBucket.upload(filePath, { + destination, + metadata: logRecord, + }); + logRecord.file = file.metadata; + } catch (e) { + console.log('error uploading to bucket, this is non-fatal', e) + } + + logger.info(`preview-image`, logRecord); + return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); +}); diff --git a/packages/content-fetch-gcf/package.json b/packages/content-fetch-gcf/package.json new file mode 100644 index 000000000..efec3ca5e --- /dev/null +++ b/packages/content-fetch-gcf/package.json @@ -0,0 +1,25 @@ +{ + "name": "@omnivore/content-fetch-gcf", + "version": "1.0.0", + "description": "Google Cloud Function that accepts URL of the article and parses its content", + "main": "index.js", + "dependencies": { + "@google-cloud/logging-winston": "^4.1.2", + "@google-cloud/storage": "^5.18.1", + "@sentry/serverless": "^6.13.3", + "axios": "^0.26.0", + "dotenv": "^8.2.0", + "jsonwebtoken": "^8.5.1", + "linkedom": "^0.14.9", + "luxon": "^2.3.1", + "winston": "^3.3.3", + "@omnivore/puppeteer-parse": "^1.0.0" + }, + "devDependencies": { + "@google-cloud/functions-framework": "^3.0.0" + }, + "scripts": { + "start": "npx functions-framework --port=9090 --target=puppeteer", + "start_preview": "npx functions-framework --target=preview" + } +} From 623bb8780cd717286edac1a98c23bd714e49c9b7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 29 Jun 2022 11:18:36 +0800 Subject: [PATCH 03/11] Call puppeteer module from content-fetch --- packages/content-fetch/app.js | 12 ++++++------ packages/content-fetch/package.json | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/content-fetch/app.js b/packages/content-fetch/app.js index 60f3031c5..0c2cc10fb 100644 --- a/packages/content-fetch/app.js +++ b/packages/content-fetch/app.js @@ -1,7 +1,7 @@ const express = require('express'); const app = express(); -const fetchContent = require('./fetch-content'); +const { fetchContent } = require("@omnivore/puppeteer-parse"); app.use(express.json()); app.use(express.urlencoded({ extended: true })); @@ -10,22 +10,22 @@ if (!process.env.VERIFICATION_TOKEN) { throw new Error('VERIFICATION_TOKEN environment variable is not set'); } -app.get('/', (req, res) => { +app.get('/', async (req, res) => { if (req.query.token !== process.env.VERIFICATION_TOKEN) { console.log('query does not include valid token') res.send(403) return } - fetchContent(req, res) + await fetchContent(req, res) }); -app.post('/', (req, res) => { +app.post('/', async (req, res) => { if (req.query.token !== process.env.VERIFICATION_TOKEN) { console.log('query does not include valid token') res.send(403) return } - fetchContent(req, res) + await fetchContent(req, res) }); const PORT = parseInt(process.env.PORT) || 8080; @@ -34,4 +34,4 @@ app.listen(PORT, () => { console.log('Press Ctrl+C to quit.'); }); -module.exports = app; \ No newline at end of file +module.exports = app; diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json index 405f1a27c..da8862d24 100644 --- a/packages/content-fetch/package.json +++ b/packages/content-fetch/package.json @@ -12,10 +12,10 @@ "linkedom": "^0.14.9", "luxon": "^2.3.1", "puppeteer-core": "^16.1.0", - "underscore": "^1.13.4" + "underscore": "^1.13.4", + "@omnivore/puppeteer-parse": "^1.0.0" }, "scripts": { - "start": "node app.js", - "test": "yarn mocha" + "start": "node app.js" } } From 00fed8a0fbc4398c5f92bf99be6af5db9b32c5a2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 7 Oct 2022 18:42:01 +0800 Subject: [PATCH 04/11] Remove content-fetch-gcf and create a Dockerfile for the cloud function --- packages/content-fetch-gcf/.gitignore | 1 - packages/content-fetch-gcf/Dockerfile | 112 ------------------ packages/content-fetch-gcf/package.json | 25 ---- .../.env.example | 10 ++ .../.gcloudignore | 0 packages/content-fetch/Dockerfile | 3 + packages/content-fetch/Dockerfile-gcf | 52 ++++++++ .../README.md | 0 .../index.js | 0 packages/content-fetch/package.json | 14 ++- packages/puppeteer-parse/.dockerignore | 4 - packages/puppeteer-parse/index.js | 26 +--- packages/puppeteer-parse/package.json | 6 - .../puppeteer-parse/test/babel-register.js | 3 - packages/puppeteer-parse/test/stub.test.ts | 13 -- 15 files changed, 82 insertions(+), 187 deletions(-) delete mode 100644 packages/content-fetch-gcf/.gitignore delete mode 100644 packages/content-fetch-gcf/Dockerfile delete mode 100644 packages/content-fetch-gcf/package.json rename packages/{content-fetch-gcf => content-fetch}/.env.example (56%) rename packages/{content-fetch-gcf => content-fetch}/.gcloudignore (100%) create mode 100644 packages/content-fetch/Dockerfile-gcf rename packages/{content-fetch-gcf => content-fetch}/README.md (100%) rename packages/{content-fetch-gcf => content-fetch}/index.js (100%) delete mode 100644 packages/puppeteer-parse/.dockerignore delete mode 100644 packages/puppeteer-parse/test/babel-register.js delete mode 100644 packages/puppeteer-parse/test/stub.test.ts diff --git a/packages/content-fetch-gcf/.gitignore b/packages/content-fetch-gcf/.gitignore deleted file mode 100644 index 9bfbc5e8b..000000000 --- a/packages/content-fetch-gcf/.gitignore +++ /dev/null @@ -1 +0,0 @@ -previewImage.* \ No newline at end of file diff --git a/packages/content-fetch-gcf/Dockerfile b/packages/content-fetch-gcf/Dockerfile deleted file mode 100644 index c0df04111..000000000 --- a/packages/content-fetch-gcf/Dockerfile +++ /dev/null @@ -1,112 +0,0 @@ -# FROM node:14-slim - -# # Taken from pu - -# # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) -# # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer -# # installs, work. -# RUN apt-get update \ -# && apt-get install -y wget gnupg \ -# && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ -# && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ -# && apt-get update \ -# && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \ -# --no-install-recommends \ -# && rm -rf /var/lib/apt/lists/* - -# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true -# ENV CHROMIUM_PATH "/usr/bin/google-chrome-stable" - -# ------------------------ - -# FROM --platform=linux/arm64 node:14.18 - -# RUN apt-get update \ -# && apt-get install -y chromium \ -# && apt-get install -y ca-certificates \ -# fonts-liberation \ -# libappindicator3-1 \ -# libasound2 \ -# libatk-bridge2.0-0 \ -# libatk1.0-0 \ -# libc6 \ -# libcairo2 \ -# libcups2 \ -# libdbus-1-3 \ -# libexpat1 \ -# libfontconfig1 \ -# libgbm1 \ -# libgcc1 \ -# libglib2.0-0 \ -# libgtk-3-0 \ -# libnspr4 \ -# libnss3 \ -# libpango-1.0-0 \ -# libpangocairo-1.0-0 \ -# libstdc++6 \ -# libx11-6 \ -# libx11-xcb1 \ -# libxcb1 \ -# libxcomposite1 \ -# libxcursor1 \ -# libxdamage1 \ -# libxext6 \ -# libxfixes3 \ -# libxi6 \ -# libxrandr2 \ -# libxrender1 \ -# libxss1 \ -# libxtst6 \ -# lsb-release \ -# wget \ -# xdg-utils - -FROM node:14.18-alpine - -# Installs latest Chromium (92) package. -RUN apk add --no-cache \ - chromium \ - nss \ - freetype \ - harfbuzz \ - ca-certificates \ - ttf-freefont \ - nodejs \ - yarn - -# Tell Puppeteer to skip installing Chrome. We'll be using the installed package. -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ - PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser - -# Puppeteer v10.0.0 works with Chromium 92. -RUN yarn add puppeteer@10.0.0 - -# Add user so we don't need --no-sandbox. -RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ - && mkdir -p /home/pptruser/Downloads /app \ - && chown -R pptruser:pptruser /home/pptruser \ - && chown -R pptruser:pptruser /app - -# Run everything after as non-privileged user. -WORKDIR /app - -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true -ENV CHROMIUM_PATH /usr/bin/chromium-browser -ENV LAUNCH_HEADLESS=true - -COPY package.json . -COPY yarn.lock . -COPY tsconfig.json . -COPY .prettierrc . -COPY .eslintrc . - -COPY /packages/content-fetch-gcf/package.json ./packages/content-fetch-gcf/package.json - -RUN yarn install --pure-lockfile - -ADD /packages/content-fetch-gcf ./packages/content-fetch-gcf - -EXPOSE 8080 - -# USER pptruser -ENTRYPOINT ["yarn", "workspace", "@omnivore/content-fetch-gcf", "start"] diff --git a/packages/content-fetch-gcf/package.json b/packages/content-fetch-gcf/package.json deleted file mode 100644 index efec3ca5e..000000000 --- a/packages/content-fetch-gcf/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "@omnivore/content-fetch-gcf", - "version": "1.0.0", - "description": "Google Cloud Function that accepts URL of the article and parses its content", - "main": "index.js", - "dependencies": { - "@google-cloud/logging-winston": "^4.1.2", - "@google-cloud/storage": "^5.18.1", - "@sentry/serverless": "^6.13.3", - "axios": "^0.26.0", - "dotenv": "^8.2.0", - "jsonwebtoken": "^8.5.1", - "linkedom": "^0.14.9", - "luxon": "^2.3.1", - "winston": "^3.3.3", - "@omnivore/puppeteer-parse": "^1.0.0" - }, - "devDependencies": { - "@google-cloud/functions-framework": "^3.0.0" - }, - "scripts": { - "start": "npx functions-framework --port=9090 --target=puppeteer", - "start_preview": "npx functions-framework --target=preview" - } -} diff --git a/packages/content-fetch-gcf/.env.example b/packages/content-fetch/.env.example similarity index 56% rename from packages/content-fetch-gcf/.env.example rename to packages/content-fetch/.env.example index 64242a22d..4b7dd61c0 100644 --- a/packages/content-fetch-gcf/.env.example +++ b/packages/content-fetch/.env.example @@ -6,3 +6,13 @@ REST_BACKEND_ENDPOINT=http://localhost:4000/api # set for local development IS_LOCAL=true + +VERIFICATION_TOKEN='123456' + +CHROMIUM_PATH=/opt/homebrew/bin/chromium +PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true +LAUNCH_HEADLESS=true + +TWITTER_BEARER_TOKEN=token + +PORT=9090 diff --git a/packages/content-fetch-gcf/.gcloudignore b/packages/content-fetch/.gcloudignore similarity index 100% rename from packages/content-fetch-gcf/.gcloudignore rename to packages/content-fetch/.gcloudignore diff --git a/packages/content-fetch/Dockerfile b/packages/content-fetch/Dockerfile index 1201c497d..b9bf619cc 100644 --- a/packages/content-fetch/Dockerfile +++ b/packages/content-fetch/Dockerfile @@ -30,12 +30,15 @@ COPY .prettierrc . COPY .eslintrc . COPY /packages/content-handler/package.json ./packages/content-handler/package.json +COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json RUN yarn install --pure-lockfile ADD /packages/content-fetch ./packages/content-fetch ADD /packages/content-handler ./packages/content-handler +ADD /packages/puppeteer-parse ./packages/puppeteer-parse RUN yarn workspace @omnivore/content-handler build +RUN yarn workspace @omnivore/puppeteer-parse build # After building, fetch the production dependencies RUN rm -rf /app/packages/content-fetch/node_modules diff --git a/packages/content-fetch/Dockerfile-gcf b/packages/content-fetch/Dockerfile-gcf new file mode 100644 index 000000000..dec0fa93a --- /dev/null +++ b/packages/content-fetch/Dockerfile-gcf @@ -0,0 +1,52 @@ +FROM node:14.18-alpine + +# Installs latest Chromium (92) package. +RUN apk add --no-cache \ + chromium \ + nss \ + freetype \ + harfbuzz \ + ca-certificates \ + ttf-freefont \ + nodejs \ + yarn + +# Add user so we don't need --no-sandbox. +RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ + && mkdir -p /home/pptruser/Downloads /app \ + && chown -R pptruser:pptruser /home/pptruser \ + && chown -R pptruser:pptruser /app + +# Run everything after as non-privileged user. +WORKDIR /app + +ENV CHROMIUM_PATH /usr/bin/chromium-browser +ENV LAUNCH_HEADLESS=true +ENV PORT 9090 + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .prettierrc . +COPY .eslintrc . + +COPY /packages/content-handler/package.json ./packages/content-handler/package.json +COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/content-handler ./packages/content-handler +ADD /packages/puppeteer-parse ./packages/puppeteer-parse +ADD /packages/content-fetch ./packages/content-fetch +RUN yarn workspace @omnivore/content-handler build +RUN yarn workspace @omnivore/puppeteer-parse build + +# After building, fetch the production dependencies +RUN rm -rf /app/packages/content-fetch/node_modules +RUN rm -rf /app/node_modules +RUN yarn install --pure-lockfile --production + +EXPOSE 9090 + +# USER pptruser +ENTRYPOINT ["yarn", "workspace", "@omnivore/content-fetch", "start_gcf"] diff --git a/packages/content-fetch-gcf/README.md b/packages/content-fetch/README.md similarity index 100% rename from packages/content-fetch-gcf/README.md rename to packages/content-fetch/README.md diff --git a/packages/content-fetch-gcf/index.js b/packages/content-fetch/index.js similarity index 100% rename from packages/content-fetch-gcf/index.js rename to packages/content-fetch/index.js diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json index da8862d24..45de11581 100644 --- a/packages/content-fetch/package.json +++ b/packages/content-fetch/package.json @@ -4,7 +4,6 @@ "description": "Service that fetches page content from a URL", "main": "index.js", "dependencies": { - "@omnivore/content-handler": "1.0.0", "axios": "^0.27.2", "dotenv": "^8.2.0", "express": "^4.17.1", @@ -13,9 +12,18 @@ "luxon": "^2.3.1", "puppeteer-core": "^16.1.0", "underscore": "^1.13.4", - "@omnivore/puppeteer-parse": "^1.0.0" + "@omnivore/puppeteer-parse": "^1.0.0", + "@google-cloud/logging-winston": "^4.1.2", + "@google-cloud/storage": "^5.18.1", + "@sentry/serverless": "^6.13.3", + "winston": "^3.3.3" + }, + "devDependencies": { + "@google-cloud/functions-framework": "^3.0.0" }, "scripts": { - "start": "node app.js" + "start": "node app.js", + "start_gcf": "npx functions-framework --port=9090 --target=puppeteer", + "start_preview": "npx functions-framework --target=preview" } } diff --git a/packages/puppeteer-parse/.dockerignore b/packages/puppeteer-parse/.dockerignore deleted file mode 100644 index 2310bc768..000000000 --- a/packages/puppeteer-parse/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -node_modules -.env* -Dockerfile -.dockerignore diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index f2f4cf02f..9dbe583c8 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -5,7 +5,7 @@ /* eslint-disable @typescript-eslint/no-require-imports */ require('dotenv').config(); const Url = require('url'); -const puppeteer = require('puppeteer-extra'); +// const puppeteer = require('puppeteer-extra'); const axios = require('axios'); const jwt = require('jsonwebtoken'); const { promisify } = require('util'); @@ -14,10 +14,8 @@ const { config, format, loggers, transports } = require('winston'); const { LoggingWinston } = require('@google-cloud/logging-winston'); const { DateTime } = require('luxon'); const os = require('os'); -const Sentry = require('@sentry/serverless'); const { Storage } = require('@google-cloud/storage'); - -const chromium = require('chrome-aws-lambda'); +const { parseHTML } = require('linkedom'); const puppeteer = require('puppeteer-core'); const { preHandleContent } = require("@omnivore/content-handler"); @@ -29,11 +27,6 @@ const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.env.PREVIEW_IMAGE_BUCKET) : undefined; -Sentry.GCPFunction.init({ - dsn: process.env.SENTRY_DSN, - tracesSampleRate: 0, -}); - const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' @@ -44,12 +37,6 @@ const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; const path = require("path"); const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; -const { parseHTML } = require('linkedom'); - -// Add stealth plugin to hide puppeteer usage -const StealthPlugin = require('puppeteer-extra-plugin-stealth'); -puppeteer.use(StealthPlugin()); - const userAgentForUrl = (url) => { try { const u = new URL(url); @@ -94,8 +81,7 @@ const enableJavascriptForUrl = (url) => { }; // launch Puppeteer -const getBrowserPromise = (async (proxyUrl, chromiumPath) => { - console.log("starting with proxy url", proxyUrl) +const getBrowserPromise = (async () => { return puppeteer.launch({ args: [ '--allow-running-insecure-content', @@ -122,8 +108,8 @@ const getBrowserPromise = (async (proxyUrl, chromiumPath) => { '--window-size=1920,1080', ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, - executablePath: chromiumPath, - headless: true, + executablePath: process.env.CHROMIUM_PATH, + headless: !!process.env.HEADLESS, timeout: 120000, // 2 minutes }); })(); @@ -237,7 +223,7 @@ async function fetchContent(req, res) { }, }; - console.log(`Article parsing request`, logRecord); + logger.info(`Article parsing request`, logRecord); if (!url) { logRecord.urlIsInvalid = true; diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index e73914fa4..cc7755c0b 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -4,13 +4,10 @@ "description": "Accepts URL of the article and parses its content", "main": "index.js", "dependencies": { - "@google-cloud/functions-framework": "^3.1.2", "@google-cloud/logging-winston": "^5.1.1", "@google-cloud/storage": "^5.18.1", "@omnivore/content-handler": "1.0.0", - "@sentry/serverless": "^6.13.3", "axios": "^0.27.2", - "chrome-aws-lambda": "^10.1.0", "dotenv": "^8.2.0", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.9", @@ -18,8 +15,5 @@ "puppeteer-core": "^16.1.0", "underscore": "^1.13.4", "winston": "^3.3.3" - }, - "scripts": { - "test": "yarn mocha" } } diff --git a/packages/puppeteer-parse/test/babel-register.js b/packages/puppeteer-parse/test/babel-register.js deleted file mode 100644 index a6f65f60a..000000000 --- a/packages/puppeteer-parse/test/babel-register.js +++ /dev/null @@ -1,3 +0,0 @@ -const register = require('@babel/register').default - -register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/puppeteer-parse/test/stub.test.ts b/packages/puppeteer-parse/test/stub.test.ts deleted file mode 100644 index 173ca4917..000000000 --- a/packages/puppeteer-parse/test/stub.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import 'mocha' -import * as chai from 'chai' -import { expect } from 'chai' -import 'chai/register-should' -import chaiString from 'chai-string' - -chai.use(chaiString) - -describe('Stub test', () => { - it('should pass', () => { - expect(true).to.be.true - }) -}) From b18af10e75fb347ecfebd7941a6f27d8d27e6b0f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 10 Oct 2022 11:42:58 +0800 Subject: [PATCH 05/11] Import puppeteer-parse in content-fetch --- packages/content-fetch/Dockerfile | 1 - packages/content-fetch/Dockerfile-gcf | 1 - packages/content-fetch/app.js | 1 + packages/content-fetch/index.js | 192 +---------------- packages/content-fetch/package.json | 19 +- packages/content-fetch/test/babel-register.js | 3 - packages/content-fetch/test/stub.test.js | 9 + packages/content-fetch/test/stub.test.ts | 13 -- packages/puppeteer-parse/index.js | 196 +++++++++++++++++- packages/puppeteer-parse/package.json | 1 - yarn.lock | 18 +- 11 files changed, 213 insertions(+), 241 deletions(-) delete mode 100644 packages/content-fetch/test/babel-register.js create mode 100644 packages/content-fetch/test/stub.test.js delete mode 100644 packages/content-fetch/test/stub.test.ts diff --git a/packages/content-fetch/Dockerfile b/packages/content-fetch/Dockerfile index b9bf619cc..9ca9f7dee 100644 --- a/packages/content-fetch/Dockerfile +++ b/packages/content-fetch/Dockerfile @@ -38,7 +38,6 @@ ADD /packages/content-fetch ./packages/content-fetch ADD /packages/content-handler ./packages/content-handler ADD /packages/puppeteer-parse ./packages/puppeteer-parse RUN yarn workspace @omnivore/content-handler build -RUN yarn workspace @omnivore/puppeteer-parse build # After building, fetch the production dependencies RUN rm -rf /app/packages/content-fetch/node_modules diff --git a/packages/content-fetch/Dockerfile-gcf b/packages/content-fetch/Dockerfile-gcf index dec0fa93a..8c355485f 100644 --- a/packages/content-fetch/Dockerfile-gcf +++ b/packages/content-fetch/Dockerfile-gcf @@ -39,7 +39,6 @@ ADD /packages/content-handler ./packages/content-handler ADD /packages/puppeteer-parse ./packages/puppeteer-parse ADD /packages/content-fetch ./packages/content-fetch RUN yarn workspace @omnivore/content-handler build -RUN yarn workspace @omnivore/puppeteer-parse build # After building, fetch the production dependencies RUN rm -rf /app/packages/content-fetch/node_modules diff --git a/packages/content-fetch/app.js b/packages/content-fetch/app.js index 0c2cc10fb..252671626 100644 --- a/packages/content-fetch/app.js +++ b/packages/content-fetch/app.js @@ -1,3 +1,4 @@ +require('dotenv').config(); const express = require('express'); const app = express(); diff --git a/packages/content-fetch/index.js b/packages/content-fetch/index.js index 2c18d99f8..a209413e3 100644 --- a/packages/content-fetch/index.js +++ b/packages/content-fetch/index.js @@ -4,84 +4,14 @@ /* eslint-disable @typescript-eslint/no-var-requires */ /* eslint-disable @typescript-eslint/no-require-imports */ require('dotenv').config(); -const { config, format, loggers, transports } = require('winston'); -const { LoggingWinston } = require('@google-cloud/logging-winston'); -const { DateTime } = require('luxon'); -const os = require('os'); const Sentry = require('@sentry/serverless'); -const { Storage } = require('@google-cloud/storage'); -const { fetchContent, getBrowserPromise, getUrl } = require("@omnivore/puppeteer-parse"); - -const storage = new Storage(); -const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; -const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.env.PREVIEW_IMAGE_BUCKET) : undefined; +const { fetchContent, preview } = require("@omnivore/puppeteer-parse"); Sentry.GCPFunction.init({ dsn: process.env.SENTRY_DSN, tracesSampleRate: 0, }); -const filePath = `${os.tmpdir()}/previewImage.png`; - -const colors = { - emerg: 'inverse underline magenta', - alert: 'underline magenta', - crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. - error: 'underline red', // Any error which is fatal to the operation, but not the service or application - warning: 'underline yellow', // Anything that can potentially cause application oddities - notice: 'underline cyan', // Normal but significant condition - info: 'underline green', // Generally useful information to log - debug: 'underline gray', -}; - -const googleConfigs = { - level: 'info', - logName: 'logger', - levels: config.syslog.levels, - resource: { - labels: { - function_name: process.env.FUNCTION_TARGET, - project_id: process.env.GCP_PROJECT, - }, - type: 'cloud_function', - }, -}; - -function localConfig(id) { - return { - level: 'debug', - format: format.combine( - format.colorize({ all: true, colors }), - format(info => - Object.assign(info, { - timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), - }), - )(), - format.printf(info => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { timestamp, message, level, ...meta } = info; - - return `[${id}@${info.timestamp}] ${info.message}${ - Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' - }`; - }), - ), - }; -} - -function buildLoggerTransport(id, options) { - return process.env.IS_LOCAL - ? new transports.Console(localConfig(id)) - : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); -} - -function buildLogger(id, options) { - return loggers.get(id, { - levels: config.syslog.levels, - transports: [buildLoggerTransport(id, options)], - }); -} - /** * Cloud Function entry point, HTTP trigger. * Loads the requested URL via Puppeteer, captures page content and sends it to backend @@ -100,122 +30,4 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(fetchContent); * * url - URL address of the page to open * @param {Object} res Cloud Function response context. */ -exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { - const functionStartTime = Date.now(); - // Grabbing execution and trace ids to attach logs to the appropriate function call - const execution_id = req.get('function-execution-id'); - const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; - const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { - trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, - labels: { - execution_id: execution_id, - }, - }); - - if (!process.env.PREVIEW_IMAGE_BUCKET) { - logger.error(`PREVIEW_IMAGE_BUCKET not set`) - return res.sendStatus(500); - } - - const url = getUrl(req); - console.log('preview request url', url); - - const logRecord = { - url, - query: req.query, - origin: req.get('Origin'), - labels: { - source: 'publicImagePreview', - }, - }; - - logger.info(`Public preview image generation request`, logRecord); - - if (!url) { - logRecord.urlIsInvalid = true; - logger.error(`Valid URL to parse is not specified`, logRecord); - return res.sendStatus(400); - } - const { origin } = new URL(url); - if (!ALLOWED_ORIGINS.some(o => o === origin)) { - logRecord.forbiddenOrigin = true; - logger.error(`This origin is not allowed: ${origin}`, logRecord); - return res.sendStatus(400); - } - - const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH); - logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; - - const page = await browser.newPage(); - const pageLoadingStart = Date.now(); - const modifiedUrl = new URL(url); - modifiedUrl.searchParams.append('fontSize', 24); - modifiedUrl.searchParams.append('adjustAspectRatio', 1.91); - try { - await page.goto(modifiedUrl); - logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart }; - } catch (error) { - console.log('error going to page: ', modifiedUrl) - console.log(error) - throw error - } - - // We lookup the destination path from our own page content and avoid trusting any passed query params - // selector - CSS selector of the element to get screenshot of - const selector = decodeURIComponent( - await page.$eval( - "head > meta[name='omnivore:preview_image_selector']", - element => element.content, - ), - ); - if (!selector) { - logRecord.selectorIsInvalid = true; - logger.error(`Valid element selector is not specified`, logRecord); - await page.close(); - return res.sendStatus(400); - } - logRecord.selector = selector; - - // destination - destination pathname for the image to save with - const destination = decodeURIComponent( - await page.$eval( - "head > meta[name='omnivore:preview_image_destination']", - element => element.content, - ), - ); - if (!destination) { - logRecord.destinationIsInvalid = true; - logger.error(`Valid file destination is not specified`, logRecord); - await page.close(); - return res.sendStatus(400); - } - logRecord.destination = destination; - - const screenshotTakingStart = Date.now(); - try { - await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load - } catch (error) { - logRecord.elementNotFound = true; - logger.error(`Element is not presented on the page`, logRecord); - await page.close(); - return res.sendStatus(400); - } - const element = await page.$(selector); - await element.screenshot({ path: filePath }); // take screenshot of the element in puppeteer - logRecord.timing = { ...logRecord.timing, screenshotTaken: Date.now() - screenshotTakingStart }; - - await page.close(); - - try { - const [file] = await previewBucket.upload(filePath, { - destination, - metadata: logRecord, - }); - logRecord.file = file.metadata; - } catch (e) { - console.log('error uploading to bucket, this is non-fatal', e) - } - - logger.info(`preview-image`, logRecord); - return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); -}); +exports.preview = Sentry.GCPFunction.wrapHttpFunction(preview); diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json index 45de11581..6facc37a9 100644 --- a/packages/content-fetch/package.json +++ b/packages/content-fetch/package.json @@ -4,26 +4,21 @@ "description": "Service that fetches page content from a URL", "main": "index.js", "dependencies": { - "axios": "^0.27.2", "dotenv": "^8.2.0", "express": "^4.17.1", - "jsonwebtoken": "^8.5.1", - "linkedom": "^0.14.9", - "luxon": "^2.3.1", - "puppeteer-core": "^16.1.0", - "underscore": "^1.13.4", "@omnivore/puppeteer-parse": "^1.0.0", - "@google-cloud/logging-winston": "^4.1.2", - "@google-cloud/storage": "^5.18.1", - "@sentry/serverless": "^6.13.3", - "winston": "^3.3.3" + "@sentry/serverless": "^6.13.3" }, "devDependencies": { - "@google-cloud/functions-framework": "^3.0.0" + "@google-cloud/functions-framework": "^3.0.0", + "chai": "^4.3.6", + "chai-string": "^1.5.0", + "mocha": "^10.0.0" }, "scripts": { "start": "node app.js", "start_gcf": "npx functions-framework --port=9090 --target=puppeteer", - "start_preview": "npx functions-framework --target=preview" + "start_preview": "npx functions-framework --target=preview", + "test": "mocha test/*.js" } } diff --git a/packages/content-fetch/test/babel-register.js b/packages/content-fetch/test/babel-register.js deleted file mode 100644 index a6f65f60a..000000000 --- a/packages/content-fetch/test/babel-register.js +++ /dev/null @@ -1,3 +0,0 @@ -const register = require('@babel/register').default - -register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/content-fetch/test/stub.test.js b/packages/content-fetch/test/stub.test.js new file mode 100644 index 000000000..317d21b52 --- /dev/null +++ b/packages/content-fetch/test/stub.test.js @@ -0,0 +1,9 @@ +const chai = require("chai"); + +const expect = chai.expect; + +describe('Stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) diff --git a/packages/content-fetch/test/stub.test.ts b/packages/content-fetch/test/stub.test.ts deleted file mode 100644 index 173ca4917..000000000 --- a/packages/content-fetch/test/stub.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import 'mocha' -import * as chai from 'chai' -import { expect } from 'chai' -import 'chai/register-should' -import chaiString from 'chai-string' - -chai.use(chaiString) - -describe('Stub test', () => { - it('should pass', () => { - expect(true).to.be.true - }) -}) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 9dbe583c8..0624eb8dd 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -3,7 +3,6 @@ /* eslint-disable @typescript-eslint/explicit-function-return-type */ /* eslint-disable @typescript-eslint/no-var-requires */ /* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); const Url = require('url'); // const puppeteer = require('puppeteer-extra'); const axios = require('axios'); @@ -27,6 +26,67 @@ const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.env.PREVIEW_IMAGE_BUCKET) : undefined; +const filePath = `${os.tmpdir()}/previewImage.png`; + +const colors = { + emerg: 'inverse underline magenta', + alert: 'underline magenta', + crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. + error: 'underline red', // Any error which is fatal to the operation, but not the service or application + warning: 'underline yellow', // Anything that can potentially cause application oddities + notice: 'underline cyan', // Normal but significant condition + info: 'underline green', // Generally useful information to log + debug: 'underline gray', +}; + +const googleConfigs = { + level: 'info', + logName: 'logger', + levels: config.syslog.levels, + resource: { + labels: { + function_name: process.env.FUNCTION_TARGET, + project_id: process.env.GCP_PROJECT, + }, + type: 'cloud_function', + }, +}; + +function localConfig(id) { + return { + level: 'debug', + format: format.combine( + format.colorize({ all: true, colors }), + format(info => + Object.assign(info, { + timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), + }), + )(), + format.printf(info => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { timestamp, message, level, ...meta } = info; + + return `[${id}@${info.timestamp}] ${info.message}${ + Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' + }`; + }), + ), + }; +} + +function buildLoggerTransport(id, options) { + return process.env.IS_LOCAL + ? new transports.Console(localConfig(id)) + : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); +} + +function buildLogger(id, options) { + return loggers.get(id, { + levels: config.syslog.levels, + transports: [buildLoggerTransport(id, options)], + }); +} + const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' @@ -208,7 +268,16 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId }; async function fetchContent(req, res) { - functionStartTime = Date.now(); + const functionStartTime = Date.now(); + // Grabbing execution and trace ids to attach logs to the appropriate function call + const execution_id = req.get('function-execution-id'); + const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; + const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { + trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, + labels: { + execution_id: execution_id, + }, + }); let url = getUrl(req); const userId = req.body.userId || req.query.userId; @@ -616,9 +685,128 @@ async function retrieveHtml(page) { return { domContent, title }; } +async function preview(req, res) { + const functionStartTime = Date.now(); + // Grabbing execution and trace ids to attach logs to the appropriate function call + const execution_id = req.get('function-execution-id'); + const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; + const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { + trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, + labels: { + execution_id: execution_id, + }, + }); + + if (!process.env.PREVIEW_IMAGE_BUCKET) { + logger.error(`PREVIEW_IMAGE_BUCKET not set`) + return res.sendStatus(500); + } + + const url = getUrl(req); + console.log('preview request url', url); + + const logRecord = { + url, + query: req.query, + origin: req.get('Origin'), + labels: { + source: 'publicImagePreview', + }, + }; + + logger.info(`Public preview image generation request`, logRecord); + + if (!url) { + logRecord.urlIsInvalid = true; + logger.error(`Valid URL to parse is not specified`, logRecord); + return res.sendStatus(400); + } + const { origin } = new URL(url); + if (!ALLOWED_ORIGINS.some(o => o === origin)) { + logRecord.forbiddenOrigin = true; + logger.error(`This origin is not allowed: ${origin}`, logRecord); + return res.sendStatus(400); + } + + const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH); + logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; + + const page = await browser.newPage(); + const pageLoadingStart = Date.now(); + const modifiedUrl = new URL(url); + modifiedUrl.searchParams.append('fontSize', 24); + modifiedUrl.searchParams.append('adjustAspectRatio', 1.91); + try { + await page.goto(modifiedUrl); + logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart }; + } catch (error) { + console.log('error going to page: ', modifiedUrl) + console.log(error) + throw error + } + + // We lookup the destination path from our own page content and avoid trusting any passed query params + // selector - CSS selector of the element to get screenshot of + const selector = decodeURIComponent( + await page.$eval( + "head > meta[name='omnivore:preview_image_selector']", + element => element.content, + ), + ); + if (!selector) { + logRecord.selectorIsInvalid = true; + logger.error(`Valid element selector is not specified`, logRecord); + await page.close(); + return res.sendStatus(400); + } + logRecord.selector = selector; + + // destination - destination pathname for the image to save with + const destination = decodeURIComponent( + await page.$eval( + "head > meta[name='omnivore:preview_image_destination']", + element => element.content, + ), + ); + if (!destination) { + logRecord.destinationIsInvalid = true; + logger.error(`Valid file destination is not specified`, logRecord); + await page.close(); + return res.sendStatus(400); + } + logRecord.destination = destination; + + const screenshotTakingStart = Date.now(); + try { + await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load + } catch (error) { + logRecord.elementNotFound = true; + logger.error(`Element is not presented on the page`, logRecord); + await page.close(); + return res.sendStatus(400); + } + const element = await page.$(selector); + await element.screenshot({ path: filePath }); // take screenshot of the element in puppeteer + logRecord.timing = { ...logRecord.timing, screenshotTaken: Date.now() - screenshotTakingStart }; + + await page.close(); + + try { + const [file] = await previewBucket.upload(filePath, { + destination, + metadata: logRecord, + }); + logRecord.file = file.metadata; + } catch (e) { + console.log('error uploading to bucket, this is non-fatal', e) + } + + logger.info(`preview-image`, logRecord); + return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); +} + module.exports = { fetchContent, - getBrowserPromise, - getUrl, + preview, }; diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index cc7755c0b..172eb4d99 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -8,7 +8,6 @@ "@google-cloud/storage": "^5.18.1", "@omnivore/content-handler": "1.0.0", "axios": "^0.27.2", - "dotenv": "^8.2.0", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.9", "luxon": "^2.3.1", diff --git a/yarn.lock b/yarn.lock index b1b9a214d..3f3923d58 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2444,7 +2444,7 @@ google-gax "^2.24.1" protobufjs "^6.8.6" -"@google-cloud/functions-framework@3.1.2", "@google-cloud/functions-framework@^3.1.2": +"@google-cloud/functions-framework@3.1.2", "@google-cloud/functions-framework@^3.0.0": version "3.1.2" resolved "https://registry.yarnpkg.com/@google-cloud/functions-framework/-/functions-framework-3.1.2.tgz#2cd92ce4307bf7f32555d028dca22e398473b410" integrity sha512-pYvEH65/Rqh1JNPdcBmorcV7Xoom2/iOSmbtYza8msro7Inl+qOYxbyMiQfySD2gwAyn38WyWPRqsDRcf/BFLg== @@ -10907,13 +10907,6 @@ chownr@^2.0.0: resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ== -chrome-aws-lambda@^10.1.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/chrome-aws-lambda/-/chrome-aws-lambda-10.1.0.tgz#ac43b4cdfc1fbb2275c62effada560858099501e" - integrity sha512-NZQVf+J4kqG4sVhRm3WNmOfzY0OtTSm+S8rg77pwePa9RCYHzhnzRs8YvNI6L9tALIW6RpmefWiPURt3vURXcw== - dependencies: - lambdafs "^2.0.3" - chrome-trace-event@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz#1015eced4741e15d06664a957dbbf50d041e26ac" @@ -17677,13 +17670,6 @@ kuler@^2.0.0: resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A== -lambdafs@^2.0.3: - version "2.1.1" - resolved "https://registry.yarnpkg.com/lambdafs/-/lambdafs-2.1.1.tgz#4bf8d3037b6c61bbb4a22ab05c73ee47964c25ed" - integrity sha512-x5k8JcoJWkWLvCVBzrl4pzvkEHSgSBqFjg3Dpsc4AcTMq7oUMym4cL/gRTZ6VM4mUMY+M0dIbQ+V1c1tsqqanQ== - dependencies: - tar-fs "^2.1.1" - language-subtag-registry@~0.3.2: version "0.3.21" resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.21.tgz#04ac218bea46f04cb039084602c6da9e788dd45a" @@ -23988,7 +23974,7 @@ tar-fs@2.0.0: pump "^3.0.0" tar-stream "^2.0.0" -tar-fs@2.1.1, tar-fs@^2.1.1: +tar-fs@2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== From d6e687d5d15295d69feeaf9653e8897132efc82c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 10 Oct 2022 11:50:04 +0800 Subject: [PATCH 06/11] Update env example --- packages/content-fetch/.env.example | 2 +- packages/content-fetch/package.json | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/content-fetch/.env.example b/packages/content-fetch/.env.example index 4b7dd61c0..1537ad8ab 100644 --- a/packages/content-fetch/.env.example +++ b/packages/content-fetch/.env.example @@ -7,7 +7,7 @@ REST_BACKEND_ENDPOINT=http://localhost:4000/api # set for local development IS_LOCAL=true -VERIFICATION_TOKEN='123456' +VERIFICATION_TOKEN=some_token CHROMIUM_PATH=/opt/homebrew/bin/chromium PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json index 6facc37a9..f2ae26203 100644 --- a/packages/content-fetch/package.json +++ b/packages/content-fetch/package.json @@ -12,7 +12,6 @@ "devDependencies": { "@google-cloud/functions-framework": "^3.0.0", "chai": "^4.3.6", - "chai-string": "^1.5.0", "mocha": "^10.0.0" }, "scripts": { From b09fb4c037804339954d783de1cb9abd26507dd2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 10 Oct 2022 11:58:35 +0800 Subject: [PATCH 07/11] Fix docker build --- packages/puppeteer-parse/index.js | 61 +++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 0624eb8dd..56ae25852 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -94,7 +94,6 @@ const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_ const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; -const path = require("path"); const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; const userAgentForUrl = (url) => { @@ -142,6 +141,7 @@ const enableJavascriptForUrl = (url) => { // launch Puppeteer const getBrowserPromise = (async () => { + console.log("starting with proxy url", process.env.PROXY_URL) return puppeteer.launch({ args: [ '--allow-running-insecure-content', @@ -169,7 +169,7 @@ const getBrowserPromise = (async () => { ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.HEADLESS, + headless: !!process.env.LAUNCH_HEADLESS, timeout: 120000, // 2 minutes }); })(); @@ -268,7 +268,7 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId }; async function fetchContent(req, res) { - const functionStartTime = Date.now(); + functionStartTime = Date.now(); // Grabbing execution and trace ids to attach logs to the appropriate function call const execution_id = req.get('function-execution-id'); const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; @@ -280,7 +280,7 @@ async function fetchContent(req, res) { }); let url = getUrl(req); - const userId = req.body.userId || req.query.userId; + const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); logRecord = { @@ -292,11 +292,11 @@ async function fetchContent(req, res) { }, }; - logger.info(`Article parsing request`, logRecord); + logger.log(`Article parsing request`, logRecord); if (!url) { logRecord.urlIsInvalid = true; - logger.error(`Valid URL to parse not specified`, logRecord); + logger.log(`Valid URL to parse not specified`, logRecord); return res.sendStatus(400); } @@ -312,20 +312,20 @@ async function fetchContent(req, res) { if (result && result.content) { content = result.content } if (result && result.contentType) { contentType = result.contentType } } catch (e) { - console.log('error with handler: ', e); + logger.log('error with handler: ', e); } let context, page, finalUrl; try { - if ((!content || !title) && contentType !== 'application/pdf') { - const result = await retrievePage(url) - if (result && result.context) { context = result.context } - if (result && result.page) { page = result.page } - if (result && result.finalUrl) { finalUrl = result.finalUrl } - if (result && result.contentType) { contentType = result.contentType } - } else { - finalUrl = url - } + if ((!content || !title) && contentType !== 'application/pdf') { + const result = await retrievePage(url) + if (result && result.context) { context = result.context } + if (result && result.page) { page = result.page } + if (result && result.finalUrl) { finalUrl = result.finalUrl } + if (result && result.contentType) { contentType = result.contentType } + } else { + finalUrl = url + } if (contentType === 'application/pdf') { const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId); @@ -369,7 +369,29 @@ async function fetchContent(req, res) { console.log('error', e) logRecord.error = e.message; console.log(`Error while retrieving page`, logRecord); - return res.sendStatus(503); + + // fallback to scrapingbee + const sbResult = await fetchContentWithScrapingBee(url); + const sbUrl = finalUrl || sbResult.url; + const content = sbResult.domContent; + logRecord.fetchContentTime = Date.now() - functionStartTime; + + const apiResponse = await sendCreateArticleMutation(userId, { + url: sbUrl, + articleSavingRequestId, + preparedDocument: { + document: content, + pageInfo: { + title: sbResult.title, + canonicalUrl: sbUrl, + }, + }, + skipParsing: !content, + }); + + logRecord.totalTime = Date.now() - functionStartTime; + logRecord.result = apiResponse.createArticle; + logger.log(`parse-page`, logRecord); } finally { if (context) { await context.close(); @@ -460,6 +482,7 @@ async function retrievePage(url) { ], }); + const path = require('path'); const download_path = path.resolve('./download_dir/'); await client.send('Page.setDownloadBehavior', { @@ -468,7 +491,7 @@ async function retrievePage(url) { downloadPath: download_path, }) - client.on('Network.requestIntercepted', async (e) => { + client.on('Network.requestIntercepted', async e => { const headers = e.responseHeaders || {}; const [contentType] = (headers['content-type'] || headers['Content-Type'] || '') @@ -581,7 +604,7 @@ async function retrieveHtml(page) { } })(); }), - await page.waitForTimeout(1000), // 1 second timeout + await page.waitForTimeout(1000), ]); logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; From cffa5bb9eaf0c5659ca0c26438ecba403ba2e6fa Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 10 Oct 2022 12:04:49 +0800 Subject: [PATCH 08/11] Build content-fetch-gcf docker image in github action --- .github/workflows/run-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 3b226c0b0..6da610fee 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -90,5 +90,5 @@ jobs: run: 'docker build --file packages/content-fetch/Dockerfile .' - name: Build the inbound-email-handler docker image run: 'docker build --file packages/inbound-email-handler/Dockerfile .' - - name: Build the puppeteer-parse docker image - run: 'docker build --file packages/puppeteer-parse/Dockerfile .' + - name: Build the content-fetch cloud function docker image + run: 'docker build --file packages/content-fetch/Dockerfile-gcf .' From 177c69499b974445c4584b313c11f46a9fd5e767 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 10 Oct 2022 12:09:17 +0800 Subject: [PATCH 09/11] Replace logger.log with logger.info --- packages/puppeteer-parse/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 56ae25852..ff84053a9 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -292,11 +292,11 @@ async function fetchContent(req, res) { }, }; - logger.log(`Article parsing request`, logRecord); + logger.info(`Article parsing request`, logRecord); if (!url) { logRecord.urlIsInvalid = true; - logger.log(`Valid URL to parse not specified`, logRecord); + logger.info(`Valid URL to parse not specified`, logRecord); return res.sendStatus(400); } @@ -312,7 +312,7 @@ async function fetchContent(req, res) { if (result && result.content) { content = result.content } if (result && result.contentType) { contentType = result.contentType } } catch (e) { - logger.log('error with handler: ', e); + logger.info('error with handler: ', e); } let context, page, finalUrl; @@ -391,7 +391,7 @@ async function fetchContent(req, res) { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; - logger.log(`parse-page`, logRecord); + logger.info(`parse-page`, logRecord); } finally { if (context) { await context.close(); From c8575a203bfb5fb9598c19c740cdf30abff539cf Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 10 Oct 2022 14:59:49 +0800 Subject: [PATCH 10/11] Fix tests --- packages/import-handler/test/csv/csv.test.ts | 1 - packages/puppeteer-parse/package.json | 7 +++++++ packages/puppeteer-parse/test/stub.test.js | 9 +++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 packages/puppeteer-parse/test/stub.test.js diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index 0265f87d0..4ea3ef705 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -1,7 +1,6 @@ import 'mocha' import * as chai from 'chai' import { expect } from 'chai' -import 'chai/register-should' import chaiString from 'chai-string' import * as fs from 'fs' import { importCsv } from '../../src/csv' diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 172eb4d99..a9537f6f2 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -14,5 +14,12 @@ "puppeteer-core": "^16.1.0", "underscore": "^1.13.4", "winston": "^3.3.3" + }, + "devDependencies": { + "chai": "^4.3.6", + "mocha": "^10.0.0" + }, + "scripts": { + "test": "mocha test/*.js" } } diff --git a/packages/puppeteer-parse/test/stub.test.js b/packages/puppeteer-parse/test/stub.test.js new file mode 100644 index 000000000..317d21b52 --- /dev/null +++ b/packages/puppeteer-parse/test/stub.test.js @@ -0,0 +1,9 @@ +const chai = require("chai"); + +const expect = chai.expect; + +describe('Stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) From f39e65c159729a183aaa5576aad052b8ad5f4304 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 27 Oct 2022 18:01:17 +0800 Subject: [PATCH 11/11] Fix rebase error --- packages/puppeteer-parse/index.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index ff84053a9..cd72936af 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -342,8 +342,7 @@ async function fetchContent(req, res) { content = result.domContent; } } else { - console.log('using prefetched content and title'); - console.log(content); + logger.info('using prefetched content and title'); } logRecord.fetchContentTime = Date.now() - functionStartTime; @@ -363,12 +362,10 @@ async function fetchContent(req, res) { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; - console.log(`parse-page`, logRecord); } } catch (e) { - console.log('error', e) logRecord.error = e.message; - console.log(`Error while retrieving page`, logRecord); + logger.error(`Error while retrieving page`, logRecord); // fallback to scrapingbee const sbResult = await fetchContentWithScrapingBee(url); @@ -391,11 +388,11 @@ async function fetchContent(req, res) { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; - logger.info(`parse-page`, logRecord); } finally { if (context) { await context.close(); } + logger.info(`parse-page`, logRecord); } return res.sendStatus(200);