From cb858484c69cda861a6c29900661c077c783e087 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 29 Jun 2022 11:17:43 +0800 Subject: [PATCH] Make puppeteer-parse a module --- packages/content-fetch/fetch-content.js | 635 ------------------------ packages/puppeteer-parse/.env.example | 8 - packages/puppeteer-parse/.gcloudignore | 21 - packages/puppeteer-parse/.gitignore | 1 - packages/puppeteer-parse/Dockerfile | 113 ----- packages/puppeteer-parse/README.md | 23 +- packages/puppeteer-parse/index.js | 393 +++++---------- packages/puppeteer-parse/package.json | 4 +- 8 files changed, 125 insertions(+), 1073 deletions(-) delete mode 100644 packages/content-fetch/fetch-content.js delete mode 100644 packages/puppeteer-parse/.env.example delete mode 100644 packages/puppeteer-parse/.gcloudignore delete mode 100644 packages/puppeteer-parse/.gitignore delete mode 100644 packages/puppeteer-parse/Dockerfile diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js deleted file mode 100644 index 7a35e1eeb..000000000 --- a/packages/content-fetch/fetch-content.js +++ /dev/null @@ -1,635 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const Url = require('url'); -const puppeteer = require('puppeteer-core'); -const axios = require('axios'); -const jwt = require('jsonwebtoken'); -const { promisify } = require('util'); -const { parseHTML } = require('linkedom'); -const { preHandleContent } = require('@omnivore/content-handler'); - -const signToken = promisify(jwt.sign); - -const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' -const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] -const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; - -const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; - -// Add stealth plugin to hide puppeteer usage -// const StealthPlugin = require('puppeteer-extra-plugin-stealth'); -// puppeteer.use(StealthPlugin()); - - -const userAgentForUrl = (url) => { - try { - const u = new URL(url); - for (const host of NON_BOT_HOSTS) { - if (u.hostname.endsWith(host)) { - return NON_BOT_DESKTOP_USER_AGENT; - } - } - } catch (e) { - console.log('error getting user agent for url', url, e) - } - return DESKTOP_USER_AGENT -}; - -const fetchContentWithScrapingBee = async (url) => { - const response = await axios.get('https://app.scrapingbee.com/api/v1', { - params: { - 'api_key': process.env.SCRAPINGBEE_API_KEY, - 'url': url, - 'render_js': 'false', - 'premium_proxy': 'true', - 'country_code':'us' - } - }) - - const dom = parseHTML(response.data).document; - return { title: dom.title, domContent: dom.documentElement.outerHTML, url: url } -} - -const enableJavascriptForUrl = (url) => { - try { - const u = new URL(url); - for (const host of NON_SCRIPT_HOSTS) { - if (u.hostname.endsWith(host)) { - return false; - } - } - } catch (e) { - console.log('error getting hostname for url', url, e) - } - return true -}; - -// launch Puppeteer -const getBrowserPromise = (async () => { - console.log("starting with proxy url", process.env.PROXY_URL) - return puppeteer.launch({ - args: [ - '--allow-running-insecure-content', - '--autoplay-policy=user-gesture-required', - '--disable-component-update', - '--disable-domain-reliability', - '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process', - '--disable-print-preview', - '--disable-setuid-sandbox', - '--disable-site-isolation-trials', - '--disable-speech-api', - '--disable-web-security', - '--disk-cache-size=33554432', - '--enable-features=SharedArrayBuffer', - '--hide-scrollbars', - '--ignore-gpu-blocklist', - '--in-process-gpu', - '--mute-audio', - '--no-default-browser-check', - '--no-pings', - '--no-sandbox', - '--no-zygote', - '--use-gl=swiftshader', - '--window-size=1920,1080', - ].filter((item) => !!item), - defaultViewport: { height: 1080, width: 1920 }, - executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, - timeout: 120000, // 2 minutes - }); -})(); - -let logRecord, functionStartTime; - -const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => { - const stream = await axios.get(contentObjUrl, { responseType: 'stream' }); - return await axios.put(uploadSignedUrl, stream.data, { - headers: { - 'Content-Type': contentType, - }, - maxBodyLength: 1000000000, - maxContentLength: 100000000, - }) -}; - -const getUploadIdAndSignedUrl = async (userId, url, articleSavingRequestId) => { - const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); - const data = JSON.stringify({ - query: `mutation UploadFileRequest($input: UploadFileRequestInput!) { - uploadFileRequest(input:$input) { - ... on UploadFileRequestError { - errorCodes - } - ... on UploadFileRequestSuccess { - id - uploadSignedUrl - } - } - }`, - variables: { - input: { - url, - contentType: 'application/pdf', - clientRequestId: articleSavingRequestId, - } - } - }); - - const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); - return response.data.data.uploadFileRequest; -}; - -const uploadPdf = async (url, userId, articleSavingRequestId) => { - validateUrlString(url); - - const uploadResult = await getUploadIdAndSignedUrl(userId, url, articleSavingRequestId); - await uploadToSignedUrl(uploadResult, 'application/pdf', url); - return uploadResult.id; -}; - -const sendCreateArticleMutation = async (userId, input) => { - const data = JSON.stringify({ - query: `mutation CreateArticle ($input: CreateArticleInput!){ - createArticle(input:$input){ - ... on CreateArticleSuccess{ - createdArticle{ - id - } - } - ... on CreateArticleError{ - errorCodes - } - } - }`, - variables: { - input: Object.assign({}, input , { source: 'puppeteer-parse' }), - }, - }); - - const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); - const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); - return response.data.data.createArticle; -}; - -const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId) => { - return sendCreateArticleMutation(userId, { - url: encodeURI(url), - articleSavingRequestId, - uploadFileId: uploadFileId, - }, - ); -}; - -async function fetchContent(req, res) { - functionStartTime = Date.now(); - - let url = getUrl(req); - const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); - const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); - - console.log('user id', userId, 'url', url) - - logRecord = { - url, - userId, - articleSavingRequestId, - labels: { - source: 'parseContent', - }, - }; - - console.log(`Article parsing request`, logRecord); - - if (!url) { - logRecord.urlIsInvalid = true; - console.log(`Valid URL to parse not specified`, logRecord); - return res.sendStatus(400); - } - - // pre handle url with custom handlers - let title, content, contentType; - try { - const result = await preHandleContent(url); - if (result && result.url) { - url = result.url - validateUrlString(url); - } - if (result && result.title) { title = result.title } - if (result && result.content) { content = result.content } - if (result && result.contentType) { contentType = result.contentType } - } catch (e) { - console.log('error with handler: ', e); - } - - let context, page, finalUrl; - try { - if ((!content || !title) && contentType !== 'application/pdf') { - const result = await retrievePage(url) - if (result && result.context) { context = result.context } - if (result && result.page) { page = result.page } - if (result && result.finalUrl) { finalUrl = result.finalUrl } - if (result && result.contentType) { contentType = result.contentType } - } else { - finalUrl = url - } - - if (contentType === 'application/pdf') { - const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId); - const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId); - } else { - if (!content || !title) { - const result = await retrieveHtml(page); - if (result.isBlocked) { - const sbResult = await fetchContentWithScrapingBee(url) - title = sbResult.title - content = sbResult.domContent - } else { - title = result.title; - content = result.domContent; - } - } else { - console.log('using prefetched content and title'); - } - - logRecord.fetchContentTime = Date.now() - functionStartTime; - - const apiResponse = await sendCreateArticleMutation(userId, { - url: finalUrl, - articleSavingRequestId, - preparedDocument: { - document: content, - pageInfo: { - title, - canonicalUrl: finalUrl, - }, - }, - skipParsing: !content, - }); - - logRecord.totalTime = Date.now() - functionStartTime; - logRecord.result = apiResponse.createArticle; - } - } catch (e) { - logRecord.error = e.message; - console.log(`Error while retrieving page`, logRecord); - - // fallback to scrapingbee - const sbResult = await fetchContentWithScrapingBee(url); - const sbUrl = finalUrl || sbResult.url; - const content = sbResult.domContent; - logRecord.fetchContentTime = Date.now() - functionStartTime; - - const apiResponse = await sendCreateArticleMutation(userId, { - url: sbUrl, - articleSavingRequestId, - preparedDocument: { - document: content, - pageInfo: { - title: sbResult.title, - canonicalUrl: sbUrl, - }, - }, - skipParsing: !content, - }); - - logRecord.totalTime = Date.now() - functionStartTime; - logRecord.result = apiResponse.createArticle; - } finally { - if (context) { - await context.close(); - } - console.log(`parse-page`, logRecord); - } - - return res.sendStatus(200); -} - -function validateUrlString(url) { - const u = new URL(url); - // Make sure the URL is http or https - if (u.protocol !== 'http:' && u.protocol !== 'https:') { - throw new Error('Invalid URL protocol check failed') - } - // Make sure the domain is not localhost - if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') { - throw new Error('Invalid URL is localhost') - } - // Make sure the domain is not a private IP - if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) { - throw new Error('Invalid URL is private ip') - } -} - -function getUrl(req) { - console.log('body', req.body) - const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined); - if (!urlStr) { - throw new Error('No URL specified'); - } - - validateUrlString(urlStr); - - const parsed = Url.parse(urlStr); - return parsed.href; -} - - -async function blockResources(client) { - const blockedResources = [ - // Assets - // '*/favicon.ico', - // '.css', - // '.jpg', - // '.jpeg', - // '.png', - // '.svg', - // '.woff', - - // Analytics and other fluff - '*.optimizely.com', - 'everesttech.net', - 'userzoom.com', - 'doubleclick.net', - 'googleadservices.com', - 'adservice.google.com/*', - 'connect.facebook.com', - 'connect.facebook.net', - 'sp.analytics.yahoo.com', - ] - - await client.send('Network.setBlockedURLs', { urls: blockedResources }); -} - -async function retrievePage(url) { - validateUrlString(url); - - const browser = await getBrowserPromise; - logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; - - const context = await browser.createIncognitoBrowserContext(); - const page = await context.newPage() - - if (!enableJavascriptForUrl(url)) { - await page.setJavaScriptEnabled(false); - } - await page.setUserAgent(userAgentForUrl(url)); - - const client = await page.target().createCDPSession(); - - // intercept request when response headers was received - await client.send('Network.setRequestInterception', { - patterns: [ - { - urlPattern: '*', - resourceType: 'Document', - interceptionStage: 'HeadersReceived', - }, - ], - }); - - const path = require('path'); - const download_path = path.resolve('./download_dir/'); - - await client.send('Page.setDownloadBehavior', { - behavior: 'allow', - userDataDir: './', - downloadPath: download_path, - }) - - client.on('Network.requestIntercepted', async e => { - const headers = e.responseHeaders || {}; - - const [contentType] = (headers['content-type'] || headers['Content-Type'] || '') - .toLowerCase() - .split(';'); - const obj = { interceptionId: e.interceptionId }; - - if (e.responseStatusCode >= 200 && e.responseStatusCode < 300) { - // We only check content-type on success responses - // as it doesn't matter what the content type is for things - // like redirects - if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) { - obj['errorReason'] = 'BlockedByClient'; - } - } - - try { - await client.send('Network.continueInterceptedRequest', obj); - // eslint-disable-next-line no-empty - } catch {} - }); - - await blockResources(client); - - /* - * Disallow MathJax from running in Puppeteer and modifying the document, - * we shall instead run it in our frontend application to transform any - * mathjax content when present. - */ - await page.setRequestInterception(true); - let requestCount = 0; - page.on('request', request => { - if (['font', 'image', 'media'].includes(request.resourceType())) { - request.abort(); - return; - } - if (requestCount++ > 100) { - request.abort(); - return; - } - if ( - request.resourceType() === 'script' && - request.url().toLowerCase().indexOf('mathjax') > -1 - ) { - request.abort(); - return - } - request.continue(); - }); - - // Puppeteer fails during download of PDf files, - // so record the failure and use those items - let lastPdfUrl = undefined; - page.on('response', response => { - if (response.headers()['content-type'] === 'application/pdf') { - lastPdfUrl = response.url(); - } - }); - - try { - const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] }); - const finalUrl = response.url(); - const contentType = response.headers()['content-type']; - - logRecord.finalUrl = response.url(); - logRecord.contentType = response.headers()['content-type']; - - return { context, page, response, finalUrl, contentType }; - } catch (error) { - if (lastPdfUrl) { - return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' }; - } - await context.close(); - throw error; - } -} - -async function retrieveHtml(page) { - let domContent = '', title; - try { - title = await page.title(); - logRecord.title = title; - - const pageScrollingStart = Date.now(); - /* scroll with a 5 second timeout */ - await Promise.race([ - new Promise(resolve => { - (async function () { - try { - await page.evaluate(`(async () => { - /* credit: https://github.com/puppeteer/puppeteer/issues/305 */ - return new Promise((resolve, reject) => { - let scrollHeight = document.body.scrollHeight; - let totalHeight = 0; - let distance = 500; - let timer = setInterval(() => { - window.scrollBy(0, distance); - totalHeight += distance; - if(totalHeight >= scrollHeight){ - clearInterval(timer); - resolve(true); - } - }, 10); - }); - })()`); - } catch (e) { - logRecord.scrollError = true; - } finally { - resolve(true); - } - })(); - }), - await page.waitForTimeout(1000), - ]); - logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; - - const iframes = {}; - const urls = []; - const framesPromises = []; - const allowedUrls = /instagram\.com/gi; - - for (const frame of page.mainFrame().childFrames()) { - if (frame.url() && allowedUrls.test(frame.url())) { - urls.push(frame.url()); - framesPromises.push(frame.evaluate(el => el.innerHTML, await frame.$('body'))); - } - } - - (await Promise.all(framesPromises)).forEach((frame, index) => (iframes[urls[index]] = frame)); - - const domContentCapturingStart = Date.now(); - // get document body with all hidden elements removed - domContent = await page.evaluate(iframes => { - const BI_SRC_REGEXP = /url\("(.+?)"\)/gi; - - Array.from(document.body.getElementsByTagName('*')).forEach(el => { - const style = window.getComputedStyle(el); - - try { - // Removing blurred images since they are mostly the copies of lazy loaded ones - if (['img', 'image'].includes(el.tagName.toLowerCase())) { - const filter = style.getPropertyValue('filter'); - if (filter && filter.startsWith('blur')) { - el.parentNode && el.parentNode.removeChild(el); - } - } - } catch (err) { - // throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*')))) - } - - // convert all nodes with background image to img nodes - if (!['', 'none'].includes(style.getPropertyValue('background-image'))) { - const filter = style.getPropertyValue('filter'); - // avoiding image nodes with a blur effect creation - if (filter && filter.startsWith('blur')) { - el && el.parentNode && el.parentNode.removeChild(el); - } else { - const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image')); - // Using "g" flag with a regex we have to manually break down lastIndex to zero after every usage - // More details here: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results - BI_SRC_REGEXP.lastIndex = 0; - - if (matchedSRC && matchedSRC[1] && !el.src) { - // Replacing element only of there are no content inside, b/c might remove important div with content. - // Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html - // DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image. - if (el.innerHTML.length < 25) { - const img = document.createElement('img'); - img.src = matchedSRC[1]; - el && el.parentNode && el.parentNode.removeChild(el); - } - } - } - } - - if (el.tagName === 'IFRAME') { - if (iframes[el.src]) { - const newNode = document.createElement('div'); - newNode.className = 'omnivore-instagram-embed'; - newNode.innerHTML = iframes[el.src]; - el && el.parentNode && el.parentNode.replaceChild(newNode, el); - } - } - }); - - if (document.querySelector('[data-translate="managed_checking_msg"]') || - document.getElementById('px-block-form-wrapper')) { - return 'IS_BLOCKED' - } - - return document.documentElement.outerHTML; - }, iframes); - logRecord.puppeteerSuccess = true; - logRecord.timing = { - ...logRecord.timing, - contenCaptured: Date.now() - domContentCapturingStart, - }; - - // [END puppeteer-block] - } catch (e) { - if (e.message.startsWith('net::ERR_BLOCKED_BY_CLIENT at ')) { - logRecord.blockedByClient = true; - } else { - logRecord.puppeteerSuccess = false; - logRecord.puppeteerError = { - message: e.message, - stack: e.stack, - }; - } - } - if (domContent === 'IS_BLOCKED') { - return { isBlocked: true }; - } - return { domContent, title }; -} - -module.exports = fetchContent; diff --git a/packages/puppeteer-parse/.env.example b/packages/puppeteer-parse/.env.example deleted file mode 100644 index 64242a22d..000000000 --- a/packages/puppeteer-parse/.env.example +++ /dev/null @@ -1,8 +0,0 @@ -# Should match with the JWT_SECRET that the api uses -JWT_SECRET=some_secret - -# Address of the backend that is running locally -REST_BACKEND_ENDPOINT=http://localhost:4000/api - -# set for local development -IS_LOCAL=true diff --git a/packages/puppeteer-parse/.gcloudignore b/packages/puppeteer-parse/.gcloudignore deleted file mode 100644 index fc644d8d3..000000000 --- a/packages/puppeteer-parse/.gcloudignore +++ /dev/null @@ -1,21 +0,0 @@ -# This file specifies files that are *not* uploaded to Google Cloud Platform -# using gcloud. It follows the same syntax as .gitignore, with the addition of -# "#!include" directives (which insert the entries of the given .gitignore-style -# file at that point). -# -# For more information, run: -# $ gcloud topic gcloudignore -# -.gcloudignore -# If you would like to upload your .git directory, .gitignore file or files -# from your .gitignore file, remove the corresponding line -# below: -.git -.gitignore - -node_modules -.env* -.secrets* -Dockerfile* -previewImage.* -*.sa.json diff --git a/packages/puppeteer-parse/.gitignore b/packages/puppeteer-parse/.gitignore deleted file mode 100644 index 9bfbc5e8b..000000000 --- a/packages/puppeteer-parse/.gitignore +++ /dev/null @@ -1 +0,0 @@ -previewImage.* \ No newline at end of file diff --git a/packages/puppeteer-parse/Dockerfile b/packages/puppeteer-parse/Dockerfile deleted file mode 100644 index a52f55122..000000000 --- a/packages/puppeteer-parse/Dockerfile +++ /dev/null @@ -1,113 +0,0 @@ -# FROM node:14-slim - -# # Taken from pu - -# # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) -# # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer -# # installs, work. -# RUN apt-get update \ -# && apt-get install -y wget gnupg \ -# && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ -# && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ -# && apt-get update \ -# && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \ -# --no-install-recommends \ -# && rm -rf /var/lib/apt/lists/* - -# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true -# ENV CHROMIUM_PATH "/usr/bin/google-chrome-stable" - -# ------------------------ - -# FROM --platform=linux/arm64 node:14.18 - -# RUN apt-get update \ -# && apt-get install -y chromium \ -# && apt-get install -y ca-certificates \ -# fonts-liberation \ -# libappindicator3-1 \ -# libasound2 \ -# libatk-bridge2.0-0 \ -# libatk1.0-0 \ -# libc6 \ -# libcairo2 \ -# libcups2 \ -# libdbus-1-3 \ -# libexpat1 \ -# libfontconfig1 \ -# libgbm1 \ -# libgcc1 \ -# libglib2.0-0 \ -# libgtk-3-0 \ -# libnspr4 \ -# libnss3 \ -# libpango-1.0-0 \ -# libpangocairo-1.0-0 \ -# libstdc++6 \ -# libx11-6 \ -# libx11-xcb1 \ -# libxcb1 \ -# libxcomposite1 \ -# libxcursor1 \ -# libxdamage1 \ -# libxext6 \ -# libxfixes3 \ -# libxi6 \ -# libxrandr2 \ -# libxrender1 \ -# libxss1 \ -# libxtst6 \ -# lsb-release \ -# wget \ -# xdg-utils - -FROM node:14.18-alpine - -# Installs latest Chromium (92) package. -RUN apk add --no-cache \ - chromium \ - nss \ - freetype \ - harfbuzz \ - ca-certificates \ - ttf-freefont \ - nodejs \ - yarn - -# Add user so we don't need --no-sandbox. -RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ - && mkdir -p /home/pptruser/Downloads /app \ - && chown -R pptruser:pptruser /home/pptruser \ - && chown -R pptruser:pptruser /app - -# Run everything after as non-privileged user. -WORKDIR /app - -ENV CHROMIUM_PATH /usr/bin/chromium-browser -ENV LAUNCH_HEADLESS=true -ENV PORT 9090 - -COPY package.json . -COPY yarn.lock . -COPY tsconfig.json . -COPY .prettierrc . -COPY .eslintrc . - -COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json -COPY /packages/content-handler/package.json ./packages/content-handler/package.json - -RUN yarn install --pure-lockfile - -ADD /packages/puppeteer-parse ./packages/puppeteer-parse -ADD /packages/content-handler ./packages/content-handler -RUN yarn workspace @omnivore/content-handler build - -# After building, fetch the production dependencies -RUN rm -rf /app/packages/puppeteer-parse/node_modules -RUN rm -rf /app/node_modules -RUN yarn install --pure-lockfile --production - -EXPOSE 9090 - -# USER pptruser -ENTRYPOINT ["yarn", "workspace", "@omnivore/puppeteer-parse", "start"] diff --git a/packages/puppeteer-parse/README.md b/packages/puppeteer-parse/README.md index 501c004ee..6155c45b0 100644 --- a/packages/puppeteer-parse/README.md +++ b/packages/puppeteer-parse/README.md @@ -1,24 +1,3 @@ # Puppeteer parsing function handler -This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer. - -## Using locally - -Copy .env.example file to .env file: `cp .env.example .env` - -Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading). - -After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) - -## Deployment - -To deploy the function use the following command: - -`gcloud functions deploy puppeteer --runtime nodejs12 --trigger-http --memory 1GB --set-env-vars REST_BACKEND_ENDPOINT=,JWT_SECRET=` - - -where: - -`` - address of the backend server (e.g "http://localhost:4000") - -`` - JWT secret that the backend server is using (e.g "some_secret") +This workspace is used to provide the module for the app to hande requests for the article parsing via Puppeteer. diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index b602ff1ef..f2f4cf02f 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -5,6 +5,7 @@ /* eslint-disable @typescript-eslint/no-require-imports */ require('dotenv').config(); const Url = require('url'); +const puppeteer = require('puppeteer-extra'); const axios = require('axios'); const jwt = require('jsonwebtoken'); const { promisify } = require('util'); @@ -38,69 +39,16 @@ const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) Apple const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] +const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; -const filePath = `${os.tmpdir()}/previewImage.png`; +const path = require("path"); const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; +const { parseHTML } = require('linkedom'); -const colors = { - emerg: 'inverse underline magenta', - alert: 'underline magenta', - crit: 'inverse underline red', // Any error that is forcing a shutdown of the service or application to prevent data loss. - error: 'underline red', // Any error which is fatal to the operation, but not the service or application - warning: 'underline yellow', // Anything that can potentially cause application oddities - notice: 'underline cyan', // Normal but significant condition - info: 'underline green', // Generally useful information to log - debug: 'underline gray', -}; - -const googleConfigs = { - level: 'info', - logName: 'logger', - levels: config.syslog.levels, - resource: { - labels: { - function_name: process.env.FUNCTION_TARGET, - project_id: process.env.GCP_PROJECT, - }, - type: 'cloud_function', - }, -}; - -function localConfig(id) { - return { - level: 'debug', - format: format.combine( - format.colorize({ all: true, colors }), - format(info => - Object.assign(info, { - timestamp: DateTime.local().toLocaleString(DateTime.TIME_24_WITH_SECONDS), - }), - )(), - format.printf(info => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { timestamp, message, level, ...meta } = info; - - return `[${id}@${info.timestamp}] ${info.message}${ - Object.keys(meta).length ? '\n' + JSON.stringify(meta, null, 4) : '' - }`; - }), - ), - }; -} - -function buildLoggerTransport(id, options) { - return process.env.IS_LOCAL - ? new transports.Console(localConfig(id)) - : new LoggingWinston({ ...googleConfigs, ...{ logName: id }, ...options }); -} - -function buildLogger(id, options) { - return loggers.get(id, { - levels: config.syslog.levels, - transports: [buildLoggerTransport(id, options)], - }); -} +// Add stealth plugin to hide puppeteer usage +const StealthPlugin = require('puppeteer-extra-plugin-stealth'); +puppeteer.use(StealthPlugin()); const userAgentForUrl = (url) => { try { @@ -116,15 +64,38 @@ const userAgentForUrl = (url) => { return DESKTOP_USER_AGENT }; +const fetchContentWithScrapingBee = async (url) => { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + 'api_key': process.env.SCRAPINGBEE_API_KEY, + 'url': url, + 'render_js': 'false', + 'premium_proxy': 'true', + 'country_code':'us' + } + }) + + const dom = parseHTML(response.data).document; + return { title: dom.title, domContent: dom.documentElement.outerHTML, url: url } +} + +const enableJavascriptForUrl = (url) => { + try { + const u = new URL(url); + for (const host of NON_SCRIPT_HOSTS) { + if (u.hostname.endsWith(host)) { + return false; + } + } + } catch (e) { + console.log('error getting hostname for url', url, e) + } + return true +}; + // launch Puppeteer -const getBrowserPromise = (async () => { - // return puppeteer.launch({ - // args: chromium.args, - // defaultViewport: chromium.defaultViewport, - // executablePath: process.env.CHROMIUM_PATH, - // headless: chromium.headless, - // ignoreHTTPSErrors: true, - // }); +const getBrowserPromise = (async (proxyUrl, chromiumPath) => { + console.log("starting with proxy url", proxyUrl) return puppeteer.launch({ args: [ '--allow-running-insecure-content', @@ -151,8 +122,8 @@ const getBrowserPromise = (async () => { '--window-size=1920,1080', ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, - executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, + executablePath: chromiumPath, + headless: true, timeout: 120000, // 2 minutes }); })(); @@ -170,7 +141,7 @@ const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentOb }) }; -const getUploadIdAndSignedUrl = async (userId, url) => { +const getUploadIdAndSignedUrl = async (userId, url, articleSavingRequestId) => { const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); const data = JSON.stringify({ query: `mutation UploadFileRequest($input: UploadFileRequestInput!) { @@ -188,17 +159,18 @@ const getUploadIdAndSignedUrl = async (userId, url) => { input: { url, contentType: 'application/pdf', + clientRequestId: articleSavingRequestId, } } }); const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + }); return response.data.data.uploadFileRequest; }; @@ -231,12 +203,12 @@ const sendCreateArticleMutation = async (userId, input) => { const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + }); return response.data.data.createArticle; }; @@ -249,24 +221,8 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId ); }; -/** - * Cloud Function entry point, HTTP trigger. - * Loads the requested URL via Puppeteer, captures page content and sends it to backend - * - * @param {Object} req Cloud Function request context. - * @param {Object} res Cloud Function response context. - */ -exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { +async function fetchContent(req, res) { functionStartTime = Date.now(); - // Grabbing execution and trace ids to attach logs to the appropriate function call - const execution_id = req.get('function-execution-id'); - const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; - const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { - trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, - labels: { - execution_id: execution_id, - }, - }); let url = getUrl(req); const userId = req.body.userId || req.query.userId; @@ -281,7 +237,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { }, }; - logger.info(`Article parsing request`, logRecord); + console.log(`Article parsing request`, logRecord); if (!url) { logRecord.urlIsInvalid = true; @@ -306,16 +262,15 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { let context, page, finalUrl; try { - if ((!content || !title) && contentType !== 'application/pdf') { - const result = await retrievePage(url) - if (result && result.context) { context = result.context } - if (result && result.page) { page = result.page } - if (result && result.finalUrl) { finalUrl = result.finalUrl } - if (result && result.contentType) { contentType = result.contentType } - console.log('context, page, finalUrl, contentType', context, page, finalUrl, contentType); - } else { - finalUrl = url - } + if ((!content || !title) && contentType !== 'application/pdf') { + const result = await retrievePage(url) + if (result && result.context) { context = result.context } + if (result && result.page) { page = result.page } + if (result && result.finalUrl) { finalUrl = result.finalUrl } + if (result && result.contentType) { contentType = result.contentType } + } else { + finalUrl = url + } if (contentType === 'application/pdf') { const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId); @@ -323,14 +278,20 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { } else { if (!content || !title) { const result = await retrieveHtml(page); - title = result.title; - content = result.domContent; + if (result.isBlocked) { + const sbResult = await fetchContentWithScrapingBee(url) + title = sbResult.title + content = sbResult.domContent + } else { + title = result.title; + content = result.domContent; + } } else { console.log('using prefetched content and title'); console.log(content); } - logRecord.contentFetchTime = Date.now() - functionStartTime; + logRecord.fetchContentTime = Date.now() - functionStartTime; const apiResponse = await sendCreateArticleMutation(userId, { url: finalUrl, @@ -347,12 +308,12 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; - logger.info(`parse-page`, logRecord); + console.log(`parse-page`, logRecord); } } catch (e) { console.log('error', e) logRecord.error = e.message; - logger.error(`Error while retrieving page`, logRecord); + console.log(`Error while retrieving page`, logRecord); return res.sendStatus(503); } finally { if (context) { @@ -361,136 +322,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { } return res.sendStatus(200); -}); - -/** - * Cloud Function entry point, HTTP trigger. - * Loads the requested URL via Puppeteer and captures a screenshot of the provided element - * - * @param {Object} req Cloud Function request context. - * Inlcudes: - * * url - URL address of the page to open - * @param {Object} res Cloud Function response context. - */ -exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { - functionStartTime = Date.now(); - // Grabbing execution and trace ids to attach logs to the appropriate function call - const execution_id = req.get('function-execution-id'); - const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; - const logger = buildLogger('cloudfunctions.googleapis.com%2Fcloud-functions', { - trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`, - labels: { - execution_id: execution_id, - }, - }); - - if (!process.env.PREVIEW_IMAGE_BUCKET) { - logger.error(`PREVIEW_IMAGE_BUCKET not set`) - return res.sendStatus(500); - } - - const url = getUrl(req); - console.log('preview request url', url); - - logRecord = { - url, - query: req.query, - origin: req.get('Origin'), - labels: { - source: 'publicImagePreview', - }, - }; - - logger.info(`Public preview image generation request`, logRecord); - - if (!url) { - logRecord.urlIsInvalid = true; - logger.error(`Valid URL to parse is not specified`, logRecord); - return res.sendStatus(400); - } - const { origin } = new URL(url); - if (!ALLOWED_ORIGINS.some(o => o === origin)) { - logRecord.forbiddenOrigin = true; - logger.error(`This origin is not allowed: ${origin}`, logRecord); - return res.sendStatus(400); - } - - const browser = await getBrowserPromise; - logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; - - const page = await browser.newPage(); - const pageLoadingStart = Date.now(); - const modifiedUrl = new URL(url); - modifiedUrl.searchParams.append('fontSize', 24); - modifiedUrl.searchParams.append('adjustAspectRatio', 1.91); - try { - await page.goto(modifiedUrl); - logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart }; - } catch (error) { - console.log('error going to page: ', modifiedUrl) - console.log(error) - throw error - } - - // We lookup the destination path from our own page content and avoid trusting any passed query params - // selector - CSS selector of the element to get screenshot of - const selector = decodeURIComponent( - await page.$eval( - "head > meta[name='omnivore:preview_image_selector']", - element => element.content, - ), - ); - if (!selector) { - logRecord.selectorIsInvalid = true; - logger.error(`Valid element selector is not specified`, logRecord); - await page.close(); - return res.sendStatus(400); - } - logRecord.selector = selector; - - // destination - destination pathname for the image to save with - const destination = decodeURIComponent( - await page.$eval( - "head > meta[name='omnivore:preview_image_destination']", - element => element.content, - ), - ); - if (!destination) { - logRecord.destinationIsInvalid = true; - logger.error(`Valid file destination is not specified`, logRecord); - await page.close(); - return res.sendStatus(400); - } - logRecord.destination = destination; - - const screenshotTakingStart = Date.now(); - try { - await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load - } catch (error) { - logRecord.elementNotFound = true; - logger.error(`Element is not presented on the page`, logRecord); - await page.close(); - return res.sendStatus(400); - } - const element = await page.$(selector); - await element.screenshot({ path: filePath }); // take screenshot of the element in puppeteer - logRecord.timing = { ...logRecord.timing, screenshotTaken: Date.now() - screenshotTakingStart }; - - await page.close(); - - try { - const [file] = await previewBucket.upload(filePath, { - destination, - metadata: logRecord, - }); - logRecord.file = file.metadata; - } catch (e) { - console.log('error uploading to bucket, this is non-fatal', e) - } - - logger.info(`preview-image`, logRecord); - return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); -}); +} function validateUrlString(url) { const u = new URL(url); @@ -509,16 +341,15 @@ function validateUrlString(url) { } function getUrl(req) { - if (req.query.url || req.body.url) { - const urlStr = req.query.url || req.body.url; - validateUrlString(urlStr); - - const url = Url.parse(urlStr); - return url.href; + const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined); + if (!urlStr) { + throw new Error('No URL specified'); } - try { - return Url.parse(JSON.parse(req.body).url).href; - } catch (e) {} + + validateUrlString(urlStr); + + const parsed = Url.parse(urlStr); + return parsed.href; } async function blockResources(client) { @@ -554,7 +385,11 @@ async function retrievePage(url) { logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; const context = await browser.createIncognitoBrowserContext(); - const page = await context.newPage(); + const page = await context.newPage() + + if (!enableJavascriptForUrl(url)) { + await page.setJavaScriptEnabled(false); + } await page.setUserAgent(userAgentForUrl(url)); const client = await page.target().createCDPSession(); @@ -570,16 +405,15 @@ async function retrievePage(url) { ], }); - const path = require('path'); const download_path = path.resolve('./download_dir/'); await client.send('Page.setDownloadBehavior', { - behavior: 'allow', - userDataDir: './', - downloadPath: download_path, + behavior: 'allow', + userDataDir: './', + downloadPath: download_path, }) - client.on('Network.requestIntercepted', async e => { + client.on('Network.requestIntercepted', async (e) => { const headers = e.responseHeaders || {}; const [contentType] = (headers['content-type'] || headers['Content-Type'] || '') @@ -605,10 +439,10 @@ async function retrievePage(url) { await blockResources(client); /* - * Disallow MathJax from running in Puppeteer and modifying the document, - * we shall instead run it in our frontend application to transform any - * mathjax content when present. - */ + * Disallow MathJax from running in Puppeteer and modifying the document, + * we shall instead run it in our frontend application to transform any + * mathjax content when present. + */ await page.setRequestInterception(true); let requestCount = 0; page.on('request', request => { @@ -630,7 +464,6 @@ async function retrievePage(url) { request.continue(); }); - // Puppeteer fails during download of PDf files, // so record the failure and use those items let lastPdfUrl = undefined; @@ -719,12 +552,16 @@ async function retrieveHtml(page) { Array.from(document.body.getElementsByTagName('*')).forEach(el => { const style = window.getComputedStyle(el); - // Removing blurred images since they are mostly the copies of lazy loaded ones - if (['img', 'image'].includes(el.tagName.toLowerCase())) { - const filter = style.getPropertyValue('filter'); - if (filter && filter.startsWith('blur')) { - el.parentNode && el.parentNode.removeChild(el); + try { + // Removing blurred images since they are mostly the copies of lazy loaded ones + if (['img', 'image'].includes(el.tagName.toLowerCase())) { + const filter = style.getPropertyValue('filter'); + if (filter && filter.startsWith('blur')) { + el.parentNode && el.parentNode.removeChild(el); + } } + } catch (err) { + // throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*')))) } // convert all nodes with background image to img nodes @@ -761,6 +598,12 @@ async function retrieveHtml(page) { } } }); + + if (document.querySelector('[data-translate="managed_checking_msg"]') || + document.getElementById('px-block-form-wrapper')) { + return 'IS_BLOCKED' + } + return document.documentElement.outerHTML; }, iframes); logRecord.puppeteerSuccess = true; @@ -781,5 +624,15 @@ async function retrieveHtml(page) { }; } } + if (domContent === 'IS_BLOCKED') { + return { isBlocked: true }; + } return { domContent, title }; } + +module.exports = { + fetchContent, + getBrowserPromise, + getUrl, +}; + diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 4ef6bb4d9..e73914fa4 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -1,7 +1,7 @@ { "name": "@omnivore/puppeteer-parse", "version": "1.0.0", - "description": "Google Cloud Function that accepts URL of the article and parses its content", + "description": "Accepts URL of the article and parses its content", "main": "index.js", "dependencies": { "@google-cloud/functions-framework": "^3.1.2", @@ -20,8 +20,6 @@ "winston": "^3.3.3" }, "scripts": { - "start": "npx functions-framework --port=9090 --target=puppeteer", - "start_preview": "npx functions-framework --target=preview", "test": "yarn mocha" } }