diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 3b226c0b0..6da610fee 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -90,5 +90,5 @@ jobs: run: 'docker build --file packages/content-fetch/Dockerfile .' - name: Build the inbound-email-handler docker image run: 'docker build --file packages/inbound-email-handler/Dockerfile .' - - name: Build the puppeteer-parse docker image - run: 'docker build --file packages/puppeteer-parse/Dockerfile .' + - name: Build the content-fetch cloud function docker image + run: 'docker build --file packages/content-fetch/Dockerfile-gcf .' diff --git a/packages/puppeteer-parse/.env.example b/packages/content-fetch/.env.example similarity index 55% rename from packages/puppeteer-parse/.env.example rename to packages/content-fetch/.env.example index 64242a22d..1537ad8ab 100644 --- a/packages/puppeteer-parse/.env.example +++ b/packages/content-fetch/.env.example @@ -6,3 +6,13 @@ REST_BACKEND_ENDPOINT=http://localhost:4000/api # set for local development IS_LOCAL=true + +VERIFICATION_TOKEN=some_token + +CHROMIUM_PATH=/opt/homebrew/bin/chromium +PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true +LAUNCH_HEADLESS=true + +TWITTER_BEARER_TOKEN=token + +PORT=9090 diff --git a/packages/puppeteer-parse/.gcloudignore b/packages/content-fetch/.gcloudignore similarity index 100% rename from packages/puppeteer-parse/.gcloudignore rename to packages/content-fetch/.gcloudignore diff --git a/packages/content-fetch/Dockerfile b/packages/content-fetch/Dockerfile index 1201c497d..9ca9f7dee 100644 --- a/packages/content-fetch/Dockerfile +++ b/packages/content-fetch/Dockerfile @@ -30,11 +30,13 @@ COPY .prettierrc . COPY .eslintrc . COPY /packages/content-handler/package.json ./packages/content-handler/package.json +COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json RUN yarn install --pure-lockfile ADD /packages/content-fetch ./packages/content-fetch ADD /packages/content-handler ./packages/content-handler +ADD /packages/puppeteer-parse ./packages/puppeteer-parse RUN yarn workspace @omnivore/content-handler build # After building, fetch the production dependencies diff --git a/packages/content-fetch/Dockerfile-gcf b/packages/content-fetch/Dockerfile-gcf new file mode 100644 index 000000000..8c355485f --- /dev/null +++ b/packages/content-fetch/Dockerfile-gcf @@ -0,0 +1,51 @@ +FROM node:14.18-alpine + +# Installs latest Chromium (92) package. +RUN apk add --no-cache \ + chromium \ + nss \ + freetype \ + harfbuzz \ + ca-certificates \ + ttf-freefont \ + nodejs \ + yarn + +# Add user so we don't need --no-sandbox. +RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ + && mkdir -p /home/pptruser/Downloads /app \ + && chown -R pptruser:pptruser /home/pptruser \ + && chown -R pptruser:pptruser /app + +# Run everything after as non-privileged user. +WORKDIR /app + +ENV CHROMIUM_PATH /usr/bin/chromium-browser +ENV LAUNCH_HEADLESS=true +ENV PORT 9090 + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .prettierrc . +COPY .eslintrc . + +COPY /packages/content-handler/package.json ./packages/content-handler/package.json +COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/content-handler ./packages/content-handler +ADD /packages/puppeteer-parse ./packages/puppeteer-parse +ADD /packages/content-fetch ./packages/content-fetch +RUN yarn workspace @omnivore/content-handler build + +# After building, fetch the production dependencies +RUN rm -rf /app/packages/content-fetch/node_modules +RUN rm -rf /app/node_modules +RUN yarn install --pure-lockfile --production + +EXPOSE 9090 + +# USER pptruser +ENTRYPOINT ["yarn", "workspace", "@omnivore/content-fetch", "start_gcf"] diff --git a/packages/content-fetch/README.md b/packages/content-fetch/README.md new file mode 100644 index 000000000..501c004ee --- /dev/null +++ b/packages/content-fetch/README.md @@ -0,0 +1,24 @@ +# Puppeteer parsing function handler + +This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer. + +## Using locally + +Copy .env.example file to .env file: `cp .env.example .env` + +Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading). + +After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) + +## Deployment + +To deploy the function use the following command: + +`gcloud functions deploy puppeteer --runtime nodejs12 --trigger-http --memory 1GB --set-env-vars REST_BACKEND_ENDPOINT=,JWT_SECRET=` + + +where: + +`` - address of the backend server (e.g "http://localhost:4000") + +`` - JWT secret that the backend server is using (e.g "some_secret") diff --git a/packages/content-fetch/app.js b/packages/content-fetch/app.js index 60f3031c5..252671626 100644 --- a/packages/content-fetch/app.js +++ b/packages/content-fetch/app.js @@ -1,7 +1,8 @@ +require('dotenv').config(); const express = require('express'); const app = express(); -const fetchContent = require('./fetch-content'); +const { fetchContent } = require("@omnivore/puppeteer-parse"); app.use(express.json()); app.use(express.urlencoded({ extended: true })); @@ -10,22 +11,22 @@ if (!process.env.VERIFICATION_TOKEN) { throw new Error('VERIFICATION_TOKEN environment variable is not set'); } -app.get('/', (req, res) => { +app.get('/', async (req, res) => { if (req.query.token !== process.env.VERIFICATION_TOKEN) { console.log('query does not include valid token') res.send(403) return } - fetchContent(req, res) + await fetchContent(req, res) }); -app.post('/', (req, res) => { +app.post('/', async (req, res) => { if (req.query.token !== process.env.VERIFICATION_TOKEN) { console.log('query does not include valid token') res.send(403) return } - fetchContent(req, res) + await fetchContent(req, res) }); const PORT = parseInt(process.env.PORT) || 8080; @@ -34,4 +35,4 @@ app.listen(PORT, () => { console.log('Press Ctrl+C to quit.'); }); -module.exports = app; \ No newline at end of file +module.exports = app; diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js deleted file mode 100644 index 7a35e1eeb..000000000 --- a/packages/content-fetch/fetch-content.js +++ /dev/null @@ -1,635 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const Url = require('url'); -const puppeteer = require('puppeteer-core'); -const axios = require('axios'); -const jwt = require('jsonwebtoken'); -const { promisify } = require('util'); -const { parseHTML } = require('linkedom'); -const { preHandleContent } = require('@omnivore/content-handler'); - -const signToken = promisify(jwt.sign); - -const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' -const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] -const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; - -const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; - -// Add stealth plugin to hide puppeteer usage -// const StealthPlugin = require('puppeteer-extra-plugin-stealth'); -// puppeteer.use(StealthPlugin()); - - -const userAgentForUrl = (url) => { - try { - const u = new URL(url); - for (const host of NON_BOT_HOSTS) { - if (u.hostname.endsWith(host)) { - return NON_BOT_DESKTOP_USER_AGENT; - } - } - } catch (e) { - console.log('error getting user agent for url', url, e) - } - return DESKTOP_USER_AGENT -}; - -const fetchContentWithScrapingBee = async (url) => { - const response = await axios.get('https://app.scrapingbee.com/api/v1', { - params: { - 'api_key': process.env.SCRAPINGBEE_API_KEY, - 'url': url, - 'render_js': 'false', - 'premium_proxy': 'true', - 'country_code':'us' - } - }) - - const dom = parseHTML(response.data).document; - return { title: dom.title, domContent: dom.documentElement.outerHTML, url: url } -} - -const enableJavascriptForUrl = (url) => { - try { - const u = new URL(url); - for (const host of NON_SCRIPT_HOSTS) { - if (u.hostname.endsWith(host)) { - return false; - } - } - } catch (e) { - console.log('error getting hostname for url', url, e) - } - return true -}; - -// launch Puppeteer -const getBrowserPromise = (async () => { - console.log("starting with proxy url", process.env.PROXY_URL) - return puppeteer.launch({ - args: [ - '--allow-running-insecure-content', - '--autoplay-policy=user-gesture-required', - '--disable-component-update', - '--disable-domain-reliability', - '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process', - '--disable-print-preview', - '--disable-setuid-sandbox', - '--disable-site-isolation-trials', - '--disable-speech-api', - '--disable-web-security', - '--disk-cache-size=33554432', - '--enable-features=SharedArrayBuffer', - '--hide-scrollbars', - '--ignore-gpu-blocklist', - '--in-process-gpu', - '--mute-audio', - '--no-default-browser-check', - '--no-pings', - '--no-sandbox', - '--no-zygote', - '--use-gl=swiftshader', - '--window-size=1920,1080', - ].filter((item) => !!item), - defaultViewport: { height: 1080, width: 1920 }, - executablePath: process.env.CHROMIUM_PATH, - headless: !!process.env.LAUNCH_HEADLESS, - timeout: 120000, // 2 minutes - }); -})(); - -let logRecord, functionStartTime; - -const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => { - const stream = await axios.get(contentObjUrl, { responseType: 'stream' }); - return await axios.put(uploadSignedUrl, stream.data, { - headers: { - 'Content-Type': contentType, - }, - maxBodyLength: 1000000000, - maxContentLength: 100000000, - }) -}; - -const getUploadIdAndSignedUrl = async (userId, url, articleSavingRequestId) => { - const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); - const data = JSON.stringify({ - query: `mutation UploadFileRequest($input: UploadFileRequestInput!) { - uploadFileRequest(input:$input) { - ... on UploadFileRequestError { - errorCodes - } - ... on UploadFileRequestSuccess { - id - uploadSignedUrl - } - } - }`, - variables: { - input: { - url, - contentType: 'application/pdf', - clientRequestId: articleSavingRequestId, - } - } - }); - - const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); - return response.data.data.uploadFileRequest; -}; - -const uploadPdf = async (url, userId, articleSavingRequestId) => { - validateUrlString(url); - - const uploadResult = await getUploadIdAndSignedUrl(userId, url, articleSavingRequestId); - await uploadToSignedUrl(uploadResult, 'application/pdf', url); - return uploadResult.id; -}; - -const sendCreateArticleMutation = async (userId, input) => { - const data = JSON.stringify({ - query: `mutation CreateArticle ($input: CreateArticleInput!){ - createArticle(input:$input){ - ... on CreateArticleSuccess{ - createdArticle{ - id - } - } - ... on CreateArticleError{ - errorCodes - } - } - }`, - variables: { - input: Object.assign({}, input , { source: 'puppeteer-parse' }), - }, - }); - - const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); - const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); - return response.data.data.createArticle; -}; - -const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId) => { - return sendCreateArticleMutation(userId, { - url: encodeURI(url), - articleSavingRequestId, - uploadFileId: uploadFileId, - }, - ); -}; - -async function fetchContent(req, res) { - functionStartTime = Date.now(); - - let url = getUrl(req); - const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); - const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); - - console.log('user id', userId, 'url', url) - - logRecord = { - url, - userId, - articleSavingRequestId, - labels: { - source: 'parseContent', - }, - }; - - console.log(`Article parsing request`, logRecord); - - if (!url) { - logRecord.urlIsInvalid = true; - console.log(`Valid URL to parse not specified`, logRecord); - return res.sendStatus(400); - } - - // pre handle url with custom handlers - let title, content, contentType; - try { - const result = await preHandleContent(url); - if (result && result.url) { - url = result.url - validateUrlString(url); - } - if (result && result.title) { title = result.title } - if (result && result.content) { content = result.content } - if (result && result.contentType) { contentType = result.contentType } - } catch (e) { - console.log('error with handler: ', e); - } - - let context, page, finalUrl; - try { - if ((!content || !title) && contentType !== 'application/pdf') { - const result = await retrievePage(url) - if (result && result.context) { context = result.context } - if (result && result.page) { page = result.page } - if (result && result.finalUrl) { finalUrl = result.finalUrl } - if (result && result.contentType) { contentType = result.contentType } - } else { - finalUrl = url - } - - if (contentType === 'application/pdf') { - const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId); - const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId); - } else { - if (!content || !title) { - const result = await retrieveHtml(page); - if (result.isBlocked) { - const sbResult = await fetchContentWithScrapingBee(url) - title = sbResult.title - content = sbResult.domContent - } else { - title = result.title; - content = result.domContent; - } - } else { - console.log('using prefetched content and title'); - } - - logRecord.fetchContentTime = Date.now() - functionStartTime; - - const apiResponse = await sendCreateArticleMutation(userId, { - url: finalUrl, - articleSavingRequestId, - preparedDocument: { - document: content, - pageInfo: { - title, - canonicalUrl: finalUrl, - }, - }, - skipParsing: !content, - }); - - logRecord.totalTime = Date.now() - functionStartTime; - logRecord.result = apiResponse.createArticle; - } - } catch (e) { - logRecord.error = e.message; - console.log(`Error while retrieving page`, logRecord); - - // fallback to scrapingbee - const sbResult = await fetchContentWithScrapingBee(url); - const sbUrl = finalUrl || sbResult.url; - const content = sbResult.domContent; - logRecord.fetchContentTime = Date.now() - functionStartTime; - - const apiResponse = await sendCreateArticleMutation(userId, { - url: sbUrl, - articleSavingRequestId, - preparedDocument: { - document: content, - pageInfo: { - title: sbResult.title, - canonicalUrl: sbUrl, - }, - }, - skipParsing: !content, - }); - - logRecord.totalTime = Date.now() - functionStartTime; - logRecord.result = apiResponse.createArticle; - } finally { - if (context) { - await context.close(); - } - console.log(`parse-page`, logRecord); - } - - return res.sendStatus(200); -} - -function validateUrlString(url) { - const u = new URL(url); - // Make sure the URL is http or https - if (u.protocol !== 'http:' && u.protocol !== 'https:') { - throw new Error('Invalid URL protocol check failed') - } - // Make sure the domain is not localhost - if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') { - throw new Error('Invalid URL is localhost') - } - // Make sure the domain is not a private IP - if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) { - throw new Error('Invalid URL is private ip') - } -} - -function getUrl(req) { - console.log('body', req.body) - const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined); - if (!urlStr) { - throw new Error('No URL specified'); - } - - validateUrlString(urlStr); - - const parsed = Url.parse(urlStr); - return parsed.href; -} - - -async function blockResources(client) { - const blockedResources = [ - // Assets - // '*/favicon.ico', - // '.css', - // '.jpg', - // '.jpeg', - // '.png', - // '.svg', - // '.woff', - - // Analytics and other fluff - '*.optimizely.com', - 'everesttech.net', - 'userzoom.com', - 'doubleclick.net', - 'googleadservices.com', - 'adservice.google.com/*', - 'connect.facebook.com', - 'connect.facebook.net', - 'sp.analytics.yahoo.com', - ] - - await client.send('Network.setBlockedURLs', { urls: blockedResources }); -} - -async function retrievePage(url) { - validateUrlString(url); - - const browser = await getBrowserPromise; - logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; - - const context = await browser.createIncognitoBrowserContext(); - const page = await context.newPage() - - if (!enableJavascriptForUrl(url)) { - await page.setJavaScriptEnabled(false); - } - await page.setUserAgent(userAgentForUrl(url)); - - const client = await page.target().createCDPSession(); - - // intercept request when response headers was received - await client.send('Network.setRequestInterception', { - patterns: [ - { - urlPattern: '*', - resourceType: 'Document', - interceptionStage: 'HeadersReceived', - }, - ], - }); - - const path = require('path'); - const download_path = path.resolve('./download_dir/'); - - await client.send('Page.setDownloadBehavior', { - behavior: 'allow', - userDataDir: './', - downloadPath: download_path, - }) - - client.on('Network.requestIntercepted', async e => { - const headers = e.responseHeaders || {}; - - const [contentType] = (headers['content-type'] || headers['Content-Type'] || '') - .toLowerCase() - .split(';'); - const obj = { interceptionId: e.interceptionId }; - - if (e.responseStatusCode >= 200 && e.responseStatusCode < 300) { - // We only check content-type on success responses - // as it doesn't matter what the content type is for things - // like redirects - if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) { - obj['errorReason'] = 'BlockedByClient'; - } - } - - try { - await client.send('Network.continueInterceptedRequest', obj); - // eslint-disable-next-line no-empty - } catch {} - }); - - await blockResources(client); - - /* - * Disallow MathJax from running in Puppeteer and modifying the document, - * we shall instead run it in our frontend application to transform any - * mathjax content when present. - */ - await page.setRequestInterception(true); - let requestCount = 0; - page.on('request', request => { - if (['font', 'image', 'media'].includes(request.resourceType())) { - request.abort(); - return; - } - if (requestCount++ > 100) { - request.abort(); - return; - } - if ( - request.resourceType() === 'script' && - request.url().toLowerCase().indexOf('mathjax') > -1 - ) { - request.abort(); - return - } - request.continue(); - }); - - // Puppeteer fails during download of PDf files, - // so record the failure and use those items - let lastPdfUrl = undefined; - page.on('response', response => { - if (response.headers()['content-type'] === 'application/pdf') { - lastPdfUrl = response.url(); - } - }); - - try { - const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] }); - const finalUrl = response.url(); - const contentType = response.headers()['content-type']; - - logRecord.finalUrl = response.url(); - logRecord.contentType = response.headers()['content-type']; - - return { context, page, response, finalUrl, contentType }; - } catch (error) { - if (lastPdfUrl) { - return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' }; - } - await context.close(); - throw error; - } -} - -async function retrieveHtml(page) { - let domContent = '', title; - try { - title = await page.title(); - logRecord.title = title; - - const pageScrollingStart = Date.now(); - /* scroll with a 5 second timeout */ - await Promise.race([ - new Promise(resolve => { - (async function () { - try { - await page.evaluate(`(async () => { - /* credit: https://github.com/puppeteer/puppeteer/issues/305 */ - return new Promise((resolve, reject) => { - let scrollHeight = document.body.scrollHeight; - let totalHeight = 0; - let distance = 500; - let timer = setInterval(() => { - window.scrollBy(0, distance); - totalHeight += distance; - if(totalHeight >= scrollHeight){ - clearInterval(timer); - resolve(true); - } - }, 10); - }); - })()`); - } catch (e) { - logRecord.scrollError = true; - } finally { - resolve(true); - } - })(); - }), - await page.waitForTimeout(1000), - ]); - logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; - - const iframes = {}; - const urls = []; - const framesPromises = []; - const allowedUrls = /instagram\.com/gi; - - for (const frame of page.mainFrame().childFrames()) { - if (frame.url() && allowedUrls.test(frame.url())) { - urls.push(frame.url()); - framesPromises.push(frame.evaluate(el => el.innerHTML, await frame.$('body'))); - } - } - - (await Promise.all(framesPromises)).forEach((frame, index) => (iframes[urls[index]] = frame)); - - const domContentCapturingStart = Date.now(); - // get document body with all hidden elements removed - domContent = await page.evaluate(iframes => { - const BI_SRC_REGEXP = /url\("(.+?)"\)/gi; - - Array.from(document.body.getElementsByTagName('*')).forEach(el => { - const style = window.getComputedStyle(el); - - try { - // Removing blurred images since they are mostly the copies of lazy loaded ones - if (['img', 'image'].includes(el.tagName.toLowerCase())) { - const filter = style.getPropertyValue('filter'); - if (filter && filter.startsWith('blur')) { - el.parentNode && el.parentNode.removeChild(el); - } - } - } catch (err) { - // throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*')))) - } - - // convert all nodes with background image to img nodes - if (!['', 'none'].includes(style.getPropertyValue('background-image'))) { - const filter = style.getPropertyValue('filter'); - // avoiding image nodes with a blur effect creation - if (filter && filter.startsWith('blur')) { - el && el.parentNode && el.parentNode.removeChild(el); - } else { - const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image')); - // Using "g" flag with a regex we have to manually break down lastIndex to zero after every usage - // More details here: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results - BI_SRC_REGEXP.lastIndex = 0; - - if (matchedSRC && matchedSRC[1] && !el.src) { - // Replacing element only of there are no content inside, b/c might remove important div with content. - // Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html - // DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image. - if (el.innerHTML.length < 25) { - const img = document.createElement('img'); - img.src = matchedSRC[1]; - el && el.parentNode && el.parentNode.removeChild(el); - } - } - } - } - - if (el.tagName === 'IFRAME') { - if (iframes[el.src]) { - const newNode = document.createElement('div'); - newNode.className = 'omnivore-instagram-embed'; - newNode.innerHTML = iframes[el.src]; - el && el.parentNode && el.parentNode.replaceChild(newNode, el); - } - } - }); - - if (document.querySelector('[data-translate="managed_checking_msg"]') || - document.getElementById('px-block-form-wrapper')) { - return 'IS_BLOCKED' - } - - return document.documentElement.outerHTML; - }, iframes); - logRecord.puppeteerSuccess = true; - logRecord.timing = { - ...logRecord.timing, - contenCaptured: Date.now() - domContentCapturingStart, - }; - - // [END puppeteer-block] - } catch (e) { - if (e.message.startsWith('net::ERR_BLOCKED_BY_CLIENT at ')) { - logRecord.blockedByClient = true; - } else { - logRecord.puppeteerSuccess = false; - logRecord.puppeteerError = { - message: e.message, - stack: e.stack, - }; - } - } - if (domContent === 'IS_BLOCKED') { - return { isBlocked: true }; - } - return { domContent, title }; -} - -module.exports = fetchContent; diff --git a/packages/content-fetch/index.js b/packages/content-fetch/index.js new file mode 100644 index 000000000..a209413e3 --- /dev/null +++ b/packages/content-fetch/index.js @@ -0,0 +1,33 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const Sentry = require('@sentry/serverless'); +const { fetchContent, preview } = require("@omnivore/puppeteer-parse"); + +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}); + +/** + * Cloud Function entry point, HTTP trigger. + * Loads the requested URL via Puppeteer, captures page content and sends it to backend + * + * @param {Object} req Cloud Function request context. + * @param {Object} res Cloud Function response context. + */ +exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(fetchContent); + +/** + * Cloud Function entry point, HTTP trigger. + * Loads the requested URL via Puppeteer and captures a screenshot of the provided element + * + * @param {Object} req Cloud Function request context. + * Inlcudes: + * * url - URL address of the page to open + * @param {Object} res Cloud Function response context. + */ +exports.preview = Sentry.GCPFunction.wrapHttpFunction(preview); diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json index 405f1a27c..f2ae26203 100644 --- a/packages/content-fetch/package.json +++ b/packages/content-fetch/package.json @@ -4,18 +4,20 @@ "description": "Service that fetches page content from a URL", "main": "index.js", "dependencies": { - "@omnivore/content-handler": "1.0.0", - "axios": "^0.27.2", "dotenv": "^8.2.0", "express": "^4.17.1", - "jsonwebtoken": "^8.5.1", - "linkedom": "^0.14.9", - "luxon": "^2.3.1", - "puppeteer-core": "^16.1.0", - "underscore": "^1.13.4" + "@omnivore/puppeteer-parse": "^1.0.0", + "@sentry/serverless": "^6.13.3" + }, + "devDependencies": { + "@google-cloud/functions-framework": "^3.0.0", + "chai": "^4.3.6", + "mocha": "^10.0.0" }, "scripts": { "start": "node app.js", - "test": "yarn mocha" + "start_gcf": "npx functions-framework --port=9090 --target=puppeteer", + "start_preview": "npx functions-framework --target=preview", + "test": "mocha test/*.js" } } diff --git a/packages/content-fetch/test/babel-register.js b/packages/content-fetch/test/babel-register.js deleted file mode 100644 index a6f65f60a..000000000 --- a/packages/content-fetch/test/babel-register.js +++ /dev/null @@ -1,3 +0,0 @@ -const register = require('@babel/register').default - -register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/content-fetch/test/stub.test.js b/packages/content-fetch/test/stub.test.js new file mode 100644 index 000000000..317d21b52 --- /dev/null +++ b/packages/content-fetch/test/stub.test.js @@ -0,0 +1,9 @@ +const chai = require("chai"); + +const expect = chai.expect; + +describe('Stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) diff --git a/packages/content-fetch/test/stub.test.ts b/packages/content-fetch/test/stub.test.ts deleted file mode 100644 index 173ca4917..000000000 --- a/packages/content-fetch/test/stub.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import 'mocha' -import * as chai from 'chai' -import { expect } from 'chai' -import 'chai/register-should' -import chaiString from 'chai-string' - -chai.use(chaiString) - -describe('Stub test', () => { - it('should pass', () => { - expect(true).to.be.true - }) -}) diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index 0265f87d0..4ea3ef705 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -1,7 +1,6 @@ import 'mocha' import * as chai from 'chai' import { expect } from 'chai' -import 'chai/register-should' import chaiString from 'chai-string' import * as fs from 'fs' import { importCsv } from '../../src/csv' diff --git a/packages/puppeteer-parse/.dockerignore b/packages/puppeteer-parse/.dockerignore deleted file mode 100644 index 2310bc768..000000000 --- a/packages/puppeteer-parse/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -node_modules -.env* -Dockerfile -.dockerignore diff --git a/packages/puppeteer-parse/.gitignore b/packages/puppeteer-parse/.gitignore deleted file mode 100644 index 9bfbc5e8b..000000000 --- a/packages/puppeteer-parse/.gitignore +++ /dev/null @@ -1 +0,0 @@ -previewImage.* \ No newline at end of file diff --git a/packages/puppeteer-parse/Dockerfile b/packages/puppeteer-parse/Dockerfile deleted file mode 100644 index a52f55122..000000000 --- a/packages/puppeteer-parse/Dockerfile +++ /dev/null @@ -1,113 +0,0 @@ -# FROM node:14-slim - -# # Taken from pu - -# # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) -# # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer -# # installs, work. -# RUN apt-get update \ -# && apt-get install -y wget gnupg \ -# && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ -# && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ -# && apt-get update \ -# && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \ -# --no-install-recommends \ -# && rm -rf /var/lib/apt/lists/* - -# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true -# ENV CHROMIUM_PATH "/usr/bin/google-chrome-stable" - -# ------------------------ - -# FROM --platform=linux/arm64 node:14.18 - -# RUN apt-get update \ -# && apt-get install -y chromium \ -# && apt-get install -y ca-certificates \ -# fonts-liberation \ -# libappindicator3-1 \ -# libasound2 \ -# libatk-bridge2.0-0 \ -# libatk1.0-0 \ -# libc6 \ -# libcairo2 \ -# libcups2 \ -# libdbus-1-3 \ -# libexpat1 \ -# libfontconfig1 \ -# libgbm1 \ -# libgcc1 \ -# libglib2.0-0 \ -# libgtk-3-0 \ -# libnspr4 \ -# libnss3 \ -# libpango-1.0-0 \ -# libpangocairo-1.0-0 \ -# libstdc++6 \ -# libx11-6 \ -# libx11-xcb1 \ -# libxcb1 \ -# libxcomposite1 \ -# libxcursor1 \ -# libxdamage1 \ -# libxext6 \ -# libxfixes3 \ -# libxi6 \ -# libxrandr2 \ -# libxrender1 \ -# libxss1 \ -# libxtst6 \ -# lsb-release \ -# wget \ -# xdg-utils - -FROM node:14.18-alpine - -# Installs latest Chromium (92) package. -RUN apk add --no-cache \ - chromium \ - nss \ - freetype \ - harfbuzz \ - ca-certificates \ - ttf-freefont \ - nodejs \ - yarn - -# Add user so we don't need --no-sandbox. -RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ - && mkdir -p /home/pptruser/Downloads /app \ - && chown -R pptruser:pptruser /home/pptruser \ - && chown -R pptruser:pptruser /app - -# Run everything after as non-privileged user. -WORKDIR /app - -ENV CHROMIUM_PATH /usr/bin/chromium-browser -ENV LAUNCH_HEADLESS=true -ENV PORT 9090 - -COPY package.json . -COPY yarn.lock . -COPY tsconfig.json . -COPY .prettierrc . -COPY .eslintrc . - -COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json -COPY /packages/content-handler/package.json ./packages/content-handler/package.json - -RUN yarn install --pure-lockfile - -ADD /packages/puppeteer-parse ./packages/puppeteer-parse -ADD /packages/content-handler ./packages/content-handler -RUN yarn workspace @omnivore/content-handler build - -# After building, fetch the production dependencies -RUN rm -rf /app/packages/puppeteer-parse/node_modules -RUN rm -rf /app/node_modules -RUN yarn install --pure-lockfile --production - -EXPOSE 9090 - -# USER pptruser -ENTRYPOINT ["yarn", "workspace", "@omnivore/puppeteer-parse", "start"] diff --git a/packages/puppeteer-parse/README.md b/packages/puppeteer-parse/README.md index 501c004ee..6155c45b0 100644 --- a/packages/puppeteer-parse/README.md +++ b/packages/puppeteer-parse/README.md @@ -1,24 +1,3 @@ # Puppeteer parsing function handler -This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer. - -## Using locally - -Copy .env.example file to .env file: `cp .env.example .env` - -Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading). - -After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) - -## Deployment - -To deploy the function use the following command: - -`gcloud functions deploy puppeteer --runtime nodejs12 --trigger-http --memory 1GB --set-env-vars REST_BACKEND_ENDPOINT=,JWT_SECRET=` - - -where: - -`` - address of the backend server (e.g "http://localhost:4000") - -`` - JWT secret that the backend server is using (e.g "some_secret") +This workspace is used to provide the module for the app to hande requests for the article parsing via Puppeteer. diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index b602ff1ef..cd72936af 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -3,8 +3,8 @@ /* eslint-disable @typescript-eslint/explicit-function-return-type */ /* eslint-disable @typescript-eslint/no-var-requires */ /* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); const Url = require('url'); +// const puppeteer = require('puppeteer-extra'); const axios = require('axios'); const jwt = require('jsonwebtoken'); const { promisify } = require('util'); @@ -13,10 +13,8 @@ const { config, format, loggers, transports } = require('winston'); const { LoggingWinston } = require('@google-cloud/logging-winston'); const { DateTime } = require('luxon'); const os = require('os'); -const Sentry = require('@sentry/serverless'); const { Storage } = require('@google-cloud/storage'); - -const chromium = require('chrome-aws-lambda'); +const { parseHTML } = require('linkedom'); const puppeteer = require('puppeteer-core'); const { preHandleContent } = require("@omnivore/content-handler"); @@ -28,20 +26,7 @@ const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.env.PREVIEW_IMAGE_BUCKET) : undefined; -Sentry.GCPFunction.init({ - dsn: process.env.SENTRY_DSN, - tracesSampleRate: 0, -}); - -const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' -const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] - const filePath = `${os.tmpdir()}/previewImage.png`; -const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; - const colors = { emerg: 'inverse underline magenta', @@ -102,6 +87,15 @@ function buildLogger(id, options) { }); } +const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' +const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' +const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' +const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' +const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] +const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; + +const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; + const userAgentForUrl = (url) => { try { const u = new URL(url); @@ -116,15 +110,38 @@ const userAgentForUrl = (url) => { return DESKTOP_USER_AGENT }; +const fetchContentWithScrapingBee = async (url) => { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + 'api_key': process.env.SCRAPINGBEE_API_KEY, + 'url': url, + 'render_js': 'false', + 'premium_proxy': 'true', + 'country_code':'us' + } + }) + + const dom = parseHTML(response.data).document; + return { title: dom.title, domContent: dom.documentElement.outerHTML, url: url } +} + +const enableJavascriptForUrl = (url) => { + try { + const u = new URL(url); + for (const host of NON_SCRIPT_HOSTS) { + if (u.hostname.endsWith(host)) { + return false; + } + } + } catch (e) { + console.log('error getting hostname for url', url, e) + } + return true +}; + // launch Puppeteer const getBrowserPromise = (async () => { - // return puppeteer.launch({ - // args: chromium.args, - // defaultViewport: chromium.defaultViewport, - // executablePath: process.env.CHROMIUM_PATH, - // headless: chromium.headless, - // ignoreHTTPSErrors: true, - // }); + console.log("starting with proxy url", process.env.PROXY_URL) return puppeteer.launch({ args: [ '--allow-running-insecure-content', @@ -170,7 +187,7 @@ const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentOb }) }; -const getUploadIdAndSignedUrl = async (userId, url) => { +const getUploadIdAndSignedUrl = async (userId, url, articleSavingRequestId) => { const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); const data = JSON.stringify({ query: `mutation UploadFileRequest($input: UploadFileRequestInput!) { @@ -188,17 +205,18 @@ const getUploadIdAndSignedUrl = async (userId, url) => { input: { url, contentType: 'application/pdf', + clientRequestId: articleSavingRequestId, } } }); const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + }); return response.data.data.uploadFileRequest; }; @@ -231,12 +249,12 @@ const sendCreateArticleMutation = async (userId, input) => { const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, - { - headers: { - Cookie: `auth=${auth};`, - 'Content-Type': 'application/json', - }, - }); + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + }); return response.data.data.createArticle; }; @@ -249,14 +267,7 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId ); }; -/** - * Cloud Function entry point, HTTP trigger. - * Loads the requested URL via Puppeteer, captures page content and sends it to backend - * - * @param {Object} req Cloud Function request context. - * @param {Object} res Cloud Function response context. - */ -exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { +async function fetchContent(req, res) { functionStartTime = Date.now(); // Grabbing execution and trace ids to attach logs to the appropriate function call const execution_id = req.get('function-execution-id'); @@ -269,7 +280,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { }); let url = getUrl(req); - const userId = req.body.userId || req.query.userId; + const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); logRecord = { @@ -285,7 +296,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { if (!url) { logRecord.urlIsInvalid = true; - logger.error(`Valid URL to parse not specified`, logRecord); + logger.info(`Valid URL to parse not specified`, logRecord); return res.sendStatus(400); } @@ -301,7 +312,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { if (result && result.content) { content = result.content } if (result && result.contentType) { contentType = result.contentType } } catch (e) { - console.log('error with handler: ', e); + logger.info('error with handler: ', e); } let context, page, finalUrl; @@ -312,7 +323,6 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { if (result && result.page) { page = result.page } if (result && result.finalUrl) { finalUrl = result.finalUrl } if (result && result.contentType) { contentType = result.contentType } - console.log('context, page, finalUrl, contentType', context, page, finalUrl, contentType); } else { finalUrl = url } @@ -323,14 +333,19 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { } else { if (!content || !title) { const result = await retrieveHtml(page); - title = result.title; - content = result.domContent; + if (result.isBlocked) { + const sbResult = await fetchContentWithScrapingBee(url) + title = sbResult.title + content = sbResult.domContent + } else { + title = result.title; + content = result.domContent; + } } else { - console.log('using prefetched content and title'); - console.log(content); + logger.info('using prefetched content and title'); } - logRecord.contentFetchTime = Date.now() - functionStartTime; + logRecord.fetchContentTime = Date.now() - functionStartTime; const apiResponse = await sendCreateArticleMutation(userId, { url: finalUrl, @@ -347,33 +362,351 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; - logger.info(`parse-page`, logRecord); } } catch (e) { - console.log('error', e) logRecord.error = e.message; logger.error(`Error while retrieving page`, logRecord); - return res.sendStatus(503); + + // fallback to scrapingbee + const sbResult = await fetchContentWithScrapingBee(url); + const sbUrl = finalUrl || sbResult.url; + const content = sbResult.domContent; + logRecord.fetchContentTime = Date.now() - functionStartTime; + + const apiResponse = await sendCreateArticleMutation(userId, { + url: sbUrl, + articleSavingRequestId, + preparedDocument: { + document: content, + pageInfo: { + title: sbResult.title, + canonicalUrl: sbUrl, + }, + }, + skipParsing: !content, + }); + + logRecord.totalTime = Date.now() - functionStartTime; + logRecord.result = apiResponse.createArticle; } finally { if (context) { await context.close(); } + logger.info(`parse-page`, logRecord); } return res.sendStatus(200); -}); +} -/** - * Cloud Function entry point, HTTP trigger. - * Loads the requested URL via Puppeteer and captures a screenshot of the provided element - * - * @param {Object} req Cloud Function request context. - * Inlcudes: - * * url - URL address of the page to open - * @param {Object} res Cloud Function response context. - */ -exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { - functionStartTime = Date.now(); +function validateUrlString(url) { + const u = new URL(url); + // Make sure the URL is http or https + if (u.protocol !== 'http:' && u.protocol !== 'https:') { + throw new Error('Invalid URL protocol check failed') + } + // Make sure the domain is not localhost + if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') { + throw new Error('Invalid URL is localhost') + } + // Make sure the domain is not a private IP + if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) { + throw new Error('Invalid URL is private ip') + } +} + +function getUrl(req) { + const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined); + if (!urlStr) { + throw new Error('No URL specified'); + } + + validateUrlString(urlStr); + + const parsed = Url.parse(urlStr); + return parsed.href; +} + +async function blockResources(client) { + const blockedResources = [ + // Assets + // '*/favicon.ico', + // '.css', + // '.jpg', + // '.jpeg', + // '.png', + // '.svg', + // '.woff', + + // Analytics and other fluff + '*.optimizely.com', + 'everesttech.net', + 'userzoom.com', + 'doubleclick.net', + 'googleadservices.com', + 'adservice.google.com/*', + 'connect.facebook.com', + 'connect.facebook.net', + 'sp.analytics.yahoo.com', + ] + + await client.send('Network.setBlockedURLs', { urls: blockedResources }); +} + +async function retrievePage(url) { + validateUrlString(url); + + const browser = await getBrowserPromise; + logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; + + const context = await browser.createIncognitoBrowserContext(); + const page = await context.newPage() + + if (!enableJavascriptForUrl(url)) { + await page.setJavaScriptEnabled(false); + } + await page.setUserAgent(userAgentForUrl(url)); + + const client = await page.target().createCDPSession(); + + // intercept request when response headers was received + await client.send('Network.setRequestInterception', { + patterns: [ + { + urlPattern: '*', + resourceType: 'Document', + interceptionStage: 'HeadersReceived', + }, + ], + }); + + const path = require('path'); + const download_path = path.resolve('./download_dir/'); + + await client.send('Page.setDownloadBehavior', { + behavior: 'allow', + userDataDir: './', + downloadPath: download_path, + }) + + client.on('Network.requestIntercepted', async e => { + const headers = e.responseHeaders || {}; + + const [contentType] = (headers['content-type'] || headers['Content-Type'] || '') + .toLowerCase() + .split(';'); + const obj = { interceptionId: e.interceptionId }; + + if (e.responseStatusCode >= 200 && e.responseStatusCode < 300) { + // We only check content-type on success responses + // as it doesn't matter what the content type is for things + // like redirects + if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) { + obj['errorReason'] = 'BlockedByClient'; + } + } + + try { + await client.send('Network.continueInterceptedRequest', obj); + // eslint-disable-next-line no-empty + } catch {} + }); + + await blockResources(client); + + /* + * Disallow MathJax from running in Puppeteer and modifying the document, + * we shall instead run it in our frontend application to transform any + * mathjax content when present. + */ + await page.setRequestInterception(true); + let requestCount = 0; + page.on('request', request => { + if (['font', 'image', 'media'].includes(request.resourceType())) { + request.abort(); + return; + } + if (requestCount++ > 100) { + request.abort(); + return; + } + if ( + request.resourceType() === 'script' && + request.url().toLowerCase().indexOf('mathjax') > -1 + ) { + request.abort(); + return + } + request.continue(); + }); + + // Puppeteer fails during download of PDf files, + // so record the failure and use those items + let lastPdfUrl = undefined; + page.on('response', response => { + if (response.headers()['content-type'] === 'application/pdf') { + lastPdfUrl = response.url(); + } + }); + + try { + const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] }); + const finalUrl = response.url(); + const contentType = response.headers()['content-type']; + + logRecord.finalUrl = response.url(); + logRecord.contentType = response.headers()['content-type']; + + return { context, page, response, finalUrl, contentType }; + } catch (error) { + if (lastPdfUrl) { + return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' }; + } + await context.close(); + throw error; + } +} + +async function retrieveHtml(page) { + let domContent = '', title; + try { + title = await page.title(); + logRecord.title = title; + + const pageScrollingStart = Date.now(); + /* scroll with a 5 second timeout */ + await Promise.race([ + new Promise(resolve => { + (async function () { + try { + await page.evaluate(`(async () => { + /* credit: https://github.com/puppeteer/puppeteer/issues/305 */ + return new Promise((resolve, reject) => { + let scrollHeight = document.body.scrollHeight; + let totalHeight = 0; + let distance = 500; + let timer = setInterval(() => { + window.scrollBy(0, distance); + totalHeight += distance; + if(totalHeight >= scrollHeight){ + clearInterval(timer); + resolve(true); + } + }, 10); + }); + })()`); + } catch (e) { + logRecord.scrollError = true; + } finally { + resolve(true); + } + })(); + }), + await page.waitForTimeout(1000), + ]); + logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; + + const iframes = {}; + const urls = []; + const framesPromises = []; + const allowedUrls = /instagram\.com/gi; + + for (const frame of page.mainFrame().childFrames()) { + if (frame.url() && allowedUrls.test(frame.url())) { + urls.push(frame.url()); + framesPromises.push(frame.evaluate(el => el.innerHTML, await frame.$('body'))); + } + } + + (await Promise.all(framesPromises)).forEach((frame, index) => (iframes[urls[index]] = frame)); + + const domContentCapturingStart = Date.now(); + // get document body with all hidden elements removed + domContent = await page.evaluate(iframes => { + const BI_SRC_REGEXP = /url\("(.+?)"\)/gi; + + Array.from(document.body.getElementsByTagName('*')).forEach(el => { + const style = window.getComputedStyle(el); + + try { + // Removing blurred images since they are mostly the copies of lazy loaded ones + if (['img', 'image'].includes(el.tagName.toLowerCase())) { + const filter = style.getPropertyValue('filter'); + if (filter && filter.startsWith('blur')) { + el.parentNode && el.parentNode.removeChild(el); + } + } + } catch (err) { + // throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*')))) + } + + // convert all nodes with background image to img nodes + if (!['', 'none'].includes(style.getPropertyValue('background-image'))) { + const filter = style.getPropertyValue('filter'); + // avoiding image nodes with a blur effect creation + if (filter && filter.startsWith('blur')) { + el && el.parentNode && el.parentNode.removeChild(el); + } else { + const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image')); + // Using "g" flag with a regex we have to manually break down lastIndex to zero after every usage + // More details here: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results + BI_SRC_REGEXP.lastIndex = 0; + + if (matchedSRC && matchedSRC[1] && !el.src) { + // Replacing element only of there are no content inside, b/c might remove important div with content. + // Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html + // DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image. + if (el.innerHTML.length < 25) { + const img = document.createElement('img'); + img.src = matchedSRC[1]; + el && el.parentNode && el.parentNode.removeChild(el); + } + } + } + } + + if (el.tagName === 'IFRAME') { + if (iframes[el.src]) { + const newNode = document.createElement('div'); + newNode.className = 'omnivore-instagram-embed'; + newNode.innerHTML = iframes[el.src]; + el && el.parentNode && el.parentNode.replaceChild(newNode, el); + } + } + }); + + if (document.querySelector('[data-translate="managed_checking_msg"]') || + document.getElementById('px-block-form-wrapper')) { + return 'IS_BLOCKED' + } + + return document.documentElement.outerHTML; + }, iframes); + logRecord.puppeteerSuccess = true; + logRecord.timing = { + ...logRecord.timing, + contenCaptured: Date.now() - domContentCapturingStart, + }; + + // [END puppeteer-block] + } catch (e) { + if (e.message.startsWith('net::ERR_BLOCKED_BY_CLIENT at ')) { + logRecord.blockedByClient = true; + } else { + logRecord.puppeteerSuccess = false; + logRecord.puppeteerError = { + message: e.message, + stack: e.stack, + }; + } + } + if (domContent === 'IS_BLOCKED') { + return { isBlocked: true }; + } + return { domContent, title }; +} + +async function preview(req, res) { + const functionStartTime = Date.now(); // Grabbing execution and trace ids to attach logs to the appropriate function call const execution_id = req.get('function-execution-id'); const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0]; @@ -392,7 +725,7 @@ exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { const url = getUrl(req); console.log('preview request url', url); - logRecord = { + const logRecord = { url, query: req.query, origin: req.get('Origin'), @@ -415,7 +748,7 @@ exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { return res.sendStatus(400); } - const browser = await getBrowserPromise; + const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH); logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; const page = await browser.newPage(); @@ -490,296 +823,10 @@ exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { logger.info(`preview-image`, logRecord); return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); -}); - -function validateUrlString(url) { - const u = new URL(url); - // Make sure the URL is http or https - if (u.protocol !== 'http:' && u.protocol !== 'https:') { - throw new Error('Invalid URL protocol check failed') - } - // Make sure the domain is not localhost - if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') { - throw new Error('Invalid URL is localhost') - } - // Make sure the domain is not a private IP - if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) { - throw new Error('Invalid URL is private ip') - } } -function getUrl(req) { - if (req.query.url || req.body.url) { - const urlStr = req.query.url || req.body.url; - validateUrlString(urlStr); +module.exports = { + fetchContent, + preview, +}; - const url = Url.parse(urlStr); - return url.href; - } - try { - return Url.parse(JSON.parse(req.body).url).href; - } catch (e) {} -} - -async function blockResources(client) { - const blockedResources = [ - // Assets - // '*/favicon.ico', - // '.css', - // '.jpg', - // '.jpeg', - // '.png', - // '.svg', - // '.woff', - - // Analytics and other fluff - '*.optimizely.com', - 'everesttech.net', - 'userzoom.com', - 'doubleclick.net', - 'googleadservices.com', - 'adservice.google.com/*', - 'connect.facebook.com', - 'connect.facebook.net', - 'sp.analytics.yahoo.com', - ] - - await client.send('Network.setBlockedURLs', { urls: blockedResources }); -} - -async function retrievePage(url) { - validateUrlString(url); - - const browser = await getBrowserPromise; - logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; - - const context = await browser.createIncognitoBrowserContext(); - const page = await context.newPage(); - await page.setUserAgent(userAgentForUrl(url)); - - const client = await page.target().createCDPSession(); - - // intercept request when response headers was received - await client.send('Network.setRequestInterception', { - patterns: [ - { - urlPattern: '*', - resourceType: 'Document', - interceptionStage: 'HeadersReceived', - }, - ], - }); - - const path = require('path'); - const download_path = path.resolve('./download_dir/'); - - await client.send('Page.setDownloadBehavior', { - behavior: 'allow', - userDataDir: './', - downloadPath: download_path, - }) - - client.on('Network.requestIntercepted', async e => { - const headers = e.responseHeaders || {}; - - const [contentType] = (headers['content-type'] || headers['Content-Type'] || '') - .toLowerCase() - .split(';'); - const obj = { interceptionId: e.interceptionId }; - - if (e.responseStatusCode >= 200 && e.responseStatusCode < 300) { - // We only check content-type on success responses - // as it doesn't matter what the content type is for things - // like redirects - if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) { - obj['errorReason'] = 'BlockedByClient'; - } - } - - try { - await client.send('Network.continueInterceptedRequest', obj); - // eslint-disable-next-line no-empty - } catch {} - }); - - await blockResources(client); - - /* - * Disallow MathJax from running in Puppeteer and modifying the document, - * we shall instead run it in our frontend application to transform any - * mathjax content when present. - */ - await page.setRequestInterception(true); - let requestCount = 0; - page.on('request', request => { - if (['font', 'image', 'media'].includes(request.resourceType())) { - request.abort(); - return; - } - if (requestCount++ > 100) { - request.abort(); - return; - } - if ( - request.resourceType() === 'script' && - request.url().toLowerCase().indexOf('mathjax') > -1 - ) { - request.abort(); - return - } - request.continue(); - }); - - - // Puppeteer fails during download of PDf files, - // so record the failure and use those items - let lastPdfUrl = undefined; - page.on('response', response => { - if (response.headers()['content-type'] === 'application/pdf') { - lastPdfUrl = response.url(); - } - }); - - try { - const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] }); - const finalUrl = response.url(); - const contentType = response.headers()['content-type']; - - logRecord.finalUrl = response.url(); - logRecord.contentType = response.headers()['content-type']; - - return { context, page, response, finalUrl, contentType }; - } catch (error) { - if (lastPdfUrl) { - return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' }; - } - await context.close(); - throw error; - } -} - -async function retrieveHtml(page) { - let domContent = '', title; - try { - title = await page.title(); - logRecord.title = title; - - const pageScrollingStart = Date.now(); - /* scroll with a 5 second timeout */ - await Promise.race([ - new Promise(resolve => { - (async function () { - try { - await page.evaluate(`(async () => { - /* credit: https://github.com/puppeteer/puppeteer/issues/305 */ - return new Promise((resolve, reject) => { - let scrollHeight = document.body.scrollHeight; - let totalHeight = 0; - let distance = 500; - let timer = setInterval(() => { - window.scrollBy(0, distance); - totalHeight += distance; - if(totalHeight >= scrollHeight){ - clearInterval(timer); - resolve(true); - } - }, 10); - }); - })()`); - } catch (e) { - logRecord.scrollError = true; - } finally { - resolve(true); - } - })(); - }), - await page.waitForTimeout(1000), // 1 second timeout - ]); - logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; - - const iframes = {}; - const urls = []; - const framesPromises = []; - const allowedUrls = /instagram\.com/gi; - - for (const frame of page.mainFrame().childFrames()) { - if (frame.url() && allowedUrls.test(frame.url())) { - urls.push(frame.url()); - framesPromises.push(frame.evaluate(el => el.innerHTML, await frame.$('body'))); - } - } - - (await Promise.all(framesPromises)).forEach((frame, index) => (iframes[urls[index]] = frame)); - - const domContentCapturingStart = Date.now(); - // get document body with all hidden elements removed - domContent = await page.evaluate(iframes => { - const BI_SRC_REGEXP = /url\("(.+?)"\)/gi; - - Array.from(document.body.getElementsByTagName('*')).forEach(el => { - const style = window.getComputedStyle(el); - - // Removing blurred images since they are mostly the copies of lazy loaded ones - if (['img', 'image'].includes(el.tagName.toLowerCase())) { - const filter = style.getPropertyValue('filter'); - if (filter && filter.startsWith('blur')) { - el.parentNode && el.parentNode.removeChild(el); - } - } - - // convert all nodes with background image to img nodes - if (!['', 'none'].includes(style.getPropertyValue('background-image'))) { - const filter = style.getPropertyValue('filter'); - // avoiding image nodes with a blur effect creation - if (filter && filter.startsWith('blur')) { - el && el.parentNode && el.parentNode.removeChild(el); - } else { - const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image')); - // Using "g" flag with a regex we have to manually break down lastIndex to zero after every usage - // More details here: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results - BI_SRC_REGEXP.lastIndex = 0; - - if (matchedSRC && matchedSRC[1] && !el.src) { - // Replacing element only of there are no content inside, b/c might remove important div with content. - // Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html - // DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image. - if (el.innerHTML.length < 25) { - const img = document.createElement('img'); - img.src = matchedSRC[1]; - el && el.parentNode && el.parentNode.removeChild(el); - } - } - } - } - - if (el.tagName === 'IFRAME') { - if (iframes[el.src]) { - const newNode = document.createElement('div'); - newNode.className = 'omnivore-instagram-embed'; - newNode.innerHTML = iframes[el.src]; - el && el.parentNode && el.parentNode.replaceChild(newNode, el); - } - } - }); - return document.documentElement.outerHTML; - }, iframes); - logRecord.puppeteerSuccess = true; - logRecord.timing = { - ...logRecord.timing, - contenCaptured: Date.now() - domContentCapturingStart, - }; - - // [END puppeteer-block] - } catch (e) { - if (e.message.startsWith('net::ERR_BLOCKED_BY_CLIENT at ')) { - logRecord.blockedByClient = true; - } else { - logRecord.puppeteerSuccess = false; - logRecord.puppeteerError = { - message: e.message, - stack: e.stack, - }; - } - } - return { domContent, title }; -} diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 4ef6bb4d9..a9537f6f2 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -1,17 +1,13 @@ { "name": "@omnivore/puppeteer-parse", "version": "1.0.0", - "description": "Google Cloud Function that accepts URL of the article and parses its content", + "description": "Accepts URL of the article and parses its content", "main": "index.js", "dependencies": { - "@google-cloud/functions-framework": "^3.1.2", "@google-cloud/logging-winston": "^5.1.1", "@google-cloud/storage": "^5.18.1", "@omnivore/content-handler": "1.0.0", - "@sentry/serverless": "^6.13.3", "axios": "^0.27.2", - "chrome-aws-lambda": "^10.1.0", - "dotenv": "^8.2.0", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.9", "luxon": "^2.3.1", @@ -19,9 +15,11 @@ "underscore": "^1.13.4", "winston": "^3.3.3" }, + "devDependencies": { + "chai": "^4.3.6", + "mocha": "^10.0.0" + }, "scripts": { - "start": "npx functions-framework --port=9090 --target=puppeteer", - "start_preview": "npx functions-framework --target=preview", - "test": "yarn mocha" + "test": "mocha test/*.js" } } diff --git a/packages/puppeteer-parse/test/babel-register.js b/packages/puppeteer-parse/test/babel-register.js deleted file mode 100644 index a6f65f60a..000000000 --- a/packages/puppeteer-parse/test/babel-register.js +++ /dev/null @@ -1,3 +0,0 @@ -const register = require('@babel/register').default - -register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/puppeteer-parse/test/stub.test.js b/packages/puppeteer-parse/test/stub.test.js new file mode 100644 index 000000000..317d21b52 --- /dev/null +++ b/packages/puppeteer-parse/test/stub.test.js @@ -0,0 +1,9 @@ +const chai = require("chai"); + +const expect = chai.expect; + +describe('Stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) diff --git a/packages/puppeteer-parse/test/stub.test.ts b/packages/puppeteer-parse/test/stub.test.ts deleted file mode 100644 index 173ca4917..000000000 --- a/packages/puppeteer-parse/test/stub.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import 'mocha' -import * as chai from 'chai' -import { expect } from 'chai' -import 'chai/register-should' -import chaiString from 'chai-string' - -chai.use(chaiString) - -describe('Stub test', () => { - it('should pass', () => { - expect(true).to.be.true - }) -}) diff --git a/yarn.lock b/yarn.lock index b1b9a214d..3f3923d58 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2444,7 +2444,7 @@ google-gax "^2.24.1" protobufjs "^6.8.6" -"@google-cloud/functions-framework@3.1.2", "@google-cloud/functions-framework@^3.1.2": +"@google-cloud/functions-framework@3.1.2", "@google-cloud/functions-framework@^3.0.0": version "3.1.2" resolved "https://registry.yarnpkg.com/@google-cloud/functions-framework/-/functions-framework-3.1.2.tgz#2cd92ce4307bf7f32555d028dca22e398473b410" integrity sha512-pYvEH65/Rqh1JNPdcBmorcV7Xoom2/iOSmbtYza8msro7Inl+qOYxbyMiQfySD2gwAyn38WyWPRqsDRcf/BFLg== @@ -10907,13 +10907,6 @@ chownr@^2.0.0: resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ== -chrome-aws-lambda@^10.1.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/chrome-aws-lambda/-/chrome-aws-lambda-10.1.0.tgz#ac43b4cdfc1fbb2275c62effada560858099501e" - integrity sha512-NZQVf+J4kqG4sVhRm3WNmOfzY0OtTSm+S8rg77pwePa9RCYHzhnzRs8YvNI6L9tALIW6RpmefWiPURt3vURXcw== - dependencies: - lambdafs "^2.0.3" - chrome-trace-event@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz#1015eced4741e15d06664a957dbbf50d041e26ac" @@ -17677,13 +17670,6 @@ kuler@^2.0.0: resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A== -lambdafs@^2.0.3: - version "2.1.1" - resolved "https://registry.yarnpkg.com/lambdafs/-/lambdafs-2.1.1.tgz#4bf8d3037b6c61bbb4a22ab05c73ee47964c25ed" - integrity sha512-x5k8JcoJWkWLvCVBzrl4pzvkEHSgSBqFjg3Dpsc4AcTMq7oUMym4cL/gRTZ6VM4mUMY+M0dIbQ+V1c1tsqqanQ== - dependencies: - tar-fs "^2.1.1" - language-subtag-registry@~0.3.2: version "0.3.21" resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.21.tgz#04ac218bea46f04cb039084602c6da9e788dd45a" @@ -23988,7 +23974,7 @@ tar-fs@2.0.0: pump "^3.0.0" tar-stream "^2.0.0" -tar-fs@2.1.1, tar-fs@^2.1.1: +tar-fs@2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==