prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #899 from omnivore-app/isolate-puppeteer

Browse source 
isolate puppeteer
This commit is contained in:
Hongbo Wu 2022-11-16 10:22:18 +08:00 committed by GitHub
commit 3f69b42c05
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 574 additions and 1209 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/run-tests.yaml vendored
View file

@ -90,5 +90,5 @@ jobs:
run: 'docker build --file packages/content-fetch/Dockerfile .'
- name: Build the inbound-email-handler docker image
run: 'docker build --file packages/inbound-email-handler/Dockerfile .'
- name: Build the puppeteer-parse docker image
run: 'docker build --file packages/puppeteer-parse/Dockerfile .'
- name: Build the content-fetch cloud function docker image
run: 'docker build --file packages/content-fetch/Dockerfile-gcf .'

10
packages/puppeteer-parse/.env.example → packages/content-fetch/.env.example
View file

@ -6,3 +6,13 @@ REST_BACKEND_ENDPOINT=http://localhost:4000/api
# set for local development
IS_LOCAL=true
VERIFICATION_TOKEN=some_token
CHROMIUM_PATH=/opt/homebrew/bin/chromium
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
LAUNCH_HEADLESS=true
TWITTER_BEARER_TOKEN=token
PORT=9090

0
packages/puppeteer-parse/.gcloudignore → packages/content-fetch/.gcloudignore
View file

2
packages/content-fetch/Dockerfile
View file

@ -30,11 +30,13 @@ COPY .prettierrc .
COPY .eslintrc .
COPY /packages/content-handler/package.json ./packages/content-handler/package.json
COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json
RUN yarn install --pure-lockfile
ADD /packages/content-fetch ./packages/content-fetch
ADD /packages/content-handler ./packages/content-handler
ADD /packages/puppeteer-parse ./packages/puppeteer-parse
RUN yarn workspace @omnivore/content-handler build
# After building, fetch the production dependencies

51
packages/content-fetch/Dockerfile-gcf Normal file
View file

@ -0,0 +1,51 @@
FROM node:14.18-alpine
# Installs latest Chromium (92) package.
RUN apk add --no-cache \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont \
nodejs \
yarn
# Add user so we don't need --no-sandbox.
RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \
&& mkdir -p /home/pptruser/Downloads /app \
&& chown -R pptruser:pptruser /home/pptruser \
&& chown -R pptruser:pptruser /app
# Run everything after as non-privileged user.
WORKDIR /app
ENV CHROMIUM_PATH /usr/bin/chromium-browser
ENV LAUNCH_HEADLESS=true
ENV PORT 9090
COPY package.json .
COPY yarn.lock .
COPY tsconfig.json .
COPY .prettierrc .
COPY .eslintrc .
COPY /packages/content-handler/package.json ./packages/content-handler/package.json
COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json
RUN yarn install --pure-lockfile
ADD /packages/content-handler ./packages/content-handler
ADD /packages/puppeteer-parse ./packages/puppeteer-parse
ADD /packages/content-fetch ./packages/content-fetch
RUN yarn workspace @omnivore/content-handler build
# After building, fetch the production dependencies
RUN rm -rf /app/packages/content-fetch/node_modules
RUN rm -rf /app/node_modules
RUN yarn install --pure-lockfile --production
EXPOSE 9090
# USER pptruser
ENTRYPOINT ["yarn", "workspace", "@omnivore/content-fetch", "start_gcf"]

24
packages/content-fetch/README.md Normal file
View file

@ -0,0 +1,24 @@
# Puppeteer parsing function handler
This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer.
## Using locally
Copy .env.example file to .env file: `cp .env.example .env`
Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading).
After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer)
## Deployment
To deploy the function use the following command:
`gcloud functions deploy puppeteer --runtime nodejs12 --trigger-http --memory 1GB --set-env-vars REST_BACKEND_ENDPOINT=<backend-address>,JWT_SECRET=<jwt-secret>`
where:
`<backend-address>` - address of the backend server (e.g "http://localhost:4000")
`<jwt-secret>` - JWT secret that the backend server is using (e.g "some_secret")

13
packages/content-fetch/app.js
View file

@ -1,7 +1,8 @@
require('dotenv').config();
const express = require('express');
const app = express();
const fetchContent = require('./fetch-content');
const { fetchContent } = require("@omnivore/puppeteer-parse");
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
@ -10,22 +11,22 @@ if (!process.env.VERIFICATION_TOKEN) {
throw new Error('VERIFICATION_TOKEN environment variable is not set');
}
app.get('/', (req, res) => {
app.get('/', async (req, res) => {
if (req.query.token !== process.env.VERIFICATION_TOKEN) {
console.log('query does not include valid token')
res.send(403)
return
}
fetchContent(req, res)
await fetchContent(req, res)
});
app.post('/', (req, res) => {
app.post('/', async (req, res) => {
if (req.query.token !== process.env.VERIFICATION_TOKEN) {
console.log('query does not include valid token')
res.send(403)
return
}
fetchContent(req, res)
await fetchContent(req, res)
});
const PORT = parseInt(process.env.PORT) || 8080;
@ -34,4 +35,4 @@ app.listen(PORT, () => {
console.log('Press Ctrl+C to quit.');
});
module.exports = app;
module.exports = app;

635
packages/content-fetch/fetch-content.js
View file

@ -1,635 +0,0 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const Url = require('url');
const puppeteer = require('puppeteer-core');
const axios = require('axios');
const jwt = require('jsonwebtoken');
const { promisify } = require('util');
const { parseHTML } = require('linkedom');
const { preHandleContent } = require('@omnivore/content-handler');
const signToken = promisify(jwt.sign);
const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com']
const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com'];
const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf'];
// Add stealth plugin to hide puppeteer usage
// const StealthPlugin = require('puppeteer-extra-plugin-stealth');
// puppeteer.use(StealthPlugin());
const userAgentForUrl = (url) => {
try {
const u = new URL(url);
for (const host of NON_BOT_HOSTS) {
if (u.hostname.endsWith(host)) {
return NON_BOT_DESKTOP_USER_AGENT;
}
}
} catch (e) {
console.log('error getting user agent for url', url, e)
}
return DESKTOP_USER_AGENT
};
const fetchContentWithScrapingBee = async (url) => {
const response = await axios.get('https://app.scrapingbee.com/api/v1', {
params: {
'api_key': process.env.SCRAPINGBEE_API_KEY,
'url': url,
'render_js': 'false',
'premium_proxy': 'true',
'country_code':'us'
}
})
const dom = parseHTML(response.data).document;
return { title: dom.title, domContent: dom.documentElement.outerHTML, url: url }
}
const enableJavascriptForUrl = (url) => {
try {
const u = new URL(url);
for (const host of NON_SCRIPT_HOSTS) {
if (u.hostname.endsWith(host)) {
return false;
}
}
} catch (e) {
console.log('error getting hostname for url', url, e)
}
return true
};
// launch Puppeteer
const getBrowserPromise = (async () => {
console.log("starting with proxy url", process.env.PROXY_URL)
return puppeteer.launch({
args: [
'--allow-running-insecure-content',
'--autoplay-policy=user-gesture-required',
'--disable-component-update',
'--disable-domain-reliability',
'--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process',
'--disable-print-preview',
'--disable-setuid-sandbox',
'--disable-site-isolation-trials',
'--disable-speech-api',
'--disable-web-security',
'--disk-cache-size=33554432',
'--enable-features=SharedArrayBuffer',
'--hide-scrollbars',
'--ignore-gpu-blocklist',
'--in-process-gpu',
'--mute-audio',
'--no-default-browser-check',
'--no-pings',
'--no-sandbox',
'--no-zygote',
'--use-gl=swiftshader',
'--window-size=1920,1080',
].filter((item) => !!item),
defaultViewport: { height: 1080, width: 1920 },
executablePath: process.env.CHROMIUM_PATH,
headless: !!process.env.LAUNCH_HEADLESS,
timeout: 120000, // 2 minutes
});
})();
let logRecord, functionStartTime;
const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => {
const stream = await axios.get(contentObjUrl, { responseType: 'stream' });
return await axios.put(uploadSignedUrl, stream.data, {
headers: {
'Content-Type': contentType,
},
maxBodyLength: 1000000000,
maxContentLength: 100000000,
})
};
const getUploadIdAndSignedUrl = async (userId, url, articleSavingRequestId) => {
const auth = await signToken({ uid: userId }, process.env.JWT_SECRET);
const data = JSON.stringify({
query: `mutation UploadFileRequest($input: UploadFileRequestInput!) {
uploadFileRequest(input:$input) {
... on UploadFileRequestError {
errorCodes
}
... on UploadFileRequestSuccess {
id
uploadSignedUrl
}
}
}`,
variables: {
input: {
url,
contentType: 'application/pdf',
clientRequestId: articleSavingRequestId,
}
}
});
const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data,
{
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
});
return response.data.data.uploadFileRequest;
};
const uploadPdf = async (url, userId, articleSavingRequestId) => {
validateUrlString(url);
const uploadResult = await getUploadIdAndSignedUrl(userId, url, articleSavingRequestId);
await uploadToSignedUrl(uploadResult, 'application/pdf', url);
return uploadResult.id;
};
const sendCreateArticleMutation = async (userId, input) => {
const data = JSON.stringify({
query: `mutation CreateArticle ($input: CreateArticleInput!){
createArticle(input:$input){
... on CreateArticleSuccess{
createdArticle{
id
}
}
... on CreateArticleError{
errorCodes
}
}
}`,
variables: {
input: Object.assign({}, input , { source: 'puppeteer-parse' }),
},
});
const auth = await signToken({ uid: userId }, process.env.JWT_SECRET);
const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data,
{
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
});
return response.data.data.createArticle;
};
const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId) => {
return sendCreateArticleMutation(userId, {
url: encodeURI(url),
articleSavingRequestId,
uploadFileId: uploadFileId,
},
);
};
async function fetchContent(req, res) {
functionStartTime = Date.now();
let url = getUrl(req);
const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined);
const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined);
console.log('user id', userId, 'url', url)
logRecord = {
url,
userId,
articleSavingRequestId,
labels: {
source: 'parseContent',
},
};
console.log(`Article parsing request`, logRecord);
if (!url) {
logRecord.urlIsInvalid = true;
console.log(`Valid URL to parse not specified`, logRecord);
return res.sendStatus(400);
}
// pre handle url with custom handlers
let title, content, contentType;
try {
const result = await preHandleContent(url);
if (result && result.url) {
url = result.url
validateUrlString(url);
}
if (result && result.title) { title = result.title }
if (result && result.content) { content = result.content }
if (result && result.contentType) { contentType = result.contentType }
} catch (e) {
console.log('error with handler: ', e);
}
let context, page, finalUrl;
try {
if ((!content || !title) && contentType !== 'application/pdf') {
const result = await retrievePage(url)
if (result && result.context) { context = result.context }
if (result && result.page) { page = result.page }
if (result && result.finalUrl) { finalUrl = result.finalUrl }
if (result && result.contentType) { contentType = result.contentType }
} else {
finalUrl = url
}
if (contentType === 'application/pdf') {
const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId);
const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId);
} else {
if (!content || !title) {
const result = await retrieveHtml(page);
if (result.isBlocked) {
const sbResult = await fetchContentWithScrapingBee(url)
title = sbResult.title
content = sbResult.domContent
} else {
title = result.title;
content = result.domContent;
}
} else {
console.log('using prefetched content and title');
}
logRecord.fetchContentTime = Date.now() - functionStartTime;
const apiResponse = await sendCreateArticleMutation(userId, {
url: finalUrl,
articleSavingRequestId,
preparedDocument: {
document: content,
pageInfo: {
title,
canonicalUrl: finalUrl,
},
},
skipParsing: !content,
});
logRecord.totalTime = Date.now() - functionStartTime;
logRecord.result = apiResponse.createArticle;
}
} catch (e) {
logRecord.error = e.message;
console.log(`Error while retrieving page`, logRecord);
// fallback to scrapingbee
const sbResult = await fetchContentWithScrapingBee(url);
const sbUrl = finalUrl || sbResult.url;
const content = sbResult.domContent;
logRecord.fetchContentTime = Date.now() - functionStartTime;
const apiResponse = await sendCreateArticleMutation(userId, {
url: sbUrl,
articleSavingRequestId,
preparedDocument: {
document: content,
pageInfo: {
title: sbResult.title,
canonicalUrl: sbUrl,
},
},
skipParsing: !content,
});
logRecord.totalTime = Date.now() - functionStartTime;
logRecord.result = apiResponse.createArticle;
} finally {
if (context) {
await context.close();
}
console.log(`parse-page`, logRecord);
}
return res.sendStatus(200);
}
function validateUrlString(url) {
const u = new URL(url);
// Make sure the URL is http or https
if (u.protocol !== 'http:' && u.protocol !== 'https:') {
throw new Error('Invalid URL protocol check failed')
}
// Make sure the domain is not localhost
if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') {
throw new Error('Invalid URL is localhost')
}
// Make sure the domain is not a private IP
if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) {
throw new Error('Invalid URL is private ip')
}
}
function getUrl(req) {
console.log('body', req.body)
const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined);
if (!urlStr) {
throw new Error('No URL specified');
}
validateUrlString(urlStr);
const parsed = Url.parse(urlStr);
return parsed.href;
}
async function blockResources(client) {
const blockedResources = [
// Assets
// '*/favicon.ico',
// '.css',
// '.jpg',
// '.jpeg',
// '.png',
// '.svg',
// '.woff',
// Analytics and other fluff
'*.optimizely.com',
'everesttech.net',
'userzoom.com',
'doubleclick.net',
'googleadservices.com',
'adservice.google.com/*',
'connect.facebook.com',
'connect.facebook.net',
'sp.analytics.yahoo.com',
]
await client.send('Network.setBlockedURLs', { urls: blockedResources });
}
async function retrievePage(url) {
validateUrlString(url);
const browser = await getBrowserPromise;
logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime };
const context = await browser.createIncognitoBrowserContext();
const page = await context.newPage()
if (!enableJavascriptForUrl(url)) {
await page.setJavaScriptEnabled(false);
}
await page.setUserAgent(userAgentForUrl(url));
const client = await page.target().createCDPSession();
// intercept request when response headers was received
await client.send('Network.setRequestInterception', {
patterns: [
{
urlPattern: '*',
resourceType: 'Document',
interceptionStage: 'HeadersReceived',
},
],
});
const path = require('path');
const download_path = path.resolve('./download_dir/');
await client.send('Page.setDownloadBehavior', {
behavior: 'allow',
userDataDir: './',
downloadPath: download_path,
})
client.on('Network.requestIntercepted', async e => {
const headers = e.responseHeaders || {};
const [contentType] = (headers['content-type'] || headers['Content-Type'] || '')
.toLowerCase()
.split(';');
const obj = { interceptionId: e.interceptionId };
if (e.responseStatusCode >= 200 && e.responseStatusCode < 300) {
// We only check content-type on success responses
// as it doesn't matter what the content type is for things
// like redirects
if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) {
obj['errorReason'] = 'BlockedByClient';
}
}
try {
await client.send('Network.continueInterceptedRequest', obj);
// eslint-disable-next-line no-empty
} catch {}
});
await blockResources(client);
/*
* Disallow MathJax from running in Puppeteer and modifying the document,
* we shall instead run it in our frontend application to transform any
* mathjax content when present.
*/
await page.setRequestInterception(true);
let requestCount = 0;
page.on('request', request => {
if (['font', 'image', 'media'].includes(request.resourceType())) {
request.abort();
return;
}
if (requestCount++ > 100) {
request.abort();
return;
}
if (
request.resourceType() === 'script' &&
request.url().toLowerCase().indexOf('mathjax') > -1
) {
request.abort();
return
}
request.continue();
});
// Puppeteer fails during download of PDf files,
// so record the failure and use those items
let lastPdfUrl = undefined;
page.on('response', response => {
if (response.headers()['content-type'] === 'application/pdf') {
lastPdfUrl = response.url();
}
});
try {
const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] });
const finalUrl = response.url();
const contentType = response.headers()['content-type'];
logRecord.finalUrl = response.url();
logRecord.contentType = response.headers()['content-type'];
return { context, page, response, finalUrl, contentType };
} catch (error) {
if (lastPdfUrl) {
return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' };
}
await context.close();
throw error;
}
}
async function retrieveHtml(page) {
let domContent = '', title;
try {
title = await page.title();
logRecord.title = title;
const pageScrollingStart = Date.now();
/* scroll with a 5 second timeout */
await Promise.race([
new Promise(resolve => {
(async function () {
try {
await page.evaluate(`(async () => {
/* credit: https://github.com/puppeteer/puppeteer/issues/305 */
return new Promise((resolve, reject) => {
let scrollHeight = document.body.scrollHeight;
let totalHeight = 0;
let distance = 500;
let timer = setInterval(() => {
window.scrollBy(0, distance);
totalHeight += distance;
if(totalHeight >= scrollHeight){
clearInterval(timer);
resolve(true);
}
}, 10);
});
})()`);
} catch (e) {
logRecord.scrollError = true;
} finally {
resolve(true);
}
})();
}),
await page.waitForTimeout(1000),
]);
logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart };
const iframes = {};
const urls = [];
const framesPromises = [];
const allowedUrls = /instagram\.com/gi;
for (const frame of page.mainFrame().childFrames()) {
if (frame.url() && allowedUrls.test(frame.url())) {
urls.push(frame.url());
framesPromises.push(frame.evaluate(el => el.innerHTML, await frame.$('body')));
}
}
(await Promise.all(framesPromises)).forEach((frame, index) => (iframes[urls[index]] = frame));
const domContentCapturingStart = Date.now();
// get document body with all hidden elements removed
domContent = await page.evaluate(iframes => {
const BI_SRC_REGEXP = /url\("(.+?)"\)/gi;
Array.from(document.body.getElementsByTagName('*')).forEach(el => {
const style = window.getComputedStyle(el);
try {
// Removing blurred images since they are mostly the copies of lazy loaded ones
if (['img', 'image'].includes(el.tagName.toLowerCase())) {
const filter = style.getPropertyValue('filter');
if (filter && filter.startsWith('blur')) {
el.parentNode && el.parentNode.removeChild(el);
}
}
} catch (err) {
// throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*'))))
}
// convert all nodes with background image to img nodes
if (!['', 'none'].includes(style.getPropertyValue('background-image'))) {
const filter = style.getPropertyValue('filter');
// avoiding image nodes with a blur effect creation
if (filter && filter.startsWith('blur')) {
el && el.parentNode && el.parentNode.removeChild(el);
} else {
const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image'));
// Using "g" flag with a regex we have to manually break down lastIndex to zero after every usage
// More details here: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
BI_SRC_REGEXP.lastIndex = 0;
if (matchedSRC && matchedSRC[1] && !el.src) {
// Replacing element only of there are no content inside, b/c might remove important div with content.
// Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html
// DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image.
if (el.innerHTML.length < 25) {
const img = document.createElement('img');
img.src = matchedSRC[1];
el && el.parentNode && el.parentNode.removeChild(el);
}
}
}
}
if (el.tagName === 'IFRAME') {
if (iframes[el.src]) {
const newNode = document.createElement('div');
newNode.className = 'omnivore-instagram-embed';
newNode.innerHTML = iframes[el.src];
el && el.parentNode && el.parentNode.replaceChild(newNode, el);
}
}
});
if (document.querySelector('[data-translate="managed_checking_msg"]') ||
document.getElementById('px-block-form-wrapper')) {
return 'IS_BLOCKED'
}
return document.documentElement.outerHTML;
}, iframes);
logRecord.puppeteerSuccess = true;
logRecord.timing = {
...logRecord.timing,
contenCaptured: Date.now() - domContentCapturingStart,
};
// [END puppeteer-block]
} catch (e) {
if (e.message.startsWith('net::ERR_BLOCKED_BY_CLIENT at ')) {
logRecord.blockedByClient = true;
} else {
logRecord.puppeteerSuccess = false;
logRecord.puppeteerError = {
message: e.message,
stack: e.stack,
};
}
}
if (domContent === 'IS_BLOCKED') {
return { isBlocked: true };
}
return { domContent, title };
}
module.exports = fetchContent;

33
packages/content-fetch/index.js Normal file
View file

@ -0,0 +1,33 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const Sentry = require('@sentry/serverless');
const { fetchContent, preview } = require("@omnivore/puppeteer-parse");
Sentry.GCPFunction.init({
dsn: process.env.SENTRY_DSN,
tracesSampleRate: 0,
});
/**
* Cloud Function entry point, HTTP trigger.
* Loads the requested URL via Puppeteer, captures page content and sends it to backend
*
* @param {Object} req Cloud Function request context.
* @param {Object} res Cloud Function response context.
*/
exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(fetchContent);
/**
* Cloud Function entry point, HTTP trigger.
* Loads the requested URL via Puppeteer and captures a screenshot of the provided element
*
* @param {Object} req Cloud Function request context.
* Inlcudes:
* * url - URL address of the page to open
* @param {Object} res Cloud Function response context.
*/
exports.preview = Sentry.GCPFunction.wrapHttpFunction(preview);

18
packages/content-fetch/package.json
View file

@ -4,18 +4,20 @@
"description": "Service that fetches page content from a URL",
"main": "index.js",
"dependencies": {
"@omnivore/content-handler": "1.0.0",
"axios": "^0.27.2",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"jsonwebtoken": "^8.5.1",
"linkedom": "^0.14.9",
"luxon": "^2.3.1",
"puppeteer-core": "^16.1.0",
"underscore": "^1.13.4"
"@omnivore/puppeteer-parse": "^1.0.0",
"@sentry/serverless": "^6.13.3"
},
"devDependencies": {
"@google-cloud/functions-framework": "^3.0.0",
"chai": "^4.3.6",
"mocha": "^10.0.0"
},
"scripts": {
"start": "node app.js",
"test": "yarn mocha"
"start_gcf": "npx functions-framework --port=9090 --target=puppeteer",
"start_preview": "npx functions-framework --target=preview",
"test": "mocha test/*.js"
}
}

3
packages/content-fetch/test/babel-register.js
View file

@ -1,3 +0,0 @@
const register = require('@babel/register').default
register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })

9
packages/content-fetch/test/stub.test.js Normal file
View file

@ -0,0 +1,9 @@
const chai = require("chai");
const expect = chai.expect;
describe('Stub test', () => {
it('should pass', () => {
expect(true).to.be.true
})
})

13
packages/content-fetch/test/stub.test.ts
View file

@ -1,13 +0,0 @@
import 'mocha'
import * as chai from 'chai'
import { expect } from 'chai'
import 'chai/register-should'
import chaiString from 'chai-string'
chai.use(chaiString)
describe('Stub test', () => {
it('should pass', () => {
expect(true).to.be.true
})
})

1
packages/import-handler/test/csv/csv.test.ts
View file

@ -1,7 +1,6 @@
import 'mocha'
import * as chai from 'chai'
import { expect } from 'chai'
import 'chai/register-should'
import chaiString from 'chai-string'
import * as fs from 'fs'
import { importCsv } from '../../src/csv'

4
packages/puppeteer-parse/.dockerignore
View file

@ -1,4 +0,0 @@
node_modules
.env*
Dockerfile
.dockerignore

1
packages/puppeteer-parse/.gitignore vendored
View file

@ -1 +0,0 @@
previewImage.*

113
packages/puppeteer-parse/Dockerfile
View file

@ -1,113 +0,0 @@
# FROM node:14-slim
# # Taken from pu
# # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others)
# # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer
# # installs, work.
# RUN apt-get update \
# && apt-get install -y wget gnupg \
# && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
# && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
# && apt-get update \
# && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \
# --no-install-recommends \
# && rm -rf /var/lib/apt/lists/*
# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
# ENV CHROMIUM_PATH "/usr/bin/google-chrome-stable"
# ------------------------
# FROM --platform=linux/arm64 node:14.18
# RUN apt-get update \
# && apt-get install -y chromium \
# && apt-get install -y ca-certificates \
# fonts-liberation \
# libappindicator3-1 \
# libasound2 \
# libatk-bridge2.0-0 \
# libatk1.0-0 \
# libc6 \
# libcairo2 \
# libcups2 \
# libdbus-1-3 \
# libexpat1 \
# libfontconfig1 \
# libgbm1 \
# libgcc1 \
# libglib2.0-0 \
# libgtk-3-0 \
# libnspr4 \
# libnss3 \
# libpango-1.0-0 \
# libpangocairo-1.0-0 \
# libstdc++6 \
# libx11-6 \
# libx11-xcb1 \
# libxcb1 \
# libxcomposite1 \
# libxcursor1 \
# libxdamage1 \
# libxext6 \
# libxfixes3 \
# libxi6 \
# libxrandr2 \
# libxrender1 \
# libxss1 \
# libxtst6 \
# lsb-release \
# wget \
# xdg-utils
FROM node:14.18-alpine
# Installs latest Chromium (92) package.
RUN apk add --no-cache \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont \
nodejs \
yarn
# Add user so we don't need --no-sandbox.
RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \
&& mkdir -p /home/pptruser/Downloads /app \
&& chown -R pptruser:pptruser /home/pptruser \
&& chown -R pptruser:pptruser /app
# Run everything after as non-privileged user.
WORKDIR /app
ENV CHROMIUM_PATH /usr/bin/chromium-browser
ENV LAUNCH_HEADLESS=true
ENV PORT 9090
COPY package.json .
COPY yarn.lock .
COPY tsconfig.json .
COPY .prettierrc .
COPY .eslintrc .
COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json
COPY /packages/content-handler/package.json ./packages/content-handler/package.json
RUN yarn install --pure-lockfile
ADD /packages/puppeteer-parse ./packages/puppeteer-parse
ADD /packages/content-handler ./packages/content-handler
RUN yarn workspace @omnivore/content-handler build
# After building, fetch the production dependencies
RUN rm -rf /app/packages/puppeteer-parse/node_modules
RUN rm -rf /app/node_modules
RUN yarn install --pure-lockfile --production
EXPOSE 9090
# USER pptruser
ENTRYPOINT ["yarn", "workspace", "@omnivore/puppeteer-parse", "start"]

23
packages/puppeteer-parse/README.md
View file

@ -1,24 +1,3 @@
# Puppeteer parsing function handler
This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer.
## Using locally
Copy .env.example file to .env file: `cp .env.example .env`
Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading).
After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer)
## Deployment
To deploy the function use the following command:
`gcloud functions deploy puppeteer --runtime nodejs12 --trigger-http --memory 1GB --set-env-vars REST_BACKEND_ENDPOINT=<backend-address>,JWT_SECRET=<jwt-secret>`
where:
`<backend-address>` - address of the backend server (e.g "http://localhost:4000")
`<jwt-secret>` - JWT secret that the backend server is using (e.g "some_secret")
This workspace is used to provide the module for the app to hande requests for the article parsing via Puppeteer.

769
packages/puppeteer-parse/index.js
View file

@ -3,8 +3,8 @@
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const Url = require('url');
// const puppeteer = require('puppeteer-extra');
const axios = require('axios');
const jwt = require('jsonwebtoken');
const { promisify } = require('util');
@ -13,10 +13,8 @@ const { config, format, loggers, transports } = require('winston');
const { LoggingWinston } = require('@google-cloud/logging-winston');
const { DateTime } = require('luxon');
const os = require('os');
const Sentry = require('@sentry/serverless');
const { Storage } = require('@google-cloud/storage');
const chromium = require('chrome-aws-lambda');
const { parseHTML } = require('linkedom');
const puppeteer = require('puppeteer-core');
const { preHandleContent } = require("@omnivore/content-handler");
@ -28,20 +26,7 @@ const storage = new Storage();
const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : [];
const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.env.PREVIEW_IMAGE_BUCKET) : undefined;
Sentry.GCPFunction.init({
dsn: process.env.SENTRY_DSN,
tracesSampleRate: 0,
});
const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com']
const filePath = `${os.tmpdir()}/previewImage.png`;
const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf'];
const colors = {
emerg: 'inverse underline magenta',
@ -102,6 +87,15 @@ function buildLogger(id, options) {
});
}
const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com']
const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com'];
const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf'];
const userAgentForUrl = (url) => {
try {
const u = new URL(url);
@ -116,15 +110,38 @@ const userAgentForUrl = (url) => {
return DESKTOP_USER_AGENT
};
const fetchContentWithScrapingBee = async (url) => {
const response = await axios.get('https://app.scrapingbee.com/api/v1', {
params: {
'api_key': process.env.SCRAPINGBEE_API_KEY,
'url': url,
'render_js': 'false',
'premium_proxy': 'true',
'country_code':'us'
}
})
const dom = parseHTML(response.data).document;
return { title: dom.title, domContent: dom.documentElement.outerHTML, url: url }
}
const enableJavascriptForUrl = (url) => {
try {
const u = new URL(url);
for (const host of NON_SCRIPT_HOSTS) {
if (u.hostname.endsWith(host)) {
return false;
}
}
} catch (e) {
console.log('error getting hostname for url', url, e)
}
return true
};
// launch Puppeteer
const getBrowserPromise = (async () => {
// return puppeteer.launch({
// args: chromium.args,
// defaultViewport: chromium.defaultViewport,
// executablePath: process.env.CHROMIUM_PATH,
// headless: chromium.headless,
// ignoreHTTPSErrors: true,
// });
console.log("starting with proxy url", process.env.PROXY_URL)
return puppeteer.launch({
args: [
'--allow-running-insecure-content',
@ -170,7 +187,7 @@ const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentOb
})
};
const getUploadIdAndSignedUrl = async (userId, url) => {
const getUploadIdAndSignedUrl = async (userId, url, articleSavingRequestId) => {
const auth = await signToken({ uid: userId }, process.env.JWT_SECRET);
const data = JSON.stringify({
query: `mutation UploadFileRequest($input: UploadFileRequestInput!) {
@ -188,17 +205,18 @@ const getUploadIdAndSignedUrl = async (userId, url) => {
input: {
url,
contentType: 'application/pdf',
clientRequestId: articleSavingRequestId,
}
}
});
const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data,
{
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
});
{
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
});
return response.data.data.uploadFileRequest;
};
@ -231,12 +249,12 @@ const sendCreateArticleMutation = async (userId, input) => {
const auth = await signToken({ uid: userId }, process.env.JWT_SECRET);
const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data,
{
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
});
{
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
});
return response.data.data.createArticle;
};
@ -249,14 +267,7 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId
);
};
/**
* Cloud Function entry point, HTTP trigger.
* Loads the requested URL via Puppeteer, captures page content and sends it to backend
*
* @param {Object} req Cloud Function request context.
* @param {Object} res Cloud Function response context.
*/
exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
async function fetchContent(req, res) {
functionStartTime = Date.now();
// Grabbing execution and trace ids to attach logs to the appropriate function call
const execution_id = req.get('function-execution-id');
@ -269,7 +280,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
});
let url = getUrl(req);
const userId = req.body.userId || req.query.userId;
const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined);
const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined);
logRecord = {
@ -285,7 +296,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
if (!url) {
logRecord.urlIsInvalid = true;
logger.error(`Valid URL to parse not specified`, logRecord);
logger.info(`Valid URL to parse not specified`, logRecord);
return res.sendStatus(400);
}
@ -301,7 +312,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
if (result && result.content) { content = result.content }
if (result && result.contentType) { contentType = result.contentType }
} catch (e) {
console.log('error with handler: ', e);
logger.info('error with handler: ', e);
}
let context, page, finalUrl;
@ -312,7 +323,6 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
if (result && result.page) { page = result.page }
if (result && result.finalUrl) { finalUrl = result.finalUrl }
if (result && result.contentType) { contentType = result.contentType }
console.log('context, page, finalUrl, contentType', context, page, finalUrl, contentType);
} else {
finalUrl = url
}
@ -323,14 +333,19 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
} else {
if (!content || !title) {
const result = await retrieveHtml(page);
title = result.title;
content = result.domContent;
if (result.isBlocked) {
const sbResult = await fetchContentWithScrapingBee(url)
title = sbResult.title
content = sbResult.domContent
} else {
title = result.title;
content = result.domContent;
}
} else {
console.log('using prefetched content and title');
console.log(content);
logger.info('using prefetched content and title');
}
logRecord.contentFetchTime = Date.now() - functionStartTime;
logRecord.fetchContentTime = Date.now() - functionStartTime;
const apiResponse = await sendCreateArticleMutation(userId, {
url: finalUrl,
@ -347,33 +362,351 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
logRecord.totalTime = Date.now() - functionStartTime;
logRecord.result = apiResponse.createArticle;
logger.info(`parse-page`, logRecord);
}
} catch (e) {
console.log('error', e)
logRecord.error = e.message;
logger.error(`Error while retrieving page`, logRecord);
return res.sendStatus(503);
// fallback to scrapingbee
const sbResult = await fetchContentWithScrapingBee(url);
const sbUrl = finalUrl || sbResult.url;
const content = sbResult.domContent;
logRecord.fetchContentTime = Date.now() - functionStartTime;
const apiResponse = await sendCreateArticleMutation(userId, {
url: sbUrl,
articleSavingRequestId,
preparedDocument: {
document: content,
pageInfo: {
title: sbResult.title,
canonicalUrl: sbUrl,
},
},
skipParsing: !content,
});
logRecord.totalTime = Date.now() - functionStartTime;
logRecord.result = apiResponse.createArticle;
} finally {
if (context) {
await context.close();
}
logger.info(`parse-page`, logRecord);
}
return res.sendStatus(200);
});
}
/**
* Cloud Function entry point, HTTP trigger.
* Loads the requested URL via Puppeteer and captures a screenshot of the provided element
*
* @param {Object} req Cloud Function request context.
* Inlcudes:
* * url - URL address of the page to open
* @param {Object} res Cloud Function response context.
*/
exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
functionStartTime = Date.now();
function validateUrlString(url) {
const u = new URL(url);
// Make sure the URL is http or https
if (u.protocol !== 'http:' && u.protocol !== 'https:') {
throw new Error('Invalid URL protocol check failed')
}
// Make sure the domain is not localhost
if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') {
throw new Error('Invalid URL is localhost')
}
// Make sure the domain is not a private IP
if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) {
throw new Error('Invalid URL is private ip')
}
}
function getUrl(req) {
const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined);
if (!urlStr) {
throw new Error('No URL specified');
}
validateUrlString(urlStr);
const parsed = Url.parse(urlStr);
return parsed.href;
}
async function blockResources(client) {
const blockedResources = [
// Assets
// '*/favicon.ico',
// '.css',
// '.jpg',
// '.jpeg',
// '.png',
// '.svg',
// '.woff',
// Analytics and other fluff
'*.optimizely.com',
'everesttech.net',
'userzoom.com',
'doubleclick.net',
'googleadservices.com',
'adservice.google.com/*',
'connect.facebook.com',
'connect.facebook.net',
'sp.analytics.yahoo.com',
]
await client.send('Network.setBlockedURLs', { urls: blockedResources });
}
async function retrievePage(url) {
validateUrlString(url);
const browser = await getBrowserPromise;
logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime };
const context = await browser.createIncognitoBrowserContext();
const page = await context.newPage()
if (!enableJavascriptForUrl(url)) {
await page.setJavaScriptEnabled(false);
}
await page.setUserAgent(userAgentForUrl(url));
const client = await page.target().createCDPSession();
// intercept request when response headers was received
await client.send('Network.setRequestInterception', {
patterns: [
{
urlPattern: '*',
resourceType: 'Document',
interceptionStage: 'HeadersReceived',
},
],
});
const path = require('path');
const download_path = path.resolve('./download_dir/');
await client.send('Page.setDownloadBehavior', {
behavior: 'allow',
userDataDir: './',
downloadPath: download_path,
})
client.on('Network.requestIntercepted', async e => {
const headers = e.responseHeaders || {};
const [contentType] = (headers['content-type'] || headers['Content-Type'] || '')
.toLowerCase()
.split(';');
const obj = { interceptionId: e.interceptionId };
if (e.responseStatusCode >= 200 && e.responseStatusCode < 300) {
// We only check content-type on success responses
// as it doesn't matter what the content type is for things
// like redirects
if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) {
obj['errorReason'] = 'BlockedByClient';
}
}
try {
await client.send('Network.continueInterceptedRequest', obj);
// eslint-disable-next-line no-empty
} catch {}
});
await blockResources(client);
/*
* Disallow MathJax from running in Puppeteer and modifying the document,
* we shall instead run it in our frontend application to transform any
* mathjax content when present.
*/
await page.setRequestInterception(true);
let requestCount = 0;
page.on('request', request => {
if (['font', 'image', 'media'].includes(request.resourceType())) {
request.abort();
return;
}
if (requestCount++ > 100) {
request.abort();
return;
}
if (
request.resourceType() === 'script' &&
request.url().toLowerCase().indexOf('mathjax') > -1
) {
request.abort();
return
}
request.continue();
});
// Puppeteer fails during download of PDf files,
// so record the failure and use those items
let lastPdfUrl = undefined;
page.on('response', response => {
if (response.headers()['content-type'] === 'application/pdf') {
lastPdfUrl = response.url();
}
});
try {
const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] });
const finalUrl = response.url();
const contentType = response.headers()['content-type'];
logRecord.finalUrl = response.url();
logRecord.contentType = response.headers()['content-type'];
return { context, page, response, finalUrl, contentType };
} catch (error) {
if (lastPdfUrl) {
return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' };
}
await context.close();
throw error;
}
}
async function retrieveHtml(page) {
let domContent = '', title;
try {
title = await page.title();
logRecord.title = title;
const pageScrollingStart = Date.now();
/* scroll with a 5 second timeout */
await Promise.race([
new Promise(resolve => {
(async function () {
try {
await page.evaluate(`(async () => {
/* credit: https://github.com/puppeteer/puppeteer/issues/305 */
return new Promise((resolve, reject) => {
let scrollHeight = document.body.scrollHeight;
let totalHeight = 0;
let distance = 500;
let timer = setInterval(() => {
window.scrollBy(0, distance);
totalHeight += distance;
if(totalHeight >= scrollHeight){
clearInterval(timer);
resolve(true);
}
}, 10);
});
})()`);
} catch (e) {
logRecord.scrollError = true;
} finally {
resolve(true);
}
})();
}),
await page.waitForTimeout(1000),
]);
logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart };
const iframes = {};
const urls = [];
const framesPromises = [];
const allowedUrls = /instagram\.com/gi;
for (const frame of page.mainFrame().childFrames()) {
if (frame.url() && allowedUrls.test(frame.url())) {
urls.push(frame.url());
framesPromises.push(frame.evaluate(el => el.innerHTML, await frame.$('body')));
}
}
(await Promise.all(framesPromises)).forEach((frame, index) => (iframes[urls[index]] = frame));
const domContentCapturingStart = Date.now();
// get document body with all hidden elements removed
domContent = await page.evaluate(iframes => {
const BI_SRC_REGEXP = /url\("(.+?)"\)/gi;
Array.from(document.body.getElementsByTagName('*')).forEach(el => {
const style = window.getComputedStyle(el);
try {
// Removing blurred images since they are mostly the copies of lazy loaded ones
if (['img', 'image'].includes(el.tagName.toLowerCase())) {
const filter = style.getPropertyValue('filter');
if (filter && filter.startsWith('blur')) {
el.parentNode && el.parentNode.removeChild(el);
}
}
} catch (err) {
// throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*'))))
}
// convert all nodes with background image to img nodes
if (!['', 'none'].includes(style.getPropertyValue('background-image'))) {
const filter = style.getPropertyValue('filter');
// avoiding image nodes with a blur effect creation
if (filter && filter.startsWith('blur')) {
el && el.parentNode && el.parentNode.removeChild(el);
} else {
const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image'));
// Using "g" flag with a regex we have to manually break down lastIndex to zero after every usage
// More details here: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
BI_SRC_REGEXP.lastIndex = 0;
if (matchedSRC && matchedSRC[1] && !el.src) {
// Replacing element only of there are no content inside, b/c might remove important div with content.
// Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html
// DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image.
if (el.innerHTML.length < 25) {
const img = document.createElement('img');
img.src = matchedSRC[1];
el && el.parentNode && el.parentNode.removeChild(el);
}
}
}
}
if (el.tagName === 'IFRAME') {
if (iframes[el.src]) {
const newNode = document.createElement('div');
newNode.className = 'omnivore-instagram-embed';
newNode.innerHTML = iframes[el.src];
el && el.parentNode && el.parentNode.replaceChild(newNode, el);
}
}
});
if (document.querySelector('[data-translate="managed_checking_msg"]') ||
document.getElementById('px-block-form-wrapper')) {
return 'IS_BLOCKED'
}
return document.documentElement.outerHTML;
}, iframes);
logRecord.puppeteerSuccess = true;
logRecord.timing = {
...logRecord.timing,
contenCaptured: Date.now() - domContentCapturingStart,
};
// [END puppeteer-block]
} catch (e) {
if (e.message.startsWith('net::ERR_BLOCKED_BY_CLIENT at ')) {
logRecord.blockedByClient = true;
} else {
logRecord.puppeteerSuccess = false;
logRecord.puppeteerError = {
message: e.message,
stack: e.stack,
};
}
}
if (domContent === 'IS_BLOCKED') {
return { isBlocked: true };
}
return { domContent, title };
}
async function preview(req, res) {
const functionStartTime = Date.now();
// Grabbing execution and trace ids to attach logs to the appropriate function call
const execution_id = req.get('function-execution-id');
const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0];
@ -392,7 +725,7 @@ exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
const url = getUrl(req);
console.log('preview request url', url);
logRecord = {
const logRecord = {
url,
query: req.query,
origin: req.get('Origin'),
@ -415,7 +748,7 @@ exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
return res.sendStatus(400);
}
const browser = await getBrowserPromise;
const browser = await getBrowserPromise(process.env.PROXY_URL, process.env.CHROMIUM_PATH);
logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime };
const page = await browser.newPage();
@ -490,296 +823,10 @@ exports.preview = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
logger.info(`preview-image`, logRecord);
return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`);
});
function validateUrlString(url) {
const u = new URL(url);
// Make sure the URL is http or https
if (u.protocol !== 'http:' && u.protocol !== 'https:') {
throw new Error('Invalid URL protocol check failed')
}
// Make sure the domain is not localhost
if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') {
throw new Error('Invalid URL is localhost')
}
// Make sure the domain is not a private IP
if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) {
throw new Error('Invalid URL is private ip')
}
}
function getUrl(req) {
if (req.query.url || req.body.url) {
const urlStr = req.query.url || req.body.url;
validateUrlString(urlStr);
module.exports = {
fetchContent,
preview,
};
const url = Url.parse(urlStr);
return url.href;
}
try {
return Url.parse(JSON.parse(req.body).url).href;
} catch (e) {}
}
async function blockResources(client) {
const blockedResources = [
// Assets
// '*/favicon.ico',
// '.css',
// '.jpg',
// '.jpeg',
// '.png',
// '.svg',
// '.woff',
// Analytics and other fluff
'*.optimizely.com',
'everesttech.net',
'userzoom.com',
'doubleclick.net',
'googleadservices.com',
'adservice.google.com/*',
'connect.facebook.com',
'connect.facebook.net',
'sp.analytics.yahoo.com',
]
await client.send('Network.setBlockedURLs', { urls: blockedResources });
}
async function retrievePage(url) {
validateUrlString(url);
const browser = await getBrowserPromise;
logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime };
const context = await browser.createIncognitoBrowserContext();
const page = await context.newPage();
await page.setUserAgent(userAgentForUrl(url));
const client = await page.target().createCDPSession();
// intercept request when response headers was received
await client.send('Network.setRequestInterception', {
patterns: [
{
urlPattern: '*',
resourceType: 'Document',
interceptionStage: 'HeadersReceived',
},
],
});
const path = require('path');
const download_path = path.resolve('./download_dir/');
await client.send('Page.setDownloadBehavior', {
behavior: 'allow',
userDataDir: './',
downloadPath: download_path,
})
client.on('Network.requestIntercepted', async e => {
const headers = e.responseHeaders || {};
const [contentType] = (headers['content-type'] || headers['Content-Type'] || '')
.toLowerCase()
.split(';');
const obj = { interceptionId: e.interceptionId };
if (e.responseStatusCode >= 200 && e.responseStatusCode < 300) {
// We only check content-type on success responses
// as it doesn't matter what the content type is for things
// like redirects
if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) {
obj['errorReason'] = 'BlockedByClient';
}
}
try {
await client.send('Network.continueInterceptedRequest', obj);
// eslint-disable-next-line no-empty
} catch {}
});
await blockResources(client);
/*
* Disallow MathJax from running in Puppeteer and modifying the document,
* we shall instead run it in our frontend application to transform any
* mathjax content when present.
*/
await page.setRequestInterception(true);
let requestCount = 0;
page.on('request', request => {
if (['font', 'image', 'media'].includes(request.resourceType())) {
request.abort();
return;
}
if (requestCount++ > 100) {
request.abort();
return;
}
if (
request.resourceType() === 'script' &&
request.url().toLowerCase().indexOf('mathjax') > -1
) {
request.abort();
return
}
request.continue();
});
// Puppeteer fails during download of PDf files,
// so record the failure and use those items
let lastPdfUrl = undefined;
page.on('response', response => {
if (response.headers()['content-type'] === 'application/pdf') {
lastPdfUrl = response.url();
}
});
try {
const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] });
const finalUrl = response.url();
const contentType = response.headers()['content-type'];
logRecord.finalUrl = response.url();
logRecord.contentType = response.headers()['content-type'];
return { context, page, response, finalUrl, contentType };
} catch (error) {
if (lastPdfUrl) {
return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' };
}
await context.close();
throw error;
}
}
async function retrieveHtml(page) {
let domContent = '', title;
try {
title = await page.title();
logRecord.title = title;
const pageScrollingStart = Date.now();
/* scroll with a 5 second timeout */
await Promise.race([
new Promise(resolve => {
(async function () {
try {
await page.evaluate(`(async () => {
/* credit: https://github.com/puppeteer/puppeteer/issues/305 */
return new Promise((resolve, reject) => {
let scrollHeight = document.body.scrollHeight;
let totalHeight = 0;
let distance = 500;
let timer = setInterval(() => {
window.scrollBy(0, distance);
totalHeight += distance;
if(totalHeight >= scrollHeight){
clearInterval(timer);
resolve(true);
}
}, 10);
});
})()`);
} catch (e) {
logRecord.scrollError = true;
} finally {
resolve(true);
}
})();
}),
await page.waitForTimeout(1000), // 1 second timeout
]);
logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart };
const iframes = {};
const urls = [];
const framesPromises = [];
const allowedUrls = /instagram\.com/gi;
for (const frame of page.mainFrame().childFrames()) {
if (frame.url() && allowedUrls.test(frame.url())) {
urls.push(frame.url());
framesPromises.push(frame.evaluate(el => el.innerHTML, await frame.$('body')));
}
}
(await Promise.all(framesPromises)).forEach((frame, index) => (iframes[urls[index]] = frame));
const domContentCapturingStart = Date.now();
// get document body with all hidden elements removed
domContent = await page.evaluate(iframes => {
const BI_SRC_REGEXP = /url\("(.+?)"\)/gi;
Array.from(document.body.getElementsByTagName('*')).forEach(el => {
const style = window.getComputedStyle(el);
// Removing blurred images since they are mostly the copies of lazy loaded ones
if (['img', 'image'].includes(el.tagName.toLowerCase())) {
const filter = style.getPropertyValue('filter');
if (filter && filter.startsWith('blur')) {
el.parentNode && el.parentNode.removeChild(el);
}
}
// convert all nodes with background image to img nodes
if (!['', 'none'].includes(style.getPropertyValue('background-image'))) {
const filter = style.getPropertyValue('filter');
// avoiding image nodes with a blur effect creation
if (filter && filter.startsWith('blur')) {
el && el.parentNode && el.parentNode.removeChild(el);
} else {
const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image'));
// Using "g" flag with a regex we have to manually break down lastIndex to zero after every usage
// More details here: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
BI_SRC_REGEXP.lastIndex = 0;
if (matchedSRC && matchedSRC[1] && !el.src) {
// Replacing element only of there are no content inside, b/c might remove important div with content.
// Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html
// DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image.
if (el.innerHTML.length < 25) {
const img = document.createElement('img');
img.src = matchedSRC[1];
el && el.parentNode && el.parentNode.removeChild(el);
}
}
}
}
if (el.tagName === 'IFRAME') {
if (iframes[el.src]) {
const newNode = document.createElement('div');
newNode.className = 'omnivore-instagram-embed';
newNode.innerHTML = iframes[el.src];
el && el.parentNode && el.parentNode.replaceChild(newNode, el);
}
}
});
return document.documentElement.outerHTML;
}, iframes);
logRecord.puppeteerSuccess = true;
logRecord.timing = {
...logRecord.timing,
contenCaptured: Date.now() - domContentCapturingStart,
};
// [END puppeteer-block]
} catch (e) {
if (e.message.startsWith('net::ERR_BLOCKED_BY_CLIENT at ')) {
logRecord.blockedByClient = true;
} else {
logRecord.puppeteerSuccess = false;
logRecord.puppeteerError = {
message: e.message,
stack: e.stack,
};
}
}
return { domContent, title };
}

14
packages/puppeteer-parse/package.json
View file

@ -1,17 +1,13 @@
{
"name": "@omnivore/puppeteer-parse",
"version": "1.0.0",
"description": "Google Cloud Function that accepts URL of the article and parses its content",
"description": "Accepts URL of the article and parses its content",
"main": "index.js",
"dependencies": {
"@google-cloud/functions-framework": "^3.1.2",
"@google-cloud/logging-winston": "^5.1.1",
"@google-cloud/storage": "^5.18.1",
"@omnivore/content-handler": "1.0.0",
"@sentry/serverless": "^6.13.3",
"axios": "^0.27.2",
"chrome-aws-lambda": "^10.1.0",
"dotenv": "^8.2.0",
"jsonwebtoken": "^8.5.1",
"linkedom": "^0.14.9",
"luxon": "^2.3.1",
@ -19,9 +15,11 @@
"underscore": "^1.13.4",
"winston": "^3.3.3"
},
"devDependencies": {
"chai": "^4.3.6",
"mocha": "^10.0.0"
},
"scripts": {
"start": "npx functions-framework --port=9090 --target=puppeteer",
"start_preview": "npx functions-framework --target=preview",
"test": "yarn mocha"
"test": "mocha test/*.js"
}
}

3
packages/puppeteer-parse/test/babel-register.js
View file

@ -1,3 +0,0 @@
const register = require('@babel/register').default
register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })

9
packages/puppeteer-parse/test/stub.test.js Normal file
View file

@ -0,0 +1,9 @@
const chai = require("chai");
const expect = chai.expect;
describe('Stub test', () => {
it('should pass', () => {
expect(true).to.be.true
})
})

13
packages/puppeteer-parse/test/stub.test.ts
View file

@ -1,13 +0,0 @@
import 'mocha'
import * as chai from 'chai'
import { expect } from 'chai'
import 'chai/register-should'
import chaiString from 'chai-string'
chai.use(chaiString)
describe('Stub test', () => {
it('should pass', () => {
expect(true).to.be.true
})
})

18
yarn.lock
View file

@ -2444,7 +2444,7 @@
google-gax "^2.24.1"
protobufjs "^6.8.6"
"@google-cloud/functions-framework@3.1.2", "@google-cloud/functions-framework@^3.1.2":
"@google-cloud/functions-framework@3.1.2", "@google-cloud/functions-framework@^3.0.0":
version "3.1.2"
resolved "https://registry.yarnpkg.com/@google-cloud/functions-framework/-/functions-framework-3.1.2.tgz#2cd92ce4307bf7f32555d028dca22e398473b410"
integrity sha512-pYvEH65/Rqh1JNPdcBmorcV7Xoom2/iOSmbtYza8msro7Inl+qOYxbyMiQfySD2gwAyn38WyWPRqsDRcf/BFLg==
@ -10907,13 +10907,6 @@ chownr@^2.0.0:
resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece"
integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==
chrome-aws-lambda@^10.1.0:
version "10.1.0"
resolved "https://registry.yarnpkg.com/chrome-aws-lambda/-/chrome-aws-lambda-10.1.0.tgz#ac43b4cdfc1fbb2275c62effada560858099501e"
integrity sha512-NZQVf+J4kqG4sVhRm3WNmOfzY0OtTSm+S8rg77pwePa9RCYHzhnzRs8YvNI6L9tALIW6RpmefWiPURt3vURXcw==
dependencies:
lambdafs "^2.0.3"
chrome-trace-event@^1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz#1015eced4741e15d06664a957dbbf50d041e26ac"
@ -17677,13 +17670,6 @@ kuler@^2.0.0:
resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3"
integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==
lambdafs@^2.0.3:
version "2.1.1"
resolved "https://registry.yarnpkg.com/lambdafs/-/lambdafs-2.1.1.tgz#4bf8d3037b6c61bbb4a22ab05c73ee47964c25ed"
integrity sha512-x5k8JcoJWkWLvCVBzrl4pzvkEHSgSBqFjg3Dpsc4AcTMq7oUMym4cL/gRTZ6VM4mUMY+M0dIbQ+V1c1tsqqanQ==
dependencies:
tar-fs "^2.1.1"
language-subtag-registry@~0.3.2:
version "0.3.21"
resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.21.tgz#04ac218bea46f04cb039084602c6da9e788dd45a"
@ -23988,7 +23974,7 @@ tar-fs@2.0.0:
pump "^3.0.0"
tar-stream "^2.0.0"
tar-fs@2.1.1, tar-fs@^2.1.1:
tar-fs@2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784"
integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==