prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #4150 from omnivore-app/fix/browser-undefined

Browse source 
create a browser singleton instance and checks browser existence before creating context
This commit is contained in:
Hongbo Wu 2024-07-04 19:37:41 +08:00 committed by GitHub
commit 76bf497717
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 91 additions and 213 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
packages/content-fetch/src/index.ts
View file

@ -1,5 +1,6 @@
import { HttpFunction } from '@google-cloud/functions-framework'
import * as Sentry from '@sentry/serverless'
import 'dotenv/config'
import { contentFetchRequestHandler } from './request_handler'
Sentry.GCPFunction.init({

87
packages/puppeteer-parse/src/browser.ts Normal file
View file

@ -0,0 +1,87 @@
import { Browser } from 'puppeteer-core'
import puppeteer from 'puppeteer-extra'
import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker'
import StealthPlugin from 'puppeteer-extra-plugin-stealth'
puppeteer.use(StealthPlugin())
puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
let browserInstance: Browser | null = null
const handleDisconnection = async () => {
console.log('Browser disconnected, reconnecting...')
browserInstance = null
await getBrowser()
}
export const getBrowser = async (): Promise<Browser> => {
if (browserInstance) {
return browserInstance
}
console.log('Starting puppeteer browser')
browserInstance = (await puppeteer.launch({
args: [
'--autoplay-policy=user-gesture-required',
'--disable-component-update',
'--disable-domain-reliability',
'--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process',
'--disable-print-preview',
'--disable-setuid-sandbox',
'--disable-site-isolation-trials',
'--disable-speech-api',
'--disk-cache-size=33554432',
'--enable-features=SharedArrayBuffer',
'--hide-scrollbars',
'--disable-gpu',
'--mute-audio',
'--no-default-browser-check',
'--no-pings',
'--no-sandbox',
'--no-zygote',
'--window-size=1920,1080',
'--disable-extensions',
'--disable-dev-shm-usage',
'--no-first-run',
'--disable-background-networking',
],
defaultViewport: {
deviceScaleFactor: 1,
hasTouch: false,
height: 1080,
isLandscape: true,
isMobile: false,
width: 1920,
},
executablePath: process.env.CHROMIUM_PATH,
headless: !!process.env.LAUNCH_HEADLESS,
timeout: 30_000, // 30 seconds
dumpio: true, // show console logs in the terminal
})) as Browser
const version = await browserInstance.version()
console.log('Browser started', version)
browserInstance.on('disconnected', () => {
void handleDisconnection()
})
return browserInstance
}
export const closeBrowser = async (): Promise<void> => {
if (browserInstance) {
console.log('Closing browser...')
try {
await browserInstance.close()
console.log('Browser closed successfully')
} catch (error) {
console.error('Error closing browser:', error)
} finally {
browserInstance = null
}
} else {
console.log('No browser instance to close')
}
}

216
packages/puppeteer-parse/src/index.ts
View file

@ -2,33 +2,13 @@
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { preHandleContent } from '@omnivore/content-handler'
import axios from 'axios'
// const { Storage } = require('@google-cloud/storage');
import { parseHTML } from 'linkedom'
import path from 'path'
import { Browser, BrowserContext, Page, Protocol } from 'puppeteer-core'
import puppeteer from 'puppeteer-extra'
import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker'
import StealthPlugin from 'puppeteer-extra-plugin-stealth'
import { BrowserContext, Page, Protocol } from 'puppeteer-core'
import { getBrowser } from './browser'
// Add stealth plugin to hide puppeteer usage
puppeteer.use(StealthPlugin())
// Add adblocker plugin to block all ads and trackers (saves bandwidth)
puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
// const storage = new Storage();
// const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS
// ? process.env.ALLOWED_ORIGINS.split(',')
// : []
// const previewBucket = process.env.PREVIEW_IMAGE_BUCKET ? storage.bucket(process.env.PREVIEW_IMAGE_BUCKET) : undefined;
// const filePath = `${os.tmpdir()}/previewImage.png`
// const MOBILE_USER_AGENT =
// 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
const DESKTOP_USER_AGENT =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
// const BOT_DESKTOP_USER_AGENT =
// 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
const NON_BOT_DESKTOP_USER_AGENT =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com']
@ -86,71 +66,6 @@ const enableJavascriptForUrl = (url: string) => {
return true
}
let browser: Browser
// launch Puppeteer
const launchBrowser = async () => {
console.log('starting puppeteer browser')
browser = (await puppeteer.launch({
args: [
'--allow-running-insecure-content',
'--autoplay-policy=user-gesture-required',
'--disable-component-update',
'--disable-domain-reliability',
'--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process',
'--disable-print-preview',
'--disable-setuid-sandbox',
'--disable-site-isolation-trials',
'--disable-speech-api',
'--disable-web-security',
'--disk-cache-size=33554432',
'--enable-features=SharedArrayBuffer',
'--hide-scrollbars',
'--disable-gpu',
'--mute-audio',
'--no-default-browser-check',
'--no-pings',
'--no-sandbox',
'--no-zygote',
'--window-size=1920,1080',
'--disable-extensions',
],
defaultViewport: {
deviceScaleFactor: 1,
hasTouch: false,
height: 1080,
isLandscape: true,
isMobile: false,
width: 1920,
},
executablePath: process.env.CHROMIUM_PATH,
headless: !!process.env.LAUNCH_HEADLESS,
timeout: 30_000, // 30 seconds
dumpio: true, // show console logs in the terminal
})) as Browser
const version = await browser.version()
console.log('browser started', version)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
browser.on('disconnected', async () => {
console.log('browser disconnected, reconnecting...')
const childProcess = browser.process()
if (childProcess) {
childProcess.kill('SIGINT')
console.log('browser child process killed')
}
await launchBrowser()
console.log('browser reconnected')
})
}
// initialize Puppeteer
;(async () => await launchBrowser())()
export const fetchContent = async (
url: string,
locale?: string,
@ -237,11 +152,6 @@ export const fetchContent = async (
} catch (e) {
console.error(`Error while retrieving page ${url}`, e)
console.error(
'pendingProtocolErrors',
browser.debugInfo.pendingProtocolErrors
)
// fallback to scrapingbee for non pdf content
if (url && contentType !== 'application/pdf') {
console.info('fallback to scrapingbee', url)
@ -330,6 +240,7 @@ async function retrievePage(
browserOpened: Date.now() - functionStartTime,
}
const browser = await getBrowser()
// create a new incognito browser context
const context = await browser.createBrowserContext()
@ -635,124 +546,3 @@ async function retrieveHtml(page: Page, logRecord: Record<string, any>) {
}
return { domContent, title }
}
// async function preview(req, res) {
// const functionStartTime = Date.now();
// // Grabbing execution and trace ids to attach logs to the appropriate function call
// const execution_id = req.get('function-execution-id');
// const traceId = (req.get('x-cloud-trace-context') || '').split('/')[0];
// const console = buildconsole('cloudfunctions.googleapis.com%2Fcloud-functions', {
// trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`,
// labels: {
// execution_id: execution_id,
// },
// });
// if (!process.env.PREVIEW_IMAGE_BUCKET) {
// console.error(`PREVIEW_IMAGE_BUCKET not set`)
// return res.sendStatus(500);
// }
// const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined);
// const url = getUrl(urlStr);
// console.log('preview request url', url);
// const logRecord = {
// url,
// query: req.query,
// origin: req.get('Origin'),
// labels: {
// source: 'publicImagePreview',
// },
// };
// console.info(`Public preview image generation request`, logRecord);
// if (!url) {
// logRecord.urlIsInvalid = true;
// console.error(`Valid URL to parse is not specified`, logRecord);
// return res.sendStatus(400);
// }
// const { origin } = new URL(url);
// if (!ALLOWED_ORIGINS.some(o => o === origin)) {
// logRecord.forbiddenOrigin = true;
// console.error(`This origin is not allowed: ${origin}`, logRecord);
// return res.sendStatus(400);
// }
// const browser = await getBrowserPromise;
// logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime };
// const page = await browser.newPage();
// const pageLoadingStart = Date.now();
// const modifiedUrl = new URL(url);
// modifiedUrl.searchParams.append('fontSize', '24');
// modifiedUrl.searchParams.append('adjustAspectRatio', '1.91');
// try {
// await page.goto(modifiedUrl.toString());
// logRecord.timing = { ...logRecord.timing, pageLoaded: Date.now() - pageLoadingStart };
// } catch (error) {
// console.log('error going to page: ', modifiedUrl)
// console.log(error)
// throw error
// }
// // We lookup the destination path from our own page content and avoid trusting any passed query params
// // selector - CSS selector of the element to get screenshot of
// const selector = decodeURIComponent(
// await page.$eval(
// "head > meta[name='omnivore:preview_image_selector']",
// element => element.content,
// ),
// );
// if (!selector) {
// logRecord.selectorIsInvalid = true;
// console.error(`Valid element selector is not specified`, logRecord);
// await page.close();
// return res.sendStatus(400);
// }
// logRecord.selector = selector;
// // destination - destination pathname for the image to save with
// const destination = decodeURIComponent(
// await page.$eval(
// "head > meta[name='omnivore:preview_image_destination']",
// element => element.content,
// ),
// );
// if (!destination) {
// logRecord.destinationIsInvalid = true;
// console.error(`Valid file destination is not specified`, logRecord);
// await page.close();
// return res.sendStatus(400);
// }
// logRecord.destination = destination;
// const screenshotTakingStart = Date.now();
// try {
// await page.waitForSelector(selector, { timeout: 3000 }); // wait for the selector to load
// } catch (error) {
// logRecord.elementNotFound = true;
// console.error(`Element is not presented on the page`, logRecord);
// await page.close();
// return res.sendStatus(400);
// }
// const element = await page.$(selector);
// await element.screenshot({ path: filePath }); // take screenshot of the element in puppeteer
// logRecord.timing = { ...logRecord.timing, screenshotTaken: Date.now() - screenshotTakingStart };
// await page.close();
// try {
// const [file] = await previewBucket.upload(filePath, {
// destination,
// metadata: logRecord,
// });
// logRecord.file = file.metadata;
// } catch (e) {
// console.log('error uploading to bucket, this is non-fatal', e)
// }
// console.info(`preview-image`, logRecord);
// return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`);
// }