diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index bf5f6837e..454a55228 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -205,6 +205,7 @@ const nullableEnvVars = [ 'INTERCOM_IOS_SECRET', 'INTERCOM_ANDROID_SECRET', 'EXPORT_TASK_HANDLER_URL', + 'LOCAL_MINIO_URL', ] // Allow some vars to be null/empty const envParser = diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 0683a93ee..13dca1ca5 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -9,7 +9,7 @@ ], "dependencies": { "@omnivore/content-handler": "1.0.0", - "puppeteer-core": "^22.12.1", + "puppeteer-core": "^23.6.1", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-adblocker": "^2.13.6", "puppeteer-extra-plugin-stealth": "^2.11.2" diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index 853c9a6ef..da57ad78d 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -144,6 +144,49 @@ function getUrl(urlStr: string) { return parsed.href } +const waitForDOMToSettle = (page: Page, timeoutMs = 60000, debounceMs = 1000) => + page.evaluate( + (timeoutMs, debounceMs) => { + const debounce = (func: (...args: unknown[]) => void, ms = 1000) => { + let timeout: NodeJS.Timeout; + return (...args: unknown[]) => { + console.log('in debounce, clearing timeout again') + clearTimeout(timeout) + timeout = setTimeout(() => { + func.apply(this, args) + }, ms) + } + } + return new Promise((resolve, reject) => { + const mainTimeout = setTimeout(() => { + observer.disconnect() + reject(new Error('Timed out whilst waiting for DOM to settle')) + }, timeoutMs) + + const debouncedResolve = debounce(() => { + observer.disconnect() + clearTimeout(mainTimeout) + resolve() + }, debounceMs) + + const observer = new MutationObserver(() => { + debouncedResolve() + }) + + const config = { + attributes: true, + childList: true, + subtree: true, + } + + observer.observe(document.body, config) + }) + }, + timeoutMs, + debounceMs + ); + + async function retrievePage( url: string, logRecord: Record, @@ -246,15 +289,21 @@ async function retrievePage( */ await page.setRequestInterception(true) let requestCount = 0 + const failedRequests = new Set() page.on('request', (request) => { ;(async () => { if (request.resourceType() === 'font') { // Disallow fonts from loading return request.abort() } - if (requestCount++ > 100) { + if (requestCount++ > 50) { return request.abort() } + + if (failedRequests.has(request.url())) { + return request.abort() + } + if ( request.resourceType() === 'script' && request.url().toLowerCase().indexOf('mathjax') > -1 @@ -267,15 +316,26 @@ async function retrievePage( }) page.on('response', (response) => { + if (!response.ok()) { + console.log('Failed request', response.url()) + failedRequests.add(response.url()) + } + if (response.headers()['content-type'] === 'application/pdf') { lastPdfUrl = response.url() } }) + console.log('Trying to load page, for 30 seconds') + const response = await page.goto(url, { - timeout: 30 * 1000, - waitUntil: ['networkidle0'], + timeout: 90 * 1000, + waitUntil: ['load'], }) + + console.log('Waited for content to load, waiting for DOM to settle.') + await waitForDOMToSettle(page) + if (!response) { throw new Error('No response from page') } diff --git a/self-hosting/docker-compose/docker-compose.yml b/self-hosting/docker-compose/docker-compose.yml index c51732a62..e4ad4090b 100644 --- a/self-hosting/docker-compose/docker-compose.yml +++ b/self-hosting/docker-compose/docker-compose.yml @@ -80,8 +80,8 @@ services: image-proxy: build: - context: ../../ - dockerfile: ./imageproxy/Dockerfile + context: ../../imageproxy + dockerfile: ./Dockerfile container_name: "omnivore-image-proxy" ports: - "7070:8080" @@ -97,6 +97,8 @@ services: - "9090:8080" env_file: - .env + environment: + DBUS_SESSION_BUS_ADDRESS: depends_on: redis: condition: service_healthy