From d816ee9563ce345780506a4f822bf5a5b57dd00d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 13 Feb 2023 16:50:39 +0800 Subject: [PATCH] Allow image and media to be loaded in puppeteer --- .../src/services/create_page_save_request.ts | 2 +- packages/api/src/utils/parser.ts | 6 ++-- packages/puppeteer-parse/index.js | 31 ++----------------- 3 files changed, 6 insertions(+), 33 deletions(-) diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 60a7e5672..b0d134fef 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -93,7 +93,7 @@ export const createPageSaveRequest = async ( url: normalizedUrl, }) if (page) { - console.log('Page already exists', page) + console.log('Page already exists', page.id, page.url) articleSavingRequestId = page.id } else { page = { diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 93c9eaffc..b1d8a1e8a 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -203,7 +203,7 @@ export const parsePreparedContent = async ( return { canonicalUrl: url, parsedContent: null, - domContent: preparedDocument.document, + domContent: document, pageType: PageType.Unknown, } } @@ -223,7 +223,7 @@ export const parsePreparedContent = async ( if (!article?.textContent && allowRetry) { const newDocument = { ...preparedDocument, - document: '' + preparedDocument.document + '', + document: '' + document + '', } return parsePreparedContent( url, @@ -337,7 +337,7 @@ export const parsePreparedContent = async ( logger.info('parse-article completed') return { - domContent: preparedDocument.document, + domContent: document, parsedContent: article, canonicalUrl, pageType: parseOriginalContent(dom), diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index fa25b4732..eb61cac93 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -409,32 +409,6 @@ function getUrl(req) { return parsed.href; } -async function blockResources(client) { - const blockedResources = [ - // Assets - // '*/favicon.ico', - // '.css', - // '.jpg', - // '.jpeg', - // '.png', - // '.svg', - // '.woff', - - // Analytics and other fluff - '*.optimizely.com', - 'everesttech.net', - 'userzoom.com', - 'doubleclick.net', - 'googleadservices.com', - 'adservice.google.com/*', - 'connect.facebook.com', - 'connect.facebook.net', - 'sp.analytics.yahoo.com', - ] - - await client.send('Network.setBlockedURLs', { urls: blockedResources }); -} - async function retrievePage(url, logRecord, functionStartTime) { validateUrlString(url); @@ -494,8 +468,6 @@ async function retrievePage(url, logRecord, functionStartTime) { } catch {} }); - await blockResources(client); - /* * Disallow MathJax from running in Puppeteer and modifying the document, * we shall instead run it in our frontend application to transform any @@ -504,7 +476,8 @@ async function retrievePage(url, logRecord, functionStartTime) { await page.setRequestInterception(true); let requestCount = 0; page.on('request', request => { - if (['font', 'image', 'media'].includes(request.resourceType())) { + if (request.resourceType() === 'font') { + // Disallow fonts from loading request.abort(); return; }