prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Allow image and media to be loaded in puppeteer

Browse source
This commit is contained in:
Hongbo Wu 2023-02-13 16:50:39 +08:00
parent 1a1617e86c
commit d816ee9563
3 changed files with 6 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/api/src/services/create_page_save_request.ts
View file

@ -93,7 +93,7 @@ export const createPageSaveRequest = async (
url: normalizedUrl,
})
if (page) {
console.log('Page already exists', page)
console.log('Page already exists', page.id, page.url)
articleSavingRequestId = page.id
} else {
page = {

6
packages/api/src/utils/parser.ts
View file

@ -203,7 +203,7 @@ export const parsePreparedContent = async (
return {
canonicalUrl: url,
parsedContent: null,
domContent: preparedDocument.document,
domContent: document,
pageType: PageType.Unknown,
}
}
@ -223,7 +223,7 @@ export const parsePreparedContent = async (
if (!article?.textContent && allowRetry) {
const newDocument = {
...preparedDocument,
document: '<html>' + preparedDocument.document + '</html>',
document: '<html>' + document + '</html>',
}
return parsePreparedContent(
url,
@ -337,7 +337,7 @@ export const parsePreparedContent = async (
logger.info('parse-article completed')
return {
domContent: preparedDocument.document,
domContent: document,
parsedContent: article,
canonicalUrl,
pageType: parseOriginalContent(dom),

31
packages/puppeteer-parse/index.js
View file

@ -409,32 +409,6 @@ function getUrl(req) {
return parsed.href;
}
async function blockResources(client) {
const blockedResources = [
// Assets
// '*/favicon.ico',
// '.css',
// '.jpg',
// '.jpeg',
// '.png',
// '.svg',
// '.woff',
// Analytics and other fluff
'*.optimizely.com',
'everesttech.net',
'userzoom.com',
'doubleclick.net',
'googleadservices.com',
'adservice.google.com/*',
'connect.facebook.com',
'connect.facebook.net',
'sp.analytics.yahoo.com',
]
await client.send('Network.setBlockedURLs', { urls: blockedResources });
}
async function retrievePage(url, logRecord, functionStartTime) {
validateUrlString(url);
@ -494,8 +468,6 @@ async function retrievePage(url, logRecord, functionStartTime) {
} catch {}
});
await blockResources(client);
/*
* Disallow MathJax from running in Puppeteer and modifying the document,
* we shall instead run it in our frontend application to transform any
@ -504,7 +476,8 @@ async function retrievePage(url, logRecord, functionStartTime) {
await page.setRequestInterception(true);
let requestCount = 0;
page.on('request', request => {
if (['font', 'image', 'media'].includes(request.resourceType())) {
if (request.resourceType() === 'font') {
// Disallow fonts from loading
request.abort();
return;
}