prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix a crash when parsing content fetches that are blocked

Browse source
This commit is contained in:
Jackson Harper 2022-06-22 14:55:37 -07:00
parent f297cf1c1d
commit feb197c731
1 changed files with 11 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
packages/content-fetch/fetch-content.js
View file

@ -321,7 +321,6 @@ async function fetchContent(req, res) {
if (!content || !title) {
const result = await retrieveHtml(page);
if (result.isBlocked) {
console.log("PAGE IS BLOCKED", url)
const sbResult = await fetchContentWithScrapingBee(url)
title = sbResult.title
content = sbResult.domContent
@ -597,12 +596,16 @@ async function retrieveHtml(page) {
Array.from(document.body.getElementsByTagName('*')).forEach(el => {
const style = window.getComputedStyle(el);
// Removing blurred images since they are mostly the copies of lazy loaded ones
if (['img', 'image'].includes(el.tagName.toLowerCase())) {
const filter = style.getPropertyValue('filter');
if (filter && filter.startsWith('blur')) {
el.parentNode && el.parentNode.removeChild(el);
try {
// Removing blurred images since they are mostly the copies of lazy loaded ones
if (['img', 'image'].includes(el.tagName.toLowerCase())) {
const filter = style.getPropertyValue('filter');
if (filter && filter.startsWith('blur')) {
el.parentNode && el.parentNode.removeChild(el);
}
}
} catch (err) {
// throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*'))))
}
// convert all nodes with background image to img nodes
@ -640,7 +643,8 @@ async function retrieveHtml(page) {
}
});
if (document.querySelector('[data-translate="managed_checking_msg"]')) {
if (document.querySelector('[data-translate="managed_checking_msg"]') ||
document.getElementById('px-block-form-wrapper')) {
return 'IS_BLOCKED'
}
@ -664,8 +668,6 @@ async function retrieveHtml(page) {
};
}
}
console.log('DOM CONTENT')
console.log(domContent)
if (domContent === 'IS_BLOCKED') {
return { isBlocked: true };
}