diff --git a/platform/mv3/extension/js/filter-lists.js b/platform/mv3/extension/js/filter-lists.js index 3f3addfe8..51791c9db 100644 --- a/platform/mv3/extension/js/filter-lists.js +++ b/platform/mv3/extension/js/filter-lists.js @@ -94,10 +94,7 @@ function rulesetStats(rulesetId) { const rulesetDetails = rulesetMap.get(rulesetId); if ( rulesetDetails === undefined ) { return; } const { rules, filters } = rulesetDetails; - let ruleCount = rules.plain + rules.regex; - if ( cachedRulesetData.hasOmnipotence ) { - ruleCount += rules.removeparam + rules.redirect + rules.modifyHeaders; - } + const ruleCount = rules.plain + rules.regex; const filterCount = filters.accepted; return { ruleCount, filterCount }; } diff --git a/platform/mv3/extension/js/ruleset-manager.js b/platform/mv3/extension/js/ruleset-manager.js index 24db8812e..95567c1c7 100644 --- a/platform/mv3/extension/js/ruleset-manager.js +++ b/platform/mv3/extension/js/ruleset-manager.js @@ -30,13 +30,13 @@ import { rulesetConfig, saveRulesetConfig, } from './config.js'; +import { ubolErr, ubolLog } from './debug.js'; import { dnr } from './ext-compat.js'; import { fetchJSON } from './fetch.js'; import { getAdminRulesets } from './admin.js'; import { hasBroadHostPermissions } from './utils.js'; import { rulesFromText } from './dnr-parser.js'; -import { ubolErr, ubolLog } from './debug.js'; /******************************************************************************/ @@ -131,9 +131,8 @@ pruneInvalidRegexRules.validated = new Map(); async function updateRegexRules(currentRules, addRules, removeRuleIds) { // Remove existing regex-related block rules for ( const rule of currentRules ) { + if ( rule.id === 0 ) { continue; } if ( rule.id >= SPECIAL_RULES_REALM ) { continue; } - const { type } = rule.action; - if ( type !== 'block' && type !== 'allow' ) { continue; } if ( rule.condition.regexFilter === undefined ) { continue; } removeRuleIds.push(rule.id); } @@ -167,136 +166,19 @@ async function updateRegexRules(currentRules, addRules, removeRuleIds) { /******************************************************************************/ -async function updateRemoveparamRules(currentRules, addRules, removeRuleIds) { - // Remove existing removeparam-related rules - for ( const rule of currentRules ) { - if ( rule.id >= SPECIAL_RULES_REALM ) { continue; } - if ( rule.action.type !== 'redirect' ) { continue; } - if ( rule.action.redirect.transform === undefined ) { continue; } - removeRuleIds.push(rule.id); - } - - const rulesetDetails = await getEnabledRulesetsDetails(); - - // Fetch removeparam rules for all enabled rulesets - const toFetch = []; - for ( const details of rulesetDetails ) { - if ( details.rules.removeparam === 0 ) { continue; } - toFetch.push(fetchJSON(`/rulesets/removeparam/${details.id}`)); - } - const removeparamRulesets = await Promise.all(toFetch); - - const allRules = []; - for ( const rules of removeparamRulesets ) { - if ( Array.isArray(rules) === false ) { continue; } - for ( const rule of rules ) { - allRules.push(rule); - } - } - if ( allRules.length === 0 ) { return; } - - const validRules = await pruneInvalidRegexRules('removeparam', allRules); - if ( validRules.length === 0 ) { return; } - - ubolLog(`Add ${validRules.length} DNR removeparam rules`); - addRules.push(...validRules); -} - -/******************************************************************************/ - -async function updateRedirectRules(currentRules, addRules, removeRuleIds) { - // Remove existing redirect-related rules - for ( const rule of currentRules ) { - if ( rule.id >= SPECIAL_RULES_REALM ) { continue; } - if ( rule.action.type !== 'redirect' ) { continue; } - if ( rule.action.redirect.extensionPath === undefined ) { - if ( rule.action.redirect.regexSubstitution === undefined ) { continue; } - } - removeRuleIds.push(rule.id); - } - - const rulesetDetails = await getEnabledRulesetsDetails(); - - // Fetch redirect rules for all enabled rulesets - const toFetch = []; - for ( const details of rulesetDetails ) { - if ( details.rules.redirect === 0 ) { continue; } - toFetch.push(fetchJSON(`/rulesets/redirect/${details.id}`)); - } - const redirectRulesets = await Promise.all(toFetch); - - const allRules = []; - for ( const rules of redirectRulesets ) { - if ( Array.isArray(rules) === false ) { continue; } - for ( const rule of rules ) { - allRules.push(rule); - } - } - if ( allRules.length === 0 ) { return; } - - const validRules = await pruneInvalidRegexRules('redirect', allRules); - if ( validRules.length === 0 ) { return; } - - ubolLog(`Add ${validRules.length} DNR redirect rules`); - addRules.push(...validRules); -} - -/******************************************************************************/ - -async function updateModifyHeadersRules(currentRules, addRules, removeRuleIds) { - // Remove existing header modification-related rules - for ( const rule of currentRules ) { - if ( rule.id >= SPECIAL_RULES_REALM ) { continue; } - if ( rule.action.type !== 'modifyHeaders' ) { continue; } - removeRuleIds.push(rule.id); - } - - const rulesetDetails = await getEnabledRulesetsDetails(); - - // Fetch modifyHeaders rules for all enabled rulesets - const toFetch = []; - for ( const details of rulesetDetails ) { - if ( details.rules.modifyHeaders === 0 ) { continue; } - toFetch.push(fetchJSON(`/rulesets/modify-headers/${details.id}`)); - } - const rulesets = await Promise.all(toFetch); - - const allRules = []; - for ( const rules of rulesets ) { - if ( Array.isArray(rules) === false ) { continue; } - for ( const rule of rules ) { - allRules.push(rule); - } - } - if ( allRules.length === 0 ) { return; } - - const validRules = await pruneInvalidRegexRules('modify-headers', allRules); - if ( validRules.length === 0 ) { return; } - - ubolLog(`Add ${validRules.length} DNR modify-headers rules`); - addRules.push(...validRules); -} - -/******************************************************************************/ - async function updateDynamicRules() { const currentRules = await dnr.getDynamicRules(); const addRules = []; const removeRuleIds = []; - // Remove potentially left-over strict-block rules from previous version + // Remove potentially left-over rules from previous version for ( const rule of currentRules ) { if ( rule.id >= SPECIAL_RULES_REALM ) { continue; } - if ( isStrictBlockRule(rule) === false ) { continue; } removeRuleIds.push(rule.id); + rule.id = 0; } - await Promise.all([ - updateRegexRules(currentRules, addRules, removeRuleIds), - updateRemoveparamRules(currentRules, addRules, removeRuleIds), - updateRedirectRules(currentRules, addRules, removeRuleIds), - updateModifyHeadersRules(currentRules, addRules, removeRuleIds), - ]); + await updateRegexRules(currentRules, addRules, removeRuleIds); if ( addRules.length === 0 && removeRuleIds.length === 0 ) { return; } dynamicRegexCount = 0; diff --git a/platform/mv3/make-rulesets.js b/platform/mv3/make-rulesets.js index 1a1c43e5c..6efc76c35 100644 --- a/platform/mv3/make-rulesets.js +++ b/platform/mv3/make-rulesets.js @@ -62,6 +62,10 @@ const outputDir = commandLineArgs.get('output') || '.'; const cacheDir = `${outputDir}/../mv3-data`; const rulesetDir = `${outputDir}/rulesets`; const scriptletDir = `${rulesetDir}/scripting`; +const envExtra = (( ) => { + const env = commandLineArgs.get('env'); + return env ? env.split('|') : []; +})(); const env = [ platform, 'native_css_has', @@ -69,6 +73,7 @@ const env = [ 'ublock', 'ubol', 'user_stylesheet', + ...envExtra, ]; if ( platform === 'edge' ) { @@ -292,31 +297,9 @@ const isRegex = rule => rule.condition !== undefined && rule.condition.regexFilter !== undefined; -const isRedirect = rule => { - if ( isUnsupported(rule) ) { return false; } - if ( rule.action.type !== 'redirect' ) { return false; } - if ( rule.action.redirect?.extensionPath !== undefined ) { return true; } - if ( rule.action.redirect?.transform?.path !== undefined ) { return true; } - if ( rule.action.redirect?.regexSubstitution !== undefined ) { return true; } - return false; -}; - -const isModifyHeaders = rule => +const isGood = rule => isUnsupported(rule) === false && - rule.action.type === 'modifyHeaders'; - -const isRemoveparam = rule => - isUnsupported(rule) === false && - rule.action.type === 'redirect' && - rule.action.redirect.transform !== undefined; - -const isSafe = rule => - isUnsupported(rule) === false && - rule.action !== undefined && ( - rule.action.type === 'block' || - rule.action.type === 'allow' || - rule.action.type === 'allowAllRequests' - ); + /^(allow|block|redirect|modifyHeaders|allowAllRequests)$/.test(rule.action?.type); const isURLSkip = rule => isUnsupported(rule) === false && @@ -525,54 +508,27 @@ async function processNetworkFilters(assetDetails, network) { } } - const plainGood = await patchRuleset( - rules.filter(rule => isSafe(rule) && isRegex(rule) === false) + const staticRules = await patchRuleset( + rules.filter(rule => isGood(rule) && isRegex(rule) === false) ); - log(`\tPlain good: ${plainGood.length}`); - log(plainGood + log(`\tStatic rules: ${staticRules.length}`); + log(staticRules .filter(rule => Array.isArray(rule._warning)) .map(rule => rule._warning.map(v => `\t\t${v}`)) .join('\n'), true ); - const regexes = await patchRuleset( - rules.filter(rule => isSafe(rule) && isRegex(rule)) + const regexRules = await patchRuleset( + rules.filter(rule => isGood(rule) && isRegex(rule)) ); - log(`\tMaybe good (regexes): ${regexes.length}`); + log(`\tMaybe good (regexes): ${regexRules.length}`); - const redirects = await patchRuleset( - rules.filter(rule => - isUnsupported(rule) === false && - isRedirect(rule) - ) - ); - redirects.forEach(rule => { - if ( rule.action.redirect.extensionPath === undefined ) { return; } + staticRules.forEach(rule => { + if ( rule.action.redirect?.extensionPath === undefined ) { return; } requiredRedirectResources.add( rule.action.redirect.extensionPath.replace(/^\/+/, '') ); }); - log(`\tredirect=: ${redirects.length}`); - - const removeparamsGood = await patchRuleset( - rules.filter(rule => - isUnsupported(rule) === false && isRemoveparam(rule) - ) - ); - const removeparamsBad = await patchRuleset( - rules.filter(rule => - isUnsupported(rule) && isRemoveparam(rule) - ) - ); - log(`\tremoveparams= (accepted/discarded): ${removeparamsGood.length}/${removeparamsBad.length}`); - - const modifyHeaders = await patchRuleset( - rules.filter(rule => - isUnsupported(rule) === false && - isModifyHeaders(rule) - ) - ); - log(`\tmodifyHeaders=: ${modifyHeaders.length}`); const urlskips = new Map(); for ( const rule of rules ) { @@ -622,35 +578,17 @@ async function processNetworkFilters(assetDetails, network) { log(bad.map(rule => rule._error.map(v => `\t\t${v}`)).join('\n'), true); writeFile(`${rulesetDir}/main/${assetDetails.id}.json`, - toJSONRuleset(plainGood) + toJSONRuleset(staticRules) ); - if ( regexes.length !== 0 ) { + if ( regexRules.length !== 0 ) { writeFile(`${rulesetDir}/regex/${assetDetails.id}.json`, - toJSONRuleset(regexes) - ); - } - - if ( removeparamsGood.length !== 0 ) { - writeFile(`${rulesetDir}/removeparam/${assetDetails.id}.json`, - toJSONRuleset(removeparamsGood) - ); - } - - if ( redirects.length !== 0 ) { - writeFile(`${rulesetDir}/redirect/${assetDetails.id}.json`, - toJSONRuleset(redirects) - ); - } - - if ( modifyHeaders.length !== 0 ) { - writeFile(`${rulesetDir}/modify-headers/${assetDetails.id}.json`, - toJSONRuleset(modifyHeaders) + toJSONRuleset(regexRules) ); } const strictBlocked = new Map(); - for ( const rule of plainGood ) { + for ( const rule of staticRules ) { toStrictBlockRule(rule, strictBlocked); } if ( strictBlocked.size !== 0 ) { @@ -668,13 +606,9 @@ async function processNetworkFilters(assetDetails, network) { return { total: rules.length, - plain: plainGood.length, - discarded: removeparamsBad.length, + plain: staticRules.length, rejected: bad.length, - regex: regexes.length, - removeparam: removeparamsGood.length, - redirect: redirects.length, - modifyHeaders: modifyHeaders.length, + regex: regexRules.length, strictblock: strictBlocked.size, urlskip: urlskips.size, };