diff --git a/platform/mv3/chromium/manifest.json b/platform/mv3/chromium/manifest.json index 72760301c..c4d8e96a3 100644 --- a/platform/mv3/chromium/manifest.json +++ b/platform/mv3/chromium/manifest.json @@ -41,5 +41,16 @@ "storage": { "managed_schema": "managed_storage.json" }, - "version": "1.0" + "version": "1.0", + "web_accessible_resources": [ + { + "resources": [ + "/strictblock.html" + ], + "matches": [ + "" + ], + "use_dynamic_url": true + } + ] } diff --git a/platform/mv3/extension/js/background.js b/platform/mv3/extension/js/background.js index ce8c0ea93..9577eeea6 100644 --- a/platform/mv3/extension/js/background.js +++ b/platform/mv3/extension/js/background.js @@ -53,6 +53,7 @@ import { patchDefaultRulesets, setStrictBlockMode, updateDynamicRules, + updateSessionRules, } from './ruleset-manager.js'; import { @@ -398,8 +399,12 @@ async function start() { await enableRulesets(rulesetConfig.enabledRulesets); // We need to update the regex rules only when ruleset version changes. - if ( isNewVersion && rulesetsUpdated === false ) { - updateDynamicRules(); + if ( rulesetsUpdated === false ) { + if ( isNewVersion ) { + updateDynamicRules(); + } else if ( process.wakeupRun === false ) { + updateSessionRules(); + } } // Permissions may have been removed while the extension was disabled diff --git a/platform/mv3/extension/js/filter-lists.js b/platform/mv3/extension/js/filter-lists.js index 2356d57ce..b0e85e148 100644 --- a/platform/mv3/extension/js/filter-lists.js +++ b/platform/mv3/extension/js/filter-lists.js @@ -216,7 +216,8 @@ export function renderFilterLists(rulesetData) { [ 'default', rulesetDetails.filter(ruleset => - ruleset.id === 'default' + ruleset.id === 'default' || + ruleset.group === 'default' ), ], [ 'malware', diff --git a/platform/mv3/extension/js/ruleset-manager.js b/platform/mv3/extension/js/ruleset-manager.js index 935cf2dc6..6667b6dea 100644 --- a/platform/mv3/extension/js/ruleset-manager.js +++ b/platform/mv3/extension/js/ruleset-manager.js @@ -41,10 +41,18 @@ import { ubolLog } from './debug.js'; /******************************************************************************/ -const STRICTBLOCK_BASE_RULE_ID = 7000000; const TRUSTED_DIRECTIVE_BASE_RULE_ID = 8000000; +const STRICTBLOCK_PRIORITY = 29; -let dynamicRuleId = 1; +/******************************************************************************/ + +const isStrictBlockRule = rule => { + if ( rule.priority !== STRICTBLOCK_PRIORITY ) { return false; } + if ( rule.action.type !== 'redirect' ) { return false; } + const substitution = rule.action.redirect.regexSubstitution; + return substitution !== undefined && + substitution.includes('/strictblock.'); +}; /******************************************************************************/ @@ -106,7 +114,15 @@ pruneInvalidRegexRules.validated = new Map(); /******************************************************************************/ -async function updateRegexRules(toAdd) { +async function updateRegexRules(currentRules, addRules, removeRuleIds) { + // Remove existing regex-related block rules + for ( const rule of currentRules ) { + const { type } = rule.action; + if ( type !== 'block' && type !== 'allow' ) { continue; } + if ( rule.condition.regexFilter === undefined ) { continue; } + removeRuleIds.push(rule.id); + } + const rulesetDetails = await getEnabledRulesetsDetails(); // Fetch regexes for all enabled rulesets @@ -122,7 +138,6 @@ async function updateRegexRules(toAdd) { for ( const rules of regexRulesets ) { if ( Array.isArray(rules) === false ) { continue; } for ( const rule of rules ) { - rule.id = dynamicRuleId++; allRules.push(rule); } } @@ -132,12 +147,19 @@ async function updateRegexRules(toAdd) { if ( validRules.length === 0 ) { return; } ubolLog(`Add ${validRules.length} DNR regex rules`); - toAdd.push(...validRules); + addRules.push(...validRules); } /******************************************************************************/ -async function updateRemoveparamRules(toAdd) { +async function updateRemoveparamRules(currentRules, addRules, removeRuleIds) { + // Remove existing removeparam-related rules + for ( const rule of currentRules ) { + if ( rule.action.type !== 'redirect' ) { continue; } + if ( rule.action.redirect.transform === undefined ) { continue; } + removeRuleIds.push(rule.id); + } + const [ hasOmnipotence, rulesetDetails, @@ -160,7 +182,6 @@ async function updateRemoveparamRules(toAdd) { for ( const rules of removeparamRulesets ) { if ( Array.isArray(rules) === false ) { continue; } for ( const rule of rules ) { - rule.id = dynamicRuleId++; allRules.push(rule); } } @@ -171,12 +192,19 @@ async function updateRemoveparamRules(toAdd) { if ( validRules.length === 0 ) { return; } ubolLog(`Add ${validRules.length} DNR removeparam rules`); - toAdd.push(...validRules); + addRules.push(...validRules); } /******************************************************************************/ -async function updateRedirectRules(toAdd) { +async function updateRedirectRules(currentRules, addRules, removeRuleIds) { + // Remove existing redirect-related rules + for ( const rule of currentRules ) { + if ( rule.action.type !== 'redirect' ) { continue; } + if ( rule.action.redirect.extensionPath === undefined ) { continue; } + removeRuleIds.push(rule.id); + } + const [ hasOmnipotence, rulesetDetails, @@ -199,7 +227,6 @@ async function updateRedirectRules(toAdd) { for ( const rules of redirectRulesets ) { if ( Array.isArray(rules) === false ) { continue; } for ( const rule of rules ) { - rule.id = dynamicRuleId++; allRules.push(rule); } } @@ -210,12 +237,18 @@ async function updateRedirectRules(toAdd) { if ( validRules.length === 0 ) { return; } ubolLog(`Add ${validRules.length} DNR redirect rules`); - toAdd.push(...validRules); + addRules.push(...validRules); } /******************************************************************************/ -async function updateModifyHeadersRules(toAdd) { +async function updateModifyHeadersRules(currentRules, addRules, removeRuleIds) { + // Remove existing header modification-related rules + for ( const rule of currentRules ) { + if ( rule.action.type !== 'modifyHeaders' ) { continue; } + removeRuleIds.push(rule.id); + } + const [ hasOmnipotence, rulesetDetails, @@ -238,7 +271,6 @@ async function updateModifyHeadersRules(toAdd) { for ( const rules of rulesets ) { if ( Array.isArray(rules) === false ) { continue; } for ( const rule of rules ) { - rule.id = dynamicRuleId++; allRules.push(rule); } } @@ -249,12 +281,65 @@ async function updateModifyHeadersRules(toAdd) { if ( validRules.length === 0 ) { return; } ubolLog(`Add ${validRules.length} DNR modify-headers rules`); - toAdd.push(...validRules); + addRules.push(...validRules); } /******************************************************************************/ -async function updateStrictBlockRules(dynamicRules, sessionRules) { +async function updateDynamicRules() { + const currentRules = await dnr.getDynamicRules(); + const addRules = []; + const removeRuleIds = []; + + // Remove potentially left-over strict-block rules from previous version + for ( const rule of currentRules ) { + if ( isStrictBlockRule(rule) === false ) { continue; } + removeRuleIds.push(rule.id); + } + + await Promise.all([ + updateRegexRules(currentRules, addRules, removeRuleIds), + updateRemoveparamRules(currentRules, addRules, removeRuleIds), + updateRedirectRules(currentRules, addRules, removeRuleIds), + updateModifyHeadersRules(currentRules, addRules, removeRuleIds), + ]); + if ( addRules.length === 0 && removeRuleIds.length === 0 ) { return; } + + const maxRegexRuleCount = dnr.MAX_NUMBER_OF_REGEX_RULES; + let regexRuleCount = 0; + let ruleId = 1; + for ( const rule of addRules ) { + if ( rule?.condition.regexFilter ) { regexRuleCount += 1; } + if ( (rule.id || 0) >= TRUSTED_DIRECTIVE_BASE_RULE_ID ) { continue; } + rule.id = ruleId++; + } + if ( regexRuleCount !== 0 ) { + ubolLog(`Using ${regexRuleCount}/${maxRegexRuleCount} dynamic regex-based DNR rules`); + } + return Promise.all([ + dnr.updateDynamicRules({ addRules, removeRuleIds }).then(( ) => { + if ( removeRuleIds.length !== 0 ) { + ubolLog(`Remove ${removeRuleIds.length} dynamic DNR rules`); + } + if ( addRules.length !== 0 ) { + ubolLog(`Add ${addRules.length} dynamic DNR rules`); + } + }).catch(reason => { + console.error(`updateDynamicRules() / ${reason}`); + }), + updateSessionRules(), + ]); +} + +/******************************************************************************/ + +async function updateStrictBlockRules(currentRules, addRules, removeRuleIds) { + // Remove existing strictblock-related rules + for ( const rule of currentRules ) { + if ( isStrictBlockRule(rule) === false ) { continue; } + removeRuleIds.push(rule.id); + } + if ( rulesetConfig.strictBlockMode === false ) { return; } const [ @@ -269,107 +354,49 @@ async function updateStrictBlockRules(dynamicRules, sessionRules) { sessionRead('excludedStrictBlockHostnames'), ]); - // Fetch strick-block hostnames + // Strict-block rules can only be enforced with omnipotence + if ( hasOmnipotence === false ) { + localRemove('excludedStrictBlockHostnames'); + sessionRemove('excludedStrictBlockHostnames'); + return; + } + + // Fetch strick-block rules const toFetch = []; for ( const details of rulesetDetails ) { if ( details.rules.strictblock === 0 ) { continue; } toFetch.push(fetchJSON(`/rulesets/strictblock/${details.id}`)); } - const strictblockRulesets = await Promise.all(toFetch); + const rulesets = await Promise.all(toFetch); - // Strict-block rules can only be enforced with omnipotence - let toStrictBlock = new Set(); - if ( hasOmnipotence ) { - for ( const hostnames of strictblockRulesets ) { - if ( Array.isArray(hostnames) === false ) { continue; } - toStrictBlock = toStrictBlock.union(new Set(hostnames)); - } - } else { - if ( permanentlyExcluded.length !== 0 ) { - localRemove('excludedStrictBlockHostnames'); - permanentlyExcluded.length = 0; - } - if ( temporarilyExcluded.length !== 0 ) { - sessionRemove('excludedStrictBlockHostnames'); - temporarilyExcluded.length = 0; + const substitution = `${runtime.getURL('/strictblock.html')}#\\0`; + const allRules = []; + for ( const rules of rulesets ) { + if ( Array.isArray(rules) === false ) { continue; } + for ( const rule of rules ) { + rule.action.redirect.regexSubstitution = substitution; + allRules.push(rule); } } - for ( const hn of permanentlyExcluded ) { - toStrictBlock.delete(hn); + + const validRules = await pruneInvalidRegexRules('strictblock', allRules); + if ( validRules.length === 0 ) { return; } + ubolLog(`Add ${validRules.length} DNR strictblock rules`); + for ( const rule of validRules ) { + addRules.push(rule); } - if ( toStrictBlock.size === 0 ) { return; } - const manifest = runtime.getManifest(); - let strictblockPath = ''; - for ( const war of manifest.web_accessible_resources ) { - if ( war.resources.length !== 1 ) { continue; } - if ( war.resources[0].startsWith('/strictblock.') === false ) { continue; } - strictblockPath = runtime.getURL(war.resources[0]); - break; - } - if ( strictblockPath === '' ) { return; } - const dynamicRule = { - id: STRICTBLOCK_BASE_RULE_ID, - action: { - type: 'redirect', - redirect: { - regexSubstitution: `${strictblockPath}#\\0`, - }, - }, + + const allExcluded = permanentlyExcluded.concat(temporarilyExcluded); + if ( allExcluded.length === 0 ) { return; } + addRules.push({ + action: { type: 'allow' }, condition: { - regexFilter: '^https?://.+', - requestDomains: Array.from(toStrictBlock), + requestDomains: allExcluded, resourceTypes: [ 'main_frame' ], }, - priority: 29, - }; - if ( permanentlyExcluded.length !== 0 ) { - dynamicRule.condition.excludedRequestDomains = permanentlyExcluded; - } - dynamicRules.push(dynamicRule); - ubolLog(`Add 1 DNR dynamic rule with ${toStrictBlock.size} strictblock domains`); - - if ( temporarilyExcluded.length === 0 ) { return; } - sessionRules.push({ - id: STRICTBLOCK_BASE_RULE_ID, - action: { - type: 'allow', - }, - condition: { - requestDomains: temporarilyExcluded, - resourceTypes: [ 'main_frame' ], - }, - priority: 29, + priority: STRICTBLOCK_PRIORITY, }); - ubolLog(`Add 1 DNR session rule with ${temporarilyExcluded.length} excluded strictblock domains`); -} - -async function commitStrictBlockRules() { - const [ - beforePermanentRules, - beforeTemporaryRules, - ] = await Promise.all([ - dnr.getDynamicRules({ ruleIds: [ STRICTBLOCK_BASE_RULE_ID ] }), - dnr.getSessionRules({ ruleIds: [ STRICTBLOCK_BASE_RULE_ID ] }), - ]); - if ( beforePermanentRules?.length ) { - ubolLog(`Remove 1 DNR dynamic strictblock rule`); - } - if ( beforeTemporaryRules?.length ) { - ubolLog(`Remove 1 DNR session strictblock rule`); - } - const afterPermanentRules = []; - const afterTemporaryRules = []; - await updateStrictBlockRules(afterPermanentRules, afterTemporaryRules) - return Promise.all([ - dnr.updateDynamicRules({ - addRules: afterPermanentRules, - removeRuleIds: beforePermanentRules.map(rule => rule.id), - }), - dnr.updateSessionRules({ - addRules: afterTemporaryRules, - removeRuleIds: beforeTemporaryRules.map(rule => rule.id), - }), - ]); + ubolLog(`Add 1 DNR session rule with ${allExcluded.length} for excluded strict-block domains`); } async function excludeFromStrictBlock(hostname, permanent) { @@ -379,7 +406,7 @@ async function excludeFromStrictBlock(hostname, permanent) { hostnames.add(hostname); const writeFn = permanent ? localWrite : sessionWrite; await writeFn('excludedStrictBlockHostnames', Array.from(hostnames)); - return commitStrictBlockRules(); + return updateSessionRules(); } async function setStrictBlockMode(state) { @@ -394,67 +421,37 @@ async function setStrictBlockMode(state) { ); } await Promise.all(promises); - return commitStrictBlockRules(); + return updateSessionRules(); } /******************************************************************************/ -async function updateDynamicRules() { - dynamicRuleId = 1; - const dynamicRules = []; - const sessionRules = []; - const [ - dynamicRuleIds, - sessionRuleIds, - ] = await Promise.all([ - dnr.getDynamicRules().then(rules => - rules.map(rule => rule.id) - .filter(id => id < TRUSTED_DIRECTIVE_BASE_RULE_ID) - ), - dnr.getSessionRules().then(rules => rules.map(rule => rule.id)), - updateRegexRules(dynamicRules), - updateRemoveparamRules(dynamicRules), - updateRedirectRules(dynamicRules), - updateModifyHeadersRules(dynamicRules), - updateStrictBlockRules(dynamicRules, sessionRules), - ]); - if ( dynamicRules.length === 0 && dynamicRuleIds.length === 0 ) { return; } - const promises = []; - if ( dynamicRules.length !== 0 || dynamicRuleIds.length !== 0 ) { - promises.push( - dnr.updateDynamicRules({ - addRules: dynamicRules, - removeRuleIds: dynamicRuleIds, - }).then(( ) => { - if ( dynamicRuleIds.length !== 0 ) { - ubolLog(`Remove ${dynamicRuleIds.length} dynamic DNR rules`); - } - if ( dynamicRules.length !== 0 ) { - ubolLog(`Add ${dynamicRules.length} dynamic DNR rules`); - } - }).catch(reason => { - console.error(`updateDynamicRules() / ${reason}`); - }) - ); +async function updateSessionRules() { + const addRules = []; + const removeRuleIds = []; + const currentRules = await dnr.getSessionRules(); + await updateStrictBlockRules(currentRules, addRules, removeRuleIds); + if ( addRules.length === 0 && removeRuleIds.length === 0 ) { return; } + const maxRegexRuleCount = dnr.MAX_NUMBER_OF_REGEX_RULES; + let regexRuleCount = 0; + let ruleId = 1; + for ( const rule of addRules ) { + if ( rule?.condition.regexFilter ) { regexRuleCount += 1; } + rule.id = ruleId++; } - if ( sessionRules.length !== 0 || sessionRuleIds.length !== 0 ) { - promises.push( - dnr.updateSessionRules({ - addRules: sessionRules, - removeRuleIds: sessionRuleIds, - }).then(( ) => { - if ( sessionRuleIds.length !== 0 ) { - ubolLog(`Remove ${sessionRuleIds.length} session DNR rules`); - } - if ( sessionRules.length !== 0 ) { - ubolLog(`Add ${sessionRules.length} session DNR rules`); - } - }).catch(reason => { - console.error(`updateSessionRules() / ${reason}`); - }) - ); + if ( regexRuleCount !== 0 ) { + ubolLog(`Using ${regexRuleCount}/${maxRegexRuleCount} session regex-based DNR rules`); } - return Promise.all(promises); + return dnr.updateSessionRules({ addRules, removeRuleIds }).then(( ) => { + if ( removeRuleIds.length !== 0 ) { + ubolLog(`Remove ${removeRuleIds.length} session DNR rules`); + } + if ( addRules.length !== 0 ) { + ubolLog(`Add ${addRules.length} session DNR rules`); + } + }).catch(reason => { + console.error(`updateSessionRules() / ${reason}`); + }); } /******************************************************************************/ @@ -747,4 +744,5 @@ export { patchDefaultRulesets, setStrictBlockMode, updateDynamicRules, + updateSessionRules, }; diff --git a/platform/mv3/extension/js/strictblock.js b/platform/mv3/extension/js/strictblock.js index 2e8c74c67..3ba92aae7 100644 --- a/platform/mv3/extension/js/strictblock.js +++ b/platform/mv3/extension/js/strictblock.js @@ -49,21 +49,12 @@ function urlToFragment(raw) { /******************************************************************************/ -async function proceed() { - await sendMessage({ - what: 'excludeFromStrictBlock', - hostname: toURL.hostname, - permanent: qs$('#disableWarning').checked, - }); - window.location.replace(toURL.href); -} - -/******************************************************************************/ - const toURL = new URL('about:blank'); +const toFinalURL = new URL('about:blank'); try { toURL.href = self.location.hash.slice(1); + toFinalURL.href = toURL.href; } catch(_) { } @@ -72,6 +63,25 @@ qs$('#theURL > p > span:first-of-type').append(urlToFragment(toURL.href)); /******************************************************************************/ +async function proceed() { + const permanent = qs$('#disableWarning').checked; + // Do not exclude current hostname from strict-block ruleset if a urlskip + // directive to another site is in effect. + // TODO: what if the urlskip directive leads to a different subdomain on + // same site? + if ( toFinalURL.hostname !== toURL.hostname && permanent !== true ) { + return window.location.replace(toFinalURL.href); + } + await sendMessage({ + what: 'excludeFromStrictBlock', + hostname: toURL.hostname, + permanent, + }); + window.location.replace(toURL.href); +} + +/******************************************************************************/ + function fragmentFromTemplate(template, placeholder, text, details) { const fragment = new DocumentFragment(); const pos = template.indexOf(placeholder); @@ -165,15 +175,7 @@ function fragmentFromTemplate(template, placeholder, text, details) { dom.on('#toggleParse', 'click', ( ) => { dom.cl.toggle('#theURL', 'collapsed'); - //vAPI.localStorage.setItem( - // 'document-blocked-expand-url', - // (dom.cl.has('#theURL', 'collapsed') === false).toString() - //); }); - - //vAPI.localStorage.getItemAsync('document-blocked-expand-url').then(value => { - // dom.cl.toggle('#theURL', 'collapsed', value !== 'true' && value !== true); - //}); })(); /******************************************************************************/ @@ -184,16 +186,30 @@ function fragmentFromTemplate(template, placeholder, text, details) { let iList = -1; const searchInList = async i => { if ( iList !== -1 ) { return; } - const hostnames = new Set( - await fetchJSON(`/rulesets/strictblock/${rulesetDetails[i].id}`) - ); + const rules = await fetchJSON(`/rulesets/strictblock/${rulesetDetails[i].id}`); if ( iList !== -1 ) { return; } - let hn = toURL.hostname; - for (;;) { - if ( hostnames.has(hn) ) { iList = i; break; } - const pos = hn.indexOf('.'); - if ( pos === -1 ) { break; } - hn = hn.slice(pos+1); + const toHref = toURL.href; + for ( const rule of rules ) { + const { regexFilter, requestDomains } = rule.condition; + let matchesDomain = requestDomains === undefined; + if ( requestDomains ) { + let hn = toURL.hostname; + for (;;) { + if ( requestDomains.includes(hn) ) { + matchesDomain = true; + break; + } + const pos = hn.indexOf('.'); + if ( pos === -1 ) { break; } + hn = hn.slice(pos+1); + } + if ( matchesDomain === false ) { continue; } + } + const re = new RegExp(regexFilter); + const matchesRegex = re.test(toHref); + if ( matchesDomain && matchesRegex ) { + iList = i; + } } }; const toFetch = []; @@ -225,12 +241,24 @@ function fragmentFromTemplate(template, placeholder, text, details) { } if ( toFetch.length === 0 ) { return; } const urlskipLists = await Promise.all(toFetch); + const toHn = toURL.hostname; + const matchesHn = hn => { + if ( hn.endsWith(toHn) === false ) { return false; } + if ( hn.length === toHn.length ) { return true; } + return toHn.charAt(toHn.length - hn.length - 1) === '.'; + }; for ( const urlskips of urlskipLists ) { for ( const urlskip of urlskips ) { const re = new RegExp(urlskip.re, urlskip.c ? undefined : 'i'); if ( re.test(toURL.href) === false ) { continue; } + if ( urlskip.hostnames ) { + if ( urlskip.hostnames.some(hn => matchesHn(hn)) === false ) { + continue; + } + } const finalURL = urlSkip(toURL.href, false, urlskip.steps); if ( finalURL === undefined ) { continue; } + toFinalURL.href = finalURL; const fragment = fragmentFromTemplate( i18n$('strictblockRedirectSentence1'), '{{url}}', urlToFragment(finalURL), diff --git a/platform/mv3/strictblock.html b/platform/mv3/extension/strictblock.html similarity index 100% rename from platform/mv3/strictblock.html rename to platform/mv3/extension/strictblock.html diff --git a/platform/mv3/firefox/manifest.json b/platform/mv3/firefox/manifest.json index 2ee265673..7847e3ad5 100644 --- a/platform/mv3/firefox/manifest.json +++ b/platform/mv3/firefox/manifest.json @@ -50,5 +50,15 @@ "storage" ], "short_name": "uBO Lite", - "version": "1.0" + "version": "1.0", + "web_accessible_resources": [ + { + "resources": [ + "/strictblock.html" + ], + "matches": [ + "" + ] + } + ] } diff --git a/platform/mv3/make-rulesets.js b/platform/mv3/make-rulesets.js index 2d9c2d380..b3495483e 100644 --- a/platform/mv3/make-rulesets.js +++ b/platform/mv3/make-rulesets.js @@ -22,9 +22,15 @@ import * as makeScriptlet from './make-scriptlets.js'; import * as sfp from './js/static-filtering-parser.js'; -import { createHash, randomBytes } from 'crypto'; +import { + createHash, + randomBytes, +} from 'crypto'; +import { + dnrRulesetFromRawLists, + mergeRules, +} from './js/static-dnr-filtering.js'; -import { dnrRulesetFromRawLists } from './js/static-dnr-filtering.js'; import fs from 'fs/promises'; import https from 'https'; import path from 'path'; @@ -158,6 +164,52 @@ const scriptletStats = new Map(); const genericDetails = new Map(); const requiredRedirectResources = new Set(); +// This will be used to sign our inserted `!#trusted on` directives +const secret = createHash('sha256').update(randomBytes(16)).digest('hex').slice(0,16); +log(`Secret: ${secret}`); + +/******************************************************************************/ + +const restrSeparator = '(?:[^%.0-9a-z_-]|$)'; + +const rePatternFromUrlFilter = s => { + let anchor = 0b000; + if ( s.startsWith('||') ) { + anchor = 0b100; + s = s.slice(2); + } else if ( s.startsWith('|') ) { + anchor = 0b010; + s = s.slice(1); + } + if ( s.endsWith('|') ) { + anchor |= 0b001; + s = s.slice(0, -1); + } + let reStr = s.replace(rePatternFromUrlFilter.rePlainChars, '\\$&') + .replace(rePatternFromUrlFilter.reSeparators, restrSeparator) + .replace(rePatternFromUrlFilter.reDanglingAsterisks, '') + .replace(rePatternFromUrlFilter.reAsterisks, '\\S*?'); + if ( anchor & 0b100 ) { + reStr = ( + reStr.startsWith('\\.') ? + rePatternFromUrlFilter.restrHostnameAnchor2 : + rePatternFromUrlFilter.restrHostnameAnchor1 + ) + reStr; + } else if ( anchor & 0b010 ) { + reStr = '^' + reStr; + } + if ( anchor & 0b001 ) { + reStr += '$'; + } + return reStr; +}; +rePatternFromUrlFilter.rePlainChars = /[.+?${}()|[\]\\]/g; +rePatternFromUrlFilter.reSeparators = /\^/g; +rePatternFromUrlFilter.reDanglingAsterisks = /^\*+|\*+$/g; +rePatternFromUrlFilter.reAsterisks = /\*+/g; +rePatternFromUrlFilter.restrHostnameAnchor1 = '^[a-z-]+://(?:[^/?#]+\\.)?'; +rePatternFromUrlFilter.restrHostnameAnchor2 = '^[a-z-]+://(?:[^/?#]+)?'; + /******************************************************************************/ async function fetchList(assetDetails) { @@ -179,7 +231,7 @@ async function fetchList(assetDetails) { } fetchedURLs.add(part.url); if ( part.url.startsWith('https://ublockorigin.github.io/uAssets/filters/') ) { - newParts.push(`!#trusted on ${assetDetails.secret}`); + newParts.push(`!#trusted on ${secret}`); } newParts.push( fetchText(part.url, cacheDir).then(details => { @@ -197,7 +249,7 @@ async function fetchList(assetDetails) { return { url, content: '' }; }) ); - newParts.push(`!#trusted off ${assetDetails.secret}`); + newParts.push(`!#trusted off ${secret}`); } parts = await Promise.all(newParts); parts = sfp.utils.preparser.expandIncludes(parts, env); @@ -330,6 +382,68 @@ function toJSONRuleset(ruleset) { /******************************************************************************/ +function toStrictBlockRule(rule, out) { + if ( rule.action.type !== 'block' ) { return; } + const { condition } = rule; + if ( condition === undefined ) { return; } + if ( condition.domainType ) { return; } + if ( condition.excludedResourceTypes ) { return; } + if ( condition.requestMethods ) { return; } + if ( condition.excludedRequestMethods ) { return; } + if ( condition.responseHeaders ) { return; } + if ( condition.excludedResponseHeaders ) { return; } + if ( condition.initiatorDomains ) { return; } + if ( condition.excludedInitiatorDomains ) { return; } + if ( condition.excludedRequestDomains ) { return; } + const { resourceTypes } = condition; + if ( resourceTypes === undefined ) { + if ( condition.requestDomains === undefined ) { return; } + } else { + if ( resourceTypes.length !== 1 ) { return; } + if ( resourceTypes[0] !== 'main_frame' ) { return; } + } + let regexFilter; + if ( condition.urlFilter ) { + regexFilter = rePatternFromUrlFilter(condition.urlFilter); + } else if ( condition.regexFilter ) { + regexFilter = condition.regexFilter; + } else { + regexFilter = '^https?://.*'; + } + if ( + regexFilter.startsWith('^') === false + ) { + regexFilter = `^.*${regexFilter}`; + } + if ( + regexFilter.endsWith('$') === false && + regexFilter.endsWith('.*') === false && + regexFilter.endsWith('.+') === false + ) { + regexFilter = `${regexFilter}.*`; + } + const strictBlockRule = { + action: { + type: 'redirect', + redirect: { + regexSubstitution: `/strictblock.html#\\0`, + }, + }, + condition: { + regexFilter, + resourceTypes: [ 'main_frame' ], + }, + priority: 29, + }; + if ( condition.requestDomains ) { + strictBlockRule.condition.requestDomains = condition.requestDomains.slice(); + } + out.set(toStrictBlockRule.ruleId++, strictBlockRule); +} +toStrictBlockRule.ruleId = 1; + +/******************************************************************************/ + async function processNetworkFilters(assetDetails, network) { const { ruleset: rules } = network; log(`Input filter count: ${network.filterCount}`); @@ -396,21 +510,46 @@ async function processNetworkFilters(assetDetails, network) { ); log(`\tmodifyHeaders=: ${modifyHeaders.length}`); - const urlskips = rules.filter(rule => isURLSkip(rule)).filter(rule => - rule.__modifierAction === 0 && - rule.condition && - rule.condition.regexFilter && - rule.condition.resourceTypes && - rule.condition.resourceTypes.includes('main_frame') - ).map(rule => { - const steps = rule.__modifierValue; - return { - re: rule.condition.regexFilter, - c: rule.condition.isUrlFilterCaseSensitive, - steps: steps.includes(' ') && steps.split(/ +/) || [ steps ], - }; - }); - log(`\turlskip=: ${urlskips.length}`); + const urlskips = new Map(); + for ( const rule of rules ) { + if ( isURLSkip(rule) === false ) { continue; } + if ( rule.__modifierAction !== 0 ) { continue; } + const { condition } = rule; + if ( condition.resourceTypes ) { + if ( condition.resourceTypes.includes('main_frame') === false ) { + continue; + } + } + const { urlFilter, regexFilter, requestDomains } = condition; + let re; + if ( urlFilter !== undefined ) { + re = rePatternFromUrlFilter(urlFilter); + } else if ( regexFilter !== undefined ) { + re = regexFilter; + } else { + re = '^'; + } + const rawSteps = rule.__modifierValue; + const steps = rawSteps.includes(' ') && rawSteps.split(/ +/) || [ rawSteps ]; + const keyEntry = { + re, + c: condition.isUrlFilterCaseSensitive, + steps, + } + const key = JSON.stringify(keyEntry); + let actualEntry = urlskips.get(key); + if ( actualEntry === undefined ) { + urlskips.set(key, keyEntry); + actualEntry = keyEntry; + } + if ( requestDomains !== undefined ) { + if ( actualEntry.hostnames === undefined ) { + actualEntry.hostnames = []; + } + actualEntry.hostnames.push(...requestDomains); + } + } + log(`\turlskip=: ${urlskips.size}`); const bad = rules.filter(rule => isUnsupported(rule) @@ -451,37 +590,26 @@ async function processNetworkFilters(assetDetails, network) { ); } - const strictBlocked = new Set(); + const strictBlocked = new Map(); for ( const rule of plainGood ) { - if ( rule.action.type !== 'block' ) { continue; } - if ( rule.condition.domainType ) { continue; } - if ( rule.condition.regexFilter ) { continue; } - if ( rule.condition.urlFilter ) { continue; } - if ( rule.condition.requestMethods ) { continue; } - if ( rule.condition.excludedRequestMethods ) { continue; } - if ( rule.condition.resourceTypes ) { continue; } - if ( rule.condition.excludedResourceTypes ) { continue; } - if ( rule.condition.responseHeaders ) { continue; } - if ( rule.condition.excludedResponseHeaders ) { continue; } - if ( rule.condition.initiatorDomains ) { continue; } - if ( rule.condition.excludedInitiatorDomains ) { continue; } - if ( rule.condition.requestDomains === undefined ) { continue; } - if ( rule.condition.excludedRequestDomains ) { continue; } - for ( const hn of rule.condition.requestDomains ) { - strictBlocked.add(hn); - } + toStrictBlockRule(rule, strictBlocked); } if ( strictBlocked.size !== 0 ) { + mergeRules(strictBlocked, 'requestDomains'); + let id = 1; + for ( const rule of strictBlocked.values() ) { + rule.id = id++; + } writeFile( `${rulesetDir}/strictblock/${assetDetails.id}.json`, - toJSONRuleset(Array.from(strictBlocked)) + toJSONRuleset(Array.from(strictBlocked.values())) ); } - if ( urlskips.length !== 0 ) { + if ( urlskips.size !== 0 ) { writeFile( `${rulesetDir}/urlskip/${assetDetails.id}.json`, - JSON.stringify(urlskips, null, 1) + JSON.stringify(Array.from(urlskips.values()), null, 1) ); } @@ -495,7 +623,7 @@ async function processNetworkFilters(assetDetails, network) { redirect: redirects.length, modifyHeaders: modifyHeaders.length, strictblock: strictBlocked.size, - urlskip: urlskips.length, + urlskip: urlskips.size, }; } @@ -1018,16 +1146,25 @@ async function rulesetFromURLs(assetDetails) { log('============================'); log(`Listset for '${assetDetails.id}':`); - if ( assetDetails.text === undefined ) { + if ( assetDetails.text === undefined && assetDetails.urls.length !== 0 ) { const text = await fetchList(assetDetails); - if ( text === '' ) { return; } assetDetails.text = text; + } else { + assetDetails.text = ''; } - if ( Array.isArray(assetDetails.filters) ) { - assetDetails.text += '\n' + assetDetails.filters.join('\n'); + if ( Array.isArray(assetDetails.filters) && assetDetails.filters.length ) { + const extra = [ + `!#trusted on ${secret}`, + ...assetDetails.filters, + `!#trusted off ${secret}`, + assetDetails.text, + ]; + assetDetails.text = extra.join('\n').trim(); } + if ( assetDetails.text === '' ) { return; } + const extensionPaths = []; for ( const [ fname, details ] of redirectResourcesMap ) { const path = `/web_accessible_resources/${fname}`; @@ -1045,7 +1182,7 @@ async function rulesetFromURLs(assetDetails) { const results = await dnrRulesetFromRawLists( [ { name: assetDetails.id, text: assetDetails.text } ], - { env, extensionPaths, secret: assetDetails.secret } + { env, extensionPaths, secret } ); const netStats = await processNetworkFilters( @@ -1181,19 +1318,13 @@ async function main() { JSON.parse(text) ); - // This will be used to sign our inserted `!#trusted on` directives - const secret = createHash('sha256').update(randomBytes(16)).digest('hex').slice(0,16); - log(`Secret: ${secret}`); - // Assemble all default lists as the default ruleset await rulesetFromURLs({ id: 'default', name: 'Ads, trackers, miners, and more' , enabled: true, - secret, urls: [ 'https://ublockorigin.github.io/uAssets/filters/filters.min.txt', - 'https://ublockorigin.github.io/uAssets/filters/badware.min.txt', 'https://ublockorigin.github.io/uAssets/filters/privacy.min.txt', 'https://ublockorigin.github.io/uAssets/filters/unbreak.min.txt', 'https://ublockorigin.github.io/uAssets/filters/quick-fixes.min.txt', @@ -1208,6 +1339,19 @@ async function main() { ], }); + await rulesetFromURLs({ + id: 'badware', + name: 'Badware risks' , + group: 'default', + enabled: true, + urls: [ + 'https://ublockorigin.github.io/uAssets/filters/badware.min.txt', + ], + homeURL: 'https://github.com/uBlockOrigin/uAssets', + filters: [ + ], + }); + // Handpicked rulesets from assets.json const handpicked = [ 'block-lan', @@ -1235,7 +1379,6 @@ async function main() { name: 'EasyList/uBO – Cookie Notices', group: 'annoyances', enabled: false, - secret, urls: [ 'https://ublockorigin.github.io/uAssets/thirdparties/easylist-cookies.txt', 'https://ublockorigin.github.io/uAssets/filters/annoyances-cookies.txt', @@ -1247,7 +1390,6 @@ async function main() { name: 'EasyList/uBO – Overlay Notices', group: 'annoyances', enabled: false, - secret, urls: [ 'https://ublockorigin.github.io/uAssets/thirdparties/easylist-newsletters.txt', 'https://ublockorigin.github.io/uAssets/filters/annoyances-others.txt', @@ -1409,14 +1551,6 @@ async function main() { manifest.declarative_net_request = { rule_resources: ruleResources }; // Patch web_accessible_resources key manifest.web_accessible_resources = manifest.web_accessible_resources || []; - // Strict-block-related resource - const strictblockDocument = `strictblock.${secret}.html`; - copyFile('./strictblock.html', `${outputDir}/${strictblockDocument}`); - manifest.web_accessible_resources.push({ - resources: [ `/${strictblockDocument}` ], - matches: [ '' ], - }); - // Secondary resources const web_accessible_resources = { resources: Array.from(requiredRedirectResources).map(path => `/${path}`), matches: [ '' ], diff --git a/src/js/static-dnr-filtering.js b/src/js/static-dnr-filtering.js index 9935fd161..2e75633eb 100644 --- a/src/js/static-dnr-filtering.js +++ b/src/js/static-dnr-filtering.js @@ -354,6 +354,94 @@ function addToDNR(context, list) { /******************************************************************************/ +// Merge rules where possible by merging arrays of a specific property. +// +// https://github.com/uBlockOrigin/uBOL-home/issues/10#issuecomment-1304822579 +// Do not merge rules which have errors. + +function mergeRules(rulesetMap, mergeTarget) { + const sorter = (_, v) => { + if ( Array.isArray(v) ) { + return typeof v[0] === 'string' ? v.sort() : v; + } + if ( v instanceof Object ) { + const sorted = {}; + for ( const kk of Object.keys(v).sort() ) { + sorted[kk] = v[kk]; + } + return sorted; + } + return v; + }; + const ruleHasher = (rule, target) => { + return JSON.stringify(rule, (k, v) => { + if ( k.startsWith('_') ) { return; } + if ( k === target ) { return; } + return sorter(k, v); + }); + }; + const extractTargetValue = (obj, target) => { + for ( const [ k, v ] of Object.entries(obj) ) { + if ( Array.isArray(v) && k === target ) { return v; } + if ( v instanceof Object ) { + const r = extractTargetValue(v, target); + if ( r !== undefined ) { return r; } + } + } + }; + const extractTargetOwner = (obj, target) => { + for ( const [ k, v ] of Object.entries(obj) ) { + if ( Array.isArray(v) && k === target ) { return obj; } + if ( v instanceof Object ) { + const r = extractTargetOwner(v, target); + if ( r !== undefined ) { return r; } + } + } + }; + const mergeMap = new Map(); + for ( const [ id, rule ] of rulesetMap ) { + if ( rule._error !== undefined ) { continue; } + const hash = ruleHasher(rule, mergeTarget); + if ( mergeMap.has(hash) === false ) { + mergeMap.set(hash, []); + } + mergeMap.get(hash).push(id); + } + for ( const ids of mergeMap.values() ) { + if ( ids.length === 1 ) { continue; } + const leftHand = rulesetMap.get(ids[0]); + const leftHandSet = new Set( + extractTargetValue(leftHand, mergeTarget) || [] + ); + for ( let i = 1; i < ids.length; i++ ) { + const rightHandId = ids[i]; + const rightHand = rulesetMap.get(rightHandId); + const rightHandArray = extractTargetValue(rightHand, mergeTarget); + if ( rightHandArray !== undefined ) { + if ( leftHandSet.size !== 0 ) { + for ( const item of rightHandArray ) { + leftHandSet.add(item); + } + } + } else { + leftHandSet.clear(); + } + rulesetMap.delete(rightHandId); + } + const leftHandOwner = extractTargetOwner(leftHand, mergeTarget); + if ( leftHandSet.size > 1 ) { + //if ( leftHandOwner === undefined ) { debugger; } + leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort(); + } else if ( leftHandSet.size === 0 ) { + if ( leftHandOwner !== undefined ) { + leftHandOwner[mergeTarget] = undefined; + } + } + } +} + +/******************************************************************************/ + function finalizeRuleset(context, network) { const ruleset = network.ruleset; @@ -365,90 +453,6 @@ function finalizeRuleset(context, network) { rulesetMap.set(ruleId++, rule); } } - // Merge rules where possible by merging arrays of a specific property. - // - // https://github.com/uBlockOrigin/uBOL-home/issues/10#issuecomment-1304822579 - // Do not merge rules which have errors. - const mergeRules = (rulesetMap, mergeTarget) => { - const mergeMap = new Map(); - const sorter = (_, v) => { - if ( Array.isArray(v) ) { - return typeof v[0] === 'string' ? v.sort() : v; - } - if ( v instanceof Object ) { - const sorted = {}; - for ( const kk of Object.keys(v).sort() ) { - sorted[kk] = v[kk]; - } - return sorted; - } - return v; - }; - const ruleHasher = (rule, target) => { - return JSON.stringify(rule, (k, v) => { - if ( k.startsWith('_') ) { return; } - if ( k === target ) { return; } - return sorter(k, v); - }); - }; - const extractTargetValue = (obj, target) => { - for ( const [ k, v ] of Object.entries(obj) ) { - if ( Array.isArray(v) && k === target ) { return v; } - if ( v instanceof Object ) { - const r = extractTargetValue(v, target); - if ( r !== undefined ) { return r; } - } - } - }; - const extractTargetOwner = (obj, target) => { - for ( const [ k, v ] of Object.entries(obj) ) { - if ( Array.isArray(v) && k === target ) { return obj; } - if ( v instanceof Object ) { - const r = extractTargetOwner(v, target); - if ( r !== undefined ) { return r; } - } - } - }; - for ( const [ id, rule ] of rulesetMap ) { - if ( rule._error !== undefined ) { continue; } - const hash = ruleHasher(rule, mergeTarget); - if ( mergeMap.has(hash) === false ) { - mergeMap.set(hash, []); - } - mergeMap.get(hash).push(id); - } - for ( const ids of mergeMap.values() ) { - if ( ids.length === 1 ) { continue; } - const leftHand = rulesetMap.get(ids[0]); - const leftHandSet = new Set( - extractTargetValue(leftHand, mergeTarget) || [] - ); - for ( let i = 1; i < ids.length; i++ ) { - const rightHandId = ids[i]; - const rightHand = rulesetMap.get(rightHandId); - const rightHandArray = extractTargetValue(rightHand, mergeTarget); - if ( rightHandArray !== undefined ) { - if ( leftHandSet.size !== 0 ) { - for ( const item of rightHandArray ) { - leftHandSet.add(item); - } - } - } else { - leftHandSet.clear(); - } - rulesetMap.delete(rightHandId); - } - const leftHandOwner = extractTargetOwner(leftHand, mergeTarget); - if ( leftHandSet.size > 1 ) { - //if ( leftHandOwner === undefined ) { debugger; } - leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort(); - } else if ( leftHandSet.size === 0 ) { - if ( leftHandOwner !== undefined ) { - leftHandOwner[mergeTarget] = undefined; - } - } - } - }; mergeRules(rulesetMap, 'resourceTypes'); mergeRules(rulesetMap, 'removeParams'); mergeRules(rulesetMap, 'initiatorDomains'); @@ -508,4 +512,4 @@ async function dnrRulesetFromRawLists(lists, options = {}) { /******************************************************************************/ -export { dnrRulesetFromRawLists }; +export { dnrRulesetFromRawLists, mergeRules };