From 5ddd3aaac679541cbda518477e702736ed51b7a2 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Sun, 18 Sep 2022 09:31:44 -0400 Subject: [PATCH] [mv3] More work toward improving declarative css/js injection --- platform/mv3/extension/css/3p-filters.css | 1 + platform/mv3/extension/css/popup.css | 2 + platform/mv3/extension/js/background.js | 20 +- .../mv3/extension/js/scripting-manager.js | 181 ++++++---------- platform/mv3/make-rulesets.js | 200 +++++++++++------- src/js/static-dnr-filtering.js | 3 + src/js/static-filtering-parser.js | 22 +- 7 files changed, 227 insertions(+), 202 deletions(-) diff --git a/platform/mv3/extension/css/3p-filters.css b/platform/mv3/extension/css/3p-filters.css index 3c588639d..741b92369 100644 --- a/platform/mv3/extension/css/3p-filters.css +++ b/platform/mv3/extension/css/3p-filters.css @@ -58,6 +58,7 @@ body { .listEntry > * { margin-left: 0; margin-right: 0; + text-transform: capitalize; unicode-bidi: embed; } .listEntry .listname { diff --git a/platform/mv3/extension/css/popup.css b/platform/mv3/extension/css/popup.css index e09228c17..e977144e4 100644 --- a/platform/mv3/extension/css/popup.css +++ b/platform/mv3/extension/css/popup.css @@ -122,7 +122,9 @@ body.needSave #revertRules { } #rulesetStats .rulesetDetails h1 { font-size: 1em; + font-weight: normal; margin: 0.5em 0; + text-transform: capitalize; } #rulesetStats .rulesetDetails p { font-size: var(--font-size-smaller); diff --git a/platform/mv3/extension/js/background.js b/platform/mv3/extension/js/background.js index a8a66c9d2..9f6e7c72d 100644 --- a/platform/mv3/extension/js/background.js +++ b/platform/mv3/extension/js/background.js @@ -434,7 +434,10 @@ function onMessage(request, sender, callback) { case 'applyRulesets': { enableRulesets(request.enabledRulesets).then(( ) => { rulesetConfig.enabledRulesets = request.enabledRulesets; - return saveRulesetConfig(); + return Promise.all([ + saveRulesetConfig(), + registerInjectable(), + ]); }).then(( ) => { callback(); }); @@ -513,14 +516,17 @@ async function start() { const currentVersion = getCurrentVersion(); if ( currentVersion !== rulesetConfig.version ) { console.log(`Version change: ${rulesetConfig.version} => ${currentVersion}`); - await Promise.all([ - updateRegexRules(), - registerInjectable(), - ]); - rulesetConfig.version = currentVersion; - saveRulesetConfig(); + updateRegexRules().then(( ) => { + rulesetConfig.version = currentVersion; + saveRulesetConfig(); + }); } + // Unsure whether the browser remembers correctly registered css/scripts + // after we quit the browser. For now uBOL will check unconditionally at + // launch time whether content css/scripts are properly registered. + registerInjectable(); + const enabledRulesets = await dnr.getEnabledRulesets(); console.log(`Enabled rulesets: ${enabledRulesets}`); diff --git a/platform/mv3/extension/js/scripting-manager.js b/platform/mv3/extension/js/scripting-manager.js index 16c9a84be..4057defe0 100644 --- a/platform/mv3/extension/js/scripting-manager.js +++ b/platform/mv3/extension/js/scripting-manager.js @@ -31,32 +31,26 @@ import { parsedURLromOrigin } from './utils.js'; /******************************************************************************/ -const CSS_TYPE = 0; -const JS_TYPE = 1; +const CSS_TYPE = '0'; +const JS_TYPE = '1'; /******************************************************************************/ -let cssDetailsPromise; -let scriptletDetailsPromise; +let scriptingDetailsPromise; -function getCSSDetails() { - if ( cssDetailsPromise !== undefined ) { - return cssDetailsPromise; +function getScriptingDetails() { + if ( scriptingDetailsPromise !== undefined ) { + return scriptingDetailsPromise; } - cssDetailsPromise = fetchJSON('/content-css/css-specific').then(rules => { - return new Map(rules); + scriptingDetailsPromise = fetchJSON('/rulesets/scripting-details').then(entries => { + const out = new Map(entries); + for ( const details of out.values() ) { + details.matches = new Map(details.matches); + details.excludeMatches = new Map(details.excludeMatches); + } + return out; }); - return cssDetailsPromise; -} - -function getScriptletDetails() { - if ( scriptletDetailsPromise !== undefined ) { - return scriptletDetailsPromise; - } - scriptletDetailsPromise = fetchJSON('/content-js/scriptlet-details').then(rules => { - return new Map(rules); - }); - return scriptletDetailsPromise; + return scriptingDetailsPromise; } /******************************************************************************/ @@ -83,6 +77,12 @@ const hostnamesFromMatches = origins => { return out; }; +const toBroaderHostname = hn => { + if ( hn === '*' ) { return ''; } + const pos = hn.indexOf('.'); + return pos !== -1 ? hn.slice(pos+1) : '*'; +}; + const arrayEq = (a, b) => { if ( a === undefined ) { return b === undefined; } if ( b === undefined ) { return false; } @@ -100,21 +100,21 @@ const toRegisterable = (fname, entry) => { id: fname, allFrames: true, }; - if ( entry.y ) { - directive.matches = matchesFromHostnames(entry.y); + if ( entry.matches ) { + directive.matches = matchesFromHostnames(entry.matches); } else { directive.matches = [ '*://*/*' ]; } - if ( entry.n ) { - directive.excludeMatches = matchesFromHostnames(entry.n); + if ( entry.excludeMatches ) { + directive.excludeMatches = matchesFromHostnames(entry.excludeMatches); } - if ( entry.type === CSS_TYPE ) { + if ( fname.at(-1) === CSS_TYPE ) { directive.css = [ - `/content-css/${fname.slice(0,1)}/${fname.slice(1,2)}/${fname.slice(2,8)}.css` + `/rulesets/css/${fname.slice(0,1)}/${fname.slice(1,2)}/${fname.slice(2)}.css` ]; - } else if ( entry.type === JS_TYPE ) { + } else if ( fname.at(-1) === JS_TYPE ) { directive.js = [ - `/content-js/${fname.slice(0,1)}/${fname.slice(1,8)}.js` + `/rulesets/js/${fname.slice(0,1)}/${fname.slice(1)}.js` ]; directive.runAt = 'document_start'; directive.world = 'MAIN'; @@ -124,11 +124,13 @@ const toRegisterable = (fname, entry) => { }; const toMaybeUpdatable = (registered, candidate) => { - const matches = candidate.y && matchesFromHostnames(candidate.y); + const matches = candidate.matches && + matchesFromHostnames(candidate.matches); if ( arrayEq(registered.matches, matches) === false ) { return toRegisterable(candidate); } - const excludeMatches = candidate.n && matchesFromHostnames(candidate.n); + const excludeMatches = candidate.excludeMatches && + matchesFromHostnames(candidate.excludeMatches); if ( arrayEq(registered.excludeMatches, excludeMatches) === false ) { return toRegisterable(candidate); } @@ -136,57 +138,28 @@ const toMaybeUpdatable = (registered, candidate) => { /******************************************************************************/ -const shouldRegister = (origins, matches) => { - if ( Array.isArray(matches) === false ) { return true; } - for ( const origin of origins ) { - if ( origin === '*' ) { return true; } - let hn = origin; - for (;;) { - if ( matches.includes(hn) ) { return true; } - if ( hn === '*' ) { break; } - const pos = hn.indexOf('.'); - hn = pos !== -1 - ? hn.slice(pos+1) - : '*'; - } - } - return false; -}; - -/******************************************************************************/ - async function getInjectableCount(origin) { const url = parsedURLromOrigin(origin); if ( url === undefined ) { return 0; } const [ rulesetIds, - cssDetails, - scriptletDetails, + scriptingDetails, ] = await Promise.all([ dnr.getEnabledRulesets(), - getCSSDetails(), - getScriptletDetails(), + getScriptingDetails(), ]); let total = 0; for ( const rulesetId of rulesetIds ) { - if ( cssDetails.has(rulesetId) ) { - const entries = cssDetails.get(rulesetId); - for ( const entry of entries ) { - if ( shouldRegister([ url.hostname ], entry[1].y) ) { - total += 1; - } - } - } - if ( scriptletDetails.has(rulesetId) ) { - const entries = cssDetails.get(rulesetId); - for ( const entry of entries ) { - if ( shouldRegister([ url.hostname ], entry[1].y) ) { - total += 1; - } - } + if ( scriptingDetails.has(rulesetId) === false ) { continue; } + const details = scriptingDetails.get(rulesetId); + let hn = url.hostname; + while ( hn !== '' ) { + const fnames = details.matches.get(hn); + total += fnames && fnames.length || 0; + hn = toBroaderHostname(hn); } } @@ -195,71 +168,53 @@ async function getInjectableCount(origin) { /******************************************************************************/ +// TODO: Mind trusted-site directives. + async function registerInjectable() { const [ - origins, + hostnames, rulesetIds, registered, - cssDetails, - scriptletDetails, + scriptingDetails, ] = await Promise.all([ browser.permissions.getAll(), dnr.getEnabledRulesets(), browser.scripting.getRegisteredContentScripts(), - getCSSDetails(), - getScriptletDetails(), + getScriptingDetails(), ]).then(results => { results[0] = new Set(hostnamesFromMatches(results[0].origins)); return results; }); - if ( origins.has('*') && origins.size > 1 ) { - origins.clear(); - origins.add('*'); + if ( hostnames.has('*') && hostnames.size > 1 ) { + hostnames.clear(); + hostnames.add('*'); } - const mergeEntries = (a, b) => { - if ( b.y !== undefined ) { - if ( a.y === undefined ) { - a.y = new Set(b.y); - } else { - b.y.forEach(v => a.y.add(v)); - } - } - if ( b.n !== undefined ) { - if ( a.n === undefined ) { - a.n = new Set(b.n); - } else { - b.n.forEach(v => a.n.add(v)); - } - } - return a; - }; - const toRegister = new Map(); - for ( const rulesetId of rulesetIds ) { - if ( cssDetails.has(rulesetId) ) { - for ( const [ fname, entry ] of cssDetails.get(rulesetId) ) { - if ( shouldRegister(origins, entry.y) === false ) { continue; } - let existing = toRegister.get(fname); - if ( existing === undefined ) { - existing = { type: CSS_TYPE }; - toRegister.set(fname, existing); - } - mergeEntries(existing, entry); + const checkRealm = (details, prop, hn) => { + const fnames = details[prop].get(hn); + if ( fnames === undefined ) { return; } + for ( const fname of fnames ) { + const existing = toRegister.get(fname); + if ( existing ) { + existing[prop].push(hn); + } else { + toRegister.set(fname, { [prop]: [ hn ] }); } } - if ( scriptletDetails.has(rulesetId) ) { - for ( const [ fname, entry ] of scriptletDetails.get(rulesetId) ) { - if ( shouldRegister(origins, entry.y) === false ) { continue; } - let existing = toRegister.get(fname); - if ( existing === undefined ) { - existing = { type: JS_TYPE }; - toRegister.set(fname, existing); - } - mergeEntries(existing, entry); + }; + + for ( const rulesetId of rulesetIds ) { + const details = scriptingDetails.get(rulesetId); + if ( details === undefined ) { continue; } + for ( let hn of hostnames ) { + while ( hn !== '' ) { + checkRealm(details, 'matches', hn); + checkRealm(details, 'excludeMatches', hn); + hn = toBroaderHostname(hn); } } } diff --git a/platform/mv3/make-rulesets.js b/platform/mv3/make-rulesets.js index b095bd5ed..b3361e048 100644 --- a/platform/mv3/make-rulesets.js +++ b/platform/mv3/make-rulesets.js @@ -54,8 +54,8 @@ const commandLineArgs = (( ) => { const outputDir = commandLineArgs.get('output') || '.'; const cacheDir = `${outputDir}/../mv3-data`; const rulesetDir = `${outputDir}/rulesets`; -const cssDir = `${outputDir}/content-css`; -const scriptletDir = `${outputDir}/content-js`; +const cssDir = `${rulesetDir}/css`; +const scriptletDir = `${rulesetDir}/js`; const env = [ 'chromium', 'ubol' ]; /******************************************************************************/ @@ -151,8 +151,7 @@ const writeOps = []; const ruleResources = []; const rulesetDetails = []; -const cssDetails = new Map(); -const scriptletDetails = new Map(); +const scriptingDetails = new Map(); /******************************************************************************/ @@ -282,6 +281,30 @@ async function processNetworkFilters(assetDetails, network) { /******************************************************************************/ +function addScriptingAPIResources(id, entry, prop, fname) { + if ( entry[prop] === undefined ) { return; } + for ( const hn of entry[prop] ) { + let details = scriptingDetails.get(id); + if ( details === undefined ) { + details = { + matches: new Map(), + excludeMatches: new Map(), + }; + scriptingDetails.set(id, details); + } + let fnames = details[prop].get(hn); + if ( fnames === undefined ) { + fnames = new Set(); + details[prop].set(hn, fnames); + } + fnames.add(fname); + } +} + +/******************************************************************************/ + +// This group together selectors which are used by a the same hostnames. + function optimizeExtendedFilters(filters) { if ( filters === undefined ) { return []; } const merge = new Map(); @@ -307,58 +330,92 @@ function optimizeExtendedFilters(filters) { const globalCSSFileSet = new Set(); -const style = [ - ' display:none!important;', - ' position:absolute!important;', - ' z-index:0!important;', - ' visibility:collapse!important;', -].join('\n'); +const cssDeclaration = +`@layer { +$selector$ { + display:none!important; + } +}`; function processCosmeticFilters(assetDetails, mapin) { if ( mapin === undefined ) { return 0; } const optimized = optimizeExtendedFilters(mapin); - const cssEntries = new Map(); + const cssContentMap = new Map(); + for ( const entry of optimized ) { - const selectors = entry.payload.join(',\n'); - const fname = createHash('sha256').update(selectors).digest('hex').slice(0,8); + const selectors = entry.payload.map(s => ` ${s}`).join(',\n'); + // ends-with 0 = css resource + const fname = createHash('sha256').update(selectors).digest('hex').slice(0,8) + '0'; + let contentDetails = cssContentMap.get(fname); + if ( contentDetails === undefined ) { + contentDetails = { selectors }; + cssContentMap.set(fname, contentDetails); + } + if ( entry.matches !== undefined ) { + if ( contentDetails.matches === undefined ) { + contentDetails.matches = new Set(); + } + for ( const hn of entry.matches ) { + contentDetails.matches.add(hn); + } + } + if ( entry.excludeMatches !== undefined ) { + if ( contentDetails.excludeMatches === undefined ) { + contentDetails.excludeMatches = new Set(); + } + for ( const hn of entry.excludeMatches ) { + contentDetails.excludeMatches.add(hn); + } + } + } + + // We do not want more than 128 CSS files per subscription, so we will + // group multiple unrelated selectors in the same file and hope this does + // not cause false positives. + + const contentPerFile = Math.ceil(cssContentMap.size / 128); + const cssContentArray = Array.from(cssContentMap).map(entry => entry[1]); + let distinctResourceCount = 0; + + for ( let i = 0; i < cssContentArray.length; i += contentPerFile ) { + const slice = cssContentArray.slice(i, i + contentPerFile); + const matches = slice.map(entry => + Array.from(entry.matches || []) + ).flat(); + const excludeMatches = slice.map(entry => + Array.from(entry.excludeMatches || []) + ).flat(); + const selectors = slice.map(entry => + entry.selectors + ).join(',\n'); + const fname = createHash('sha256').update(selectors).digest('hex').slice(0,8) + '0'; if ( globalCSSFileSet.has(fname) === false ) { globalCSSFileSet.add(fname); - const fpath = `${fname.slice(0,1)}/${fname.slice(1,2)}/${fname.slice(2,8)}`; + const fpath = `${fname.slice(0,1)}/${fname.slice(1,2)}/${fname.slice(2)}`; writeFile( `${cssDir}/${fpath}.css`, - `${selectors} {\n${style}\n}\n` + cssDeclaration.replace('$selector$', selectors) ); + distinctResourceCount += 1; } - const existing = cssEntries.get(fname); - if ( existing === undefined ) { - cssEntries.set(fname, { - y: entry.matches, - n: entry.excludeMatches, - }); - continue; - } - if ( entry.matches ) { - for ( const hn of entry.matches ) { - if ( existing.y.includes(hn) ) { continue; } - existing.y.push(hn); - } - } - if ( entry.excludeMatches ) { - for ( const hn of entry.excludeMatches ) { - if ( existing.n.includes(hn) ) { continue; } - existing.n.push(hn); - } - } + addScriptingAPIResources( + assetDetails.id, + { matches }, + 'matches', + fname + ); + addScriptingAPIResources( + assetDetails.id, + { excludeMatches }, + 'excludeMatches', + fname + ); } - log(`CSS entries: ${cssEntries.size}`); + log(`CSS entries: ${distinctResourceCount}`); - if ( cssEntries.size !== 0 ) { - cssDetails.set(assetDetails.id, Array.from(cssEntries)); - } - - return cssEntries.size; + return distinctResourceCount; } /******************************************************************************/ @@ -461,47 +518,38 @@ async function processScriptletFilters(assetDetails, mapin) { }; // Generate distinct scriptlet files according to patched scriptlets - const scriptletEntries = new Map(); + let distinctResourceCount = 0; for ( const [ rawFilter, entry ] of mapin ) { const normalized = parseFilter(rawFilter); if ( normalized === undefined ) { continue; } const json = JSON.stringify(normalized); - const fname = createHash('sha256').update(json).digest('hex').slice(0,8); + // ends-with 1 = scriptlet resource + const fname = createHash('sha256').update(json).digest('hex').slice(0,8) + '1'; if ( globalPatchedScriptletsSet.has(fname) === false ) { globalPatchedScriptletsSet.add(fname); const scriptlet = patchScriptlet(normalized); - const fpath = `${fname.slice(0,1)}/${fname.slice(1,8)}`; + const fpath = `${fname.slice(0,1)}/${fname.slice(1)}`; writeFile(`${scriptletDir}/${fpath}.js`, scriptlet); + distinctResourceCount += 1; } - const existing = scriptletEntries.get(fname); - if ( existing === undefined ) { - scriptletEntries.set(fname, { - y: entry.matches, - n: entry.excludeMatches, - }); - continue; - } - if ( entry.matches ) { - for ( const hn of entry.matches ) { - if ( existing.y.includes(hn) ) { continue; } - existing.y.push(hn); - } - } - if ( entry.excludeMatches ) { - for ( const hn of entry.excludeMatches ) { - if ( existing.n.includes(hn) ) { continue; } - existing.n.push(hn); - } - } + addScriptingAPIResources( + assetDetails.id, + entry, + 'matches', + fname + ); + addScriptingAPIResources( + assetDetails.id, + entry, + 'excludeMatches', + fname + ); } - log(`Scriptlet entries: ${scriptletEntries.size}`); + log(`Scriptlet entries: ${distinctResourceCount}`); - if ( scriptletEntries.size !== 0 ) { - scriptletDetails.set(assetDetails.id, Array.from(scriptletEntries)); - } - return scriptletEntries.size; + return distinctResourceCount; } /******************************************************************************/ @@ -666,14 +714,16 @@ async function main() { `${JSON.stringify(rulesetDetails, null, 1)}\n` ); - writeFile( - `${cssDir}/css-specific.json`, - `${JSON.stringify(Array.from(cssDetails))}\n` - ); + const replacer = (k, v) => { + if ( v instanceof Set || v instanceof Map ) { + return Array.from(v); + } + return v; + }; writeFile( - `${scriptletDir}/scriptlet-details.json`, - `${JSON.stringify(Array.from(scriptletDetails))}\n` + `${rulesetDir}/scripting-details.json`, + `${JSON.stringify(scriptingDetails, replacer)}\n` ); await Promise.all(writeOps); diff --git a/src/js/static-dnr-filtering.js b/src/js/static-dnr-filtering.js index 91ed11819..acfeddcf0 100644 --- a/src/js/static-dnr-filtering.js +++ b/src/js/static-dnr-filtering.js @@ -93,12 +93,15 @@ function addExtendedToDNR(context, parser) { // https://github.com/chrisaljoudi/uBlock/issues/151 // Negated hostname means the filter applies to all non-negated hostnames // of same filter OR globally if there is no non-negated hostnames. + // Drop selectors which can potentially lead to the hiding of + // html/body elements. for ( const { hn, not, bad } of parser.extOptions() ) { if ( bad ) { continue; } if ( hn.endsWith('.*') ) { continue; } const { compiled, exception } = parser.result; if ( compiled.startsWith('{') ) { continue; } if ( exception ) { continue; } + if ( /(^|[^\w#.\-\[])(html|body)(,|$)/i.test(compiled) ) { continue; } let details = context.cosmeticFilters.get(compiled); if ( details === undefined ) { details = {}; diff --git a/src/js/static-filtering-parser.js b/src/js/static-filtering-parser.js index 70ebc5bd8..e36328c31 100644 --- a/src/js/static-filtering-parser.js +++ b/src/js/static-filtering-parser.js @@ -1374,11 +1374,15 @@ Parser.prototype.SelectorCompiler = class { return document.createElement('div'); })(); - this.reProceduralOperator = new RegExp([ - '^(?:', - Array.from(parser.proceduralOperatorTokens.keys()).join('|'), - ')\\(' - ].join('')); + const allProceduralOperators = Array.from( + parser.proceduralOperatorTokens.keys() + ); + this.reProceduralOperator = new RegExp( + `^(?:${allProceduralOperators.join('|')})\\(` + ); + this.reHasProceduralOperator = new RegExp( + `:(?:${allProceduralOperators.filter(s => s !== 'not').join('|')})\\(` + ); this.reEatBackslashes = /\\([()])/g; this.reEscapeRegex = /[.*+?^${}()|[\]\\]/g; this.reDropScope = /^\s*:scope\s*(?=[+>~])/; @@ -1502,7 +1506,9 @@ Parser.prototype.SelectorCompiler = class { // assign new text. sheetSelectable(s) { if ( this.reCommonSelector.test(s) ) { return true; } - if ( this.cssValidatorElement === null ) { return false; } + if ( this.cssValidatorElement === null ) { + return this.reHasProceduralOperator.test(s) === false; + } let valid = false; try { this.cssValidatorElement.childNodes[0].nodeValue = `_z + ${s}{color:red;} _z{color:red;}`; @@ -1521,7 +1527,9 @@ Parser.prototype.SelectorCompiler = class { // - opening comment `/*` querySelectable(s) { if ( this.reCommonSelector.test(s) ) { return true; } - if ( this.div === null ) { return false; } + if ( this.div === null ) { + return this.reHasProceduralOperator.test(s) === false; + } try { this.div.querySelector(`${s},${s}:not(#foo)`); if ( s.includes('/*') ) { return false; }