From 2cfeaddbedeed59c073e83ba3b4df4b4f93cc9e0 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Fri, 13 Nov 2020 09:23:25 -0500 Subject: [PATCH] Fine tune various static filtering code Notably, make `queryprune` option available only to filter list authors, until there are guards against bad filters in some future and until the option syntax and behavior is fully settled. Instances of `queryprune` in filter lists will be compiled, however instances of `queryprune` in _"My filters"_ will be ignored unless users indicated they are a filter list author. --- src/js/1p-filters.js | 1 + src/js/codemirror/ubo-static-filtering.js | 3 ++ src/js/messaging.js | 1 + src/js/static-filtering-parser.js | 27 ++++++++----- src/js/static-net-filtering.js | 48 +++++++++++++---------- src/js/storage.js | 28 ++++++------- 6 files changed, 64 insertions(+), 44 deletions(-) diff --git a/src/js/1p-filters.js b/src/js/1p-filters.js index c30122958..adf179f88 100644 --- a/src/js/1p-filters.js +++ b/src/js/1p-filters.js @@ -57,6 +57,7 @@ vAPI.messaging.send('dashboard', { if ( mode.setHints instanceof Function ) { mode.setHints(response); } + mode.parser.expertMode = response.expertMode !== false; }); let cachedUserFilters = ''; diff --git a/src/js/codemirror/ubo-static-filtering.js b/src/js/codemirror/ubo-static-filtering.js index 6f7ac7088..b4a74239c 100644 --- a/src/js/codemirror/ubo-static-filtering.js +++ b/src/js/codemirror/ubo-static-filtering.js @@ -364,6 +364,9 @@ CodeMirror.defineMode('ubo-static-filtering', function() { preparseDirectiveHints.push(...details.preparseDirectiveHints); initHints(); }, + get parser() { + return parser; + }, }; }); diff --git a/src/js/messaging.js b/src/js/messaging.js index 010f18045..06cad9fdb 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -1184,6 +1184,7 @@ const onMessage = function(request, sender, callback) { redirectResources: µb.redirectEngine.getResourceDetails(), preparseDirectiveTokens: µb.preparseDirectives.getTokens(), preparseDirectiveHints: µb.preparseDirectives.getHints(), + expertMode: µb.hiddenSettings.filterAuthorMode, }; break; diff --git a/src/js/static-filtering-parser.js b/src/js/static-filtering-parser.js index 764c6b38f..002178bdf 100644 --- a/src/js/static-filtering-parser.js +++ b/src/js/static-filtering-parser.js @@ -102,6 +102,7 @@ const Parser = class { this.netOptionsIterator = new NetOptionsIterator(this); this.extOptionsIterator = new ExtOptionsIterator(this); this.maxTokenLength = Number.MAX_SAFE_INTEGER; + this.expertMode = options.expertMode !== false; this.reIsLocalhostRedirect = /(?:0\.0\.0\.0|(?:broadcast|local)host|local|ip6-\w+)(?:[^\w.-]|$)/; this.reHostname = /^[^\x00-\x24\x26-\x29\x2B\x2C\x2F\x3A-\x40\x5B-\x5E\x60\x7B-\x7F]+/; this.reHostsSink = /^[\w-.:\[\]]+$/; @@ -2281,14 +2282,19 @@ const NetOptionsIterator = class { if ( descriptor === undefined || hasBits(descriptor, OPTNotSupported) || - ltok !== lopt && hasNoBits(descriptor, OPTCanNegate) || - this.exception && hasBits(descriptor, OPTBlockOnly) || - this.exception === false && hasBits(descriptor, OPTAllowOnly) || - assigned && hasNoBits(descriptor, OPTMustAssign) || - assigned === false && hasBits(descriptor, OPTMustAssign) && ( - this.exception === false || - hasNoBits(descriptor, OPTAllowMayAssign) - ) + ltok !== lopt && + hasNoBits(descriptor, OPTCanNegate) || + this.exception && + hasBits(descriptor, OPTBlockOnly) || + this.exception === false && + hasBits(descriptor, OPTAllowOnly) || + assigned && + hasNoBits(descriptor, OPTMustAssign) || + assigned === false && + hasBits(descriptor, OPTMustAssign) && ( + this.exception === false || + hasNoBits(descriptor, OPTAllowMayAssign) + ) ) { descriptor = OPTTokenInvalid; } @@ -2398,7 +2404,10 @@ const NetOptionsIterator = class { { const i = this.tokenPos[OPTTokenQueryprune]; if ( i !== -1 ) { - if ( hasBits(allBits, OPTNonNetworkType) ) { + if ( + this.parser.expertMode === false || + hasBits(allBits, OPTNonNetworkType) + ) { optSlices[i] = OPTTokenInvalid; if ( this.interactive ) { this.parser.errorSlices(optSlices[i+1], optSlices[i+5]); diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 7d8cda908..2339f1d16 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -2660,6 +2660,17 @@ const FilterParser = class { return i === 1 ? out[0] : out.join('|'); } + parseModifierOption(modifier, value) { + if ( this.modifyType !== undefined ) { return false; } + this.modifyType = modifier; + if ( value !== undefined ) { + this.modifyValue = value; + } else if ( this.action === AllowAction ) { + this.modifyValue = ''; + } + return true; + } + parseOptions(parser) { for ( let { id, val, not } of parser.netOptions() ) { switch ( id ) { @@ -2677,13 +2688,11 @@ const FilterParser = class { this.badFilter = true; break; case parser.OPTTokenCsp: - if ( this.modifyType !== undefined ) { return false; } - this.modifyType = parser.OPTTokenCsp; - if ( val !== undefined ) { - if ( this.reBadCSP.test(val) ) { return false; } - this.modifyValue = val; - } else if ( this.action === AllowAction ) { - this.modifyValue = ''; + if ( this.parseModifierOption(id, val) === false ) { + return false; + } + if ( val !== undefined && this.reBadCSP.test(val) ) { + return false; } break; // https://github.com/gorhill/uBlock/issues/2294 @@ -2725,15 +2734,10 @@ const FilterParser = class { this.modifyValue = 'noopmp4-1s'; break; case parser.OPTTokenQueryprune: - // TODO: validate value case parser.OPTTokenRedirect: case parser.OPTTokenRedirectRule: - if ( this.modifyType !== undefined ) { return false; } - this.modifyType = id; - if ( val !== undefined ) { - this.modifyValue = val; - } else if ( this.action === AllowAction ) { - this.modifyValue = ''; + if ( this.parseModifierOption(id, val) === false ) { + return false; } break; case parser.OPTTokenInvalid: @@ -2877,7 +2881,7 @@ const FilterParser = class { if ( this.isRegex ) { return this.extractTokenFromRegex(); } - const match = this.findGoodToken(parser); + const match = this.extractTokenFromPattern(parser); if ( match === null ) { return; } this.token = match.token; this.tokenHash = urlTokenizer.tokenHashFromString(this.token); @@ -2885,15 +2889,15 @@ const FilterParser = class { } // Note: a one-char token is better than a documented bad token. - findGoodToken(parser) { + extractTokenFromPattern(parser) { let bestMatch = null; - let bestBadness = 0; + let bestBadness = 0x7FFFFFFF; for ( const match of parser.patternTokens() ) { const badness = match.token.length > 1 ? this.badTokens.get(match.token) || 0 : 1; if ( badness === 0 ) { return match; } - if ( bestBadness === 0 || badness < bestBadness ) { + if ( badness < bestBadness ) { bestMatch = match; bestBadness = badness; } @@ -2910,7 +2914,7 @@ const FilterParser = class { extractTokenFromRegex() { this.reRegexToken.lastIndex = 0; const s = this.pattern; - let bestBadness = 0; + let bestBadness = 0x7FFFFFFF; for (;;) { const matches = this.reRegexToken.exec(s); if ( matches === null ) { break; } @@ -2941,7 +2945,7 @@ const FilterParser = class { const badness = token.length > 1 ? this.badTokens.get(token) || 0 : 1; - if ( bestBadness === 0 || badness < bestBadness ) { + if ( badness < bestBadness ) { this.token = token.toLowerCase(); this.tokenHash = urlTokenizer.tokenHashFromString(this.token); this.tokenBeg = matches.index; @@ -3170,6 +3174,8 @@ FilterContainer.prototype.freeze = function() { /******************************************************************************/ FilterContainer.prototype.optimize = function() { + const t0 = Date.now(); + for ( let bits = 0, n = this.categories.length; bits < n; bits++ ) { const bucket = this.categories[bits]; if ( bucket === undefined ) { continue; } @@ -3192,6 +3198,8 @@ FilterContainer.prototype.optimize = function() { for ( let i = filterUnitWritePtr, n = filterUnits.length; i < n; i++ ) { filterUnits[i] = null; } + + log.info(`staticNetFilteringEngine.optimize() took ${Date.now()-t0} ms`); }; /******************************************************************************/ diff --git a/src/js/storage.js b/src/js/storage.js index e539da0c8..1fcde7527 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -768,10 +768,9 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { // fetch the compiled content in case it has become available. const compiledDetails = await this.assets.get(compiledPath); if ( compiledDetails.content === '' ) { - compiledDetails.content = this.compileFilters( - rawDetails.content, - { assetKey: assetKey } - ); + compiledDetails.content = this.compileFilters(rawDetails.content, { + assetKey + }); this.assets.put(compiledPath, compiledDetails.content); } @@ -830,24 +829,24 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { /******************************************************************************/ -µBlock.compileFilters = function(rawText, details) { +µBlock.compileFilters = function(rawText, details = {}) { const writer = new this.CompiledLineIO.Writer(); // Populate the writer with information potentially useful to the // client compilers. - if ( details ) { - if ( details.assetKey ) { - writer.properties.set('assetKey', details.assetKey); - } + if ( details.assetKey ) { + writer.properties.set('assetKey', details.assetKey); } - + const expertMode = + details.assetKey !== this.userFiltersPath || + this.hiddenSettings.filterAuthorMode !== false; // Useful references: // https://adblockplus.org/en/filter-cheatsheet // https://adblockplus.org/en/filters const staticNetFilteringEngine = this.staticNetFilteringEngine; const staticExtFilteringEngine = this.staticExtFilteringEngine; const lineIter = new this.LineIterator(this.preparseDirectives.prune(rawText)); - const parser = new vAPI.StaticFilteringParser(); + const parser = new vAPI.StaticFilteringParser({ expertMode }); parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH); @@ -1386,10 +1385,9 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { if ( this.badLists.has(details.assetKey) === false ) { this.assets.put( 'compiled/' + details.assetKey, - this.compileFilters( - details.content, - { assetKey: details.assetKey } - ) + this.compileFilters(details.content, { + assetKey: details.assetKey + }) ); } }