diff --git a/platform/chromium/vapi-background-ext.js b/platform/chromium/vapi-background-ext.js index 34367a76b..17526ba9c 100644 --- a/platform/chromium/vapi-background-ext.js +++ b/platform/chromium/vapi-background-ext.js @@ -212,7 +212,7 @@ vAPI.scriptletsInjector = (( ) => { const parts = [ '(', function(details) { - if ( typeof self.uBO_scriptletsInjected === 'string' ) { return; } + if ( self.uBO_scriptletsInjected !== undefined ) { return; } const doc = document; const { location } = doc; if ( location === null ) { return; } diff --git a/platform/firefox/vapi-background-ext.js b/platform/firefox/vapi-background-ext.js index 9e0d8d357..a5a80b8d6 100644 --- a/platform/firefox/vapi-background-ext.js +++ b/platform/firefox/vapi-background-ext.js @@ -355,7 +355,7 @@ vAPI.scriptletsInjector = (( ) => { const parts = [ '(', function(details) { - if ( typeof self.uBO_scriptletsInjected === 'string' ) { return; } + if ( self.uBO_scriptletsInjected !== undefined ) { return; } const doc = document; const { location } = doc; if ( location === null ) { return; } diff --git a/src/js/background.js b/src/js/background.js index 058eee5e5..47d113ed5 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -180,8 +180,8 @@ const µBlock = { // jshint ignore:line // Read-only systemSettings: { - compiledMagic: 57, // Increase when compiled format changes - selfieMagic: 59, // Increase when selfie format changes + compiledMagic: 60, // Increase when compiled format changes + selfieMagic: 60, // Increase when selfie format changes }, // https://github.com/uBlockOrigin/uBlock-issues/issues/759#issuecomment-546654501 diff --git a/src/js/codemirror/ubo-static-filtering.js b/src/js/codemirror/ubo-static-filtering.js index 386ed52f3..e8e3119db 100644 --- a/src/js/codemirror/ubo-static-filtering.js +++ b/src/js/codemirror/ubo-static-filtering.js @@ -25,6 +25,7 @@ import * as sfp from '../static-filtering-parser.js'; import { dom, qs$ } from '../dom.js'; +import { tokenizableStrFromRegex } from '../regex-analyzer.js'; /******************************************************************************/ @@ -67,6 +68,8 @@ const uBOStaticFilteringMode = (( ) => { ) !== 0; }; + const reGoodRegexToken = /[^\x01%0-9A-Za-z][%0-9A-Za-z]{7,}|[^\x01%0-9A-Za-z][%0-9A-Za-z]{1,6}[^\x01%0-9A-Za-z]/; + const colorFromAstNode = mode => { if ( mode.astParser.nodeIsEmptyString(mode.currentWalkerNode) ) { return '+'; } if ( nodeHasError(mode) ) { return 'error'; } @@ -119,7 +122,9 @@ const uBOStaticFilteringMode = (( ) => { case sfp.NODE_TYPE_NET_PATTERN: if ( mode.astWalker.canGoDown() ) { break; } if ( mode.astParser.isRegexPattern() ) { - if ( mode.astParser.getNodeFlags(mode.currentWalkerNode, sfp.NODE_FLAG_PATTERN_UNTOKENIZABLE) !== 0 ) { + const s = mode.astParser.getNodeString(mode.currentWalkerNode); + const tokenizable = tokenizableStrFromRegex(s); + if ( reGoodRegexToken.test(tokenizable) === false ) { return 'variable warning'; } return 'variable notice'; diff --git a/src/js/contentscript.js b/src/js/contentscript.js index 1f6a97111..0874918e7 100644 --- a/src/js/contentscript.js +++ b/src/js/contentscript.js @@ -1312,7 +1312,7 @@ vAPI.DOMFilterer = class { vAPI.messaging.send('contentscript', { what: 'retrieveContentScriptParameters', url: vAPI.effectiveSelf.location.href, - needScriptlets: typeof self.uBO_scriptletsInjected !== 'string', + needScriptlets: self.uBO_scriptletsInjected === undefined, }).then(response => { onResponseReady(response); }); diff --git a/src/js/cosmetic-filtering.js b/src/js/cosmetic-filtering.js index 663fa0df9..24d2ab925 100644 --- a/src/js/cosmetic-filtering.js +++ b/src/js/cosmetic-filtering.js @@ -235,7 +235,7 @@ const CosmeticFilteringEngine = function() { }); // specific filters - this.specificFilters = new StaticExtFilteringHostnameDB(2); + this.specificFilters = new StaticExtFilteringHostnameDB(); // low generic cosmetic filters: map of hash => stringified selector list this.lowlyGeneric = new Map(); @@ -254,16 +254,6 @@ const CosmeticFilteringEngine = function() { str: '', mru: new MRUCache(16) }; - - // Short-lived: content is valid only during one function call. These - // is to prevent repeated allocation/deallocation overheads -- the - // constructors/destructors of javascript Set/Map is assumed to be costlier - // than just calling clear() on these. - this.$specificSet = new Set(); - this.$exceptionSet = new Set(); - this.$proceduralSet = new Set(); - this.$dummySet = new Set(); - this.reset(); }; @@ -429,7 +419,7 @@ CosmeticFilteringEngine.prototype.compileGenericUnhideSelector = function( // hostnames). No distinction is made between declarative and // procedural selectors, since they really exist only to cancel // out other cosmetic filters. - writer.push([ 8, '', 0b001, compiled ]); + writer.push([ 8, '', `-${compiled}` ]); }; /******************************************************************************/ @@ -452,23 +442,8 @@ CosmeticFilteringEngine.prototype.compileSpecificSelector = function( } writer.select('COSMETIC_FILTERS:SPECIFIC'); - - // https://github.com/chrisaljoudi/uBlock/issues/145 - let unhide = exception ? 1 : 0; - if ( not ) { unhide ^= 1; } - - let kind = 0; - if ( unhide === 1 ) { - kind |= 0b001; // Exception - } - if ( compiled.charCodeAt(0) === 0x7B /* '{' */ ) { - kind |= 0b010; // Procedural - } - if ( hostname === '*' ) { - kind |= 0b100; // Applies everywhere - } - - writer.push([ 8, hostname, kind, compiled ]); + const prefix = ((exception ? 1 : 0) ^ (not ? 1 : 0)) ? '-' : '+'; + writer.push([ 8, hostname, `${prefix}${compiled}` ]); }; /******************************************************************************/ @@ -500,15 +475,16 @@ CosmeticFilteringEngine.prototype.fromCompiledContent = function(reader, options // not to be injected conditionally through the DOM surveyor. // hash, *, .promoted-tweet case 8: - if ( args[2] === 0b100 ) { - if ( this.reSimpleHighGeneric.test(args[3]) ) - this.highlyGeneric.simple.dict.add(args[3]); + if ( args[1] === '*' ) { + const selector = args[2].slice(1); + if ( this.reSimpleHighGeneric.test(selector) ) + this.highlyGeneric.simple.dict.add(selector); else { - this.highlyGeneric.complex.dict.add(args[3]); + this.highlyGeneric.complex.dict.add(selector); } break; } - this.specificFilters.store(args[1], args[2] & 0b011, args[3]); + this.specificFilters.store(args[1], args[2]); break; default: this.discardedCount += 1; @@ -590,9 +566,7 @@ CosmeticFilteringEngine.prototype.toSelfie = function() { CosmeticFilteringEngine.prototype.fromSelfie = function(selfie) { if ( selfie.version !== this.selfieVersion ) { - throw new Error( - `cosmeticFilteringEngine: mismatched selfie version, ${selfie.version}, expected ${this.selfieVersion}` - ); + throw new TypeError('Bad selfie'); } this.acceptedCount = selfie.acceptedCount; this.discardedCount = selfie.discardedCount; @@ -630,16 +604,11 @@ CosmeticFilteringEngine.prototype.removeFromSelectorCache = function( type = undefined ) { const targetHostnameLength = targetHostname.length; - for ( let entry of this.selectorCache ) { - let hostname = entry[0]; - let item = entry[1]; + for ( const [ hostname, item ] of this.selectorCache ) { if ( targetHostname !== '*' ) { if ( hostname.endsWith(targetHostname) === false ) { continue; } - if ( - hostname.length !== targetHostnameLength && - hostname.charAt(hostname.length - targetHostnameLength - 1) !== '.' - ) { - continue; + if ( hostname.length !== targetHostnameLength ) { + if ( hostname.at(-1) !== '.' ) { continue; } } } item.remove(type); @@ -791,12 +760,8 @@ CosmeticFilteringEngine.prototype.retrieveSpecificSelectors = function( options.noSpecificCosmeticFiltering !== true || options.noGenericCosmeticFiltering !== true ) { - const specificSet = this.$specificSet; - const proceduralSet = this.$proceduralSet; - const exceptionSet = this.$exceptionSet; - const dummySet = this.$dummySet; - // Cached cosmetic filters: these are always declarative. + const specificSet = new Set(); if ( cacheEntry !== undefined ) { cacheEntry.retrieveCosmetic(specificSet, out.genericCosmeticHashes = []); if ( cacheEntry.disableSurveyor ) { @@ -804,35 +769,30 @@ CosmeticFilteringEngine.prototype.retrieveSpecificSelectors = function( } } + const allSet = new Set(); // Retrieve filters with a non-empty hostname - const retrieveSets = [ specificSet, exceptionSet, proceduralSet, exceptionSet ]; - const discardSets = [ dummySet, exceptionSet ]; - this.specificFilters.retrieve( - hostname, - options.noSpecificCosmeticFiltering ? discardSets : retrieveSets, - 1 - ); - // Retrieve filters with a regex-based hostname value - this.specificFilters.retrieve( - hostname, - options.noSpecificCosmeticFiltering ? discardSets : retrieveSets, - 3 - ); + this.specificFilters.retrieveSpecifics(allSet, hostname); // Retrieve filters with a entity-based hostname value const entity = entityFromHostname(hostname, request.domain); - if ( entity !== '' ) { - this.specificFilters.retrieve( - entity, - options.noSpecificCosmeticFiltering ? discardSets : retrieveSets, - 1 - ); - } + this.specificFilters.retrieveSpecifics(allSet, entity); + // Retrieve filters with a regex-based hostname value + this.specificFilters.retrieveSpecificsByRegex(allSet, hostname, request.url); // Retrieve filters with an empty hostname - this.specificFilters.retrieve( - hostname, - options.noGenericCosmeticFiltering ? discardSets : retrieveSets, - 2 - ); + this.specificFilters.retrieveGenerics(allSet); + + // Split filters in different groups + const proceduralSet = new Set(); + const exceptionSet = new Set(); + for ( const s of allSet ) { + const selector = s.slice(1); + if ( s.charCodeAt(0) === 0x2D /* - */ ) { + exceptionSet.add(selector); + } else if ( selector.charCodeAt(0) === 0x7B /* { */ ) { + proceduralSet.add(selector); + } else { + specificSet.add(selector); + } + } // Apply exceptions to specific filterset if ( exceptionSet.size !== 0 ) { @@ -855,12 +815,12 @@ CosmeticFilteringEngine.prototype.retrieveSpecificSelectors = function( // filters, so we extract and inject them immediately. if ( proceduralSet.size !== 0 ) { for ( const json of proceduralSet ) { - const pfilter = JSON.parse(json); if ( exceptionSet.has(json) ) { proceduralSet.delete(json); out.exceptedFilters.push(json); continue; } + const pfilter = JSON.parse(json); if ( exceptionSet.has(pfilter.raw) ) { proceduralSet.delete(json); out.exceptedFilters.push(pfilter.raw); @@ -914,12 +874,6 @@ CosmeticFilteringEngine.prototype.retrieveSpecificSelectors = function( } } } - - // Important: always clear used registers before leaving. - specificSet.clear(); - proceduralSet.clear(); - exceptionSet.clear(); - dummySet.clear(); } const details = { diff --git a/src/js/html-filtering.js b/src/js/html-filtering.js index ac6dee23c..9b5a50465 100644 --- a/src/js/html-filtering.js +++ b/src/js/html-filtering.js @@ -20,7 +20,7 @@ */ import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js'; -import { entityFromDomain } from './uri-utils.js'; +import { entityFromHostname } from './uri-utils.js'; import logger from './logger.js'; import { sessionFirewall } from './filtering-engines.js'; import µb from './background.js'; @@ -30,7 +30,7 @@ import µb from './background.js'; const pselectors = new Map(); const duplicates = new Set(); -const filterDB = new StaticExtFilteringHostnameDB(2); +const filterDB = new StaticExtFilteringHostnameDB(); let acceptedCount = 0; let discardedCount = 0; @@ -260,7 +260,7 @@ function logOne(details, exception, selector) { .setDocOriginFromURL(details.url) .setFilter({ source: 'extended', - raw: `${exception === 0 ? '##' : '#@#'}^${selector}` + raw: `${exception === 0 ? '##' : '#@#'}^${selector}`, }) .toLogger(); } @@ -338,16 +338,11 @@ htmlFilteringEngine.compile = function(parser, writer) { let hasOnlyNegated = true; for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { if ( bad ) { continue; } - let kind = isException ? 0b01 : 0b00; - if ( not ) { - kind ^= 0b01; - } else { + const prefix = ((isException ? 1 : 0) ^ (not ? 1 : 0)) ? '-' : '+'; + if ( not === false ) { hasOnlyNegated = false; } - if ( compiled.charCodeAt(0) === 0x7B /* '{' */ ) { - kind |= 0b10; - } - compiledFilters.push([ 64, hn, kind, compiled ]); + compiledFilters.push([ 64, hn, `${prefix}${compiled}` ]); } // Not allowed since it's equivalent to forbidden generic HTML filters @@ -373,59 +368,45 @@ htmlFilteringEngine.fromCompiledContent = function(reader) { } duplicates.add(fingerprint); const args = reader.args(); - filterDB.store(args[1], args[2], args[3]); + filterDB.store(args[1], args[2]); } }; htmlFilteringEngine.retrieve = function(fctxt) { - const plains = new Set(); - const procedurals = new Set(); - const exceptions = new Set(); - const retrieveSets = [ plains, exceptions, procedurals, exceptions ]; - + const all = new Set(); const hostname = fctxt.getHostname(); - filterDB.retrieve(hostname, retrieveSets); - - const domain = fctxt.getDomain(); - const entity = entityFromDomain(domain); - const hostnameEntity = entity !== '' - ? `${hostname.slice(0, -domain.length)}${entity}` - : '*'; - filterDB.retrieve(hostnameEntity, retrieveSets, 1); - - if ( plains.size === 0 && procedurals.size === 0 ) { return; } + filterDB.retrieveSpecifics(all, hostname); + const entity = entityFromHostname(hostname, fctxt.getDomain()); + filterDB.retrieveSpecifics(all, entity); + filterDB.retrieveSpecificsByRegex(all, hostname, fctxt.url); + filterDB.retrieveGenerics(all); + if ( all.size === 0 ) { return; } // https://github.com/gorhill/uBlock/issues/2835 // Do not filter if the site is under an `allow` rule. - if ( - µb.userSettings.advancedUserEnabled && - sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 - ) { - return; + if ( µb.userSettings.advancedUserEnabled ) { + if ( sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 ) { return; } } - const out = { plains, procedurals }; - - if ( exceptions.size === 0 ) { - return out; - } - - for ( const selector of exceptions ) { - if ( plains.has(selector) ) { - plains.delete(selector); - logOne(fctxt, 1, selector); - continue; - } - if ( procedurals.has(selector) ) { - procedurals.delete(selector); - logOne(fctxt, 1, JSON.parse(selector).raw); - continue; + // Split filters in different groups + const plains = new Set(); + const procedurals = new Set(); + for ( const s of all ) { + if ( s.charCodeAt(0) === 0x2D /* - */ ) { continue; } + const selector = s.slice(1); + const isProcedural = selector.startsWith('{'); + if ( all.has(`-${selector}`) ) { + logOne(fctxt, 1, isProcedural ? JSON.parse(selector).raw : selector); + } else if ( isProcedural ) { + procedurals.add(selector); + } else { + plains.add(selector); } } - if ( plains.size !== 0 || procedurals.size !== 0 ) { - return out; - } + if ( plains.size === 0 && procedurals.size === 0 ) { return; } + + return { plains, procedurals }; }; htmlFilteringEngine.apply = function(doc, details, selectors) { diff --git a/src/js/httpheader-filtering.js b/src/js/httpheader-filtering.js index 20f83219b..9d09543ab 100644 --- a/src/js/httpheader-filtering.js +++ b/src/js/httpheader-filtering.js @@ -20,7 +20,7 @@ */ import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js'; -import { entityFromDomain } from './uri-utils.js'; +import { entityFromHostname } from './uri-utils.js'; import logger from './logger.js'; import { sessionFirewall } from './filtering-engines.js'; import µb from './background.js'; @@ -28,10 +28,7 @@ import µb from './background.js'; /******************************************************************************/ const duplicates = new Set(); -const filterDB = new StaticExtFilteringHostnameDB(1); - -const $headers = new Set(); -const $exceptions = new Set(); +const filterDB = new StaticExtFilteringHostnameDB(); let acceptedCount = 0; let discardedCount = 0; @@ -52,7 +49,7 @@ const logOne = function(isException, token, fctxt) { modifier: true, result: isException ? 2 : 1, source: 'extended', - raw: `${(isException ? '#@#' : '##')}^responseheader(${token})` + raw: `${(isException ? '#@#' : '##')}^responseheader(${token})`, }) .toLogger(); }; @@ -101,14 +98,8 @@ httpheaderFilteringEngine.compile = function(parser, writer) { for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { if ( bad ) { continue; } - let kind = 0; - if ( isException ) { - if ( not ) { continue; } - kind |= 1; - } else if ( not ) { - kind |= 1; - } - writer.push([ 64, hn, kind, headerName ]); + const prefix = ((isException ? 1 : 0) ^ (not ? 1 : 0)) ? '-' : '+'; + writer.push([ 64, hn, `${prefix}${headerName}` ]); } }; @@ -129,8 +120,7 @@ httpheaderFilteringEngine.fromCompiledContent = function(reader) { } duplicates.add(fingerprint); const args = reader.args(); - if ( args.length < 4 ) { continue; } - filterDB.store(args[1], args[2], args[3]); + filterDB.store(args[1], args[2]); } }; @@ -140,37 +130,39 @@ httpheaderFilteringEngine.apply = function(fctxt, headers) { const hostname = fctxt.getHostname(); if ( hostname === '' ) { return; } - const domain = fctxt.getDomain(); - let entity = entityFromDomain(domain); - if ( entity !== '' ) { - entity = `${hostname.slice(0, -domain.length)}${entity}`; - } else { - entity = '*'; - } - - $headers.clear(); - $exceptions.clear(); - - filterDB.retrieve(hostname, [ $headers, $exceptions ]); - filterDB.retrieve(entity, [ $headers, $exceptions ], 1); - if ( $headers.size === 0 ) { return; } + const all = new Set(); + filterDB.retrieveSpecifics(all, hostname); + const entity = entityFromHostname(hostname, fctxt.getDomain()); + filterDB.retrieveSpecifics(all, entity); + filterDB.retrieveSpecificsByRegex(all, hostname, fctxt.url); + filterDB.retrieveGenerics(all); + if ( all.size === 0 ) { return; } // https://github.com/gorhill/uBlock/issues/2835 // Do not filter response headers if the site is under an `allow` rule. - if ( - µb.userSettings.advancedUserEnabled && - sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 - ) { - return; + if ( µb.userSettings.advancedUserEnabled ) { + if ( sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 ) { return; } } - const hasGlobalException = $exceptions.has(''); + // Split filters in different groups + const filters = new Map(); + const exceptions = new Map(); + for ( const s of all ) { + const selector = s.slice(1); + if ( s.charCodeAt(0) === 0x2D /* - */ ) { + exceptions.add(selector); + } else { + filters.add(selector); + } + } + + const hasGlobalException = exceptions.has(''); let modified = false; let i = 0; - for ( const name of $headers ) { - const isExcepted = hasGlobalException || $exceptions.has(name); + for ( const name of filters ) { + const isExcepted = hasGlobalException || exceptions.has(name); if ( isExcepted ) { if ( logger.enabled ) { logOne(true, hasGlobalException ? '' : name, fctxt); diff --git a/src/js/logger-ui.js b/src/js/logger-ui.js index 4a62b92ad..468add174 100644 --- a/src/js/logger-ui.js +++ b/src/js/logger-ui.js @@ -1592,9 +1592,8 @@ dom.on(document, 'keydown', ev => { } let bestMatchFilter = ''; for ( const filter in response ) { - if ( filter.length > bestMatchFilter.length ) { - bestMatchFilter = filter; - } + if ( filter.length <= bestMatchFilter.length ) { continue; } + bestMatchFilter = filter; } if ( bestMatchFilter !== '' && @@ -1619,14 +1618,14 @@ dom.on(document, 'keydown', ev => { if ( dom.cl.has(targetRow, 'networkRealm') ) { const response = await messaging.send('loggerUI', { what: 'listsFromNetFilter', - rawFilter: rawFilter, + rawFilter, }); handleResponse(response); } else if ( dom.cl.has(targetRow, 'extendedRealm') ) { const response = await messaging.send('loggerUI', { what: 'listsFromCosmeticFilter', url: targetRow.children[COLUMN_URL].textContent, - rawFilter: rawFilter, + rawFilter, }); handleResponse(response); } diff --git a/src/js/messaging.js b/src/js/messaging.js index d7c862d10..10488e98e 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -51,7 +51,7 @@ import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js'; import punycode from '../lib/punycode.js'; import { redirectEngine } from './redirect-engine.js'; import scriptletFilteringEngine from './scriptlet-filtering.js'; -import staticFilteringReverseLookup from './reverselookup.js'; +import { staticFilteringReverseLookup } from './reverselookup.js'; import staticNetFilteringEngine from './static-net-filtering.js'; import webRequest from './traffic.js'; import µb from './background.js'; diff --git a/src/js/regex-analyzer.js b/src/js/regex-analyzer.js new file mode 100644 index 000000000..d41915ba7 --- /dev/null +++ b/src/js/regex-analyzer.js @@ -0,0 +1,202 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2020-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +import Regex from '../lib/regexanalyzer/regex.js'; + +/******************************************************************************/ + +// Depends on: +// https://github.com/foo123/RegexAnalyzer +const RegexAnalyzer = Regex && Regex.Analyzer || null; + +export function isRE2(reStr) { + if ( RegexAnalyzer === null ) { return true; } + try { + return _isRE2(RegexAnalyzer(reStr, false).tree()); + } catch { + } + return false; +} + +export function tokenizableStrFromRegex(reStr) { + return _literalStrFromRegex(reStr); +} + +/******************************************************************************/ + +function _isRE2(node) { + if ( node instanceof Object === false ) { return true; } + if ( node.flags instanceof Object ) { + if ( node.flags.LookAhead === 1 ) { return false; } + if ( node.flags.NegativeLookAhead === 1 ) { return false; } + if ( node.flags.LookBehind === 1 ) { return false; } + if ( node.flags.NegativeLookBehind === 1 ) { return false; } + } + if ( Array.isArray(node.val) ) { + for ( const entry of node.val ) { + if ( _isRE2(entry) === false ) { return false; } + } + } + if ( node.val instanceof Object ) { + return _isRE2(node.val); + } + return true; +} + +function _literalStrFromRegex(reStr) { + if ( RegexAnalyzer === null ) { return ''; } + let s = ''; + try { + s = tokenizableStrFromNode( + RegexAnalyzer(reStr, false).tree() + ); + } catch { + } + // Process optional sequences + const reOptional = /[\x02\x03]+/; + for (;;) { + const match = reOptional.exec(s); + if ( match === null ) { break; } + const left = s.slice(0, match.index); + const middle = match[0]; + const right = s.slice(match.index + middle.length); + s = left; + s += firstCharCodeClass(right) === 1 || firstCharCodeClass(middle) === 1 + ? '\x01' + : '\x00'; + s += lastCharCodeClass(left) === 1 || lastCharCodeClass(middle) === 1 + ? '\x01' + : '\x00'; + s += right; + } + return s; +} + +function firstCharCodeClass(s) { + if ( s.length === 0 ) { return 0; } + const c = s.charCodeAt(0); + if ( c === 1 || c === 3 ) { return 1; } + return reCharCodeClass.test(s.charAt(0)) ? 1 : 0; +} + +function lastCharCodeClass(s) { + const i = s.length - 1; + if ( i === -1 ) { return 0; } + const c = s.charCodeAt(i); + if ( c === 1 || c === 3 ) { return 1; } + return reCharCodeClass.test(s.charAt(i)) ? 1 : 0; +} + +const reCharCodeClass = /[%0-9A-Za-z]/; + +function tokenizableStrFromNode(node) { + switch ( node.type ) { + case 1: /* T_SEQUENCE, 'Sequence' */ { + let s = ''; + for ( let i = 0; i < node.val.length; i++ ) { + s += tokenizableStrFromNode(node.val[i]); + } + return s; + } + case 2: /* T_ALTERNATION, 'Alternation' */ + case 8: /* T_CHARGROUP, 'CharacterGroup' */ { + if ( node.flags.NegativeMatch ) { return '\x01'; } + let firstChar = 0; + let lastChar = 0; + for ( let i = 0; i < node.val.length; i++ ) { + const s = tokenizableStrFromNode(node.val[i]); + if ( firstChar === 0 && firstCharCodeClass(s) === 1 ) { + firstChar = 1; + } + if ( lastChar === 0 && lastCharCodeClass(s) === 1 ) { + lastChar = 1; + } + if ( firstChar === 1 && lastChar === 1 ) { break; } + } + return String.fromCharCode(firstChar, lastChar); + } + case 4: /* T_GROUP, 'Group' */ { + if ( + node.flags.NegativeLookAhead === 1 || + node.flags.NegativeLookBehind === 1 + ) { + return ''; + } + return tokenizableStrFromNode(node.val); + } + case 16: /* T_QUANTIFIER, 'Quantifier' */ { + if ( node.flags.max === 0 ) { return ''; } + const s = tokenizableStrFromNode(node.val); + const first = firstCharCodeClass(s); + const last = lastCharCodeClass(s); + if ( node.flags.min !== 0 ) { + return String.fromCharCode(first, last); + } + return String.fromCharCode(first+2, last+2); + } + case 64: /* T_HEXCHAR, 'HexChar' */ { + if ( + node.flags.Code === '01' || + node.flags.Code === '02' || + node.flags.Code === '03' + ) { + return '\x00'; + } + return node.flags.Char; + } + case 128: /* T_SPECIAL, 'Special' */ { + const flags = node.flags; + if ( + flags.EndCharGroup === 1 || // dangling `]` + flags.EndGroup === 1 || // dangling `)` + flags.EndRepeats === 1 // dangling `}` + ) { + throw new Error('Unmatched bracket'); + } + return flags.MatchEnd === 1 || + flags.MatchStart === 1 || + flags.MatchWordBoundary === 1 + ? '\x00' + : '\x01'; + } + case 256: /* T_CHARS, 'Characters' */ { + for ( let i = 0; i < node.val.length; i++ ) { + if ( firstCharCodeClass(node.val[i]) === 1 ) { + return '\x01'; + } + } + return '\x00'; + } + // Ranges are assumed to always involve token-related characters. + case 512: /* T_CHARRANGE, 'CharacterRange' */ { + return '\x01'; + } + case 1024: /* T_STRING, 'String' */ { + return node.val; + } + case 2048: /* T_COMMENT, 'Comment' */ { + return ''; + } + default: + break; + } + return '\x01'; +} diff --git a/src/js/reverselookup-worker.js b/src/js/reverselookup-worker.js index 1cad3cf02..35b86ad57 100644 --- a/src/js/reverselookup-worker.js +++ b/src/js/reverselookup-worker.js @@ -115,7 +115,8 @@ const fromExtendedFilter = function(details) { const exception = prefix.charAt(1) === '@'; const selector = details.rawFilter.slice(prefix.length); const isHtmlFilter = prefix.endsWith('^'); - const hostname = details.hostname; + const url = new URL(details.url || 'about:blank'); + const { hostname, pathname } = url; // The longer the needle, the lower the number of false positives. // https://github.com/uBlockOrigin/uBlock-issues/issues/1139 @@ -147,17 +148,43 @@ const fromExtendedFilter = function(details) { } } - const hostnameMatches = hn => { - if ( hn === '' ) { return true; } - if ( hn.charCodeAt(0) === 0x2F /* / */ ) { - return (new RegExp(hn.slice(1,-1))).test(hostname); - } - if ( reHostname.test(hn) ) { return true; } + const hostnameTargetMatchesHostname = target => { + if ( target === '' ) { return true; } + if ( reHostname.test(target) ) { return true; } if ( reEntity === undefined ) { return false; } - if ( reEntity.test(hn) ) { return true; } + if ( reEntity.test(target) ) { return true; } return false; }; + const regexTargetMatchesHostname = target => { + return (new RegExp(target)).test(hostname); + }; + + const regexTargetMatchesURL = target => { + const pathPos = target.indexOf('\\/'); + if ( pathPos === -1 ) { + return regexTargetMatchesHostname(target.slice(1, -1)); + } + return regexTargetMatchesHostname(`${target.slice(1, pathPos)}$`) && + (new RegExp(`^${target.slice(pathPos, -1)}`)).test(pathname); + }; + + const pathTargetMatchesURL = target => { + const pathPos = target.indexOf('/'); + return hostnameTargetMatchesHostname(target.slice(0, pathPos)) && + pathname.startsWith(target.slice(pathPos)); + }; + + const targetMatchesURL = target => { + if ( target.charCodeAt(0) === 0x2F /* / */ ) { + return regexTargetMatchesURL(target.slice(1, -1)); + } + if ( target.includes('/') ) { + return pathTargetMatchesURL(target); + } + return hostnameTargetMatchesHostname(target); + }; + const response = Object.create(null); for ( const assetKey in listEntries ) { @@ -212,24 +239,25 @@ const fromExtendedFilter = function(details) { // Response header filtering /* fallthrough */ case 64: { - if ( exception !== ((fargs[2] & 0b001) !== 0) ) { break; } + if ( exception !== (fargs[2].charCodeAt(0) === 0x2D /* - */) ) { break; } + const candidate = fargs[2].slice(1); if ( /^responseheader\(.+\)$/.test(selector) ) { - if ( fargs[3] !== needle ) { break; } - if ( hostnameMatches(fargs[1]) === false ) { break; } + if ( candidate !== needle ) { break; } + if ( targetMatchesURL(fargs[1]) === false ) { break; } found = fargs[1] + prefix + selector; break; } - const isProcedural = (fargs[2] & 0b010) !== 0; + const isProcedural = candidate.charCodeAt(0) === 0x7B /* { */; if ( - isProcedural === false && fargs[3] !== selector || - isProcedural && JSON.parse(fargs[3]).raw !== selector + isProcedural === false && candidate !== selector || + isProcedural && JSON.parse(candidate).raw !== selector ) { break; } - if ( hostnameMatches(fargs[1]) === false ) { break; } + if ( targetMatchesURL(fargs[1]) === false ) { break; } // https://www.reddit.com/r/uBlockOrigin/comments/d6vxzj/ // Ignore match if specific cosmetic filters are disabled - if ( + if ( filterType === 8 && exception === false && details.ignoreSpecific @@ -240,13 +268,15 @@ const fromExtendedFilter = function(details) { break; } // Scriptlet injection - case 32: - if ( exception !== ((fargs[2] & 0b001) !== 0) ) { break; } - if ( fargs[3] !== details.needle ) { break; } - if ( hostnameMatches(fargs[1]) ) { + case 32: { + if ( exception !== (fargs[2].charCodeAt(0) === 0x2D /* - */) ) { break; } + const candidate = fargs[2].slice(1); + if ( candidate !== details.needle ) { break; } + if (targetMatchesURL(fargs[1]) ) { found = fargs[1] + prefix + selector; } break; + } default: break; } diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js index 2d2709c1e..ade8c5a7f 100644 --- a/src/js/reverselookup.js +++ b/src/js/reverselookup.js @@ -177,8 +177,8 @@ const fromExtendedFilter = async function(details) { worker.postMessage({ what: 'fromExtendedFilter', id, + url: details.url, domain: domainFromHostname(hostname), - hostname, ignoreGeneric: staticNetFilteringEngine.matchRequestReverse( 'generichide', @@ -208,13 +208,11 @@ const resetLists = function() { /******************************************************************************/ -const staticFilteringReverseLookup = { +export const staticFilteringReverseLookup = { fromNetFilter, fromExtendedFilter, resetLists, shutdown: stopWorker }; -export default staticFilteringReverseLookup; - /******************************************************************************/ diff --git a/src/js/scriptlet-filtering-core.js b/src/js/scriptlet-filtering-core.js index 00888da1c..eda887634 100644 --- a/src/js/scriptlet-filtering-core.js +++ b/src/js/scriptlet-filtering-core.js @@ -25,11 +25,6 @@ import { redirectEngine as reng } from './redirect-engine.js'; /******************************************************************************/ -// Increment when internal representation changes -const VERSION = 1; - -const $scriptlets = new Set(); -const $exceptions = new Set(); const $mainWorldMap = new Map(); const $isolatedWorldMap = new Map(); @@ -115,10 +110,10 @@ const requote = s => { return `'${s.replace(/'/g, "\\'")}'`; }; -const decompile = json => { +const decompile = (json, isException) => { + const prefix = isException ? '#@#' : '##'; const args = JSON.parse(json); - if ( args.length === 0 ) { return '+js()'; } - return `+js(${args.map(s => requote(s)).join(', ')})`; + return `${prefix}+js(${args.map(s => requote(s)).join(', ')})`; }; /******************************************************************************/ @@ -127,7 +122,7 @@ export class ScriptletFilteringEngine { constructor() { this.acceptedCount = 0; this.discardedCount = 0; - this.scriptletDB = new StaticExtFilteringHostnameDB(1, VERSION); + this.scriptletDB = new StaticExtFilteringHostnameDB(); this.duplicates = new Set(); } @@ -164,7 +159,7 @@ export class ScriptletFilteringEngine { if ( parser.hasOptions() === false ) { if ( isException ) { - writer.push([ 32, '', 1, normalized ]); + writer.push([ 32, '', `-${normalized}` ]); } return; } @@ -175,14 +170,8 @@ export class ScriptletFilteringEngine { for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { if ( bad ) { continue; } - let kind = 0; - if ( isException ) { - if ( not ) { continue; } - kind |= 1; - } else if ( not ) { - kind |= 1; - } - writer.push([ 32, hn, kind, normalized ]); + const prefix = ((isException ? 1 : 0) ^ (not ? 1 : 0)) ? '-' : '+'; + writer.push([ 32, hn, `${prefix}${normalized}` ]); } } @@ -199,8 +188,7 @@ export class ScriptletFilteringEngine { } this.duplicates.add(fingerprint); const args = reader.args(); - if ( args.length < 4 ) { continue; } - this.scriptletDB.store(args[1], args[2], args[3]); + this.scriptletDB.store(args[1], args[2]); } } @@ -209,8 +197,6 @@ export class ScriptletFilteringEngine { } fromSelfie(selfie) { - if ( typeof selfie !== 'object' || selfie === null ) { return false; } - if ( selfie.version !== VERSION ) { return false; } this.scriptletDB.fromSelfie(selfie); return true; } @@ -218,42 +204,46 @@ export class ScriptletFilteringEngine { retrieve(request, options = {}) { if ( this.scriptletDB.size === 0 ) { return; } - $scriptlets.clear(); - $exceptions.clear(); - + const all = new Set(); const { ancestors = [], domain, hostname } = request; - this.scriptletDB.retrieve(hostname, [ $scriptlets, $exceptions ]); + this.scriptletDB.retrieveSpecifics(all, hostname); const entity = entityFromHostname(hostname, domain); - if ( entity !== '' ) { - this.scriptletDB.retrieve(entity, [ $scriptlets, $exceptions ], 1); - } else { - this.scriptletDB.retrieve('*', [ $scriptlets, $exceptions ], 1); - } + this.scriptletDB.retrieveSpecifics(all, entity); + this.scriptletDB.retrieveSpecificsByRegex(all, hostname, request.url); + this.scriptletDB.retrieveGenerics(all); + const visitedAncestors = []; for ( const ancestor of ancestors ) { const { domain, hostname } = ancestor; - this.scriptletDB.retrieve(`${hostname}>>`, [ $scriptlets, $exceptions ], 1); + if ( visitedAncestors.includes(hostname) ) { continue; } + visitedAncestors.push(hostname); + this.scriptletDB.retrieveSpecifics(all, `${hostname}>>`); const entity = entityFromHostname(hostname, domain); if ( entity !== '' ) { - this.scriptletDB.retrieve(`${entity}>>`, [ $scriptlets, $exceptions ], 1); + this.scriptletDB.retrieveSpecifics(all, `${entity}>>`); } } - if ( $scriptlets.size === 0 ) { return; } + if ( all.size === 0 ) { return; } // Wholly disable scriptlet injection? - if ( $exceptions.has('[]') ) { - return { filters: '#@#+js()' }; + if ( all.has('-[]') ) { + return { filters: [ '#@#+js()' ] }; } - for ( const token of $exceptions ) { - if ( $scriptlets.has(token) ) { - $scriptlets.delete(token); + // Split filters in different groups + const scriptlets = new Set(); + const exceptions = new Set(); + for ( const s of all ) { + if ( s.charCodeAt(0) === 0x2D /* - */ ) { continue; } + const selector = s.slice(1); + if ( all.has(`-${selector}`) ) { + exceptions.add(selector); } else { - $exceptions.delete(token); + scriptlets.add(selector); } } - for ( const token of $scriptlets ) { + for ( const token of scriptlets ) { lookupScriptlet(token, $mainWorldMap, $isolatedWorldMap, options.debug); } @@ -273,9 +263,9 @@ export class ScriptletFilteringEngine { mainWorld: mainWorldCode.join('\n\n'), isolatedWorld: isolatedWorldCode.join('\n\n'), filters: [ - ...Array.from($scriptlets).map(s => `##${decompile(s)}`), - ...Array.from($exceptions).map(s => `#@#${decompile(s)}`), - ].join('\n'), + ...Array.from(scriptlets).map(a => decompile(a, false)), + ...Array.from(exceptions).map(a => decompile(a, true)), + ], }; $mainWorldMap.clear(); $isolatedWorldMap.clear(); @@ -286,6 +276,8 @@ export class ScriptletFilteringEngine { scriptletGlobals.canDebug = true; } + const scriptletGlobalsJSON = JSON.stringify(scriptletGlobals, null, 4); + return { mainWorld: scriptletDetails.mainWorld === '' ? '' : [ '(function() {', @@ -294,7 +286,7 @@ export class ScriptletFilteringEngine { options.debugScriptlets ? 'debugger;' : ';', '', // For use by scriptlets to share local data among themselves - `const scriptletGlobals = ${JSON.stringify(scriptletGlobals, null, 4)};`, + `const scriptletGlobals = ${scriptletGlobalsJSON};`, '', scriptletDetails.mainWorld, '', @@ -308,7 +300,7 @@ export class ScriptletFilteringEngine { options.debugScriptlets ? 'debugger;' : ';', '', // For use by scriptlets to share local data among themselves - `const scriptletGlobals = ${JSON.stringify(scriptletGlobals, null, 4)};`, + `const scriptletGlobals = ${scriptletGlobalsJSON};`, '', scriptletDetails.isolatedWorld, '', diff --git a/src/js/scriptlet-filtering.js b/src/js/scriptlet-filtering.js index 0d127f6a5..89a5ec92c 100644 --- a/src/js/scriptlet-filtering.js +++ b/src/js/scriptlet-filtering.js @@ -37,10 +37,8 @@ import µb from './background.js'; /******************************************************************************/ -const contentScriptRegisterer = new (class { - constructor() { - this.hostnameToDetails = new Map(); - } +const contentScriptRegisterer = { + hostnameToDetails: new Map(), register(hostname, code) { if ( browser.contentScripts === undefined ) { return false; } if ( hostname === '' ) { return false; } @@ -66,7 +64,7 @@ const contentScriptRegisterer = new (class { }); this.hostnameToDetails.set(hostname, { handle: promise, code }); return false; - } + }, unregister(hostname) { if ( hostname === '' ) { return; } if ( this.hostnameToDetails.size === 0 ) { return; } @@ -74,7 +72,7 @@ const contentScriptRegisterer = new (class { if ( details === undefined ) { return; } this.hostnameToDetails.delete(hostname); this.unregisterHandle(details.handle); - } + }, flush(hostname) { if ( hostname === '' ) { return; } if ( hostname === '*' ) { return this.reset(); } @@ -84,14 +82,14 @@ const contentScriptRegisterer = new (class { if ( pos !== 0 && hn.charCodeAt(pos-1) !== 0x2E /* . */ ) { continue; } this.unregister(hn); } - } + }, reset() { if ( this.hostnameToDetails.size === 0 ) { return; } for ( const details of this.hostnameToDetails.values() ) { this.unregisterHandle(details.handle); } this.hostnameToDetails.clear(); - } + }, unregisterHandle(handle) { if ( handle instanceof Promise ) { handle.then(handle => { @@ -100,8 +98,8 @@ const contentScriptRegisterer = new (class { } else { handle.unregister(); } - } -})(); + }, +}; /******************************************************************************/ @@ -370,17 +368,16 @@ export class ScriptletFilteringEngineEx extends ScriptletFilteringEngine { toLogger(request, details) { if ( details === undefined ) { return; } if ( logger.enabled !== true ) { return; } - if ( typeof details.filters !== 'string' ) { return; } - const fctxt = µb.filteringContext + if ( Array.isArray(details.filters) === false ) { return; } + µb.filteringContext .duplicate() .fromTabId(request.tabId) .setRealm('extended') .setType('scriptlet') .setURL(request.url) - .setDocOriginFromURL(request.url); - for ( const raw of details.filters.split('\n') ) { - fctxt.setFilter({ source: 'extended', raw }).toLogger(); - } + .setDocOriginFromURL(request.url) + .setFilter(details.filters.map(a => ({ source: 'extended', raw: a }))) + .toLogger(); } } diff --git a/src/js/scriptlets/cosmetic-report.js b/src/js/scriptlets/cosmetic-report.js index 83b8b62bf..93023df66 100644 --- a/src/js/scriptlets/cosmetic-report.js +++ b/src/js/scriptlets/cosmetic-report.js @@ -125,8 +125,8 @@ if ( Array.isArray(allSelectors.exceptions) ) { } } -if ( typeof self.uBO_scriptletsInjected === 'string' ) { - matchedSelectors.push(...self.uBO_scriptletsInjected.split('\n')); +if ( self.uBO_scriptletsInjected !== undefined ) { + matchedSelectors.push(...self.uBO_scriptletsInjected); } if ( matchedSelectors.length === 0 ) { return; } diff --git a/src/js/start.js b/src/js/start.js index bb6b1850e..bb310b2f0 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -48,7 +48,7 @@ import { filteringBehaviorChanged } from './broadcast.js'; import io from './assets.js'; import { redirectEngine } from './redirect-engine.js'; import staticExtFilteringEngine from './static-ext-filtering.js'; -import staticFilteringReverseLookup from './reverselookup.js'; +import { staticFilteringReverseLookup } from './reverselookup.js'; import staticNetFilteringEngine from './static-net-filtering.js'; import { ubolog } from './console.js'; import webRequest from './traffic.js'; diff --git a/src/js/static-ext-filtering-db.js b/src/js/static-ext-filtering-db.js index 2d25c83bd..775c93229 100644 --- a/src/js/static-ext-filtering-db.js +++ b/src/js/static-ext-filtering-db.js @@ -21,149 +21,231 @@ /******************************************************************************/ -const StaticExtFilteringHostnameDB = class { - constructor(nBits, version = 0) { - this.version = version; - this.nBits = nBits; - this.strToIdMap = new Map(); - this.hostnameToSlotIdMap = new Map(); - this.regexToSlotIdMap = new Map(); - this.regexMap = new Map(); - // Array of integer pairs - this.hostnameSlots = []; - // Array of strings (selectors and pseudo-selectors) - this.strSlots = []; +// example.com: domain => no slash +// example.com/toto: domain + path => slash +// /example\d+\.com$/: domain regex: no literal slash in regex +// /example\d+\.com\/toto\d+/: domain + path => literal slash in regex + +/******************************************************************************/ + +const naivePathnameFromURL = url => { + if ( typeof url !== 'string' ) { return; } + const hnPos = url.indexOf('://'); + if ( hnPos === -1 ) { return; } + const pathPos = url.indexOf('/', hnPos+3); + if ( pathPos === -1 ) { return; } + return url.slice(pathPos); +}; + +const extractSubTargets = target => { + const isRegex = target.charCodeAt(0) === 0x2F /* / */; + if ( isRegex === false ) { + const pathPos = target.indexOf('/'); + if ( pathPos !== -1 ) { + return { + isRegex, + hn: target.slice(0, pathPos), + pn: target.slice(pathPos), + }; + } + return { isRegex, hn: target }; + } + const pathPos = target.indexOf('\\/'); + if ( pathPos !== -1 ) { + return { + isRegex, + hn: `${target.slice(1, pathPos)}$`, + pn: `^${target.slice(pathPos, -1)}`, + }; + } + return { isRegex, hn: target.slice(1, -1) }; +}; + +/******************************************************************************/ + +export class StaticExtFilteringHostnameDB { + static VERSION = 1; + constructor() { this.size = 0; - this.cleanupTimer = vAPI.defer.create(( ) => { - this.strToIdMap.clear(); - }); } - store(hn, bits, s) { + #hostnameToStringListMap = new Map(); + #matcherMap = new Map(); + #hostnameToMatcherListMap = new Map(); + #strSlots = [ '' ]; // Array of strings (selectors and pseudo-selectors) + #matcherSlots = [ null ]; + #linkedLists = [ 0, 0 ];// Array of integer pairs + #regexMap = new Map(); + #strToSlotMap = new Map(); + #cleanupTimer = vAPI.defer.create(( ) => { + this.#strToSlotMap.clear(); + }); + + store(target, s) { this.size += 1; - let iStr = this.strToIdMap.get(s); + let iStr = this.#strToSlotMap.get(s); if ( iStr === undefined ) { - iStr = this.strSlots.length; - this.strSlots.push(s); - this.strToIdMap.set(s, iStr); - if ( this.cleanupTimer.ongoing() === false ) { + iStr = this.#strSlots.length; + this.#strSlots.push(s); + this.#strToSlotMap.set(s, iStr); + if ( this.#cleanupTimer.ongoing() === false ) { this.collectGarbage(true); } } - const strId = iStr << this.nBits | bits; - const hnIsNotRegex = hn.charCodeAt(0) !== 0x2F /* / */; - let iHn = hnIsNotRegex - ? this.hostnameToSlotIdMap.get(hn) - : this.regexToSlotIdMap.get(hn); - if ( iHn === undefined ) { - if ( hnIsNotRegex ) { - this.hostnameToSlotIdMap.set(hn, this.hostnameSlots.length); + if ( target.includes('/') ) { + return this.#storeMatcher(target, iStr); + } + const iList = this.#hostnameToStringListMap.get(target); + this.#hostnameToStringListMap.set(target, this.#linkedLists.length); + this.#linkedLists.push(iStr, iList !== undefined ? iList : 0); + } + + #storeMatcher(target, iStr) { + const iMatcher = this.#matcherMap.get(target) || + this.#matcherSlots.length; + if ( iMatcher === this.#matcherSlots.length ) { + const { isRegex, hn, pn } = extractSubTargets(target); + this.#matcherSlots.push({ isRegex, hn, pn, iList: 0 }); + this.#matcherMap.set(target, iMatcher); + if ( isRegex === false ) { + const iMatcherList = this.#hostnameToMatcherListMap.get(hn) || 0; + this.#hostnameToMatcherListMap.set(hn, this.#linkedLists.length); + this.#linkedLists.push(iMatcher, iMatcherList); } else { - this.regexToSlotIdMap.set(hn, this.hostnameSlots.length); + const iMatcherList = this.#hostnameToMatcherListMap.get('') || 0; + this.#hostnameToMatcherListMap.set('', this.#linkedLists.length); + this.#linkedLists.push(iMatcher, iMatcherList); } - this.hostnameSlots.push(strId, 0); - return; } - // Add as last item. - while ( this.hostnameSlots[iHn+1] !== 0 ) { - iHn = this.hostnameSlots[iHn+1]; - } - this.hostnameSlots[iHn+1] = this.hostnameSlots.length; - this.hostnameSlots.push(strId, 0); + const matcher = this.#matcherSlots[iMatcher]; + const iList = matcher.iList; + matcher.iList = this.#linkedLists.length; + this.#linkedLists.push(iStr, iList); } clear() { - this.hostnameToSlotIdMap.clear(); - this.regexToSlotIdMap.clear(); - this.hostnameSlots.length = 0; - this.strSlots.length = 0; - this.strToIdMap.clear(); - this.regexMap.clear(); + this.#hostnameToStringListMap.clear(); + this.#matcherMap.clear(); + this.#hostnameToMatcherListMap.clear(); + this.#strSlots = [ '' ]; + this.#matcherSlots = [ null ]; + this.#linkedLists = [ 0, 0 ]; + this.#regexMap.clear(); + this.#strToSlotMap.clear(); this.size = 0; } collectGarbage(later = false) { if ( later ) { - return this.cleanupTimer.onidle(5000, { timeout: 5000 }); + return this.#cleanupTimer.onidle(5000, { timeout: 5000 }); } - this.cleanupTimer.off(); - this.strToIdMap.clear(); + this.#cleanupTimer.off(); + this.#strToSlotMap.clear(); } - // modifiers = 0: all items - // modifiers = 1: only specific items - // modifiers = 2: only generic items - // modifiers = 3: only regex-based items - // - retrieve(hostname, out, modifiers = 0) { + retrieveSpecifics(out, hostname) { let hn = hostname; - if ( modifiers === 2 ) { hn = ''; } + if ( hn === '' ) { return; } for (;;) { - const hnSlot = this.hostnameToSlotIdMap.get(hn); - if ( hnSlot !== undefined ) { - this.retrieveFromSlot(hnSlot, out); + const iList = this.#hostnameToStringListMap.get(hn); + if ( iList !== undefined ) { + this.#retrieveFromSlot(out, iList); } - if ( hn === '' ) { break; } const pos = hn.indexOf('.'); - if ( pos === -1 ) { - if ( modifiers === 1 ) { break; } - hn = ''; - } else { - hn = hn.slice(pos + 1); - } - } - if ( modifiers !== 0 && modifiers !== 3 ) { return; } - if ( this.regexToSlotIdMap.size === 0 ) { return; } - // TODO: consider using a combined regex to test once for whether - // iterating is worth it. - for ( const restr of this.regexToSlotIdMap.keys() ) { - let re = this.regexMap.get(restr); - if ( re === undefined ) { - this.regexMap.set(restr, (re = new RegExp(restr.slice(1,-1)))); - } - if ( re.test(hostname) === false ) { continue; } - this.retrieveFromSlot(this.regexToSlotIdMap.get(restr), out); + if ( pos === -1 ) { break; } + hn = hn.slice(pos + 1); + if ( hn === '*' ) { break; } } } - retrieveFromSlot(hnSlot, out) { - if ( hnSlot === undefined ) { return; } - const mask = out.length - 1; // out.length must be power of two + retrieveGenerics(out) { + let iList = this.#hostnameToStringListMap.get(''); + if ( iList ) { this.#retrieveFromSlot(out, iList); } + iList = this.#hostnameToStringListMap.get('*'); + if ( iList ) { this.#retrieveFromSlot(out, iList); } + } + + retrieveSpecificsByRegex(out, hostname, url) { + let hn = hostname; + if ( hn === '' ) { return; } + const pathname = naivePathnameFromURL(url) ?? ''; + for (;;) { + this.#retrieveSpecificsByRegex(hn, out, hostname, pathname); + const pos = hn.indexOf('.'); + if ( pos === -1 ) { break; } + hn = hn.slice(pos + 1); + } + this.#retrieveSpecificsByRegex('', out, hostname, pathname); + } + + #retrieveSpecificsByRegex(hn, out, hostname, pathname) { + let iMatchList = this.#hostnameToMatcherListMap.get(hn) || 0; + while ( iMatchList !== 0 ) { + const iMatchSlot = this.#linkedLists[iMatchList+0]; + const matcher = this.#matcherSlots[iMatchSlot]; + if ( this.#matcherTest(matcher, hostname, pathname) ) { + this.#retrieveFromSlot(out, matcher.iList); + } + iMatchList = this.#linkedLists[iMatchList+1]; + } + } + + #matcherTest(matcher, hn, pn) { + if ( matcher.isRegex ) { + if ( this.#restrTest(matcher.hn, hn) === false ) { return false; } + if ( matcher.pn === undefined ) { return true; } + return this.#restrTest(matcher.pn, pn); + } + if ( hn.endsWith(matcher.hn) === false ) { return false; } + if ( hn.length !== matcher.hn.length ) { + if ( hn.at(-1) !== '.' ) { return false; } + } + return pn.startsWith(matcher.pn); + } + + #restrTest(restr, s) { + let re = this.#regexMap.get(restr); + if ( re === undefined ) { + this.#regexMap.set(restr, (re = new RegExp(restr))); + } + return re.test(s); + } + + #retrieveFromSlot(out, iList) { + if ( iList === undefined ) { return; } do { - const strId = this.hostnameSlots[hnSlot+0]; - out[strId & mask].add(this.strSlots[strId >>> this.nBits]); - hnSlot = this.hostnameSlots[hnSlot+1]; - } while ( hnSlot !== 0 ); + const iStr = this.#linkedLists[iList+0]; + out.add(this.#strSlots[iStr]); + iList = this.#linkedLists[iList+1]; + } while ( iList !== 0 ); } toSelfie() { return { - version: this.version, - hostnameToSlotIdMap: this.hostnameToSlotIdMap, - regexToSlotIdMap: this.regexToSlotIdMap, - hostnameSlots: this.hostnameSlots, - strSlots: this.strSlots, + VERSION: StaticExtFilteringHostnameDB.VERSION, + hostnameToStringListMap: this.#hostnameToStringListMap, + matcherMap: this.#matcherMap, + hostnameToMatcherListMap: this.#hostnameToMatcherListMap, + strSlots: this.#strSlots, + matcherSlots: this.#matcherSlots, + linkedLists: this.#linkedLists, size: this.size }; } fromSelfie(selfie) { if ( typeof selfie !== 'object' || selfie === null ) { return; } - this.hostnameToSlotIdMap = selfie.hostnameToSlotIdMap; - // Regex-based lookup available in uBO 1.47.0 and above - if ( selfie.regexToSlotIdMap ) { - this.regexToSlotIdMap = selfie.regexToSlotIdMap; + if ( selfie.VERSION !== StaticExtFilteringHostnameDB.VERSION ) { + throw new TypeError('Bad selfie'); } - this.hostnameSlots = selfie.hostnameSlots; - this.strSlots = selfie.strSlots; + this.#hostnameToStringListMap = selfie.hostnameToStringListMap; + this.#matcherMap = selfie.matcherMap; + this.#hostnameToMatcherListMap = selfie.hostnameToMatcherListMap; + this.#strSlots = selfie.strSlots; + this.#matcherSlots = selfie.matcherSlots; + this.#linkedLists = selfie.linkedLists; this.size = selfie.size; } -}; - -/******************************************************************************/ - -export { - StaticExtFilteringHostnameDB, -}; +} /******************************************************************************/ diff --git a/src/js/static-filtering-parser.js b/src/js/static-filtering-parser.js index 3ba2c4653..66d2853ae 100644 --- a/src/js/static-filtering-parser.js +++ b/src/js/static-filtering-parser.js @@ -22,7 +22,6 @@ import * as cssTree from '../lib/csstree/css-tree.js'; import { ArglistParser } from './arglist-parser.js'; import { JSONPath } from './jsonpath.js'; -import Regex from '../lib/regexanalyzer/regex.js'; /******************************************************************************* * @@ -338,12 +337,13 @@ export const nodeNameFromNodeType = new Map([ // Local constants -const DOMAIN_CAN_USE_WILDCARD = 0b000001; -const DOMAIN_CAN_USE_ENTITY = 0b000010; -const DOMAIN_CAN_USE_SINGLE_WILDCARD = 0b000100; -const DOMAIN_CAN_BE_NEGATED = 0b001000; -const DOMAIN_CAN_BE_REGEX = 0b010000; -const DOMAIN_CAN_BE_ANCESTOR = 0b100000; +const DOMAIN_CAN_USE_WILDCARD = 0b0000001; +const DOMAIN_CAN_USE_ENTITY = 0b0000010; +const DOMAIN_CAN_USE_SINGLE_WILDCARD = 0b0000100; +const DOMAIN_CAN_BE_NEGATED = 0b0001000; +const DOMAIN_CAN_BE_REGEX = 0b0010000; +const DOMAIN_CAN_BE_ANCESTOR = 0b0100000; +const DOMAIN_CAN_HAVE_PATH = 0b1000000; const DOMAIN_FROM_FROMTO_LIST = DOMAIN_CAN_USE_ENTITY | DOMAIN_CAN_BE_NEGATED | @@ -353,7 +353,8 @@ const DOMAIN_FROM_EXT_LIST = DOMAIN_CAN_USE_ENTITY | DOMAIN_CAN_USE_SINGLE_WILDCARD | DOMAIN_CAN_BE_NEGATED | DOMAIN_CAN_BE_REGEX | - DOMAIN_CAN_BE_ANCESTOR; + DOMAIN_CAN_BE_ANCESTOR | + DOMAIN_CAN_HAVE_PATH; /******************************************************************************/ @@ -804,7 +805,6 @@ export class AstFilterParser { this.reHostsRedirect = /(?:0\.0\.0\.0|broadcasthost|local|localhost(?:\.localdomain)?|ip6-\w+)(?:[^\w.-]|$)/; this.reNetOptionComma = /,(?:~?[13a-z-]+(?:=.*?)?|_+)(?:,|$)/; this.rePointlessLeftAnchor = /^\|\|?\*+/; - this.reIsTokenChar = /^[%0-9A-Za-z]/; this.rePointlessLeadingWildcards = /^(\*+)[^%0-9A-Za-z\u{a0}-\u{10FFFF}]/u; this.rePointlessTrailingSeparator = /\*(\^\**)$/; this.rePointlessTrailingWildcards = /(?:[^%0-9A-Za-z]|[%0-9A-Za-z]{7,})(\*+)$/; @@ -826,13 +826,14 @@ export class AstFilterParser { this.reHostnameLabel = /[^.]+/g; this.reResponseheaderPattern = /^\^responseheader\(.*\)$/; this.rePatternScriptletJsonArgs = /^\{.*\}$/; - this.reGoodRegexToken = /[^\x01%0-9A-Za-z][%0-9A-Za-z]{7,}|[^\x01%0-9A-Za-z][%0-9A-Za-z]{1,6}[^\x01%0-9A-Za-z]/; this.reBadCSP = /(?:^|[;,])\s*report-(?:to|uri)\b/i; this.reBadPP = /(?:^|[;,])\s*report-to\b/i; this.reNetOption = /^(~?)([134a-z_-]+)(=?)/; this.reNoopOption = /^_+$/; + this.reAdvancedDomainSyntax = /^([^>]+?)(>>)?(\/.*)?$/; this.netOptionValueParser = new ArglistParser(','); this.scriptletArgListParser = new ArglistParser(','); + this.domainRegexValueParser = new ArglistParser('/'); } finish() { @@ -1637,12 +1638,6 @@ export class AstFilterParser { if ( normal !== pattern ) { this.setNodeTransform(next, normal); } - if ( this.interactive ) { - const tokenizable = utils.regex.toTokenizableStr(normal); - if ( this.reGoodRegexToken.test(tokenizable) === false ) { - this.addNodeFlags(next, NODE_FLAG_PATTERN_UNTOKENIZABLE); - } - } } else { this.astTypeFlavor = AST_TYPE_NETWORK_PATTERN_BAD; this.astError = AST_ERROR_REGEX; @@ -2102,11 +2097,20 @@ export class AstFilterParser { ); this.parseDomain(next, parseDetails); end = beg + parseDetails.len; + const badSeparator = end < listEnd && s.charCodeAt(end) !== separatorCode; + if ( badSeparator ) { + end = s.indexOf(separator, end); + if ( end === -1 ) { end = listEnd; } + } this.nodes[next+NODE_END_INDEX] = parentBeg + end; if ( end !== beg ) { domainNode = next; this.linkDown(domainNode, parseDetails.node); prev = this.linkRight(prev, domainNode); + if ( badSeparator ) { + this.addNodeFlags(domainNode, NODE_FLAG_ERROR); + this.addFlags(AST_FLAG_HAS_ERROR); + } } else { domainNode = 0; if ( separatorNode !== 0 ) { @@ -2157,22 +2161,22 @@ export class AstFilterParser { } const c0 = this.charCodeAt(beg); let end = beg; - let type = 0; + let isRegex = false; if ( c0 === 0x2F /* / */ ) { - end = this.indexOf('/', beg + 1, parentEnd); - if ( end !== -1 ) { end += 1; } - type = 1; + this.domainRegexValueParser.nextArg(this.raw, beg+1); + end = this.domainRegexValueParser.separatorEnd; + isRegex = true; } else if ( c0 === 0x5B /* [ */ && this.startsWith('[$domain=/', beg) ) { end = this.indexOf('/]', beg + 10, parentEnd); if ( end !== -1 ) { end += 2; } - type = 2; + isRegex = true; } else { end = this.indexOf(parseDetails.separator, end, parentEnd); } if ( end === -1 ) { end = parentEnd; } if ( beg !== end ) { next = this.allocTypedNode(NODE_TYPE_OPTION_VALUE_DOMAIN, beg, end); - const hn = this.normalizeDomainValue(next, type, parseDetails.mode); + const hn = this.normalizeDomainValue(next, isRegex, parseDetails.mode); if ( hn !== undefined ) { if ( hn !== '' ) { this.setNodeTransform(next, hn); @@ -2195,28 +2199,38 @@ export class AstFilterParser { parseDetails.len = end - parentBeg; } - normalizeDomainValue(node, type, modeBits) { + normalizeDomainValue(node, isRegex, modeBits) { const raw = this.getNodeString(node); - const isAncestor = raw.endsWith('>>'); - if ( isAncestor ) { - if ( (modeBits & DOMAIN_CAN_BE_ANCESTOR) === 0 ) { return ''; } - } - const before = isAncestor ? raw.slice(0, -2) : raw; - let after; - if ( type === 0 ) { - after = this.normalizeHostnameValue(before, modeBits) ?? before; - if ( after === '' ) { return ''; } - } else { + if ( isRegex ) { if ( (modeBits & DOMAIN_CAN_BE_REGEX) === 0 ) { return ''; } - const regex = type === 1 ? before : `/${before.slice(10, -2)}/`; - const source = this.normalizeRegexPattern(regex); - if ( source === '' ) { return ''; } - after = type === 2 || source !== regex ? `/${source}/` : before; + return this.normalizeDomainRegexValue(raw); } - if ( isAncestor ) { - after = `${after}>>`; - } - if ( after === raw ) { return; } + // Common: Assume plain hostname + const r1 = this.normalizeHostnameValue(raw, modeBits); + if ( r1 === undefined ) { return; } + if ( r1 !== '' ) { return r1; } + // Rare: Maybe advanced syntax is used + const match = this.reAdvancedDomainSyntax.exec(raw); + if ( match === null ) { return '' }; + const isAncestor = match[2] !== undefined; + if ( isAncestor && (modeBits & DOMAIN_CAN_BE_ANCESTOR) === 0 ) { return ''; } + const hasPath = match[3] !== undefined; + if ( hasPath && (modeBits & DOMAIN_CAN_HAVE_PATH) === 0 ) { return ''; } + if ( isAncestor && hasPath ) { return ''; } + const r2 = this.normalizeHostnameValue(match[1], modeBits); + if ( r2 === undefined ) { return; } + if ( r2 === '' ) { return ''; } + return `${r2}${match[2] ?? ''}${match[3] ?? ''}`; + } + + normalizeDomainRegexValue(before) { + const regex = before.startsWith('[$domain=/') + ? `/${before.slice(9, -1)}/` + : before; + const source = this.normalizeRegexPattern(regex); + if ( source === '' ) { return ''; } + const after = `/${source}/`; + if ( after === before ) { return; } return after; } @@ -4145,188 +4159,6 @@ export const proceduralOperatorTokens = new Map([ export const utils = (( ) => { - // Depends on: - // https://github.com/foo123/RegexAnalyzer - const regexAnalyzer = Regex && Regex.Analyzer || null; - - class regex { - static firstCharCodeClass(s) { - return /^[\x01\x03%0-9A-Za-z]/.test(s) ? 1 : 0; - } - - static lastCharCodeClass(s) { - return /[\x01\x03%0-9A-Za-z]$/.test(s) ? 1 : 0; - } - - static tokenizableStrFromNode(node) { - switch ( node.type ) { - case 1: /* T_SEQUENCE, 'Sequence' */ { - let s = ''; - for ( let i = 0; i < node.val.length; i++ ) { - s += this.tokenizableStrFromNode(node.val[i]); - } - return s; - } - case 2: /* T_ALTERNATION, 'Alternation' */ - case 8: /* T_CHARGROUP, 'CharacterGroup' */ { - if ( node.flags.NegativeMatch ) { return '\x01'; } - let firstChar = 0; - let lastChar = 0; - for ( let i = 0; i < node.val.length; i++ ) { - const s = this.tokenizableStrFromNode(node.val[i]); - if ( firstChar === 0 && this.firstCharCodeClass(s) === 1 ) { - firstChar = 1; - } - if ( lastChar === 0 && this.lastCharCodeClass(s) === 1 ) { - lastChar = 1; - } - if ( firstChar === 1 && lastChar === 1 ) { break; } - } - return String.fromCharCode(firstChar, lastChar); - } - case 4: /* T_GROUP, 'Group' */ { - if ( - node.flags.NegativeLookAhead === 1 || - node.flags.NegativeLookBehind === 1 - ) { - return ''; - } - return this.tokenizableStrFromNode(node.val); - } - case 16: /* T_QUANTIFIER, 'Quantifier' */ { - if ( node.flags.max === 0 ) { return ''; } - const s = this.tokenizableStrFromNode(node.val); - const first = this.firstCharCodeClass(s); - const last = this.lastCharCodeClass(s); - if ( node.flags.min !== 0 ) { - return String.fromCharCode(first, last); - } - return String.fromCharCode(first+2, last+2); - } - case 64: /* T_HEXCHAR, 'HexChar' */ { - if ( - node.flags.Code === '01' || - node.flags.Code === '02' || - node.flags.Code === '03' - ) { - return '\x00'; - } - return node.flags.Char; - } - case 128: /* T_SPECIAL, 'Special' */ { - const flags = node.flags; - if ( - flags.EndCharGroup === 1 || // dangling `]` - flags.EndGroup === 1 || // dangling `)` - flags.EndRepeats === 1 // dangling `}` - ) { - throw new Error('Unmatched bracket'); - } - return flags.MatchEnd === 1 || - flags.MatchStart === 1 || - flags.MatchWordBoundary === 1 - ? '\x00' - : '\x01'; - } - case 256: /* T_CHARS, 'Characters' */ { - for ( let i = 0; i < node.val.length; i++ ) { - if ( this.firstCharCodeClass(node.val[i]) === 1 ) { - return '\x01'; - } - } - return '\x00'; - } - // Ranges are assumed to always involve token-related characters. - case 512: /* T_CHARRANGE, 'CharacterRange' */ { - return '\x01'; - } - case 1024: /* T_STRING, 'String' */ { - return node.val; - } - case 2048: /* T_COMMENT, 'Comment' */ { - return ''; - } - default: - break; - } - return '\x01'; - } - - static isValid(reStr) { - try { - void new RegExp(reStr); - if ( regexAnalyzer !== null ) { - void this.tokenizableStrFromNode( - regexAnalyzer(reStr, false).tree() - ); - } - } catch { - return false; - } - return true; - } - - static isRE2(reStr) { - if ( regexAnalyzer === null ) { return true; } - let tree; - try { - tree = regexAnalyzer(reStr, false).tree(); - } catch { - return; - } - const isRE2 = node => { - if ( node instanceof Object === false ) { return true; } - if ( node.flags instanceof Object ) { - if ( node.flags.LookAhead === 1 ) { return false; } - if ( node.flags.NegativeLookAhead === 1 ) { return false; } - if ( node.flags.LookBehind === 1 ) { return false; } - if ( node.flags.NegativeLookBehind === 1 ) { return false; } - } - if ( Array.isArray(node.val) ) { - for ( const entry of node.val ) { - if ( isRE2(entry) === false ) { return false; } - } - } - if ( node.val instanceof Object ) { - return isRE2(node.val); - } - return true; - }; - return isRE2(tree); - } - - static toTokenizableStr(reStr) { - if ( regexAnalyzer === null ) { return ''; } - let s = ''; - try { - s = this.tokenizableStrFromNode( - regexAnalyzer(reStr, false).tree() - ); - } catch { - } - // Process optional sequences - const reOptional = /[\x02\x03]+/; - for (;;) { - const match = reOptional.exec(s); - if ( match === null ) { break; } - const left = s.slice(0, match.index); - const middle = match[0]; - const right = s.slice(match.index + middle.length); - s = left; - s += this.firstCharCodeClass(right) === 1 || - this.firstCharCodeClass(middle) === 1 - ? '\x01' - : '\x00'; - s += this.lastCharCodeClass(left) === 1 || - this.lastCharCodeClass(middle) === 1 - ? '\x01' - : '\x00'; - s += right; - } - return s; - } - } - const preparserTokens = new Map([ [ 'ext_ublock', 'ublock' ], [ 'ext_ubol', 'ubol' ], @@ -4544,7 +4376,6 @@ export const utils = (( ) => { return { preparser, - regex, }; })(); diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 24961d5f7..3aad7fb10 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -23,6 +23,7 @@ import * as sfp from './static-filtering-parser.js'; import { domainFromHostname, hostnameFromNetworkURL } from './uri-utils.js'; import { dropTask, queueTask } from './tasks.js'; +import { isRE2, tokenizableStrFromRegex } from './regex-analyzer.js'; import BidiTrieContainer from './biditrie.js'; import { CompiledListReader } from './static-filtering-io.js'; @@ -1250,7 +1251,7 @@ class FilterRegex { if ( rule.condition === undefined ) { rule.condition = {}; } - if ( sfp.utils.regex.isRE2(args[1]) === false ) { + if ( isRE2(args[1]) === false ) { dnrAddRuleError(rule, `regexFilter is not RE2-compatible: ${args[1]}`); } rule.condition.regexFilter = args[1]; @@ -3246,7 +3247,7 @@ class FilterCompiler { if ( other !== undefined ) { return Object.assign(this, other); } - this.reToken = /[%0-9A-Za-z]+/g; + this.reTokens = /[%0-9A-Za-z]+/g; this.optionValues = new Map(); this.tokenIdToNormalizedType = new Map([ [ sfp.NODE_TYPE_NET_OPTION_NAME_CNAME, bitFromType('cname') ], @@ -3797,11 +3798,11 @@ class FilterCompiler { // Note: a one-char token is better than a documented bad token. extractTokenFromPattern(pattern) { - this.reToken.lastIndex = 0; + this.reTokens.lastIndex = 0; let bestMatch = null; let bestBadness = 0x7FFFFFFF; for (;;) { - const match = this.reToken.exec(pattern); + const match = this.reTokens.exec(pattern); if ( match === null ) { break; } const token = match[0]; const badness = token.length > 1 ? this.badTokens.get(token) || 0 : 1; @@ -3811,7 +3812,7 @@ class FilterCompiler { if ( c === 0x2A /* '*' */ ) { continue; } } if ( token.length < MAX_TOKEN_LENGTH ) { - const lastIndex = this.reToken.lastIndex; + const lastIndex = this.reTokens.lastIndex; if ( lastIndex < pattern.length ) { const c = pattern.charCodeAt(lastIndex); if ( c === 0x2A /* '*' */ ) { continue; } @@ -3835,18 +3836,18 @@ class FilterCompiler { // Mind `\b` directives: `/\bads\b/` should result in token being `ads`, // not `bads`. extractTokenFromRegex(pattern) { - pattern = sfp.utils.regex.toTokenizableStr(pattern); - this.reToken.lastIndex = 0; + pattern = tokenizableStrFromRegex(pattern); + this.reTokens.lastIndex = 0; let bestToken; let bestBadness = 0x7FFFFFFF; for (;;) { - const matches = this.reToken.exec(pattern); + const matches = this.reTokens.exec(pattern); if ( matches === null ) { break; } const { 0: token, index } = matches; if ( index === 0 || pattern.charAt(index - 1) === '\x01' ) { continue; } - const { lastIndex } = this.reToken; + const { lastIndex } = this.reTokens; if ( token.length < MAX_TOKEN_LENGTH && ( lastIndex === pattern.length || diff --git a/src/js/storage.js b/src/js/storage.js index 7dc6e8638..758ae9bd3 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -42,7 +42,7 @@ import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js'; import punycode from '../lib/punycode.js'; import { redirectEngine } from './redirect-engine.js'; import staticExtFilteringEngine from './static-ext-filtering.js'; -import staticFilteringReverseLookup from './reverselookup.js'; +import { staticFilteringReverseLookup } from './reverselookup.js'; import staticNetFilteringEngine from './static-net-filtering.js'; import µb from './background.js'; diff --git a/tools/make-nodejs.sh b/tools/make-nodejs.sh index fb0a7a65f..59350e7bd 100755 --- a/tools/make-nodejs.sh +++ b/tools/make-nodejs.sh @@ -16,6 +16,7 @@ cp src/js/hnswitches.js "$DES/js" cp src/js/hntrie.js "$DES/js" cp src/js/jsonpath.js "$DES/js" cp src/js/redirect-resources.js "$DES/js" +cp src/js/regex-analyzer.js "$DES/js" cp src/js/s14e-serializer.js "$DES/js" cp src/js/static-dnr-filtering.js "$DES/js" cp src/js/static-filtering-parser.js "$DES/js"