diff --git a/platform/mv3/extension/js/develop.js b/platform/mv3/extension/js/develop.js index 5068d3831..f04316504 100644 --- a/platform/mv3/extension/js/develop.js +++ b/platform/mv3/extension/js/develop.js @@ -21,9 +21,6 @@ import { dom, qs$, qsa$ } from './dom.js'; import { localRead, localWrite, sendMessage } from './ext.js'; -import { ModeEditor } from './mode-editor.js'; -import { ReadOnlyDNREditor } from './ro-dnr-editor.js'; -import { ReadWriteDNREditor } from './rw-dnr-editor.js'; import { faIconsInit } from './fa-icons.js'; import { i18n } from './i18n.js'; @@ -39,14 +36,21 @@ class Editor { this.ioPanel = self.cm6.createViewPanel(); this.summaryPanel = self.cm6.createViewPanel(); this.panels = []; + this.editors = {}; } async init() { - this.editors = { - 'modes': new ModeEditor(this), - 'dnr.rw': new ReadWriteDNREditor(this), - 'dnr.ro': new ReadOnlyDNREditor(this), - }; + await Promise.all([ + import('./mode-editor.js').then(module => { + this.editors['modes'] = new module.ModeEditor(this); + }), + import('./ro-dnr-editor.js').then(module => { + this.editors['dnr.ro'] = new module.ReadOnlyDNREditor(this); + }), + import('./rw-dnr-editor.js').then(module => { + this.editors['dnr.rw'] = new module.ReadWriteDNREditor(this); + }), + ]); const rulesetDetails = await sendMessage({ what: 'getRulesetDetails' }); const parent = qs$('#editors optgroup'); for ( const details of rulesetDetails ) { diff --git a/platform/mv3/extension/js/rw-dnr-editor.js b/platform/mv3/extension/js/rw-dnr-editor.js index 30e90f6b5..05a56c1f4 100644 --- a/platform/mv3/extension/js/rw-dnr-editor.js +++ b/platform/mv3/extension/js/rw-dnr-editor.js @@ -29,6 +29,7 @@ import { import { dom, qs$ } from './dom.js'; import { i18n, i18n$ } from './i18n.js'; import { DNREditor } from './dnr-editor.js'; +import { parseFilters } from './ubo-parser.js'; import { textFromRules } from './dnr-parser.js'; /******************************************************************************/ @@ -368,15 +369,28 @@ export class ReadWriteDNREditor extends DNREditor { const lineFrom = newDoc.lineAt(from); if ( lineFrom.from !== from ) { return; } // Paste position must match a rule boundary + let separatorBefore = false; if ( lineFrom.number !== 1 ) { const lineBefore = newDoc.line(lineFrom.number-1); if ( /^---\s*$/.test(lineBefore.text) === false ) { return; } + separatorBefore = true; } const pastedText = newDoc.sliceString(from, to); - const rules = this.rulesFromJSON(pastedText); - if ( rules === undefined ) { return; } - const yamlText = textFromRules(rules); + let prepend; + let rules = this.rulesFromJSON(pastedText); + if ( Boolean(rules?.length) === false ) { + rules = parseFilters(pastedText); + if ( Boolean(rules?.length) === false ) { return; } + prepend = pastedText.trim().split(/\n/).map(a => `# ${a}`).join('\n'); + } + let yamlText = textFromRules(rules); if ( yamlText === undefined ) { return; } + if ( prepend ) { + yamlText = yamlText.replace('---\n', `---\n${prepend}\n`); + } + if ( separatorBefore && yamlText.startsWith('---\n') ) { + yamlText = yamlText.slice(4); + } editor.view.dispatch({ changes: { from, to, insert: yamlText } }); self.cm6.foldAll(editor.view); return true; diff --git a/platform/mv3/extension/js/ubo-parser.js b/platform/mv3/extension/js/ubo-parser.js new file mode 100644 index 000000000..dbfb91994 --- /dev/null +++ b/platform/mv3/extension/js/ubo-parser.js @@ -0,0 +1,457 @@ +/******************************************************************************* + + uBlock Origin Lite - a comprehensive, MV3-compliant content blocker + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +import * as sfp from './static-filtering-parser.js'; +import punycode from './punycode.js'; +import redirectResourceMap from './redirect-resources.js'; + +/******************************************************************************/ + +const validResourceTypes = [ + 'main_frame', + 'sub_frame', + 'stylesheet', + 'script', + 'image', + 'font', + 'object', + 'xmlhttprequest', + 'ping', + 'csp_report', + 'media', + 'websocket', + 'webtransport', + 'webbundle', + 'other', +]; + +/******************************************************************************/ + +const validRedirectResources = (( ) => { + const out = new Map(); + for ( const [ name, resource ] of redirectResourceMap ) { + out.set(name, name); + if ( resource.alias === undefined ) { continue; } + if ( typeof resource.alias === 'string' ) { + out.set(resource.alias, name); + continue; + } + if ( Array.isArray(resource.alias) ) { + for ( const alias of resource.alias ) { + out.set(alias, name); + } + } + } + return out; +})(); + +/******************************************************************************/ + +function parseHostnameList(iter) { + const out = { + included: { + good: [], + bad: [], + }, + excluded: { + good: [], + bad: [], + }, + }; + for ( let { hn, not, bad } of iter ) { + bad ||= hn.includes('/') || hn.includes('*'); + const hnAscii = bad === false && hn.startsWith('xn--') + ? punycode.toASCII(hn) + : hn; + const destination = not ? out.excluded : out.included; + if ( bad ) { + destination.bad.push(hnAscii); + } else { + destination.good.push(hnAscii); + } + } + return out; +} + +/******************************************************************************/ + +function mergeIncludeExclude(rules) { + const includeExcludes = [ + { includeName: 'requestDomains', excludeName: 'excludedRequestDomains' }, + { includeName: 'initiatorDomains', excludeName: 'excludedInitiatorDomains' }, + { includeName: 'resourceTypes', excludeName: 'excludedResourceTypes' }, + { includeName: 'requestMethods', excludeName: 'excludedRequestMethods' }, + ]; + for ( const { includeName, excludeName } of includeExcludes ) { + const out = []; + const distinctRules = new Map(); + for ( const rule of rules ) { + const { condition } = rule; + if ( Boolean(condition[includeName]?.length) === false ) { + if ( Boolean(condition[excludeName]?.length) === false ) { + out.push(rule); + continue; + } + } + const included = condition[includeName] || []; + condition[includeName] = undefined; + const excluded = condition[excludeName] || []; + condition[excludeName] = undefined; + const hash = JSON.stringify(rule); + const details = distinctRules.get(hash) || + { included: new Set(), excluded: new Set() }; + if ( details.included.size === 0 && details.excluded.size === 0 ) { + distinctRules.set(hash, details); + } + for ( const hn of included ) { + details.included.add(hn); + } + for ( const hn of excluded ) { + if ( details.included.has(hn) ) { continue; } + details.excluded.add(hn); + } + } + for ( const [ hash, details ] of distinctRules ) { + const rule = JSON.parse(hash); + if ( details.included.size !== 0 ) { + rule.condition[includeName] = Array.from(details.included); + } + if ( details.excluded.size !== 0 ) { + rule.condition[excludeName] = Array.from(details.excluded); + } + out.push(rule); + } + rules = out; + } + return rules; +} + +/******************************************************************************/ + +function parseNetworkFilter(parser) { + if ( parser.isNetworkFilter() === false ) { return; } + if ( parser.hasError() ) { return; } + + const rule = { + action: { type: 'block' }, + condition: { }, + }; + if ( parser.isException() ) { + rule.action.type = 'allow'; + } + + let pattern = parser.getNetPattern(); + if ( parser.isHostnamePattern() ) { + rule.condition.requestDomains = [ pattern ]; + } else if ( parser.isGenericPattern() ) { + if ( parser.isLeftHnAnchored() ) { + pattern = `||${pattern}`; + } else if ( parser.isLeftAnchored() ) { + pattern = `|${pattern}`; + } + if ( parser.isRightAnchored() ) { + pattern = `${pattern}|`; + } + rule.condition.urlFilter = pattern; + } else if ( parser.isRegexPattern() ) { + rule.condition.regexFilter = pattern; + } else if ( parser.isAnyPattern() === false ) { + rule.condition.urlFilter = pattern; + } + + const initiatorDomains = new Set(); + const excludedInitiatorDomains = new Set(); + const requestDomains = new Set(); + const excludedRequestDomains = new Set(); + const requestMethods = new Set(); + const excludedRequestMethods = new Set(); + const resourceTypes = new Set(); + const excludedResourceTypes = new Set(); + + const processResourceType = (resourceType, nodeType) => { + const not = parser.isNegatedOption(nodeType) + if ( validResourceTypes.includes(resourceType) === false ) { + if ( not ) { return; } + } + if ( not ) { + excludedResourceTypes.add(resourceType); + } else { + resourceTypes.add(resourceType); + } + }; + + let priority = 0; + + for ( const type of parser.getNodeTypes() ) { + switch ( type ) { + case sfp.NODE_TYPE_NET_OPTION_NAME_1P: + rule.domainType = parser.isNegatedOption(type) + ? 'thirdParty' + : 'firstParty'; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT1P: + case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT3P: + case sfp.NODE_TYPE_NET_OPTION_NAME_BADFILTER: + case sfp.NODE_TYPE_NET_OPTION_NAME_CNAME: + case sfp.NODE_TYPE_NET_OPTION_NAME_EHIDE: + case sfp.NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK: + case sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE: + case sfp.NODE_TYPE_NET_OPTION_NAME_IPADDRESS: + case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE: + case sfp.NODE_TYPE_NET_OPTION_NAME_REPLACE: + case sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE: + case sfp.NODE_TYPE_NET_OPTION_NAME_URLSKIP: + return; + case sfp.NODE_TYPE_NET_OPTION_NAME_INLINEFONT: + case sfp.NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT: + case sfp.NODE_TYPE_NET_OPTION_NAME_POPUNDER: + case sfp.NODE_TYPE_NET_OPTION_NAME_POPUP: + case sfp.NODE_TYPE_NET_OPTION_NAME_WEBRTC: + processResourceType('', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_3P: + rule.condition.domainType = parser.isNegatedOption(type) + ? 'firstParty' + : 'thirdParty'; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_ALL: + validResourceTypes.forEach(a => resourceTypes.add(a)); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_CSP: + if ( rule.action.responseHeaders ) { return; } + rule.action.type = 'modifyHeaders'; + rule.action.responseHeaders = [ { + header: 'content-security-policy', + operation: 'append', + value: parser.getNetOptionValue(type), + } ]; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_CSS: + processResourceType('stylesheet', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_DENYALLOW: { + const { included, excluded } = parseHostnameList( + parser.getNetFilterDenyallowOptionIterator() + ); + if ( excluded.good.length !== 0 || excluded.bad.length !== 0 ) { return; } + if ( included.bad.length !== 0 ) { return; } + if ( included.good.length === 0 ) { return; } + for ( const hn of included.good ) { + excludedRequestDomains.add(hn); + } + break; + } + case sfp.NODE_TYPE_NET_OPTION_NAME_DOC: + processResourceType('main_frame', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_FONT: + processResourceType('font', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_FRAME: + processResourceType('sub_frame', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_FROM: { + const { included, excluded } = parseHostnameList( + parser.getNetFilterFromOptionIterator() + ); + if ( included.good.length === 0 ) { + if ( included.bad.length !== 0 ) { return; } + } + if ( excluded.bad.length !== 0 ) { return; } + for ( const hn of included.good ) { + initiatorDomains.add(hn); + } + for ( const hn of excluded.good ) { + excludedInitiatorDomains.add(hn); + } + break; + } + case sfp.NODE_TYPE_NET_OPTION_NAME_HEADER: { + const details = sfp.parseHeaderValue(parser.getNetOptionValue(type)); + const headerInfo = { + header: details.name, + }; + if ( details.value !== '' ) { + if ( details.isRegex ) { return; } + headerInfo.values = [ details.value ]; + } + rule.condition.responseHeaders = [ headerInfo ]; + break; + } + case sfp.NODE_TYPE_NET_OPTION_NAME_IMAGE: + processResourceType('image', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_IMPORTANT: + priority += 30; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_MATCHCASE: + rule.condition.isUrlFilterCaseSensitive = true; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_MEDIA: + processResourceType('media', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_METHOD: { + const value = parser.getNetOptionValue(type); + for ( const method of value.toUpperCase().split('|') ) { + const not = method.charCodeAt(0) === 0x7E /* '~' */; + if ( not ) { + excludedRequestMethods.add(method.slice(1)); + } else { + requestMethods.add(method); + } + } + break; + } + case sfp.NODE_TYPE_NET_OPTION_NAME_OBJECT: + processResourceType('object', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_OTHER: + processResourceType('other', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_PERMISSIONS: + if ( rule.action.responseHeaders ) { return; } + rule.action.type = 'modifyHeaders'; + rule.action.responseHeaders = [ { + header: 'permissions-policy', + operation: 'append', + value: parser.getNetOptionValue(type), + } ]; + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_PING: + processResourceType('ping', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_REASON: + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT: { + if ( rule.action.type !== 'block' ) { return; } + let value = parser.getNetOptionValue(type); + const match = /:(\d+)$/.exec(value); + if ( match ) { + const subpriority = parseInt(match[1], 10); + priority += Math.min(subpriority, 8); + value = value.slice(0, match.index); + } + if ( validRedirectResources.has(value) === false ) { return; } + rule.action.type = 'redirect'; + rule.action.redirect = { + extensionPath: `/web_accessible_resources/${validRedirectResources.get(value)}`, + }; + priority += 11; + break; + } + case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM: { + const details = sfp.parseQueryPruneValue(parser.getNetOptionValue(type)); + if ( details.bad ) { return; } + if ( details.not ) { return; } + if ( details.re ) { return; } + const removeParams = []; + if ( details.name ) { + removeParams.push(details.name); + } + rule.action.type = 'redirect'; + rule.action.redirect = { + transform: { queryTransform: { removeParams } } + }; + break; + } + case sfp.NODE_TYPE_NET_OPTION_NAME_SCRIPT: + processResourceType('script', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_TO: { + const { included, excluded } = parseHostnameList( + parser.getNetFilterToOptionIterator() + ); + if ( included.good.length === 0 ) { + if ( included.bad.length !== 0 ) { return; } + } + if ( excluded.bad.length !== 0 ) { return; } + for ( const hn of included.good ) { + requestDomains.add(hn); + } + for ( const hn of excluded.good ) { + excludedRequestDomains.add(hn); + } + break; + } + case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: + if ( this.processOptionWithValue(parser, type) === false ) { + return this.FILTER_INVALID; + } + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_XHR: + processResourceType('xmlhttprequest', type); + break; + case sfp.NODE_TYPE_NET_OPTION_NAME_WEBSOCKET: + processResourceType('websocket', type); + break; + default: + break; + } + } + if ( initiatorDomains.size !== 0 ) { + rule.condition.initiatorDomains = Array.from(initiatorDomains); + } + if ( excludedInitiatorDomains.size !== 0 ) { + rule.condition.excludedInitiatorDomains = Array.from(excludedInitiatorDomains); + } + if ( requestDomains.size !== 0 ) { + rule.condition.requestDomains = Array.from(requestDomains); + } + if ( excludedRequestDomains.size !== 0 ) { + rule.condition.excludedRequestDomains = Array.from(excludedRequestDomains); + } + if ( requestMethods.size !== 0 ) { + rule.condition.requestMethods = Array.from(requestMethods); + } + if ( excludedRequestMethods.size !== 0 ) { + rule.condition.excludedRequestMethods = Array.from(excludedRequestMethods); + } + if ( resourceTypes.size !== 0 ) { + const types = Array.from(resourceTypes).filter(a => a !== ''); + if ( types.length === 0 ) { return; } + rule.condition.resourceTypes = types; + } + if ( excludedResourceTypes.size !== 0 ) { + if ( resourceTypes.size !== 0 ) { + if ( excludedResourceTypes.size !== 0 ) { return; } + } + rule.condition.excludedResourceTypes = Array.from(excludedResourceTypes); + } + if ( priority !== 0 ) { + rule.priority = priority; + } + return rule; +} + +/******************************************************************************/ + +export function parseFilters(text) { + const rules = []; + const parser = new sfp.AstFilterParser({ trustedSource: true }); + for ( const line of text.split(/\n/) ) { + parser.parse(line); + if ( parser.isNetworkFilter() === false ) { continue; } + const rule = parseNetworkFilter(parser); + if ( rule === undefined ) { continue; } + rules.push(rule); + } + return mergeIncludeExclude(rules); +} diff --git a/tools/make-mv3.sh b/tools/make-mv3.sh index f42052931..25c42ae4c 100755 --- a/tools/make-mv3.sh +++ b/tools/make-mv3.sh @@ -80,9 +80,13 @@ cp "$UBO_DIR"/src/css/common.css "$UBOL_DIR"/css/ cp "$UBO_DIR"/src/css/dashboard-common.css "$UBOL_DIR"/css/ cp "$UBO_DIR"/src/css/fa-icons.css "$UBOL_DIR"/css/ +cp "$UBO_DIR"/src/js/arglist-parser.js "$UBOL_DIR"/js/ cp "$UBO_DIR"/src/js/dom.js "$UBOL_DIR"/js/ cp "$UBO_DIR"/src/js/fa-icons.js "$UBOL_DIR"/js/ cp "$UBO_DIR"/src/js/i18n.js "$UBOL_DIR"/js/ +cp "$UBO_DIR"/src/js/jsonpath.js "$UBOL_DIR"/js/ +cp "$UBO_DIR"/src/js/redirect-resources.js "$UBOL_DIR"/js/ +cp "$UBO_DIR"/src/js/static-filtering-parser.js "$UBOL_DIR"/js/ cp "$UBO_DIR"/src/js/urlskip.js "$UBOL_DIR"/js/ cp "$UBO_DIR"/src/lib/punycode.js "$UBOL_DIR"/js/ @@ -111,6 +115,8 @@ cp platform/mv3/extension/lib/codemirror/codemirror.LICENSE \ "$UBOL_DIR"/lib/codemirror/ cp platform/mv3/extension/lib/codemirror/codemirror-ubol/LICENSE \ "$UBOL_DIR"/lib/codemirror/codemirror-quickstart.LICENSE +mkdir -p "$UBOL_DIR"/lib/csstree +cp "$UBO_DIR"/src/lib/csstree/* "$UBOL_DIR"/lib/csstree/ echo "*** uBOLite.mv3: Generating rulesets" UBOL_BUILD_DIR=$(mktemp -d)