[mv3] Extend strict-blocking coverage; improve URL-skip behavior

This extends coverage of strict-blocking to pattern-based filters
with `doc` filter option.

When proceeding with a URL-skip URL present, no temporary bypass
will be created when the "Don't warn me again about this site" is
left unchecked. The idea is to avoid the intermediate redirects
if we navigate again on the same strict-blocked site, while a
temporary bypass would prevent this.

uBO's "Badware risks" list has been spinned off as its own list.
The idea is that should a site be strict-blocked from that list,
we would want to know the strict-block is due to the
"Badware risks" list.

platform/mv3/chromium/manifest.json

@@ -41,5 +41,16 @@
   "storage": {
     "managed_schema": "managed_storage.json"
   },
-  "version": "1.0"
+  "version": "1.0",
+  "web_accessible_resources": [
+    {
+      "resources": [
+        "/strictblock.html"
+      ],
+      "matches": [
+        "<all_urls>"
+      ],
+      "use_dynamic_url": true
+    }
+  ]
 }

platform/mv3/extension/js/background.js

@@ -53,6 +53,7 @@ import {
     patchDefaultRulesets,
     setStrictBlockMode,
     updateDynamicRules,
+    updateSessionRules,
 } from './ruleset-manager.js';
 
 import {
@@ -398,8 +399,12 @@ async function start() {
         await enableRulesets(rulesetConfig.enabledRulesets);
 
     // We need to update the regex rules only when ruleset version changes.
-    if ( isNewVersion && rulesetsUpdated === false ) {
-        updateDynamicRules();
+    if ( rulesetsUpdated === false ) {
+        if ( isNewVersion ) {
+            updateDynamicRules();
+        } else if ( process.wakeupRun === false ) {
+            updateSessionRules();
+        }
     }
 
     // Permissions may have been removed while the extension was disabled

platform/mv3/extension/js/filter-lists.js

@@ -216,7 +216,8 @@ export function renderFilterLists(rulesetData) {
         [
             'default',
             rulesetDetails.filter(ruleset =>
-                ruleset.id === 'default' 
+                ruleset.id === 'default' ||
+                ruleset.group === 'default'
             ),
         ], [
             'malware',

platform/mv3/extension/js/ruleset-manager.js

@@ -41,10 +41,18 @@ import { ubolLog } from './debug.js';
 
 /******************************************************************************/
 
-const STRICTBLOCK_BASE_RULE_ID = 7000000;
 const TRUSTED_DIRECTIVE_BASE_RULE_ID = 8000000;
+const STRICTBLOCK_PRIORITY = 29;
 
-let dynamicRuleId = 1;
+/******************************************************************************/
+
+const isStrictBlockRule = rule => {
+    if ( rule.priority !== STRICTBLOCK_PRIORITY ) { return false; }
+    if ( rule.action.type !== 'redirect' ) { return false; }
+    const substitution = rule.action.redirect.regexSubstitution;
+    return substitution !== undefined &&
+        substitution.includes('/strictblock.');
+};
 
 /******************************************************************************/
 
@@ -106,7 +114,15 @@ pruneInvalidRegexRules.validated = new Map();
 
 /******************************************************************************/
 
-async function updateRegexRules(toAdd) {
+async function updateRegexRules(currentRules, addRules, removeRuleIds) {
+    // Remove existing regex-related block rules
+    for ( const rule of currentRules ) {
+        const { type } = rule.action;
+        if ( type !== 'block' && type !== 'allow' ) { continue; }
+        if ( rule.condition.regexFilter === undefined ) { continue; }
+        removeRuleIds.push(rule.id);
+    }
+
     const rulesetDetails = await getEnabledRulesetsDetails();
 
     // Fetch regexes for all enabled rulesets
@@ -122,7 +138,6 @@ async function updateRegexRules(toAdd) {
     for ( const rules of regexRulesets ) {
         if ( Array.isArray(rules) === false ) { continue; }
         for ( const rule of rules ) {
-            rule.id = dynamicRuleId++;
             allRules.push(rule);
         }
     }
@@ -132,12 +147,19 @@ async function updateRegexRules(toAdd) {
     if ( validRules.length === 0 ) { return; }
 
     ubolLog(`Add ${validRules.length} DNR regex rules`);
-    toAdd.push(...validRules);
+    addRules.push(...validRules);
 }
 
 /******************************************************************************/
 
-async function updateRemoveparamRules(toAdd) {
+async function updateRemoveparamRules(currentRules, addRules, removeRuleIds) {
+    // Remove existing removeparam-related rules
+    for ( const rule of currentRules ) {
+        if ( rule.action.type !== 'redirect' ) { continue; }
+        if ( rule.action.redirect.transform === undefined ) { continue; }
+        removeRuleIds.push(rule.id);
+    }
+
     const [
         hasOmnipotence,
         rulesetDetails,
@@ -160,7 +182,6 @@ async function updateRemoveparamRules(toAdd) {
         for ( const rules of removeparamRulesets ) {
             if ( Array.isArray(rules) === false ) { continue; }
             for ( const rule of rules ) {
-                rule.id = dynamicRuleId++;
                 allRules.push(rule);
             }
         }
@@ -171,12 +192,19 @@ async function updateRemoveparamRules(toAdd) {
     if ( validRules.length === 0 ) { return; }
 
     ubolLog(`Add ${validRules.length} DNR removeparam rules`);
-    toAdd.push(...validRules);
+    addRules.push(...validRules);
 }
 
 /******************************************************************************/
 
-async function updateRedirectRules(toAdd) {
+async function updateRedirectRules(currentRules, addRules, removeRuleIds) {
+    // Remove existing redirect-related rules
+    for ( const rule of currentRules ) {
+        if ( rule.action.type !== 'redirect' ) { continue; }
+        if ( rule.action.redirect.extensionPath === undefined ) { continue; }
+        removeRuleIds.push(rule.id);
+    }
+
     const [
         hasOmnipotence,
         rulesetDetails,
@@ -199,7 +227,6 @@ async function updateRedirectRules(toAdd) {
         for ( const rules of redirectRulesets ) {
             if ( Array.isArray(rules) === false ) { continue; }
             for ( const rule of rules ) {
-                rule.id = dynamicRuleId++;
                 allRules.push(rule);
             }
         }
@@ -210,12 +237,18 @@ async function updateRedirectRules(toAdd) {
     if ( validRules.length === 0 ) { return; }
 
     ubolLog(`Add ${validRules.length} DNR redirect rules`);
-    toAdd.push(...validRules);
+    addRules.push(...validRules);
 }
 
 /******************************************************************************/
 
-async function updateModifyHeadersRules(toAdd) {
+async function updateModifyHeadersRules(currentRules, addRules, removeRuleIds) {
+    // Remove existing header modification-related rules
+    for ( const rule of currentRules ) {
+        if ( rule.action.type !== 'modifyHeaders' ) { continue; }
+        removeRuleIds.push(rule.id);
+    }
+
     const [
         hasOmnipotence,
         rulesetDetails,
@@ -238,7 +271,6 @@ async function updateModifyHeadersRules(toAdd) {
         for ( const rules of rulesets ) {
             if ( Array.isArray(rules) === false ) { continue; }
             for ( const rule of rules ) {
-                rule.id = dynamicRuleId++;
                 allRules.push(rule);
             }
         }
@@ -249,12 +281,65 @@ async function updateModifyHeadersRules(toAdd) {
     if ( validRules.length === 0 ) { return; }
 
     ubolLog(`Add ${validRules.length} DNR modify-headers rules`);
-    toAdd.push(...validRules);
+    addRules.push(...validRules);
 }
 
 /******************************************************************************/
 
-async function updateStrictBlockRules(dynamicRules, sessionRules) {
+async function updateDynamicRules() {
+    const currentRules = await dnr.getDynamicRules();
+    const addRules = [];
+    const removeRuleIds = [];
+
+    // Remove potentially left-over strict-block rules from previous version
+    for ( const rule of currentRules ) {
+        if ( isStrictBlockRule(rule) === false ) { continue; }
+        removeRuleIds.push(rule.id);
+    }
+
+    await Promise.all([
+        updateRegexRules(currentRules, addRules, removeRuleIds),
+        updateRemoveparamRules(currentRules, addRules, removeRuleIds),
+        updateRedirectRules(currentRules, addRules, removeRuleIds),
+        updateModifyHeadersRules(currentRules, addRules, removeRuleIds),
+    ]);
+    if ( addRules.length === 0 && removeRuleIds.length === 0 ) { return; }
+
+    const maxRegexRuleCount = dnr.MAX_NUMBER_OF_REGEX_RULES;
+    let regexRuleCount = 0;
+    let ruleId = 1;
+    for ( const rule of addRules ) {
+        if ( rule?.condition.regexFilter ) { regexRuleCount += 1; }
+        if ( (rule.id || 0) >= TRUSTED_DIRECTIVE_BASE_RULE_ID ) { continue; }
+        rule.id = ruleId++;
+    }
+    if ( regexRuleCount !== 0 ) {
+        ubolLog(`Using ${regexRuleCount}/${maxRegexRuleCount} dynamic regex-based DNR rules`);
+    }
+    return Promise.all([
+        dnr.updateDynamicRules({ addRules, removeRuleIds }).then(( ) => {
+            if ( removeRuleIds.length !== 0 ) {
+                ubolLog(`Remove ${removeRuleIds.length} dynamic DNR rules`);
+            }
+            if ( addRules.length !== 0 ) {
+                ubolLog(`Add ${addRules.length} dynamic DNR rules`);
+            }
+        }).catch(reason => {
+            console.error(`updateDynamicRules() / ${reason}`);
+        }),
+        updateSessionRules(),
+    ]);
+}
+
+/******************************************************************************/
+
+async function updateStrictBlockRules(currentRules, addRules, removeRuleIds) {
+    // Remove existing strictblock-related rules
+    for ( const rule of currentRules ) {
+        if ( isStrictBlockRule(rule) === false ) { continue; }
+        removeRuleIds.push(rule.id);
+    }
+
     if ( rulesetConfig.strictBlockMode === false ) { return; }
 
     const [
@@ -269,107 +354,49 @@ async function updateStrictBlockRules(dynamicRules, sessionRules) {
         sessionRead('excludedStrictBlockHostnames'),
     ]);
 
-    // Fetch strick-block hostnames
+    // Strict-block rules can only be enforced with omnipotence
+    if ( hasOmnipotence === false ) {
+        localRemove('excludedStrictBlockHostnames');
+        sessionRemove('excludedStrictBlockHostnames');
+        return;
+    }
+
+    // Fetch strick-block rules
     const toFetch = [];
     for ( const details of rulesetDetails ) {
         if ( details.rules.strictblock === 0 ) { continue; }
         toFetch.push(fetchJSON(`/rulesets/strictblock/${details.id}`));
     }
-    const strictblockRulesets = await Promise.all(toFetch);
+    const rulesets = await Promise.all(toFetch);
 
-    // Strict-block rules can only be enforced with omnipotence
-    let toStrictBlock = new Set();
-    if ( hasOmnipotence ) {
-        for ( const hostnames of strictblockRulesets ) {
-            if ( Array.isArray(hostnames) === false ) { continue; }
-            toStrictBlock = toStrictBlock.union(new Set(hostnames));
-        }
-    } else {
-        if ( permanentlyExcluded.length !== 0 ) {
-            localRemove('excludedStrictBlockHostnames');
-            permanentlyExcluded.length = 0;
-        }
-        if ( temporarilyExcluded.length !== 0 ) {
-            sessionRemove('excludedStrictBlockHostnames');
-            temporarilyExcluded.length = 0;
+    const substitution = `${runtime.getURL('/strictblock.html')}#\\0`;
+    const allRules = [];
+    for ( const rules of rulesets ) {
+        if ( Array.isArray(rules) === false ) { continue; }
+        for ( const rule of rules ) {
+            rule.action.redirect.regexSubstitution = substitution;
+            allRules.push(rule);
         }
     }
-    for ( const hn of permanentlyExcluded ) {
-        toStrictBlock.delete(hn);
+
+    const validRules = await pruneInvalidRegexRules('strictblock', allRules);
+    if ( validRules.length === 0 ) { return; }
+    ubolLog(`Add ${validRules.length} DNR strictblock rules`);
+    for ( const rule of validRules ) {
+        addRules.push(rule);
     }
-    if ( toStrictBlock.size === 0 ) { return; }
-    const manifest = runtime.getManifest();
-    let strictblockPath = '';
-    for ( const war of manifest.web_accessible_resources ) {
-        if ( war.resources.length !== 1 ) { continue; }
-        if ( war.resources[0].startsWith('/strictblock.') === false ) { continue; }
-        strictblockPath = runtime.getURL(war.resources[0]);
-        break;
-    }
-    if ( strictblockPath === '' ) { return; }
-    const dynamicRule = {
-        id: STRICTBLOCK_BASE_RULE_ID,
-        action: {
-            type: 'redirect',
-            redirect: {
-                regexSubstitution: `${strictblockPath}#\\0`,
-            },
-        },
+
+    const allExcluded = permanentlyExcluded.concat(temporarilyExcluded);
+    if ( allExcluded.length === 0 ) { return; }
+    addRules.push({
+        action: { type: 'allow' },
         condition: {
-            regexFilter: '^https?://.+',
-            requestDomains: Array.from(toStrictBlock),
+            requestDomains: allExcluded,
             resourceTypes: [ 'main_frame' ],
         },
-        priority: 29,
-    };
-    if ( permanentlyExcluded.length !== 0 ) {
-        dynamicRule.condition.excludedRequestDomains = permanentlyExcluded;
-    }
-    dynamicRules.push(dynamicRule);
-    ubolLog(`Add 1 DNR dynamic rule with ${toStrictBlock.size} strictblock domains`);
-
-    if ( temporarilyExcluded.length === 0 ) { return; }
-    sessionRules.push({
-        id: STRICTBLOCK_BASE_RULE_ID,
-        action: {
-            type: 'allow',
-        },
-        condition: {
-            requestDomains: temporarilyExcluded,
-            resourceTypes: [ 'main_frame' ],
-        },
-        priority: 29,
+        priority: STRICTBLOCK_PRIORITY,
     });
-    ubolLog(`Add 1 DNR session rule with ${temporarilyExcluded.length} excluded strictblock domains`);
-}
-
-async function commitStrictBlockRules() {
-    const [
-        beforePermanentRules,
-        beforeTemporaryRules,
-    ] = await Promise.all([
-        dnr.getDynamicRules({ ruleIds: [ STRICTBLOCK_BASE_RULE_ID ] }),
-        dnr.getSessionRules({ ruleIds: [ STRICTBLOCK_BASE_RULE_ID ] }),
-    ]);
-    if ( beforePermanentRules?.length ) {
-        ubolLog(`Remove 1 DNR dynamic strictblock rule`);
-    }
-    if ( beforeTemporaryRules?.length ) {
-        ubolLog(`Remove 1 DNR session strictblock rule`);
-    }
-    const afterPermanentRules = [];
-    const afterTemporaryRules = [];
-    await updateStrictBlockRules(afterPermanentRules, afterTemporaryRules)
-    return Promise.all([
-        dnr.updateDynamicRules({
-            addRules: afterPermanentRules,
-            removeRuleIds: beforePermanentRules.map(rule => rule.id),
-        }),
-        dnr.updateSessionRules({
-            addRules: afterTemporaryRules,
-            removeRuleIds: beforeTemporaryRules.map(rule => rule.id),
-        }),
-    ]);
+    ubolLog(`Add 1 DNR session rule with ${allExcluded.length} for excluded strict-block domains`);
 }
 
 async function excludeFromStrictBlock(hostname, permanent) {
@@ -379,7 +406,7 @@ async function excludeFromStrictBlock(hostname, permanent) {
     hostnames.add(hostname);
     const writeFn = permanent ? localWrite : sessionWrite;
     await writeFn('excludedStrictBlockHostnames', Array.from(hostnames));
-    return commitStrictBlockRules();
+    return updateSessionRules();
 }
 
 async function setStrictBlockMode(state) {
@@ -394,67 +421,37 @@ async function setStrictBlockMode(state) {
         );
     }
     await Promise.all(promises);
-    return commitStrictBlockRules();
+    return updateSessionRules();
 }
 
 /******************************************************************************/
 
-async function updateDynamicRules() {
-    dynamicRuleId = 1;
-    const dynamicRules = [];
-    const sessionRules = [];
-    const [
-        dynamicRuleIds,
-        sessionRuleIds,
-    ] = await Promise.all([
-        dnr.getDynamicRules().then(rules =>
-            rules.map(rule => rule.id)
-                .filter(id => id < TRUSTED_DIRECTIVE_BASE_RULE_ID)
-        ),
-        dnr.getSessionRules().then(rules => rules.map(rule => rule.id)),
-        updateRegexRules(dynamicRules),
-        updateRemoveparamRules(dynamicRules),
-        updateRedirectRules(dynamicRules),
-        updateModifyHeadersRules(dynamicRules),
-        updateStrictBlockRules(dynamicRules, sessionRules),
-    ]);
-    if ( dynamicRules.length === 0 && dynamicRuleIds.length === 0 ) { return; }
-    const promises = [];
-    if ( dynamicRules.length !== 0 || dynamicRuleIds.length !== 0 ) {
-        promises.push(
-            dnr.updateDynamicRules({
-                addRules: dynamicRules,
-                removeRuleIds: dynamicRuleIds,
-            }).then(( ) => {
-                if ( dynamicRuleIds.length !== 0 ) {
-                    ubolLog(`Remove ${dynamicRuleIds.length} dynamic DNR rules`);
-                }
-                if ( dynamicRules.length !== 0 ) {
-                    ubolLog(`Add ${dynamicRules.length} dynamic DNR rules`);
-                }
-            }).catch(reason => {
-                console.error(`updateDynamicRules() / ${reason}`);
-            })
-        );
+async function updateSessionRules() {
+    const addRules = [];
+    const removeRuleIds = [];
+    const currentRules = await dnr.getSessionRules();
+    await updateStrictBlockRules(currentRules, addRules, removeRuleIds);
+    if ( addRules.length === 0 && removeRuleIds.length === 0 ) { return; }
+    const maxRegexRuleCount = dnr.MAX_NUMBER_OF_REGEX_RULES;
+    let regexRuleCount = 0;
+    let ruleId = 1;
+    for ( const rule of addRules ) {
+        if ( rule?.condition.regexFilter ) { regexRuleCount += 1; }
+        rule.id = ruleId++;
     }
-    if ( sessionRules.length !== 0 || sessionRuleIds.length !== 0 ) {
-        promises.push(
-            dnr.updateSessionRules({
-                addRules: sessionRules,
-                removeRuleIds: sessionRuleIds,
-            }).then(( ) => {
-                if ( sessionRuleIds.length !== 0 ) {
-                    ubolLog(`Remove ${sessionRuleIds.length} session DNR rules`);
-                }
-                if ( sessionRules.length !== 0 ) {
-                    ubolLog(`Add ${sessionRules.length} session DNR rules`);
-                }
-            }).catch(reason => {
-                console.error(`updateSessionRules() / ${reason}`);
-            })
-        );
+    if ( regexRuleCount !== 0 ) {
+        ubolLog(`Using ${regexRuleCount}/${maxRegexRuleCount} session regex-based DNR rules`);
     }
-    return Promise.all(promises);
+    return dnr.updateSessionRules({ addRules, removeRuleIds }).then(( ) => {
+        if ( removeRuleIds.length !== 0 ) {
+            ubolLog(`Remove ${removeRuleIds.length} session DNR rules`);
+        }
+        if ( addRules.length !== 0 ) {
+            ubolLog(`Add ${addRules.length} session DNR rules`);
+        }
+    }).catch(reason => {
+        console.error(`updateSessionRules() / ${reason}`);
+    });
 }
 
 /******************************************************************************/
@@ -747,4 +744,5 @@ export {
     patchDefaultRulesets,
     setStrictBlockMode,
     updateDynamicRules,
+    updateSessionRules,
 };

platform/mv3/extension/js/strictblock.js

@@ -49,21 +49,12 @@ function urlToFragment(raw) {
 
 /******************************************************************************/
 
-async function proceed() {
-    await sendMessage({
-        what: 'excludeFromStrictBlock',
-        hostname: toURL.hostname,
-        permanent: qs$('#disableWarning').checked,
-    });
-    window.location.replace(toURL.href);
-}
-
-/******************************************************************************/
-
 const toURL = new URL('about:blank');
+const toFinalURL = new URL('about:blank');
 
 try {
     toURL.href = self.location.hash.slice(1);
+    toFinalURL.href = toURL.href;
 } catch(_) {
 }
 
@@ -72,6 +63,25 @@ qs$('#theURL > p > span:first-of-type').append(urlToFragment(toURL.href));
 
 /******************************************************************************/
 
+async function proceed() {
+    const permanent = qs$('#disableWarning').checked;
+    // Do not exclude current hostname from strict-block ruleset if a urlskip
+    // directive to another site is in effect.
+    // TODO: what if the urlskip directive leads to a different subdomain on
+    //       same site?
+    if ( toFinalURL.hostname !== toURL.hostname && permanent !== true ) {
+        return window.location.replace(toFinalURL.href);
+    }
+    await sendMessage({
+        what: 'excludeFromStrictBlock',
+        hostname: toURL.hostname,
+        permanent,
+    });
+    window.location.replace(toURL.href);
+}
+
+/******************************************************************************/
+
 function fragmentFromTemplate(template, placeholder, text, details) {
     const fragment = new DocumentFragment();
     const pos = template.indexOf(placeholder);
@@ -165,15 +175,7 @@ function fragmentFromTemplate(template, placeholder, text, details) {
 
     dom.on('#toggleParse', 'click', ( ) => {
         dom.cl.toggle('#theURL', 'collapsed');
-        //vAPI.localStorage.setItem(
-        //    'document-blocked-expand-url',
-        //    (dom.cl.has('#theURL', 'collapsed') === false).toString()
-        //);
     });
-
-    //vAPI.localStorage.getItemAsync('document-blocked-expand-url').then(value => {
-    //    dom.cl.toggle('#theURL', 'collapsed', value !== 'true' && value !== true);
-    //});
 })();
 
 /******************************************************************************/
@@ -184,16 +186,30 @@ function fragmentFromTemplate(template, placeholder, text, details) {
     let iList = -1;
     const searchInList = async i => {
         if ( iList !== -1 ) { return; }
-        const hostnames = new Set(
-            await fetchJSON(`/rulesets/strictblock/${rulesetDetails[i].id}`)
-        );
+        const rules = await fetchJSON(`/rulesets/strictblock/${rulesetDetails[i].id}`);
         if ( iList !== -1 ) { return; }
-        let hn = toURL.hostname;
-        for (;;) {
-            if ( hostnames.has(hn) ) { iList = i; break; }
-            const pos = hn.indexOf('.');
-            if ( pos === -1 ) { break; }
-            hn = hn.slice(pos+1);
+        const toHref = toURL.href;
+        for ( const rule of rules ) {
+            const { regexFilter, requestDomains } = rule.condition;
+            let matchesDomain = requestDomains === undefined;
+            if ( requestDomains ) {
+                let hn = toURL.hostname;
+                for (;;) {
+                    if ( requestDomains.includes(hn) ) {
+                        matchesDomain = true;
+                        break;
+                    }
+                    const pos = hn.indexOf('.');
+                    if ( pos === -1 ) { break; }
+                    hn = hn.slice(pos+1);
+                }
+                if ( matchesDomain === false ) { continue; }
+            }
+            const re = new RegExp(regexFilter);
+            const matchesRegex = re.test(toHref);
+            if ( matchesDomain && matchesRegex ) {
+                iList = i;
+            }
         }
     };
     const toFetch = [];
@@ -225,12 +241,24 @@ function fragmentFromTemplate(template, placeholder, text, details) {
     }
     if ( toFetch.length === 0 ) { return; }
     const urlskipLists = await Promise.all(toFetch);
+    const toHn = toURL.hostname;
+    const matchesHn = hn => {
+        if ( hn.endsWith(toHn) === false ) { return false; }
+        if ( hn.length === toHn.length ) { return true; }
+        return toHn.charAt(toHn.length - hn.length - 1) === '.';
+    };
     for ( const urlskips of urlskipLists ) {
         for ( const urlskip of urlskips ) {
             const re = new RegExp(urlskip.re, urlskip.c ? undefined : 'i');
             if ( re.test(toURL.href) === false ) { continue; }
+            if ( urlskip.hostnames ) {
+                if ( urlskip.hostnames.some(hn => matchesHn(hn)) === false ) {
+                    continue;
+                }
+            }
             const finalURL = urlSkip(toURL.href, false, urlskip.steps);
             if ( finalURL === undefined ) { continue; }
+            toFinalURL.href = finalURL;
             const fragment = fragmentFromTemplate(
                 i18n$('strictblockRedirectSentence1'),
                 '{{url}}', urlToFragment(finalURL),

platform/mv3/firefox/manifest.json

@@ -50,5 +50,15 @@
     "storage"
   ],
   "short_name": "uBO Lite",
-  "version": "1.0"
+  "version": "1.0",
+  "web_accessible_resources": [
+    {
+      "resources": [
+        "/strictblock.html"
+      ],
+      "matches": [
+        "<all_urls>"
+      ]
+    }
+  ]
 }

platform/mv3/make-rulesets.js

@@ -22,9 +22,15 @@
 import * as makeScriptlet from './make-scriptlets.js';
 import * as sfp from './js/static-filtering-parser.js';
 
-import { createHash, randomBytes } from 'crypto';
+import {
+    createHash,
+    randomBytes,
+} from 'crypto';
+import {
+    dnrRulesetFromRawLists,
+    mergeRules,
+} from './js/static-dnr-filtering.js';
 
-import { dnrRulesetFromRawLists } from './js/static-dnr-filtering.js';
 import fs from 'fs/promises';
 import https from 'https';
 import path from 'path';
@@ -158,6 +164,52 @@ const scriptletStats = new Map();
 const genericDetails = new Map();
 const requiredRedirectResources = new Set();
 
+// This will be used to sign our inserted `!#trusted on` directives
+const secret = createHash('sha256').update(randomBytes(16)).digest('hex').slice(0,16);
+log(`Secret: ${secret}`);
+
+/******************************************************************************/
+
+const restrSeparator = '(?:[^%.0-9a-z_-]|$)';
+
+const rePatternFromUrlFilter = s => {
+    let anchor = 0b000;
+    if ( s.startsWith('||') ) {
+        anchor = 0b100;
+        s = s.slice(2);
+    } else if ( s.startsWith('|') ) {
+        anchor = 0b010;
+        s = s.slice(1);
+    }
+    if ( s.endsWith('|') ) {
+        anchor |= 0b001;
+        s = s.slice(0, -1);
+    }
+    let reStr = s.replace(rePatternFromUrlFilter.rePlainChars, '\\$&')
+                 .replace(rePatternFromUrlFilter.reSeparators, restrSeparator)
+                 .replace(rePatternFromUrlFilter.reDanglingAsterisks, '')
+                 .replace(rePatternFromUrlFilter.reAsterisks, '\\S*?');
+    if ( anchor & 0b100 ) {
+        reStr = (
+            reStr.startsWith('\\.') ?
+                rePatternFromUrlFilter.restrHostnameAnchor2 :
+                rePatternFromUrlFilter.restrHostnameAnchor1
+        ) + reStr;
+    } else if ( anchor & 0b010 ) {
+        reStr = '^' + reStr;
+    }
+    if ( anchor & 0b001 ) {
+        reStr += '$';
+    }
+    return reStr;
+};
+rePatternFromUrlFilter.rePlainChars = /[.+?${}()|[\]\\]/g;
+rePatternFromUrlFilter.reSeparators = /\^/g;
+rePatternFromUrlFilter.reDanglingAsterisks = /^\*+|\*+$/g;
+rePatternFromUrlFilter.reAsterisks = /\*+/g;
+rePatternFromUrlFilter.restrHostnameAnchor1 = '^[a-z-]+://(?:[^/?#]+\\.)?';
+rePatternFromUrlFilter.restrHostnameAnchor2 = '^[a-z-]+://(?:[^/?#]+)?';
+
 /******************************************************************************/
 
 async function fetchList(assetDetails) {
@@ -179,7 +231,7 @@ async function fetchList(assetDetails) {
             }
             fetchedURLs.add(part.url);
             if ( part.url.startsWith('https://ublockorigin.github.io/uAssets/filters/') ) {
-                newParts.push(`!#trusted on ${assetDetails.secret}`);
+                newParts.push(`!#trusted on ${secret}`);
             }
             newParts.push(
                 fetchText(part.url, cacheDir).then(details => {
@@ -197,7 +249,7 @@ async function fetchList(assetDetails) {
                     return { url, content: '' };
                 })
             );
-            newParts.push(`!#trusted off ${assetDetails.secret}`);
+            newParts.push(`!#trusted off ${secret}`);
         }
         parts = await Promise.all(newParts);
         parts = sfp.utils.preparser.expandIncludes(parts, env);
@@ -330,6 +382,68 @@ function toJSONRuleset(ruleset) {
 
 /******************************************************************************/
 
+function toStrictBlockRule(rule, out) {
+    if ( rule.action.type !== 'block' ) { return; }
+    const { condition } = rule;
+    if ( condition === undefined ) { return; }
+    if ( condition.domainType ) { return; }
+    if ( condition.excludedResourceTypes ) { return; }
+    if ( condition.requestMethods ) { return; }
+    if ( condition.excludedRequestMethods ) { return; }
+    if ( condition.responseHeaders ) { return; }
+    if ( condition.excludedResponseHeaders ) { return; }
+    if ( condition.initiatorDomains ) { return; }
+    if ( condition.excludedInitiatorDomains ) { return; }
+    if ( condition.excludedRequestDomains ) { return; }
+    const { resourceTypes } = condition;
+    if ( resourceTypes === undefined ) {
+        if ( condition.requestDomains === undefined ) { return; }
+    } else {
+        if ( resourceTypes.length !== 1 ) { return; }
+        if ( resourceTypes[0] !== 'main_frame' ) { return; }
+    }
+    let regexFilter;
+    if ( condition.urlFilter ) {
+        regexFilter = rePatternFromUrlFilter(condition.urlFilter);
+    } else if ( condition.regexFilter ) {
+        regexFilter = condition.regexFilter;
+    } else {
+        regexFilter = '^https?://.*';
+    }
+    if (
+        regexFilter.startsWith('^') === false
+    ) {
+        regexFilter = `^.*${regexFilter}`;
+    }
+    if (
+        regexFilter.endsWith('$') === false &&
+        regexFilter.endsWith('.*') === false &&
+        regexFilter.endsWith('.+') === false
+    ) {
+        regexFilter = `${regexFilter}.*`;
+    }
+    const strictBlockRule = {
+        action: {
+            type: 'redirect',
+            redirect: {
+                regexSubstitution: `/strictblock.html#\\0`,
+            },
+        },
+        condition: {
+            regexFilter,
+            resourceTypes: [ 'main_frame' ],
+        },
+        priority: 29,
+    };
+    if ( condition.requestDomains ) {
+        strictBlockRule.condition.requestDomains = condition.requestDomains.slice();
+    }
+    out.set(toStrictBlockRule.ruleId++, strictBlockRule);
+}
+toStrictBlockRule.ruleId = 1;
+
+/******************************************************************************/
+
 async function processNetworkFilters(assetDetails, network) {
     const { ruleset: rules } = network;
     log(`Input filter count: ${network.filterCount}`);
@@ -396,21 +510,46 @@ async function processNetworkFilters(assetDetails, network) {
     );
     log(`\tmodifyHeaders=: ${modifyHeaders.length}`);
 
-    const urlskips = rules.filter(rule => isURLSkip(rule)).filter(rule =>
-        rule.__modifierAction === 0 &&
-        rule.condition &&
-        rule.condition.regexFilter &&
-        rule.condition.resourceTypes &&
-        rule.condition.resourceTypes.includes('main_frame')
-    ).map(rule => {
-        const steps = rule.__modifierValue;
-        return {
-            re: rule.condition.regexFilter,
-            c: rule.condition.isUrlFilterCaseSensitive,
-            steps: steps.includes(' ') && steps.split(/ +/) || [ steps ],
-        };
-    });
-    log(`\turlskip=: ${urlskips.length}`);
+    const urlskips = new Map();
+    for ( const rule of rules ) {
+        if ( isURLSkip(rule) === false ) { continue; }
+        if ( rule.__modifierAction !== 0 ) { continue; }
+        const { condition } = rule;
+        if ( condition.resourceTypes ) {
+            if ( condition.resourceTypes.includes('main_frame') === false ) {
+                continue;
+            }
+        }
+        const { urlFilter, regexFilter, requestDomains } = condition;
+        let re;
+        if ( urlFilter !== undefined ) {
+            re = rePatternFromUrlFilter(urlFilter);
+        } else if ( regexFilter !== undefined ) {
+            re = regexFilter;
+        } else {
+            re = '^';
+        }
+        const rawSteps = rule.__modifierValue;
+        const steps = rawSteps.includes(' ') && rawSteps.split(/ +/) || [ rawSteps ];
+        const keyEntry = {
+            re,
+            c: condition.isUrlFilterCaseSensitive,
+            steps,
+        }
+        const key = JSON.stringify(keyEntry);
+        let actualEntry = urlskips.get(key);
+        if ( actualEntry === undefined ) {
+            urlskips.set(key, keyEntry);
+            actualEntry = keyEntry;
+        }
+        if ( requestDomains !== undefined ) {
+            if ( actualEntry.hostnames === undefined ) {
+                actualEntry.hostnames = [];
+            }
+            actualEntry.hostnames.push(...requestDomains);
+        }
+    }
+    log(`\turlskip=: ${urlskips.size}`);
 
     const bad = rules.filter(rule =>
         isUnsupported(rule)
@@ -451,37 +590,26 @@ async function processNetworkFilters(assetDetails, network) {
         );
     }
 
-    const strictBlocked = new Set();
+    const strictBlocked = new Map();
     for ( const rule of plainGood ) {
-        if ( rule.action.type !== 'block' ) { continue; }
-        if ( rule.condition.domainType ) { continue; }
-        if ( rule.condition.regexFilter ) { continue; }
-        if ( rule.condition.urlFilter ) { continue; }
-        if ( rule.condition.requestMethods ) { continue; }
-        if ( rule.condition.excludedRequestMethods ) { continue; }
-        if ( rule.condition.resourceTypes ) { continue; }
-        if ( rule.condition.excludedResourceTypes ) { continue; }
-        if ( rule.condition.responseHeaders ) { continue; }
-        if ( rule.condition.excludedResponseHeaders ) { continue; }
-        if ( rule.condition.initiatorDomains ) { continue; }
-        if ( rule.condition.excludedInitiatorDomains ) { continue; }
-        if ( rule.condition.requestDomains === undefined ) { continue; }
-        if ( rule.condition.excludedRequestDomains ) { continue; }
-        for ( const hn of rule.condition.requestDomains ) {
-            strictBlocked.add(hn);
-        }
+        toStrictBlockRule(rule, strictBlocked);
     }
     if ( strictBlocked.size !== 0 ) {
+        mergeRules(strictBlocked, 'requestDomains');
+        let id = 1;
+        for ( const rule of strictBlocked.values() ) {
+            rule.id = id++;
+        }
         writeFile(
             `${rulesetDir}/strictblock/${assetDetails.id}.json`,
-            toJSONRuleset(Array.from(strictBlocked))
+            toJSONRuleset(Array.from(strictBlocked.values()))
         );
     }
 
-    if ( urlskips.length !== 0 ) {
+    if ( urlskips.size !== 0 ) {
         writeFile(
             `${rulesetDir}/urlskip/${assetDetails.id}.json`,
-            JSON.stringify(urlskips, null, 1)
+            JSON.stringify(Array.from(urlskips.values()), null, 1)
         );
     }
 
@@ -495,7 +623,7 @@ async function processNetworkFilters(assetDetails, network) {
         redirect: redirects.length,
         modifyHeaders: modifyHeaders.length,
         strictblock: strictBlocked.size,
-        urlskip: urlskips.length,
+        urlskip: urlskips.size,
     };
 }
 
@@ -1018,16 +1146,25 @@ async function rulesetFromURLs(assetDetails) {
     log('============================');
     log(`Listset for '${assetDetails.id}':`);
 
-    if ( assetDetails.text === undefined ) {
+    if ( assetDetails.text === undefined && assetDetails.urls.length !== 0 ) {
         const text = await fetchList(assetDetails);
-        if ( text === '' ) { return; }
         assetDetails.text = text;
+    } else {
+        assetDetails.text = '';
     }
 
-    if ( Array.isArray(assetDetails.filters) ) {
-        assetDetails.text += '\n' + assetDetails.filters.join('\n');
+    if ( Array.isArray(assetDetails.filters) && assetDetails.filters.length ) {
+        const extra = [
+            `!#trusted on ${secret}`,
+            ...assetDetails.filters,
+            `!#trusted off ${secret}`,
+            assetDetails.text,
+        ];
+        assetDetails.text = extra.join('\n').trim();
     }
 
+    if ( assetDetails.text === '' ) { return; }
+
     const extensionPaths = [];
     for ( const [ fname, details ] of redirectResourcesMap ) {
         const path = `/web_accessible_resources/${fname}`;
@@ -1045,7 +1182,7 @@ async function rulesetFromURLs(assetDetails) {
 
     const results = await dnrRulesetFromRawLists(
         [ { name: assetDetails.id, text: assetDetails.text } ],
-        { env, extensionPaths, secret: assetDetails.secret }
+        { env, extensionPaths, secret }
     );
 
     const netStats = await processNetworkFilters(
@@ -1181,19 +1318,13 @@ async function main() {
         JSON.parse(text)
     );
 
-    // This will be used to sign our inserted `!#trusted on` directives
-    const secret = createHash('sha256').update(randomBytes(16)).digest('hex').slice(0,16);
-    log(`Secret: ${secret}`);
-
     // Assemble all default lists as the default ruleset
     await rulesetFromURLs({
         id: 'default',
         name: 'Ads, trackers, miners, and more' ,
         enabled: true,
-        secret,
         urls: [
             'https://ublockorigin.github.io/uAssets/filters/filters.min.txt',
-            'https://ublockorigin.github.io/uAssets/filters/badware.min.txt',
             'https://ublockorigin.github.io/uAssets/filters/privacy.min.txt',
             'https://ublockorigin.github.io/uAssets/filters/unbreak.min.txt',
             'https://ublockorigin.github.io/uAssets/filters/quick-fixes.min.txt',
@@ -1208,6 +1339,19 @@ async function main() {
         ],
     });
 
+    await rulesetFromURLs({
+        id: 'badware',
+        name: 'Badware risks' ,
+        group: 'default',
+        enabled: true,
+        urls: [
+            'https://ublockorigin.github.io/uAssets/filters/badware.min.txt',
+        ],
+        homeURL: 'https://github.com/uBlockOrigin/uAssets',
+        filters: [
+        ],
+    });
+
     // Handpicked rulesets from assets.json
     const handpicked = [
         'block-lan',
@@ -1235,7 +1379,6 @@ async function main() {
         name: 'EasyList/uBO – Cookie Notices',
         group: 'annoyances',
         enabled: false,
-        secret,
         urls: [
             'https://ublockorigin.github.io/uAssets/thirdparties/easylist-cookies.txt',
             'https://ublockorigin.github.io/uAssets/filters/annoyances-cookies.txt',
@@ -1247,7 +1390,6 @@ async function main() {
         name: 'EasyList/uBO – Overlay Notices',
         group: 'annoyances',
         enabled: false,
-        secret,
         urls: [
             'https://ublockorigin.github.io/uAssets/thirdparties/easylist-newsletters.txt',
             'https://ublockorigin.github.io/uAssets/filters/annoyances-others.txt',
@@ -1409,14 +1551,6 @@ async function main() {
     manifest.declarative_net_request = { rule_resources: ruleResources };
     // Patch web_accessible_resources key
     manifest.web_accessible_resources = manifest.web_accessible_resources || [];
-    // Strict-block-related resource
-    const strictblockDocument = `strictblock.${secret}.html`;
-    copyFile('./strictblock.html', `${outputDir}/${strictblockDocument}`);
-    manifest.web_accessible_resources.push({
-      resources: [ `/${strictblockDocument}` ],
-      matches: [ '<all_urls>' ],
-    });
-    // Secondary resources
     const web_accessible_resources = {
         resources: Array.from(requiredRedirectResources).map(path => `/${path}`),
         matches: [ '<all_urls>' ],

src/js/static-dnr-filtering.js

@@ -354,6 +354,94 @@ function addToDNR(context, list) {
 
 /******************************************************************************/
 
+// Merge rules where possible by merging arrays of a specific property.
+//
+// https://github.com/uBlockOrigin/uBOL-home/issues/10#issuecomment-1304822579
+//   Do not merge rules which have errors.
+
+function mergeRules(rulesetMap, mergeTarget) {
+    const sorter = (_, v) => {
+        if ( Array.isArray(v) ) {
+            return typeof v[0] === 'string' ? v.sort() : v;
+        }
+        if ( v instanceof Object ) {
+            const sorted = {};
+            for ( const kk of Object.keys(v).sort() ) {
+                sorted[kk] = v[kk];
+            }
+            return sorted;
+        }
+        return v;
+    };
+    const ruleHasher = (rule, target) => {
+        return JSON.stringify(rule, (k, v) => {
+            if ( k.startsWith('_') ) { return; }
+            if ( k === target ) { return; }
+            return sorter(k, v);
+        });
+    };
+    const extractTargetValue = (obj, target) => {
+        for ( const [ k, v ] of Object.entries(obj) ) {
+            if ( Array.isArray(v) && k === target ) { return v; }
+            if ( v instanceof Object ) {
+                const r = extractTargetValue(v, target);
+                if ( r !== undefined ) { return r; }
+            }
+        }
+    };
+    const extractTargetOwner = (obj, target) => {
+        for ( const [ k, v ] of Object.entries(obj) ) {
+            if ( Array.isArray(v) && k === target ) { return obj; }
+            if ( v instanceof Object ) {
+                const r = extractTargetOwner(v, target);
+                if ( r !== undefined ) { return r; }
+            }
+        }
+    };
+    const mergeMap = new Map();
+    for ( const [ id, rule ] of rulesetMap ) {
+        if ( rule._error !== undefined ) { continue; }
+        const hash = ruleHasher(rule, mergeTarget);
+        if ( mergeMap.has(hash) === false ) {
+            mergeMap.set(hash, []);
+        }
+        mergeMap.get(hash).push(id);
+    }
+    for ( const ids of mergeMap.values() ) {
+        if ( ids.length === 1 ) { continue; }
+        const leftHand = rulesetMap.get(ids[0]);
+        const leftHandSet = new Set(
+            extractTargetValue(leftHand, mergeTarget) || []
+        );
+        for ( let i = 1; i < ids.length; i++ ) {
+            const rightHandId = ids[i];
+            const rightHand = rulesetMap.get(rightHandId);
+            const rightHandArray =  extractTargetValue(rightHand, mergeTarget);
+            if ( rightHandArray !== undefined ) {
+                if ( leftHandSet.size !== 0 ) {
+                    for ( const item of rightHandArray ) {
+                        leftHandSet.add(item);
+                    }
+                }
+            } else {
+                leftHandSet.clear();
+            }
+            rulesetMap.delete(rightHandId);
+        }
+        const leftHandOwner = extractTargetOwner(leftHand, mergeTarget);
+        if ( leftHandSet.size > 1 ) {
+            //if ( leftHandOwner === undefined ) { debugger; }
+            leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort();
+        } else if ( leftHandSet.size === 0 ) {
+            if ( leftHandOwner !== undefined ) {
+                leftHandOwner[mergeTarget] = undefined;
+            }
+        }
+    }
+}
+
+/******************************************************************************/
+
 function finalizeRuleset(context, network) {
     const ruleset = network.ruleset;
 
@@ -365,90 +453,6 @@ function finalizeRuleset(context, network) {
             rulesetMap.set(ruleId++, rule);
         }
     }
-    // Merge rules where possible by merging arrays of a specific property.
-    //
-    // https://github.com/uBlockOrigin/uBOL-home/issues/10#issuecomment-1304822579
-    //   Do not merge rules which have errors.
-    const mergeRules = (rulesetMap, mergeTarget) => {
-        const mergeMap = new Map();
-        const sorter = (_, v) => {
-            if ( Array.isArray(v) ) {
-                return typeof v[0] === 'string' ? v.sort() : v;
-            }
-            if ( v instanceof Object ) {
-                const sorted = {};
-                for ( const kk of Object.keys(v).sort() ) {
-                    sorted[kk] = v[kk];
-                }
-                return sorted;
-            }
-            return v;
-        };
-        const ruleHasher = (rule, target) => {
-            return JSON.stringify(rule, (k, v) => {
-                if ( k.startsWith('_') ) { return; }
-                if ( k === target ) { return; }
-                return sorter(k, v);
-            });
-        };
-        const extractTargetValue = (obj, target) => {
-            for ( const [ k, v ] of Object.entries(obj) ) {
-                if ( Array.isArray(v) && k === target ) { return v; }
-                if ( v instanceof Object ) {
-                    const r = extractTargetValue(v, target);
-                    if ( r !== undefined ) { return r; }
-                }
-            }
-        };
-        const extractTargetOwner = (obj, target) => {
-            for ( const [ k, v ] of Object.entries(obj) ) {
-                if ( Array.isArray(v) && k === target ) { return obj; }
-                if ( v instanceof Object ) {
-                    const r = extractTargetOwner(v, target);
-                    if ( r !== undefined ) { return r; }
-                }
-            }
-        };
-        for ( const [ id, rule ] of rulesetMap ) {
-            if ( rule._error !== undefined ) { continue; }
-            const hash = ruleHasher(rule, mergeTarget);
-            if ( mergeMap.has(hash) === false ) {
-                mergeMap.set(hash, []);
-            }
-            mergeMap.get(hash).push(id);
-        }
-        for ( const ids of mergeMap.values() ) {
-            if ( ids.length === 1 ) { continue; }
-            const leftHand = rulesetMap.get(ids[0]);
-            const leftHandSet = new Set(
-                extractTargetValue(leftHand, mergeTarget) || []
-            );
-            for ( let i = 1; i < ids.length; i++ ) {
-                const rightHandId = ids[i];
-                const rightHand = rulesetMap.get(rightHandId);
-                const rightHandArray =  extractTargetValue(rightHand, mergeTarget);
-                if ( rightHandArray !== undefined ) {
-                    if ( leftHandSet.size !== 0 ) {
-                        for ( const item of rightHandArray ) {
-                            leftHandSet.add(item);
-                        }
-                    }
-                } else {
-                    leftHandSet.clear();
-                }
-                rulesetMap.delete(rightHandId);
-            }
-            const leftHandOwner = extractTargetOwner(leftHand, mergeTarget);
-            if ( leftHandSet.size > 1 ) {
-                //if ( leftHandOwner === undefined ) { debugger; }
-                leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort();
-            } else if ( leftHandSet.size === 0 ) {
-                if ( leftHandOwner !== undefined ) {
-                    leftHandOwner[mergeTarget] = undefined;
-                }
-            }
-        }
-    };
     mergeRules(rulesetMap, 'resourceTypes');
     mergeRules(rulesetMap, 'removeParams');
     mergeRules(rulesetMap, 'initiatorDomains');
@@ -508,4 +512,4 @@ async function dnrRulesetFromRawLists(lists, options = {}) {
 
 /******************************************************************************/
 
-export { dnrRulesetFromRawLists };
+export { dnrRulesetFromRawLists, mergeRules };