[mv3] Separate EasyList, EasyPrivacy, PGL lists from uBlock filters

Additionally, remember `badfilter` filters across conversion of
filter lists to DNR rulesets.

platform/mv3/extension/js/admin.js

@@ -87,11 +87,32 @@ const adminSettings = {
 /******************************************************************************/
 
 export async function getAdminRulesets() {
-    const adminList = await adminReadEx('rulesets');
+    const [
+        adminList,
+        rulesetDetails,
+    ] = await Promise.all([
+        adminReadEx('rulesets'),
+        getRulesetDetails(),
+    ]);
     const adminRulesets = new Set(Array.isArray(adminList) && adminList || []);
+    if ( adminRulesets.has('-default') ) {
+        adminRulesets.delete('-default');
+        for ( const ruleset of rulesetDetails.values() ) {
+            if ( ruleset.enabled !== true ) { continue; }
+            if ( adminRulesets.has(`+${ruleset.id}`) ) { continue; }
+            adminRulesets.add(`-${ruleset.id}`);
+        }
+    }
+    if ( adminRulesets.has('+default') ) {
+        adminRulesets.delete('+default');
+        for ( const ruleset of rulesetDetails.values() ) {
+            if ( ruleset.enabled !== true ) { continue; }
+            if ( adminRulesets.has(`-${ruleset.id}`) ) { continue; }
+            adminRulesets.add(`+${ruleset.id}`);
+        }
+    }
     if ( adminRulesets.has('-*') ) {
         adminRulesets.delete('-*');
-        const rulesetDetails = await getRulesetDetails();
         for ( const ruleset of rulesetDetails.values() ) {
             if ( ruleset.enabled ) { continue; }
             if ( adminRulesets.has(`+${ruleset.id}`) ) { continue; }

platform/mv3/extension/js/debug.js

@@ -27,10 +27,8 @@ import { browser } from './ext.js';
 const isModern = dnr.onRuleMatchedDebug instanceof Object;
 
 export const isSideloaded = (( ) => {
-    if ( isModern ) { return true; }
+    const { permissions } = browser.runtime.getManifest();
-    if ( typeof dnr.getMatchedRules === 'function' ) { return true; }
+    return permissions?.includes('declarativeNetRequestFeedback') ?? false;
-    const manifest = browser.runtime.getManifest();
-    return manifest.permissions?.includes('declarativeNetRequestFeedback') ?? false;
 })();
 
 /******************************************************************************/
@@ -119,6 +117,7 @@ export const getMatchedRules = (( ) => {
     }
 
     return async tabId => {
+        if ( typeof dnr.getMatchedRules !== 'function' ) { return []; }
         const matchedRules = await dnr.getMatchedRules({ tabId });
         if ( matchedRules instanceof Object === false ) { return []; }
         const out = [];

platform/mv3/extension/js/filter-lists.js

@@ -135,7 +135,7 @@ export function renderFilterLists(rulesetData) {
         if ( ruleset.homeURL ) {
             dom.attr(qs$(listEntry, 'a.support'), 'href', ruleset.homeURL);
         }
-        dom.cl.toggle(listEntry, 'isDefault', ruleset.id === 'default');
+        dom.cl.toggle(listEntry, 'isDefault', ruleset.enabled === true);
         const stats = rulesetStats(ruleset.id);
         if ( stats === undefined ) { return; }
         listEntry.title = listStatsTemplate
@@ -215,9 +215,13 @@ export function renderFilterLists(rulesetData) {
         [
             'default',
             rulesetDetails.filter(ruleset =>
-                ruleset.id === 'default' ||
                 ruleset.group === 'default'
             ),
+        ], [
+            'privacy',
+            rulesetDetails.filter(ruleset =>
+                ruleset.group === 'privacy'
+            ),
         ], [
             'malware',
             rulesetDetails.filter(ruleset =>
@@ -231,7 +235,6 @@ export function renderFilterLists(rulesetData) {
         ], [
             'misc',
             rulesetDetails.filter(ruleset =>
-                ruleset.id !== 'default' &&
                 ruleset.group === undefined &&
                 typeof ruleset.lang !== 'string' 
             ),

platform/mv3/extension/js/ruleset-manager.js

@@ -484,19 +484,16 @@ async function defaultRulesetsFromLanguage() {
         `\\b(${Array.from(langSet).join('|')})\\b`
     );
 
-    const manifest = runtime.getManifest();
-    const rulesets = manifest.declarative_net_request.rule_resources;
     const rulesetDetails = await getRulesetDetails();
     const out = [];
-    for ( const ruleset of rulesets ) {
+    for ( const ruleset of rulesetDetails.values() ) {
         const { id, enabled } = ruleset;
         if ( enabled ) {
             out.push(id);
             continue;
         }
-        const details = rulesetDetails.get(id);
+        if ( typeof ruleset.lang !== 'string' ) { continue; }
-        if ( typeof details.lang !== 'string' ) { continue; }
+        if ( reTargetLang.test(ruleset.lang) === false ) { continue; }
-        if ( reTargetLang.test(details.lang) === false ) { continue; }
         out.push(id);
     }
     return out;
@@ -508,15 +505,12 @@ async function patchDefaultRulesets() {
     const [
         oldDefaultIds = [],
         newDefaultIds,
+        staticRulesetIds,
     ] = await Promise.all([
         localRead('defaultRulesetIds'),
         defaultRulesetsFromLanguage(),
+        getStaticRulesets().then(r => r.map(a => a.id)),
     ]);
-
-    const manifest = runtime.getManifest();
-    const validIds = new Set(
-        manifest.declarative_net_request.rule_resources.map(r => r.id)
-    );
     const toAdd = [];
     const toRemove = [];
     for ( const id of newDefaultIds ) {
@@ -528,7 +522,7 @@ async function patchDefaultRulesets() {
         toRemove.push(id);
     }
     for ( const id of rulesetConfig.enabledRulesets ) {
-        if ( validIds.has(id) ) { continue; }
+        if ( staticRulesetIds.includes(id) ) { continue; }
         toRemove.push(id);
     }
     localWrite('defaultRulesetIds', newDefaultIds);
@@ -545,7 +539,11 @@ async function patchDefaultRulesets() {
 
 async function enableRulesets(ids) {
     const afterIds = new Set(ids);
-    const [ beforeIds, adminIds, rulesetDetails ] = await Promise.all([
+    const [
+        beforeIds,
+        adminIds,
+        rulesetDetails,
+    ] = await Promise.all([
         dnr.getEnabledRulesets().then(ids => new Set(ids)),
         getAdminRulesets(),
         getRulesetDetails(),
@@ -614,6 +612,13 @@ async function enableRulesets(ids) {
 
 /******************************************************************************/
 
+async function getStaticRulesets() {
+    const manifest = runtime.getManifest();
+    return manifest.declarative_net_request.rule_resources;
+}
+
+/******************************************************************************/
+
 async function getEnabledRulesetsDetails() {
     const [
         ids,
@@ -638,8 +643,8 @@ export {
     enableRulesets,
     excludeFromStrictBlock,
     filteringModesToDNR,
-    getRulesetDetails,
     getEnabledRulesetsDetails,
+    getRulesetDetails,
     patchDefaultRulesets,
     setStrictBlockMode,
     updateDynamicRules,

platform/mv3/make-rulesets.js

@@ -175,6 +175,7 @@ const rulesetDetails = [];
 const scriptletStats = new Map();
 const genericDetails = new Map();
 const requiredRedirectResources = new Set();
+let networkBad = new Set();
 
 // This will be used to sign our inserted `!#trusted on` directives
 const secret = createHash('sha256').update(randomBytes(16)).digest('hex').slice(0,16);
@@ -1272,8 +1273,9 @@ async function rulesetFromURLs(assetDetails) {
 
     const results = await dnrRulesetFromRawLists(
         [ { name: assetDetails.id, text: assetDetails.text } ],
-        { env, extensionPaths, secret }
+        { env, extensionPaths, secret, networkBad }
     );
+    networkBad = results.networkBad;
 
     // Release memory used by filter list content
     assetDetails.text = undefined;

platform/mv3/rulesets.json

@@ -1,24 +1,51 @@
 [
   {
-    "id": "default",
+    "id": "ublock-filters",
-    "name": "Ads, trackers, miners, and more",
+    "name": "uBlock filters – Ads, trackers, and more",
+    "group": "default",
     "enabled": true,
     "urls": [
+      "https://ublockorigin.github.io/uAssets/filters/quick-fixes.min.txt",
+      "https://ublockorigin.github.io/uAssets/filters/unbreak.min.txt",
       "https://ublockorigin.github.io/uAssets/filters/filters.min.txt",
       "https://ublockorigin.github.io/uAssets/filters/privacy.min.txt",
-      "https://ublockorigin.github.io/uAssets/filters/unbreak.min.txt",
+      "https://ublockorigin.github.io/uAssets/filters/ubol-filters.txt"
-      "https://ublockorigin.github.io/uAssets/filters/quick-fixes.min.txt",
-      "https://ublockorigin.github.io/uAssets/filters/ubol-filters.txt",
-      "https://ublockorigin.github.io/uAssets/thirdparties/easylist.txt",
-      "https://ublockorigin.github.io/uAssets/thirdparties/easyprivacy.txt",
-      "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext"
     ],
-    "dnrURL": "https://ublockorigin.github.io/uAssets/dnr/default.json",
     "homeURL": "https://github.com/uBlockOrigin/uAssets"
   },
   {
-    "id": "badware",
+    "id": "easylist",
-    "name": "Badware risks",
+    "name": "EasyList",
+    "group": "default",
+    "enabled": true,
+    "urls": [
+      "https://ublockorigin.github.io/uAssets/thirdparties/easylist.txt"
+    ],
+    "homeURL": "https://easylist.to/"
+  },
+  {
+    "id": "easyprivacy",
+    "name": "EasyPrivacy",
+    "group": "default",
+    "enabled": true,
+    "urls": [
+      "https://ublockorigin.github.io/uAssets/thirdparties/easyprivacy.txt"
+    ],
+    "homeURL": "https://easylist.to/"
+  },
+  {
+    "id": "pgl",
+    "name": "Peter Lowe – Ads, trackers, and more",
+    "group": "default",
+    "enabled": true,
+    "urls": [
+      "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext"
+    ],
+    "homeURL": "https://pgl.yoyo.org/adservers/"
+  },
+  {
+    "id": "ublock-badware",
+    "name": "uBlock filters – Badware risks",
     "group": "malware",
     "enabled": true,
     "urls": [
@@ -49,6 +76,7 @@
   {
     "id": "block-lan",
     "name": "Block Outsider Intrusion into LAN",
+    "group": "privacy",
     "enabled": false,
     "urls": [
       "https://ublockorigin.github.io/uAssets/filters/lan-block.txt"
@@ -67,6 +95,7 @@
   {
     "id": "adguard-spyware-url",
     "name": "AdGuard URL Tracking Protection",
+    "group": "privacy",
     "enabled": false,
     "urls": [
       "https://filters.adtidy.org/extension/ublock/filters/17.txt"

src/js/static-dnr-filtering.js

@@ -475,6 +475,7 @@ function finalizeRuleset(context, network) {
 
 async function dnrRulesetFromRawLists(lists, options = {}) {
     const context = Object.assign({}, options);
+    context.bad = options.networkBad;
     staticNetFilteringEngine.dnrFromCompiled('begin', context);
     context.extensionPaths = new Map(context.extensionPaths || []);
     const toLoad = [];
@@ -489,6 +490,7 @@ async function dnrRulesetFromRawLists(lists, options = {}) {
     await Promise.all(toLoad);
     const result = {
         network: staticNetFilteringEngine.dnrFromCompiled('end', context),
+        networkBad: context.bad,
         genericCosmeticFilters: context.genericCosmeticFilters,
         genericCosmeticExceptions: context.genericCosmeticExceptions,
         specificCosmetic: context.specificCosmeticFilters,

src/js/static-net-filtering.js

@@ -4324,7 +4324,7 @@ StaticNetFilteringEngine.prototype.dnrFromCompiled = function(op, context, ...ar
     if ( op === 'begin' ) {
         Object.assign(context, {
             good: new Set(),
-            bad: new Set(),
+            bad: new Set(context.bad),
             invalid: new Set(),
             filterCount: 0,
             acceptedFilterCount: 0,