mirror of
https://github.com/gorhill/uBlock.git
synced 2026-03-11 09:04:36 +00:00
46ea5519c1
Related issues: - https://github.com/uBlockOrigin/uBlock-issues/issues/3161 - https://github.com/uBlockOrigin/uBlock-issues/discussions/2895#discussioncomment-8504374 Two checkboxes have been added to the "My filters "pane: 1. A checkbox to wholly disable/enable "My filters". This is equivalent to the checkbox for "My filters" in "Filter lists" pane. 2. A checkbox to enable/disable the trustworthiness of the content of "My filters". Default to untrusted. Since toggling these checkboxes requires reloading all filter lists, their new state must be committed through the "Apply changes" button. Additionally: a "book" icon has been added to the top-right of the dashboard, which is a link to the wiki according to whichever pane is currently active.
1737 lines
60 KiB
JavaScript
1737 lines
60 KiB
JavaScript
/*******************************************************************************
|
|
|
|
uBlock Origin - a comprehensive, efficient content blocker
|
|
Copyright (C) 2014-present Raymond Hill
|
|
|
|
This program is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program. If not, see {http://www.gnu.org/licenses/}.
|
|
|
|
Home: https://github.com/gorhill/uBlock
|
|
*/
|
|
|
|
'use strict';
|
|
|
|
/******************************************************************************/
|
|
|
|
import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js';
|
|
import punycode from '../lib/punycode.js';
|
|
|
|
import io from './assets.js';
|
|
import { broadcast, filteringBehaviorChanged, onBroadcast } from './broadcast.js';
|
|
import cosmeticFilteringEngine from './cosmetic-filtering.js';
|
|
import logger from './logger.js';
|
|
import lz4Codec from './lz4.js';
|
|
import staticExtFilteringEngine from './static-ext-filtering.js';
|
|
import staticFilteringReverseLookup from './reverselookup.js';
|
|
import staticNetFilteringEngine from './static-net-filtering.js';
|
|
import µb from './background.js';
|
|
import { hostnameFromURI } from './uri-utils.js';
|
|
import { i18n, i18n$ } from './i18n.js';
|
|
import { redirectEngine } from './redirect-engine.js';
|
|
import { ubolog, ubologSet } from './console.js';
|
|
import * as sfp from './static-filtering-parser.js';
|
|
|
|
import {
|
|
permanentFirewall,
|
|
permanentSwitches,
|
|
permanentURLFiltering,
|
|
} from './filtering-engines.js';
|
|
|
|
import {
|
|
CompiledListReader,
|
|
CompiledListWriter,
|
|
} from './static-filtering-io.js';
|
|
|
|
import {
|
|
LineIterator,
|
|
orphanizeString,
|
|
} from './text-utils.js';
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.getBytesInUse = async function() {
|
|
const promises = [];
|
|
let bytesInUse;
|
|
|
|
// Not all platforms implement this method.
|
|
promises.push(
|
|
vAPI.storage.getBytesInUse instanceof Function
|
|
? vAPI.storage.getBytesInUse(null)
|
|
: undefined
|
|
);
|
|
|
|
if (
|
|
navigator.storage instanceof Object &&
|
|
navigator.storage.estimate instanceof Function
|
|
) {
|
|
promises.push(navigator.storage.estimate());
|
|
}
|
|
|
|
const results = await Promise.all(promises);
|
|
|
|
const processCount = count => {
|
|
if ( typeof count !== 'number' ) { return; }
|
|
if ( bytesInUse === undefined ) { bytesInUse = 0; }
|
|
bytesInUse += count;
|
|
return bytesInUse;
|
|
};
|
|
|
|
processCount(results[0]);
|
|
if ( results.length > 1 && results[1] instanceof Object ) {
|
|
processCount(results[1].usage);
|
|
}
|
|
µb.storageUsed = bytesInUse;
|
|
return bytesInUse;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
{
|
|
const requestStats = µb.requestStats;
|
|
let requestStatsDisabled = false;
|
|
|
|
µb.loadLocalSettings = async ( ) => {
|
|
requestStatsDisabled = µb.hiddenSettings.requestStatsDisabled;
|
|
if ( requestStatsDisabled ) { return; }
|
|
return Promise.all([
|
|
vAPI.sessionStorage.get('requestStats'),
|
|
vAPI.storage.get('requestStats'),
|
|
vAPI.storage.get([ 'blockedRequestCount', 'allowedRequestCount' ]),
|
|
]).then(([ a, b, c ]) => {
|
|
if ( a instanceof Object && a.requestStats ) { return a.requestStats; }
|
|
if ( b instanceof Object && b.requestStats ) { return b.requestStats; }
|
|
if ( c instanceof Object && Object.keys(c).length === 2 ) {
|
|
return {
|
|
blockedCount: c.blockedRequestCount,
|
|
allowedCount: c.allowedRequestCount,
|
|
};
|
|
}
|
|
return { blockedCount: 0, allowedCount: 0 };
|
|
}).then(({ blockedCount, allowedCount }) => {
|
|
requestStats.blockedCount += blockedCount;
|
|
requestStats.allowedCount += allowedCount;
|
|
});
|
|
};
|
|
|
|
const SAVE_DELAY_IN_MINUTES = 3.6;
|
|
const QUICK_SAVE_DELAY_IN_SECONDS = 23;
|
|
|
|
const stopTimers = ( ) => {
|
|
vAPI.alarms.clear('saveLocalSettings');
|
|
quickSaveTimer.off();
|
|
saveTimer.off();
|
|
};
|
|
|
|
const saveTimer = vAPI.defer.create(( ) => {
|
|
µb.saveLocalSettings();
|
|
});
|
|
|
|
const quickSaveTimer = vAPI.defer.create(( ) => {
|
|
if ( vAPI.sessionStorage.unavailable !== true ) {
|
|
vAPI.sessionStorage.set({ requestStats: requestStats });
|
|
}
|
|
if ( requestStatsDisabled ) { return; }
|
|
saveTimer.on({ min: SAVE_DELAY_IN_MINUTES });
|
|
vAPI.alarms.createIfNotPresent('saveLocalSettings', {
|
|
delayInMinutes: SAVE_DELAY_IN_MINUTES + 0.5
|
|
});
|
|
});
|
|
|
|
µb.incrementRequestStats = (blocked, allowed) => {
|
|
requestStats.blockedCount += blocked;
|
|
requestStats.allowedCount += allowed;
|
|
quickSaveTimer.on({ sec: QUICK_SAVE_DELAY_IN_SECONDS });
|
|
};
|
|
|
|
µb.saveLocalSettings = ( ) => {
|
|
stopTimers();
|
|
if ( requestStatsDisabled ) { return; }
|
|
return vAPI.storage.set({ requestStats: µb.requestStats });
|
|
};
|
|
|
|
onBroadcast(msg => {
|
|
if ( msg.what !== 'hiddenSettingsChanged' ) { return; }
|
|
const newState = µb.hiddenSettings.requestStatsDisabled;
|
|
if ( requestStatsDisabled === newState ) { return; }
|
|
requestStatsDisabled = newState;
|
|
if ( newState ) {
|
|
stopTimers();
|
|
µb.requestStats.blockedCount = µb.requestStats.allowedCount = 0;
|
|
} else {
|
|
µb.loadLocalSettings();
|
|
}
|
|
});
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.loadUserSettings = async function() {
|
|
const usDefault = this.userSettingsDefault;
|
|
|
|
const results = await Promise.all([
|
|
vAPI.storage.get(Object.assign(usDefault)),
|
|
vAPI.adminStorage.get('userSettings'),
|
|
]);
|
|
|
|
const usUser = results[0] instanceof Object && results[0] ||
|
|
Object.assign(usDefault);
|
|
|
|
if ( Array.isArray(results[1]) ) {
|
|
const adminSettings = results[1];
|
|
for ( const entry of adminSettings ) {
|
|
if ( entry.length < 1 ) { continue; }
|
|
const name = entry[0];
|
|
if ( usDefault.hasOwnProperty(name) === false ) { continue; }
|
|
const value = entry.length < 2
|
|
? usDefault[name]
|
|
: this.settingValueFromString(usDefault, name, entry[1]);
|
|
if ( value === undefined ) { continue; }
|
|
usUser[name] = usDefault[name] = value;
|
|
}
|
|
}
|
|
|
|
return usUser;
|
|
};
|
|
|
|
µb.saveUserSettings = function() {
|
|
// `externalLists` will be deprecated in some future, it is kept around
|
|
// for forward compatibility purpose, and should reflect the content of
|
|
// `importedLists`.
|
|
//
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/1803
|
|
// Do this before computing modified settings.
|
|
this.userSettings.externalLists =
|
|
this.userSettings.importedLists.join('\n');
|
|
|
|
const toSave = this.getModifiedSettings(
|
|
this.userSettings,
|
|
this.userSettingsDefault
|
|
);
|
|
|
|
const toRemove = [];
|
|
for ( const key in this.userSettings ) {
|
|
if ( this.userSettings.hasOwnProperty(key) === false ) { continue; }
|
|
if ( toSave.hasOwnProperty(key) ) { continue; }
|
|
toRemove.push(key);
|
|
}
|
|
if ( toRemove.length !== 0 ) {
|
|
vAPI.storage.remove(toRemove);
|
|
}
|
|
vAPI.storage.set(toSave);
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
// Admin hidden settings have precedence over user hidden settings.
|
|
|
|
µb.loadHiddenSettings = async function() {
|
|
const hsDefault = this.hiddenSettingsDefault;
|
|
const hsAdmin = this.hiddenSettingsAdmin;
|
|
const hsUser = this.hiddenSettings;
|
|
|
|
const results = await Promise.all([
|
|
vAPI.adminStorage.get([
|
|
'advancedSettings',
|
|
'disableDashboard',
|
|
'disabledPopupPanelParts',
|
|
]),
|
|
vAPI.storage.get('hiddenSettings'),
|
|
]);
|
|
|
|
if ( results[0] instanceof Object ) {
|
|
const {
|
|
advancedSettings,
|
|
disableDashboard,
|
|
disabledPopupPanelParts
|
|
} = results[0];
|
|
if ( Array.isArray(advancedSettings) ) {
|
|
for ( const entry of advancedSettings ) {
|
|
if ( entry.length < 1 ) { continue; }
|
|
const name = entry[0];
|
|
if ( hsDefault.hasOwnProperty(name) === false ) { continue; }
|
|
const value = entry.length < 2
|
|
? hsDefault[name]
|
|
: this.hiddenSettingValueFromString(name, entry[1]);
|
|
if ( value === undefined ) { continue; }
|
|
hsDefault[name] = hsAdmin[name] = hsUser[name] = value;
|
|
}
|
|
}
|
|
µb.noDashboard = disableDashboard === true;
|
|
if ( Array.isArray(disabledPopupPanelParts) ) {
|
|
const partNameToBit = new Map([
|
|
[ 'globalStats', 0b00010 ],
|
|
[ 'basicTools', 0b00100 ],
|
|
[ 'extraTools', 0b01000 ],
|
|
[ 'overviewPane', 0b10000 ],
|
|
]);
|
|
let bits = hsDefault.popupPanelDisabledSections;
|
|
for ( const part of disabledPopupPanelParts ) {
|
|
const bit = partNameToBit.get(part);
|
|
if ( bit === undefined ) { continue; }
|
|
bits |= bit;
|
|
}
|
|
hsDefault.popupPanelDisabledSections =
|
|
hsAdmin.popupPanelDisabledSections =
|
|
hsUser.popupPanelDisabledSections = bits;
|
|
}
|
|
}
|
|
|
|
const hs = results[1] instanceof Object && results[1].hiddenSettings || {};
|
|
if ( Object.keys(hsAdmin).length === 0 && Object.keys(hs).length === 0 ) {
|
|
return;
|
|
}
|
|
|
|
for ( const key in hsDefault ) {
|
|
if ( hsDefault.hasOwnProperty(key) === false ) { continue; }
|
|
if ( hsAdmin.hasOwnProperty(name) ) { continue; }
|
|
if ( typeof hs[key] !== typeof hsDefault[key] ) { continue; }
|
|
this.hiddenSettings[key] = hs[key];
|
|
}
|
|
broadcast({ what: 'hiddenSettingsChanged' });
|
|
};
|
|
|
|
// Note: Save only the settings which values differ from the default ones.
|
|
// This way the new default values in the future will properly apply for
|
|
// those which were not modified by the user.
|
|
|
|
µb.saveHiddenSettings = function() {
|
|
vAPI.storage.set({
|
|
hiddenSettings: this.getModifiedSettings(
|
|
this.hiddenSettings,
|
|
this.hiddenSettingsDefault
|
|
)
|
|
});
|
|
};
|
|
|
|
onBroadcast(msg => {
|
|
if ( msg.what !== 'hiddenSettingsChanged' ) { return; }
|
|
const µbhs = µb.hiddenSettings;
|
|
ubologSet(µbhs.consoleLogLevel === 'info');
|
|
vAPI.net.setOptions({
|
|
cnameIgnoreList: µbhs.cnameIgnoreList,
|
|
cnameIgnore1stParty: µbhs.cnameIgnore1stParty,
|
|
cnameIgnoreExceptions: µbhs.cnameIgnoreExceptions,
|
|
cnameIgnoreRootDocument: µbhs.cnameIgnoreRootDocument,
|
|
cnameMaxTTL: µbhs.cnameMaxTTL,
|
|
cnameReplayFullURL: µbhs.cnameReplayFullURL,
|
|
cnameUncloakProxied: µbhs.cnameUncloakProxied,
|
|
});
|
|
});
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.hiddenSettingsFromString = function(raw) {
|
|
const out = Object.assign({}, this.hiddenSettingsDefault);
|
|
const lineIter = new LineIterator(raw);
|
|
while ( lineIter.eot() === false ) {
|
|
const line = lineIter.next();
|
|
const matches = /^\s*(\S+)\s+(.+)$/.exec(line);
|
|
if ( matches === null || matches.length !== 3 ) { continue; }
|
|
const name = matches[1];
|
|
if ( out.hasOwnProperty(name) === false ) { continue; }
|
|
if ( this.hiddenSettingsAdmin.hasOwnProperty(name) ) { continue; }
|
|
const value = this.hiddenSettingValueFromString(name, matches[2]);
|
|
if ( value !== undefined ) {
|
|
out[name] = value;
|
|
}
|
|
}
|
|
return out;
|
|
};
|
|
|
|
µb.hiddenSettingValueFromString = function(name, value) {
|
|
if ( typeof name !== 'string' || typeof value !== 'string' ) { return; }
|
|
const hsDefault = this.hiddenSettingsDefault;
|
|
if ( hsDefault.hasOwnProperty(name) === false ) { return; }
|
|
let r;
|
|
switch ( typeof hsDefault[name] ) {
|
|
case 'boolean':
|
|
if ( value === 'true' ) {
|
|
r = true;
|
|
} else if ( value === 'false' ) {
|
|
r = false;
|
|
}
|
|
break;
|
|
case 'string':
|
|
r = value.trim();
|
|
break;
|
|
case 'number':
|
|
if ( value.startsWith('0b') ) {
|
|
r = parseInt(value.slice(2), 2);
|
|
} else if ( value.startsWith('0x') ) {
|
|
r = parseInt(value.slice(2), 16);
|
|
} else {
|
|
r = parseInt(value, 10);
|
|
}
|
|
if ( isNaN(r) ) { r = undefined; }
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
return r;
|
|
};
|
|
|
|
µb.stringFromHiddenSettings = function() {
|
|
const out = [];
|
|
for ( const key of Object.keys(this.hiddenSettings).sort() ) {
|
|
out.push(key + ' ' + this.hiddenSettings[key]);
|
|
}
|
|
return out.join('\n');
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.savePermanentFirewallRules = function() {
|
|
vAPI.storage.set({
|
|
dynamicFilteringString: permanentFirewall.toString()
|
|
});
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.savePermanentURLFilteringRules = function() {
|
|
vAPI.storage.set({
|
|
urlFilteringString: permanentURLFiltering.toString()
|
|
});
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.saveHostnameSwitches = function() {
|
|
vAPI.storage.set({
|
|
hostnameSwitchesString: permanentSwitches.toString()
|
|
});
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.saveWhitelist = function() {
|
|
vAPI.storage.set({
|
|
netWhitelist: this.arrayFromWhitelist(this.netWhitelist)
|
|
});
|
|
this.netWhitelistModifyTime = Date.now();
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.isTrustedList = function(assetKey) {
|
|
if ( assetKey === this.userFiltersPath ) {
|
|
if ( this.userSettings.userFiltersTrusted ) { return true; }
|
|
}
|
|
if ( this.parsedTrustedListPrefixes.length === 0 ) {
|
|
this.parsedTrustedListPrefixes =
|
|
µb.hiddenSettings.trustedListPrefixes.split(/ +/).map(prefix => {
|
|
if ( prefix === '' ) { return; }
|
|
if ( prefix.startsWith('http://') ) { return; }
|
|
if ( prefix.startsWith('file:///') ) { return prefix; }
|
|
if ( prefix.startsWith('https://') === false ) {
|
|
return prefix.includes('://') ? undefined : prefix;
|
|
}
|
|
try {
|
|
const url = new URL(prefix);
|
|
if ( url.hostname.length > 0 ) { return url.href; }
|
|
} catch(_) {
|
|
}
|
|
}).filter(prefix => prefix !== undefined);
|
|
}
|
|
for ( const prefix of this.parsedTrustedListPrefixes ) {
|
|
if ( assetKey.startsWith(prefix) ) { return true; }
|
|
}
|
|
return false;
|
|
};
|
|
|
|
onBroadcast(msg => {
|
|
if ( msg.what !== 'hiddenSettingsChanged' ) { return; }
|
|
µb.parsedTrustedListPrefixes = [];
|
|
});
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.loadSelectedFilterLists = async function() {
|
|
const bin = await vAPI.storage.get('selectedFilterLists');
|
|
if ( bin instanceof Object && Array.isArray(bin.selectedFilterLists) ) {
|
|
this.selectedFilterLists = bin.selectedFilterLists;
|
|
return;
|
|
}
|
|
|
|
// https://github.com/gorhill/uBlock/issues/747
|
|
// Select default filter lists if first-time launch.
|
|
const lists = await io.metadata();
|
|
this.saveSelectedFilterLists(this.autoSelectRegionalFilterLists(lists));
|
|
};
|
|
|
|
µb.saveSelectedFilterLists = function(newKeys, append = false) {
|
|
const oldKeys = this.selectedFilterLists.slice();
|
|
if ( append ) {
|
|
newKeys = newKeys.concat(oldKeys);
|
|
}
|
|
const newSet = new Set(newKeys);
|
|
// Purge unused filter lists from cache.
|
|
for ( const oldKey of oldKeys ) {
|
|
if ( newSet.has(oldKey) === false ) {
|
|
this.removeFilterList(oldKey);
|
|
}
|
|
}
|
|
newKeys = Array.from(newSet);
|
|
this.selectedFilterLists = newKeys;
|
|
return vAPI.storage.set({ selectedFilterLists: newKeys });
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.applyFilterListSelection = function(details) {
|
|
let selectedListKeySet = new Set(this.selectedFilterLists);
|
|
let importedLists = this.userSettings.importedLists.slice();
|
|
|
|
// Filter lists to select
|
|
if ( Array.isArray(details.toSelect) ) {
|
|
if ( details.merge ) {
|
|
for ( let i = 0, n = details.toSelect.length; i < n; i++ ) {
|
|
selectedListKeySet.add(details.toSelect[i]);
|
|
}
|
|
} else {
|
|
selectedListKeySet = new Set(details.toSelect);
|
|
}
|
|
}
|
|
|
|
// Imported filter lists to remove
|
|
if ( Array.isArray(details.toRemove) ) {
|
|
for ( let i = 0, n = details.toRemove.length; i < n; i++ ) {
|
|
const assetKey = details.toRemove[i];
|
|
selectedListKeySet.delete(assetKey);
|
|
const pos = importedLists.indexOf(assetKey);
|
|
if ( pos !== -1 ) {
|
|
importedLists.splice(pos, 1);
|
|
}
|
|
this.removeFilterList(assetKey);
|
|
}
|
|
}
|
|
|
|
// Filter lists to import
|
|
if ( typeof details.toImport === 'string' ) {
|
|
// https://github.com/gorhill/uBlock/issues/1181
|
|
// Try mapping the URL of an imported filter list to the assetKey
|
|
// of an existing stock list.
|
|
const assetKeyFromURL = url => {
|
|
const needle = url.replace(/^https?:/, '');
|
|
const assets = this.availableFilterLists;
|
|
for ( const assetKey in assets ) {
|
|
const asset = assets[assetKey];
|
|
if ( asset.content !== 'filters' ) { continue; }
|
|
if ( typeof asset.contentURL === 'string' ) {
|
|
if ( asset.contentURL.endsWith(needle) ) { return assetKey; }
|
|
continue;
|
|
}
|
|
if ( Array.isArray(asset.contentURL) === false ) { continue; }
|
|
for ( let i = 0, n = asset.contentURL.length; i < n; i++ ) {
|
|
if ( asset.contentURL[i].endsWith(needle) ) {
|
|
return assetKey;
|
|
}
|
|
}
|
|
}
|
|
return url;
|
|
};
|
|
const importedSet = new Set(this.listKeysFromCustomFilterLists(importedLists));
|
|
const toImportSet = new Set(this.listKeysFromCustomFilterLists(details.toImport));
|
|
for ( const urlKey of toImportSet ) {
|
|
if ( importedSet.has(urlKey) ) {
|
|
selectedListKeySet.add(urlKey);
|
|
continue;
|
|
}
|
|
const assetKey = assetKeyFromURL(urlKey);
|
|
if ( assetKey === urlKey ) {
|
|
importedSet.add(urlKey);
|
|
}
|
|
selectedListKeySet.add(assetKey);
|
|
}
|
|
importedLists = Array.from(importedSet).sort();
|
|
}
|
|
|
|
const result = Array.from(selectedListKeySet);
|
|
if ( importedLists.join() !== this.userSettings.importedLists.join() ) {
|
|
this.userSettings.importedLists = importedLists;
|
|
this.saveUserSettings();
|
|
}
|
|
this.saveSelectedFilterLists(result);
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.listKeysFromCustomFilterLists = function(raw) {
|
|
const urls = typeof raw === 'string'
|
|
? raw.trim().split(/[\n\r]+/)
|
|
: raw;
|
|
const out = new Set();
|
|
const reIgnore = /^[!#]/;
|
|
const reValid = /^[a-z-]+:\/\/\S+/;
|
|
for ( const url of urls ) {
|
|
if ( reIgnore.test(url) || !reValid.test(url) ) { continue; }
|
|
// Ignore really bad lists.
|
|
if ( this.badLists.get(url) === true ) { continue; }
|
|
out.add(url);
|
|
}
|
|
return Array.from(out);
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.saveUserFilters = function(content) {
|
|
// https://github.com/gorhill/uBlock/issues/1022
|
|
// Be sure to end with an empty line.
|
|
content = content.trim();
|
|
if ( content !== '' ) { content += '\n'; }
|
|
this.removeCompiledFilterList(this.userFiltersPath);
|
|
return io.put(this.userFiltersPath, content);
|
|
};
|
|
|
|
µb.loadUserFilters = function() {
|
|
return io.get(this.userFiltersPath);
|
|
};
|
|
|
|
µb.appendUserFilters = async function(filters, options) {
|
|
filters = filters.trim();
|
|
if ( filters.length === 0 ) { return; }
|
|
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/372
|
|
// Auto comment using user-defined template.
|
|
let comment = '';
|
|
if (
|
|
options instanceof Object &&
|
|
options.autoComment === true &&
|
|
this.hiddenSettings.autoCommentFilterTemplate.indexOf('{{') !== -1
|
|
) {
|
|
const d = new Date();
|
|
// Date in YYYY-MM-DD format - https://stackoverflow.com/a/50130338
|
|
const ISO8601Date = new Date(d.getTime() +
|
|
(d.getTimezoneOffset()*60000)).toISOString().split('T')[0];
|
|
const url = new URL(options.docURL);
|
|
comment =
|
|
'! ' +
|
|
this.hiddenSettings.autoCommentFilterTemplate
|
|
.replace('{{date}}', ISO8601Date)
|
|
.replace('{{time}}', d.toLocaleTimeString())
|
|
.replace('{{hostname}}', url.hostname)
|
|
.replace('{{origin}}', url.origin)
|
|
.replace('{{url}}', url.href);
|
|
}
|
|
|
|
const details = await this.loadUserFilters();
|
|
if ( details.error ) { return; }
|
|
|
|
// The comment, if any, will be applied if and only if it is different
|
|
// from the last comment found in the user filter list.
|
|
if ( comment !== '' ) {
|
|
const beg = details.content.lastIndexOf(comment);
|
|
const end = beg === -1 ? -1 : beg + comment.length;
|
|
if (
|
|
end === -1 ||
|
|
details.content.startsWith('\n', end) === false ||
|
|
details.content.includes('\n!', end)
|
|
) {
|
|
filters = '\n' + comment + '\n' + filters;
|
|
}
|
|
}
|
|
|
|
// https://github.com/chrisaljoudi/uBlock/issues/976
|
|
// If we reached this point, the filter quite probably needs to be
|
|
// added for sure: do not try to be too smart, trying to avoid
|
|
// duplicates at this point may lead to more issues.
|
|
await this.saveUserFilters(details.content.trim() + '\n' + filters);
|
|
|
|
const compiledFilters = this.compileFilters(filters, {
|
|
assetKey: this.userFiltersPath,
|
|
trustedSource: true,
|
|
});
|
|
const snfe = staticNetFilteringEngine;
|
|
const cfe = cosmeticFilteringEngine;
|
|
const acceptedCount = snfe.acceptedCount + cfe.acceptedCount;
|
|
const discardedCount = snfe.discardedCount + cfe.discardedCount;
|
|
this.applyCompiledFilters(compiledFilters, true);
|
|
const entry = this.availableFilterLists[this.userFiltersPath];
|
|
const deltaEntryCount =
|
|
snfe.acceptedCount +
|
|
cfe.acceptedCount - acceptedCount;
|
|
const deltaEntryUsedCount =
|
|
deltaEntryCount -
|
|
(snfe.discardedCount + cfe.discardedCount - discardedCount);
|
|
entry.entryCount += deltaEntryCount;
|
|
entry.entryUsedCount += deltaEntryUsedCount;
|
|
vAPI.storage.set({ 'availableFilterLists': this.availableFilterLists });
|
|
staticNetFilteringEngine.freeze();
|
|
redirectEngine.freeze();
|
|
staticExtFilteringEngine.freeze();
|
|
this.selfieManager.destroy();
|
|
|
|
// https://www.reddit.com/r/uBlockOrigin/comments/cj7g7m/
|
|
// https://www.reddit.com/r/uBlockOrigin/comments/cnq0bi/
|
|
filteringBehaviorChanged();
|
|
broadcast({ what: 'userFiltersUpdated' });
|
|
};
|
|
|
|
µb.createUserFilters = function(details) {
|
|
this.appendUserFilters(details.filters, details);
|
|
// https://github.com/gorhill/uBlock/issues/1786
|
|
if ( details.docURL === undefined ) { return; }
|
|
cosmeticFilteringEngine.removeFromSelectorCache(
|
|
hostnameFromURI(details.docURL)
|
|
);
|
|
staticFilteringReverseLookup.resetLists();
|
|
};
|
|
|
|
µb.userFiltersAreEnabled = function() {
|
|
return this.selectedFilterLists.includes(this.userFiltersPath);
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.autoSelectRegionalFilterLists = function(lists) {
|
|
const selectedListKeys = [ this.userFiltersPath ];
|
|
for ( const key in lists ) {
|
|
if ( lists.hasOwnProperty(key) === false ) { continue; }
|
|
const list = lists[key];
|
|
if ( list.content !== 'filters' ) { continue; }
|
|
if ( list.off !== true ) {
|
|
selectedListKeys.push(key);
|
|
continue;
|
|
}
|
|
if ( this.listMatchesEnvironment(list) ) {
|
|
selectedListKeys.push(key);
|
|
list.off = false;
|
|
}
|
|
}
|
|
return selectedListKeys;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.hasInMemoryFilter = function(raw) {
|
|
return this.inMemoryFilters.includes(raw);
|
|
};
|
|
|
|
µb.addInMemoryFilter = async function(raw) {
|
|
if ( this.inMemoryFilters.includes(raw) ){ return true; }
|
|
this.inMemoryFilters.push(raw);
|
|
this.inMemoryFiltersCompiled = '';
|
|
await this.loadFilterLists();
|
|
return true;
|
|
};
|
|
|
|
µb.removeInMemoryFilter = async function(raw) {
|
|
const pos = this.inMemoryFilters.indexOf(raw);
|
|
if ( pos === -1 ) { return false; }
|
|
this.inMemoryFilters.splice(pos, 1);
|
|
this.inMemoryFiltersCompiled = '';
|
|
await this.loadFilterLists();
|
|
return false;
|
|
};
|
|
|
|
µb.clearInMemoryFilters = async function() {
|
|
if ( this.inMemoryFilters.length === 0 ) { return; }
|
|
this.inMemoryFilters = [];
|
|
this.inMemoryFiltersCompiled = '';
|
|
await this.loadFilterLists();
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.getAvailableLists = async function() {
|
|
const newAvailableLists = {};
|
|
|
|
// User filter list
|
|
newAvailableLists[this.userFiltersPath] = {
|
|
content: 'filters',
|
|
group: 'user',
|
|
title: i18n$('1pPageName'),
|
|
};
|
|
|
|
// Custom filter lists
|
|
const importedListKeys = new Set(
|
|
this.listKeysFromCustomFilterLists(this.userSettings.importedLists)
|
|
);
|
|
for ( const listKey of importedListKeys ) {
|
|
const asset = {
|
|
content: 'filters',
|
|
contentURL: listKey,
|
|
external: true,
|
|
group: 'custom',
|
|
submitter: 'user',
|
|
title: '',
|
|
};
|
|
newAvailableLists[listKey] = asset;
|
|
io.registerAssetSource(listKey, asset);
|
|
}
|
|
|
|
// Load previously saved available lists -- these contains data
|
|
// computed at run-time, we will reuse this data if possible
|
|
const [ bin, registeredAssets, badlists ] = await Promise.all([
|
|
Object.keys(this.availableFilterLists).length !== 0
|
|
? { availableFilterLists: this.availableFilterLists }
|
|
: vAPI.storage.get('availableFilterLists'),
|
|
io.metadata(),
|
|
this.badLists.size === 0 ? io.get('ublock-badlists') : false,
|
|
]);
|
|
|
|
if ( badlists instanceof Object ) {
|
|
for ( const line of badlists.content.split(/\s*[\n\r]+\s*/) ) {
|
|
if ( line === '' || line.startsWith('#') ) { continue; }
|
|
const fields = line.split(/\s+/);
|
|
const remove = fields.length === 2;
|
|
this.badLists.set(fields[0], remove);
|
|
}
|
|
}
|
|
|
|
const oldAvailableLists = bin && bin.availableFilterLists || {};
|
|
|
|
for ( const [ assetKey, asset ] of Object.entries(registeredAssets) ) {
|
|
if ( asset.content !== 'filters' ) { continue; }
|
|
newAvailableLists[assetKey] = Object.assign({}, asset);
|
|
}
|
|
|
|
// Load set of currently selected filter lists
|
|
const selectedListset = new Set(this.selectedFilterLists);
|
|
|
|
// Remove imported filter lists which are already present in stock lists
|
|
for ( const [ stockAssetKey, stockEntry ] of Object.entries(newAvailableLists) ) {
|
|
if ( stockEntry.content !== 'filters' ) { continue; }
|
|
if ( stockEntry.group === 'user' ) { continue; }
|
|
if ( stockEntry.submitter === 'user' ) { continue; }
|
|
if ( stockAssetKey.includes('://') ) { continue; }
|
|
const contentURLs = Array.isArray(stockEntry.contentURL)
|
|
? stockEntry.contentURL
|
|
: [ stockEntry.contentURL ];
|
|
for ( const importedAssetKey of contentURLs ) {
|
|
const importedEntry = newAvailableLists[importedAssetKey];
|
|
if ( importedEntry === undefined ) { continue; }
|
|
delete newAvailableLists[importedAssetKey];
|
|
io.unregisterAssetSource(importedAssetKey);
|
|
this.removeFilterList(importedAssetKey);
|
|
if ( selectedListset.has(importedAssetKey) ) {
|
|
selectedListset.add(stockAssetKey);
|
|
selectedListset.delete(importedAssetKey);
|
|
}
|
|
importedListKeys.delete(importedAssetKey);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Unregister lists in old listset not present in new listset.
|
|
// Convert a no longer existing stock list into an imported list, except
|
|
// when the removed stock list is deemed a "bad list".
|
|
for ( const [ assetKey, oldEntry ] of Object.entries(oldAvailableLists) ) {
|
|
if ( newAvailableLists[assetKey] !== undefined ) { continue; }
|
|
const on = selectedListset.delete(assetKey);
|
|
this.removeFilterList(assetKey);
|
|
io.unregisterAssetSource(assetKey);
|
|
if ( assetKey.includes('://') ) { continue; }
|
|
if ( on === false ) { continue; }
|
|
const listURL = Array.isArray(oldEntry.contentURL)
|
|
? oldEntry.contentURL[0]
|
|
: oldEntry.contentURL;
|
|
if ( this.badLists.has(listURL) ) { continue; }
|
|
const newEntry = {
|
|
content: 'filters',
|
|
contentURL: listURL,
|
|
external: true,
|
|
group: 'custom',
|
|
submitter: 'user',
|
|
title: oldEntry.title || ''
|
|
};
|
|
newAvailableLists[listURL] = newEntry;
|
|
io.registerAssetSource(listURL, newEntry);
|
|
importedListKeys.add(listURL);
|
|
selectedListset.add(listURL);
|
|
}
|
|
|
|
// Remove unreferenced imported filter lists
|
|
for ( const [ assetKey, asset ] of Object.entries(newAvailableLists) ) {
|
|
if ( asset.submitter !== 'user' ) { continue; }
|
|
if ( importedListKeys.has(assetKey) ) { continue; }
|
|
selectedListset.delete(assetKey);
|
|
delete newAvailableLists[assetKey];
|
|
this.removeFilterList(assetKey);
|
|
io.unregisterAssetSource(assetKey);
|
|
}
|
|
|
|
// Mark lists as disabled/enabled according to selected listset
|
|
for ( const [ assetKey, asset ] of Object.entries(newAvailableLists) ) {
|
|
asset.off = selectedListset.has(assetKey) === false;
|
|
}
|
|
|
|
// Reuse existing metadata
|
|
for ( const [ assetKey, oldEntry ] of Object.entries(oldAvailableLists) ) {
|
|
const newEntry = newAvailableLists[assetKey];
|
|
if ( newEntry === undefined ) { continue; }
|
|
if ( oldEntry.entryCount !== undefined ) {
|
|
newEntry.entryCount = oldEntry.entryCount;
|
|
}
|
|
if ( oldEntry.entryUsedCount !== undefined ) {
|
|
newEntry.entryUsedCount = oldEntry.entryUsedCount;
|
|
}
|
|
// This may happen if the list name was pulled from the list content
|
|
// https://github.com/chrisaljoudi/uBlock/issues/982
|
|
// There is no guarantee the title was successfully extracted from
|
|
// the list content
|
|
if (
|
|
newEntry.title === '' &&
|
|
typeof oldEntry.title === 'string' &&
|
|
oldEntry.title !== ''
|
|
) {
|
|
newEntry.title = oldEntry.title;
|
|
}
|
|
}
|
|
|
|
if ( Array.from(importedListKeys).join('\n') !== this.userSettings.importedLists.join('\n') ) {
|
|
this.userSettings.importedLists = Array.from(importedListKeys);
|
|
this.saveUserSettings();
|
|
}
|
|
|
|
if ( Array.from(selectedListset).join() !== this.selectedFilterLists.join() ) {
|
|
this.saveSelectedFilterLists(Array.from(selectedListset));
|
|
}
|
|
|
|
return newAvailableLists;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
{
|
|
const loadedListKeys = [];
|
|
let loadingPromise;
|
|
let t0 = 0;
|
|
|
|
const elapsed = ( ) => `${Date.now() - t0} ms`;
|
|
|
|
const onDone = ( ) => {
|
|
ubolog(`loadFilterLists() All filters in memory at ${elapsed()}`);
|
|
|
|
staticNetFilteringEngine.freeze();
|
|
staticExtFilteringEngine.freeze();
|
|
redirectEngine.freeze();
|
|
vAPI.net.unsuspend();
|
|
filteringBehaviorChanged();
|
|
|
|
ubolog(`loadFilterLists() All filters ready at ${elapsed()}`);
|
|
|
|
logger.writeOne({
|
|
realm: 'message',
|
|
type: 'info',
|
|
text: `Reloading all filter lists: done, took ${elapsed()}`
|
|
});
|
|
|
|
vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists });
|
|
|
|
broadcast({
|
|
what: 'staticFilteringDataChanged',
|
|
parseCosmeticFilters: µb.userSettings.parseAllABPHideFilters,
|
|
ignoreGenericCosmeticFilters: µb.userSettings.ignoreGenericCosmeticFilters,
|
|
listKeys: loadedListKeys
|
|
});
|
|
|
|
µb.selfieManager.destroy();
|
|
lz4Codec.relinquish();
|
|
µb.compiledFormatChanged = false;
|
|
|
|
loadingPromise = undefined;
|
|
};
|
|
|
|
const applyCompiledFilters = (assetKey, compiled) => {
|
|
ubolog(`loadFilterLists() Loading filters from ${assetKey} at ${elapsed()}`);
|
|
const snfe = staticNetFilteringEngine;
|
|
const sxfe = staticExtFilteringEngine;
|
|
let acceptedCount = snfe.acceptedCount + sxfe.acceptedCount;
|
|
let discardedCount = snfe.discardedCount + sxfe.discardedCount;
|
|
µb.applyCompiledFilters(compiled, assetKey === µb.userFiltersPath);
|
|
if ( µb.availableFilterLists.hasOwnProperty(assetKey) ) {
|
|
const entry = µb.availableFilterLists[assetKey];
|
|
entry.entryCount = snfe.acceptedCount + sxfe.acceptedCount -
|
|
acceptedCount;
|
|
entry.entryUsedCount = entry.entryCount -
|
|
(snfe.discardedCount + sxfe.discardedCount - discardedCount);
|
|
}
|
|
loadedListKeys.push(assetKey);
|
|
};
|
|
|
|
const onFilterListsReady = lists => {
|
|
logger.writeOne({
|
|
realm: 'message',
|
|
type: 'info',
|
|
text: 'Reloading all filter lists: start'
|
|
});
|
|
|
|
µb.availableFilterLists = lists;
|
|
|
|
if ( vAPI.Net.canSuspend() ) {
|
|
vAPI.net.suspend();
|
|
}
|
|
redirectEngine.reset();
|
|
staticExtFilteringEngine.reset();
|
|
staticNetFilteringEngine.reset();
|
|
µb.selfieManager.destroy();
|
|
staticFilteringReverseLookup.resetLists();
|
|
|
|
ubolog(`loadFilterLists() All filters removed at ${elapsed()}`);
|
|
|
|
// We need to build a complete list of assets to pull first: this is
|
|
// because it *may* happens that some load operations are synchronous:
|
|
// This happens for assets which do not exist, or assets with no
|
|
// content.
|
|
const toLoad = [];
|
|
for ( const assetKey in lists ) {
|
|
if ( lists.hasOwnProperty(assetKey) === false ) { continue; }
|
|
if ( lists[assetKey].off ) { continue; }
|
|
toLoad.push(
|
|
µb.getCompiledFilterList(assetKey).then(details => {
|
|
applyCompiledFilters(details.assetKey, details.content);
|
|
})
|
|
);
|
|
}
|
|
|
|
if ( µb.inMemoryFilters.length !== 0 ) {
|
|
if ( µb.inMemoryFiltersCompiled === '' ) {
|
|
µb.inMemoryFiltersCompiled =
|
|
µb.compileFilters(µb.inMemoryFilters.join('\n'), {
|
|
assetKey: 'in-memory',
|
|
trustedSource: true,
|
|
});
|
|
}
|
|
if ( µb.inMemoryFiltersCompiled !== '' ) {
|
|
toLoad.push(
|
|
µb.applyCompiledFilters(µb.inMemoryFiltersCompiled, true)
|
|
);
|
|
}
|
|
}
|
|
|
|
return Promise.all(toLoad);
|
|
};
|
|
|
|
µb.loadFilterLists = function() {
|
|
if ( loadingPromise instanceof Promise ) { return loadingPromise; }
|
|
ubolog('loadFilterLists() Start');
|
|
t0 = Date.now();
|
|
loadedListKeys.length = 0;
|
|
loadingPromise = Promise.all([
|
|
this.getAvailableLists().then(lists => onFilterListsReady(lists)),
|
|
this.loadRedirectResources().then(( ) => {
|
|
ubolog(`loadFilterLists() Redirects/scriptlets ready at ${elapsed()}`);
|
|
}),
|
|
]).then(( ) => {
|
|
onDone();
|
|
});
|
|
return loadingPromise;
|
|
};
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.getCompiledFilterList = async function(assetKey) {
|
|
const compiledPath = `compiled/${assetKey}`;
|
|
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/1365
|
|
// Verify that the list version matches that of the current compiled
|
|
// format.
|
|
if (
|
|
this.compiledFormatChanged === false &&
|
|
this.badLists.has(assetKey) === false
|
|
) {
|
|
const content = await io.fromCache(compiledPath);
|
|
const compilerVersion = `${this.systemSettings.compiledMagic}\n`;
|
|
if ( content.startsWith(compilerVersion) ) {
|
|
return { assetKey, content };
|
|
}
|
|
}
|
|
|
|
// Skip downloading really bad lists.
|
|
if ( this.badLists.get(assetKey) ) {
|
|
return { assetKey, content: '' };
|
|
}
|
|
|
|
const rawDetails = await io.get(assetKey, {
|
|
favorLocal: this.readyToFilter !== true,
|
|
silent: true,
|
|
});
|
|
// Compiling an empty string results in an empty string.
|
|
if ( rawDetails.content === '' ) {
|
|
rawDetails.assetKey = assetKey;
|
|
return rawDetails;
|
|
}
|
|
|
|
this.extractFilterListMetadata(assetKey, rawDetails.content);
|
|
|
|
// Skip compiling bad lists.
|
|
if ( this.badLists.has(assetKey) ) {
|
|
return { assetKey, content: '' };
|
|
}
|
|
|
|
const compiledContent = this.compileFilters(rawDetails.content, {
|
|
assetKey,
|
|
trustedSource: this.isTrustedList(assetKey),
|
|
});
|
|
io.toCache(compiledPath, compiledContent);
|
|
|
|
return { assetKey, content: compiledContent };
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.extractFilterListMetadata = function(assetKey, raw) {
|
|
const listEntry = this.availableFilterLists[assetKey];
|
|
if ( listEntry === undefined ) { return; }
|
|
// https://github.com/gorhill/uBlock/issues/313
|
|
// Always try to fetch the name if this is an external filter list.
|
|
if ( listEntry.group !== 'custom' ) { return; }
|
|
const data = io.extractMetadataFromList(raw, [ 'Title', 'Homepage' ]);
|
|
const props = {};
|
|
if ( data.title && data.title !== listEntry.title ) {
|
|
props.title = listEntry.title = orphanizeString(data.title);
|
|
}
|
|
if ( data.homepage && /^https?:\/\/\S+/.test(data.homepage) ) {
|
|
if ( data.homepage !== listEntry.supportURL ) {
|
|
props.supportURL = listEntry.supportURL = orphanizeString(data.homepage);
|
|
}
|
|
}
|
|
io.registerAssetSource(assetKey, props);
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.removeCompiledFilterList = function(assetKey) {
|
|
io.remove(`compiled/${assetKey}`);
|
|
};
|
|
|
|
µb.removeFilterList = function(assetKey) {
|
|
this.removeCompiledFilterList(assetKey);
|
|
io.remove(assetKey);
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.compileFilters = function(rawText, details = {}) {
|
|
const writer = new CompiledListWriter();
|
|
|
|
// Populate the writer with information potentially useful to the
|
|
// client compilers.
|
|
const trustedSource = details.trustedSource === true;
|
|
if ( details.assetKey ) {
|
|
writer.properties.set('name', details.assetKey);
|
|
writer.properties.set('trustedSource', trustedSource);
|
|
}
|
|
const assetName = details.assetKey ? details.assetKey : '?';
|
|
const parser = new sfp.AstFilterParser({
|
|
trustedSource,
|
|
maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH,
|
|
nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
|
|
});
|
|
const compiler = staticNetFilteringEngine.createCompiler(parser);
|
|
const lineIter = new LineIterator(
|
|
sfp.utils.preparser.prune(rawText, vAPI.webextFlavor.env)
|
|
);
|
|
|
|
compiler.start(writer);
|
|
|
|
while ( lineIter.eot() === false ) {
|
|
let line = lineIter.next();
|
|
|
|
while ( line.endsWith(' \\') ) {
|
|
if ( lineIter.peek(4) !== ' ' ) { break; }
|
|
line = line.slice(0, -2).trim() + lineIter.next().trim();
|
|
}
|
|
|
|
parser.parse(line);
|
|
|
|
if ( parser.isFilter() === false ) { continue; }
|
|
if ( parser.hasError() ) {
|
|
logger.writeOne({
|
|
realm: 'message',
|
|
type: 'error',
|
|
text: `Invalid filter (${assetName}): ${parser.raw}`
|
|
});
|
|
continue;
|
|
}
|
|
|
|
if ( parser.isExtendedFilter() ) {
|
|
staticExtFilteringEngine.compile(parser, writer);
|
|
continue;
|
|
}
|
|
|
|
if ( parser.isNetworkFilter() === false ) { continue; }
|
|
|
|
if ( compiler.compile(parser, writer) ) { continue; }
|
|
if ( compiler.error !== undefined ) {
|
|
logger.writeOne({
|
|
realm: 'message',
|
|
type: 'error',
|
|
text: compiler.error
|
|
});
|
|
}
|
|
}
|
|
|
|
compiler.finish(writer);
|
|
parser.finish();
|
|
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/1365
|
|
// Embed version into compiled list itself: it is encoded in as the
|
|
// first digits followed by a whitespace.
|
|
const compiledContent
|
|
= `${this.systemSettings.compiledMagic}\n` + writer.toString();
|
|
|
|
return compiledContent;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
// https://github.com/gorhill/uBlock/issues/1395
|
|
// Added `firstparty` argument: to avoid discarding cosmetic filters when
|
|
// applying 1st-party filters.
|
|
|
|
µb.applyCompiledFilters = function(rawText, firstparty) {
|
|
if ( rawText === '' ) { return; }
|
|
const reader = new CompiledListReader(rawText);
|
|
staticNetFilteringEngine.fromCompiled(reader);
|
|
staticExtFilteringEngine.fromCompiledContent(reader, {
|
|
skipGenericCosmetic: this.userSettings.ignoreGenericCosmeticFilters,
|
|
skipCosmetic: !firstparty && !this.userSettings.parseAllABPHideFilters
|
|
});
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.loadRedirectResources = async function() {
|
|
try {
|
|
const success = await redirectEngine.resourcesFromSelfie(io);
|
|
if ( success === true ) {
|
|
ubolog('Loaded redirect/scriptlets resources from selfie');
|
|
return true;
|
|
}
|
|
|
|
const fetcher = (path, options = undefined) => {
|
|
if ( path.startsWith('/web_accessible_resources/') ) {
|
|
path += `?secret=${vAPI.warSecret.short()}`;
|
|
return io.fetch(path, options);
|
|
}
|
|
return io.fetchText(path);
|
|
};
|
|
|
|
const fetchPromises = [
|
|
redirectEngine.loadBuiltinResources(fetcher)
|
|
];
|
|
|
|
const userResourcesLocation = this.hiddenSettings.userResourcesLocation;
|
|
if ( userResourcesLocation !== 'unset' ) {
|
|
for ( const url of userResourcesLocation.split(/\s+/) ) {
|
|
fetchPromises.push(io.fetchText(url));
|
|
}
|
|
}
|
|
|
|
const results = await Promise.all(fetchPromises);
|
|
if ( Array.isArray(results) === false ) { return results; }
|
|
|
|
const content = [];
|
|
for ( let i = 1; i < results.length; i++ ) {
|
|
const result = results[i];
|
|
if ( result instanceof Object === false ) { continue; }
|
|
if ( typeof result.content !== 'string' ) { continue; }
|
|
if ( result.content === '' ) { continue; }
|
|
content.push(result.content);
|
|
}
|
|
if ( content.length !== 0 ) {
|
|
redirectEngine.resourcesFromString(content.join('\n\n'));
|
|
}
|
|
redirectEngine.selfieFromResources(io);
|
|
} catch(ex) {
|
|
ubolog(ex);
|
|
return false;
|
|
}
|
|
return true;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.loadPublicSuffixList = async function() {
|
|
const psl = publicSuffixList;
|
|
|
|
// WASM is nice but not critical
|
|
if ( vAPI.canWASM && this.hiddenSettings.disableWebAssembly !== true ) {
|
|
const wasmModuleFetcher = function(path) {
|
|
return fetch( `${path}.wasm`, {
|
|
mode: 'same-origin'
|
|
}).then(
|
|
WebAssembly.compileStreaming
|
|
).catch(reason => {
|
|
ubolog(reason);
|
|
});
|
|
};
|
|
let result = false;
|
|
try {
|
|
result = await psl.enableWASM(wasmModuleFetcher,
|
|
'./lib/publicsuffixlist/wasm/'
|
|
);
|
|
} catch(reason) {
|
|
ubolog(reason);
|
|
}
|
|
if ( result ) {
|
|
ubolog(`WASM PSL ready ${Date.now()-vAPI.T0} ms after launch`);
|
|
}
|
|
}
|
|
|
|
try {
|
|
const selfie = await io.fromCache(`selfie/${this.pslAssetKey}`);
|
|
if ( psl.fromSelfie(selfie) ) {
|
|
ubolog('Loaded PSL from selfie');
|
|
return;
|
|
}
|
|
} catch (reason) {
|
|
ubolog(reason);
|
|
}
|
|
|
|
const result = await io.get(this.pslAssetKey);
|
|
if ( result.content !== '' ) {
|
|
this.compilePublicSuffixList(result.content);
|
|
}
|
|
};
|
|
|
|
µb.compilePublicSuffixList = function(content) {
|
|
const psl = publicSuffixList;
|
|
psl.parse(content, punycode.toASCII);
|
|
ubolog(`Loaded PSL from ${this.pslAssetKey}`);
|
|
return io.toCache(`selfie/${this.pslAssetKey}`, psl.toSelfie());
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
// This is to be sure the selfie is generated in a sane manner: the selfie will
|
|
// be generated if the user doesn't change his filter lists selection for
|
|
// some set time.
|
|
|
|
{
|
|
// As of 2018-05-31:
|
|
// JSON.stringify-ing ourselves results in a better baseline
|
|
// memory usage at selfie-load time. For some reasons.
|
|
|
|
const create = async function() {
|
|
vAPI.alarms.clear('createSelfie');
|
|
createTimer.off();
|
|
if ( µb.inMemoryFilters.length !== 0 ) { return; }
|
|
if ( Object.keys(µb.availableFilterLists).length === 0 ) { return; }
|
|
await Promise.all([
|
|
io.toCache('selfie/staticMain', {
|
|
magic: µb.systemSettings.selfieMagic,
|
|
availableFilterLists: µb.availableFilterLists,
|
|
}),
|
|
io.toCache('selfie/staticExtFilteringEngine',
|
|
staticExtFilteringEngine.toSelfie()
|
|
),
|
|
io.toCache('selfie/staticNetFilteringEngine',
|
|
staticNetFilteringEngine.toSelfie()
|
|
),
|
|
]);
|
|
lz4Codec.relinquish();
|
|
µb.selfieIsInvalid = false;
|
|
ubolog('Filtering engine selfie created');
|
|
};
|
|
|
|
const loadMain = async function() {
|
|
const selfie = await io.fromCache('selfie/staticMain');
|
|
if ( selfie instanceof Object === false ) { return false; }
|
|
if ( selfie.magic !== µb.systemSettings.selfieMagic ) { return false; }
|
|
if ( selfie.availableFilterLists instanceof Object === false ) { return false; }
|
|
if ( Object.keys(selfie.availableFilterLists).length === 0 ) { return false; }
|
|
µb.availableFilterLists = selfie.availableFilterLists;
|
|
return true;
|
|
};
|
|
|
|
const load = async function() {
|
|
if ( µb.selfieIsInvalid ) { return false; }
|
|
try {
|
|
const results = await Promise.all([
|
|
loadMain(),
|
|
io.fromCache('selfie/staticExtFilteringEngine').then(selfie =>
|
|
staticExtFilteringEngine.fromSelfie(selfie)
|
|
),
|
|
io.fromCache('selfie/staticNetFilteringEngine').then(selfie =>
|
|
staticNetFilteringEngine.fromSelfie(selfie)
|
|
),
|
|
]);
|
|
if ( results.every(v => v) ) {
|
|
return µb.loadRedirectResources();
|
|
}
|
|
}
|
|
catch (reason) {
|
|
ubolog(reason);
|
|
}
|
|
ubolog('Filtering engine selfie not available');
|
|
destroy();
|
|
return false;
|
|
};
|
|
|
|
const destroy = function(options = {}) {
|
|
if ( µb.selfieIsInvalid === false ) {
|
|
io.remove(/^selfie\/static/, options);
|
|
µb.selfieIsInvalid = true;
|
|
ubolog('Filtering engine selfie marked for invalidation');
|
|
}
|
|
vAPI.alarms.create('createSelfie', {
|
|
delayInMinutes: (µb.hiddenSettings.selfieDelayInSeconds + 17) / 60,
|
|
});
|
|
createTimer.offon({ sec: µb.hiddenSettings.selfieDelayInSeconds });
|
|
};
|
|
|
|
const createTimer = vAPI.defer.create(create);
|
|
|
|
µb.selfieManager = { load, create, destroy };
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
// https://github.com/gorhill/uBlock/issues/531
|
|
// Overwrite user settings with admin settings if present.
|
|
//
|
|
// Admin settings match layout of a uBlock backup. Not all data is
|
|
// necessarily present, i.e. administrators may removed entries which
|
|
// values are left to the user's choice.
|
|
|
|
µb.restoreAdminSettings = async function() {
|
|
let toOverwrite = {};
|
|
let data;
|
|
try {
|
|
const store = await vAPI.adminStorage.get([
|
|
'adminSettings',
|
|
'toOverwrite',
|
|
]) || {};
|
|
if ( store.toOverwrite instanceof Object ) {
|
|
toOverwrite = store.toOverwrite;
|
|
}
|
|
const json = store.adminSettings;
|
|
if ( typeof json === 'string' && json !== '' ) {
|
|
data = JSON.parse(json);
|
|
} else if ( json instanceof Object ) {
|
|
data = json;
|
|
}
|
|
} catch (ex) {
|
|
console.error(ex);
|
|
}
|
|
|
|
if ( data instanceof Object === false ) { data = {}; }
|
|
|
|
const bin = {};
|
|
let binNotEmpty = false;
|
|
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/666
|
|
// Allows an admin to set their own 'assets.json' file, with their
|
|
// own set of stock assets.
|
|
if (
|
|
typeof data.assetsBootstrapLocation === 'string' &&
|
|
data.assetsBootstrapLocation !== ''
|
|
) {
|
|
µb.assetsBootstrapLocation = data.assetsBootstrapLocation;
|
|
}
|
|
|
|
if ( typeof data.userSettings === 'object' ) {
|
|
const µbus = this.userSettings;
|
|
const adminus = data.userSettings;
|
|
for ( const name in µbus ) {
|
|
if ( µbus.hasOwnProperty(name) === false ) { continue; }
|
|
if ( adminus.hasOwnProperty(name) === false ) { continue; }
|
|
bin[name] = adminus[name];
|
|
binNotEmpty = true;
|
|
}
|
|
}
|
|
|
|
// 'selectedFilterLists' is an array of filter list tokens. Each token
|
|
// is a reference to an asset in 'assets.json', or a URL for lists not
|
|
// present in 'assets.json'.
|
|
if (
|
|
Array.isArray(toOverwrite.filterLists) &&
|
|
toOverwrite.filterLists.length !== 0
|
|
) {
|
|
const importedLists = [];
|
|
for ( const list of toOverwrite.filterLists ) {
|
|
if ( /^[a-z-]+:\/\//.test(list) === false ) { continue; }
|
|
importedLists.push(list);
|
|
}
|
|
if ( importedLists.length !== 0 ) {
|
|
bin.importedLists = importedLists;
|
|
bin.externalLists = importedLists.join('\n');
|
|
}
|
|
bin.selectedFilterLists = toOverwrite.filterLists;
|
|
binNotEmpty = true;
|
|
} else if ( Array.isArray(data.selectedFilterLists) ) {
|
|
bin.selectedFilterLists = data.selectedFilterLists;
|
|
binNotEmpty = true;
|
|
}
|
|
|
|
if (
|
|
Array.isArray(toOverwrite.trustedSiteDirectives) &&
|
|
toOverwrite.trustedSiteDirectives.length !== 0
|
|
) {
|
|
µb.netWhitelistDefault = toOverwrite.trustedSiteDirectives.slice();
|
|
bin.netWhitelist = toOverwrite.trustedSiteDirectives.slice();
|
|
binNotEmpty = true;
|
|
} else if ( Array.isArray(data.whitelist) ) {
|
|
bin.netWhitelist = data.whitelist;
|
|
binNotEmpty = true;
|
|
} else if ( typeof data.netWhitelist === 'string' ) {
|
|
bin.netWhitelist = data.netWhitelist.split('\n');
|
|
binNotEmpty = true;
|
|
}
|
|
|
|
if ( typeof data.dynamicFilteringString === 'string' ) {
|
|
bin.dynamicFilteringString = data.dynamicFilteringString;
|
|
binNotEmpty = true;
|
|
}
|
|
|
|
if ( typeof data.urlFilteringString === 'string' ) {
|
|
bin.urlFilteringString = data.urlFilteringString;
|
|
binNotEmpty = true;
|
|
}
|
|
|
|
if ( typeof data.hostnameSwitchesString === 'string' ) {
|
|
bin.hostnameSwitchesString = data.hostnameSwitchesString;
|
|
binNotEmpty = true;
|
|
}
|
|
|
|
if ( binNotEmpty ) {
|
|
vAPI.storage.set(bin);
|
|
}
|
|
|
|
if (
|
|
Array.isArray(toOverwrite.filters) &&
|
|
toOverwrite.filters.length !== 0
|
|
) {
|
|
this.saveUserFilters(toOverwrite.filters.join('\n'));
|
|
} else if ( typeof data.userFilters === 'string' ) {
|
|
this.saveUserFilters(data.userFilters);
|
|
}
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
// https://github.com/gorhill/uBlock/issues/2344
|
|
// Support multiple locales per filter list.
|
|
// https://github.com/gorhill/uBlock/issues/3210
|
|
// Support ability to auto-enable a filter list based on user agent.
|
|
// https://github.com/gorhill/uBlock/pull/3860
|
|
// Get current language using extensions API (instead of `navigator.language`)
|
|
|
|
µb.listMatchesEnvironment = function(details) {
|
|
// Matches language?
|
|
if ( typeof details.lang === 'string' ) {
|
|
let re = this.listMatchesEnvironment.reLang;
|
|
if ( re === undefined ) {
|
|
const match = /^[a-z]+/.exec(i18n.getUILanguage());
|
|
if ( match !== null ) {
|
|
re = new RegExp('\\b' + match[0] + '\\b');
|
|
this.listMatchesEnvironment.reLang = re;
|
|
}
|
|
}
|
|
if ( re !== undefined && re.test(details.lang) ) { return true; }
|
|
}
|
|
// Matches user agent?
|
|
if ( typeof details.ua === 'string' ) {
|
|
let re = new RegExp('\\b' + this.escapeRegex(details.ua) + '\\b', 'i');
|
|
if ( re.test(self.navigator.userAgent) ) { return true; }
|
|
}
|
|
return false;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
{
|
|
let next = 0;
|
|
|
|
const launchTimer = vAPI.defer.create(fetchDelay => {
|
|
next = 0;
|
|
io.updateStart({ fetchDelay, auto: true });
|
|
});
|
|
|
|
µb.scheduleAssetUpdater = async function(details = {}) {
|
|
launchTimer.off();
|
|
vAPI.alarms.clear('assetUpdater');
|
|
|
|
if ( details.now ) {
|
|
next = 0;
|
|
io.updateStart(details);
|
|
return;
|
|
}
|
|
|
|
if ( µb.userSettings.autoUpdate === false ) {
|
|
if ( Boolean(details.updateDelay) === false ) {
|
|
next = 0;
|
|
return;
|
|
}
|
|
}
|
|
|
|
let updateDelay = details.updateDelay ||
|
|
this.hiddenSettings.autoUpdatePeriod * 3600000;
|
|
|
|
const now = Date.now();
|
|
|
|
// Use the new schedule if and only if it is earlier than the previous
|
|
// one.
|
|
if ( next !== 0 ) {
|
|
updateDelay = Math.min(updateDelay, Math.max(next - now, 1));
|
|
}
|
|
|
|
next = now + updateDelay;
|
|
|
|
const fetchDelay = details.fetchDelay ||
|
|
this.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 ||
|
|
60000;
|
|
|
|
launchTimer.on(updateDelay, fetchDelay);
|
|
vAPI.alarms.create('assetUpdater', {
|
|
delayInMinutes: Math.ceil(updateDelay / 60000) + 0.25
|
|
});
|
|
};
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
µb.assetObserver = function(topic, details) {
|
|
// Do not update filter list if not in use.
|
|
// Also, ignore really bad lists, i.e. those which should not even be
|
|
// fetched from a remote server.
|
|
if ( topic === 'before-asset-updated' ) {
|
|
if ( details.type === 'filters' ) {
|
|
if (
|
|
this.availableFilterLists.hasOwnProperty(details.assetKey) === false ||
|
|
this.selectedFilterLists.indexOf(details.assetKey) === -1 ||
|
|
this.badLists.get(details.assetKey)
|
|
) {
|
|
return;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Compile the list while we have the raw version in memory
|
|
if ( topic === 'after-asset-updated' ) {
|
|
// Skip selfie-related content.
|
|
if ( details.assetKey.startsWith('selfie/') ) { return; }
|
|
const cached = typeof details.content === 'string' && details.content !== '';
|
|
if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) {
|
|
if ( cached ) {
|
|
if ( this.selectedFilterLists.indexOf(details.assetKey) !== -1 ) {
|
|
this.extractFilterListMetadata(
|
|
details.assetKey,
|
|
details.content
|
|
);
|
|
if ( this.badLists.has(details.assetKey) === false ) {
|
|
io.toCache(`compiled/${details.assetKey}`,
|
|
this.compileFilters(details.content, {
|
|
assetKey: details.assetKey,
|
|
trustedSource: this.isTrustedList(details.assetKey),
|
|
})
|
|
);
|
|
}
|
|
}
|
|
} else {
|
|
this.removeCompiledFilterList(details.assetKey);
|
|
}
|
|
} else if ( details.assetKey === this.pslAssetKey ) {
|
|
if ( cached ) {
|
|
this.compilePublicSuffixList(details.content);
|
|
}
|
|
} else if ( details.assetKey === 'ublock-badlists' ) {
|
|
this.badLists = new Map();
|
|
}
|
|
broadcast({
|
|
what: 'assetUpdated',
|
|
key: details.assetKey,
|
|
cached,
|
|
});
|
|
// https://github.com/gorhill/uBlock/issues/2585
|
|
// Whenever an asset is overwritten, the current selfie is quite
|
|
// likely no longer valid.
|
|
this.selfieManager.destroy();
|
|
return;
|
|
}
|
|
|
|
// Update failed.
|
|
if ( topic === 'asset-update-failed' ) {
|
|
broadcast({
|
|
what: 'assetUpdated',
|
|
key: details.assetKey,
|
|
failed: true,
|
|
});
|
|
return;
|
|
}
|
|
|
|
// Reload all filter lists if needed.
|
|
if ( topic === 'after-assets-updated' ) {
|
|
if ( details.assetKeys.length !== 0 ) {
|
|
// https://github.com/gorhill/uBlock/pull/2314#issuecomment-278716960
|
|
if (
|
|
this.hiddenSettings.userResourcesLocation !== 'unset' ||
|
|
vAPI.webextFlavor.soup.has('devbuild')
|
|
) {
|
|
redirectEngine.invalidateResourcesSelfie(io);
|
|
}
|
|
this.loadFilterLists();
|
|
}
|
|
this.scheduleAssetUpdater();
|
|
broadcast({
|
|
what: 'assetsUpdated',
|
|
assetKeys: details.assetKeys
|
|
});
|
|
return;
|
|
}
|
|
|
|
// New asset source became available, if it's a filter list, should we
|
|
// auto-select it?
|
|
if ( topic === 'builtin-asset-source-added' ) {
|
|
if ( details.entry.content === 'filters' ) {
|
|
if (
|
|
details.entry.off === true &&
|
|
this.listMatchesEnvironment(details.entry)
|
|
) {
|
|
this.saveSelectedFilterLists([ details.assetKey ], true);
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
|
|
if ( topic === 'assets.json-updated' ) {
|
|
const { newDict, oldDict } = details;
|
|
if ( newDict['assets.json'] === undefined ) { return; }
|
|
if ( oldDict['assets.json'] === undefined ) { return; }
|
|
const newDefaultListset = new Set(newDict['assets.json'].defaultListset || []);
|
|
const oldDefaultListset = new Set(oldDict['assets.json'].defaultListset || []);
|
|
if ( newDefaultListset.size === 0 ) { return; }
|
|
if ( oldDefaultListset.size === 0 ) {
|
|
Array.from(Object.entries(oldDict))
|
|
.filter(a =>
|
|
a[1].content === 'filters' &&
|
|
a[1].off === undefined &&
|
|
/^https?:\/\//.test(a[0]) === false
|
|
)
|
|
.map(a => a[0])
|
|
.forEach(a => oldDefaultListset.add(a));
|
|
if ( oldDefaultListset.size === 0 ) { return; }
|
|
}
|
|
const selectedListset = new Set(this.selectedFilterLists);
|
|
let selectedListModified = false;
|
|
for ( const assetKey of oldDefaultListset ) {
|
|
if ( newDefaultListset.has(assetKey) ) { continue; }
|
|
selectedListset.delete(assetKey);
|
|
selectedListModified = true;
|
|
}
|
|
for ( const assetKey of newDefaultListset ) {
|
|
if ( oldDefaultListset.has(assetKey) ) { continue; }
|
|
selectedListset.add(assetKey);
|
|
selectedListModified = true;
|
|
}
|
|
if ( selectedListModified ) {
|
|
this.saveSelectedFilterLists(Array.from(selectedListset));
|
|
}
|
|
return;
|
|
}
|
|
};
|