mirror of
https://github.com/gorhill/uBlock.git
synced 2026-03-11 09:04:36 +00:00
0aa0d81caf
To minimize delay to First Contentful Paint. The idea is to avoid complex data structures such as Map in order to speed up first content script execution. Use stringified arrays where practical and instanciate those arrays from their stringified representation only when there are actual cosmetic filters to apply. Related commit: https://github.com/gorhill/uBlock/commit/6039ef2b6d
1184 lines
39 KiB
JavaScript
1184 lines
39 KiB
JavaScript
/*******************************************************************************
|
|
|
|
uBlock Origin - a comprehensive, efficient content blocker
|
|
Copyright (C) 2022-present Raymond Hill
|
|
|
|
This program is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program. If not, see {http://www.gnu.org/licenses/}.
|
|
|
|
Home: https://github.com/gorhill/uBlock
|
|
*/
|
|
|
|
import * as makeScriptlet from './make-scriptlets.js';
|
|
import * as sfp from './js/static-filtering-parser.js';
|
|
|
|
import {
|
|
createHash,
|
|
randomBytes,
|
|
} from 'crypto';
|
|
import {
|
|
dnrRulesetFromRawLists,
|
|
mergeRules,
|
|
} from './js/static-dnr-filtering.js';
|
|
|
|
import { execSync } from 'node:child_process';
|
|
import fs from 'fs/promises';
|
|
import path from 'path';
|
|
import process from 'process';
|
|
import redirectResourcesMap from './js/redirect-resources.js';
|
|
import { safeReplace } from './safe-replace.js';
|
|
|
|
/******************************************************************************/
|
|
|
|
const commandLineArgs = (( ) => {
|
|
const args = new Map();
|
|
let name, value;
|
|
for ( const arg of process.argv.slice(2) ) {
|
|
const pos = arg.indexOf('=');
|
|
if ( pos === -1 ) {
|
|
name = arg;
|
|
value = '';
|
|
} else {
|
|
name = arg.slice(0, pos);
|
|
value = arg.slice(pos+1);
|
|
}
|
|
args.set(name, value);
|
|
}
|
|
return args;
|
|
})();
|
|
|
|
const platform = commandLineArgs.get('platform') || 'chromium';
|
|
const outputDir = commandLineArgs.get('output') || '.';
|
|
const cacheDir = `${outputDir}/../mv3-data`;
|
|
const rulesetDir = `${outputDir}/rulesets`;
|
|
const scriptletDir = `${rulesetDir}/scripting`;
|
|
const envExtra = (( ) => {
|
|
const env = commandLineArgs.get('env');
|
|
return env ? env.split('|') : [];
|
|
})();
|
|
const env = [
|
|
platform,
|
|
'native_css_has',
|
|
'mv3',
|
|
'ublock',
|
|
'ubol',
|
|
'user_stylesheet',
|
|
...envExtra,
|
|
];
|
|
|
|
if ( platform === 'edge' ) {
|
|
env.push('chromium');
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
const jsonSetMapReplacer = (k, v) => {
|
|
if ( v instanceof Set || v instanceof Map ) {
|
|
if ( v.size === 0 ) { return; }
|
|
return Array.from(v);
|
|
}
|
|
return v;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
const consoleLog = console.log;
|
|
const stdOutput = [];
|
|
|
|
const log = (text, silent = true) => {
|
|
silent = silent && text.startsWith('!!!') === false;
|
|
stdOutput.push(text);
|
|
if ( silent === false ) {
|
|
consoleLog(text);
|
|
}
|
|
};
|
|
|
|
console.log = log;
|
|
|
|
const logProgress = text => {
|
|
process?.stdout?.clearLine?.();
|
|
process?.stdout?.cursorTo?.(0);
|
|
process?.stdout?.write?.(text.length > 120 ? `${text.slice(0, 119)}… ` : `${text} `);
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
async function fetchText(url, cacheDir) {
|
|
logProgress(`Reading locally cached ${path.basename(url)}`);
|
|
const fname = url
|
|
.replace(/^https?:\/\//, '')
|
|
.replace(/\//g, '_');(url);
|
|
const content = await fs.readFile(
|
|
`${cacheDir}/${fname}`,
|
|
{ encoding: 'utf8' }
|
|
).catch(( ) => { });
|
|
if ( content !== undefined ) {
|
|
log(`\tFetched local ${url}`);
|
|
return { url, content };
|
|
}
|
|
logProgress(`Fetching remote ${path.basename(url)}`);
|
|
log(`\tFetching remote ${url}`);
|
|
const response = await fetch(url).catch(( ) => { });
|
|
if ( response === undefined ) {
|
|
return { url, error: `Fetching failed: ${url}` };
|
|
}
|
|
let text;
|
|
if ( response.ok ) {
|
|
text = await response.text().catch(( ) => { });
|
|
} else {
|
|
text = await fallbackFetchText(url).catch(( ) => { });
|
|
}
|
|
if ( text === undefined ) {
|
|
return { url, error: `Fetching text content failed: ${url}` };
|
|
}
|
|
writeFile(`${cacheDir}/${fname}`, text);
|
|
return { url, content: text };
|
|
}
|
|
|
|
async function fallbackFetchText(url) {
|
|
const match = /^https:\/\/raw\.githubusercontent\.com\/([^/]+)\/([^/]+)\/master\/([^?]+)/.exec(url);
|
|
if ( match === null ) { return; }
|
|
logProgress(`\tGitHub CLI-fetching remote ${path.basename(url)}`);
|
|
// https://docs.github.com/en/rest/repos/contents
|
|
const content = execSync(`gh api \
|
|
-H "Accept: application/vnd.github.raw+json" \
|
|
-H "X-GitHub-Api-Version: 2022-11-28" \
|
|
/repos/${match[1]}/${match[2]}/contents/${match[3]} \
|
|
`, { encoding: 'utf8' });
|
|
return content;
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
const writeFile = async (fname, data) => {
|
|
const dir = path.dirname(fname);
|
|
await fs.mkdir(dir, { recursive: true });
|
|
const promise = fs.writeFile(fname, data);
|
|
writeOps.push(promise);
|
|
return promise;
|
|
};
|
|
|
|
const copyFile = async (from, to) => {
|
|
const dir = path.dirname(to);
|
|
await fs.mkdir(dir, { recursive: true });
|
|
const promise = fs.copyFile(from, to);
|
|
writeOps.push(promise);
|
|
return promise;
|
|
};
|
|
|
|
const writeOps = [];
|
|
|
|
/******************************************************************************/
|
|
|
|
const ruleResources = [];
|
|
const rulesetDetails = [];
|
|
const scriptletStats = new Map();
|
|
const genericDetails = new Map();
|
|
const requiredRedirectResources = new Set();
|
|
let networkBad = new Set();
|
|
|
|
// This will be used to sign our inserted `!#trusted on` directives
|
|
const secret = createHash('sha256').update(randomBytes(16)).digest('hex').slice(0,16);
|
|
log(`Secret: ${secret}`, false);
|
|
|
|
/******************************************************************************/
|
|
|
|
const restrSeparator = '(?:[^%.0-9a-z_-]|$)';
|
|
|
|
const rePatternFromUrlFilter = s => {
|
|
let anchor = 0b000;
|
|
if ( s.startsWith('||') ) {
|
|
anchor = 0b100;
|
|
s = s.slice(2);
|
|
} else if ( s.startsWith('|') ) {
|
|
anchor = 0b010;
|
|
s = s.slice(1);
|
|
}
|
|
if ( s.endsWith('|') ) {
|
|
anchor |= 0b001;
|
|
s = s.slice(0, -1);
|
|
}
|
|
let reStr = s.replace(rePatternFromUrlFilter.rePlainChars, '\\$&')
|
|
.replace(rePatternFromUrlFilter.reSeparators, restrSeparator)
|
|
.replace(rePatternFromUrlFilter.reDanglingAsterisks, '')
|
|
.replace(rePatternFromUrlFilter.reAsterisks, '\\S*?');
|
|
if ( anchor & 0b100 ) {
|
|
reStr = (
|
|
reStr.startsWith('\\.') ?
|
|
rePatternFromUrlFilter.restrHostnameAnchor2 :
|
|
rePatternFromUrlFilter.restrHostnameAnchor1
|
|
) + reStr;
|
|
} else if ( anchor & 0b010 ) {
|
|
reStr = '^' + reStr;
|
|
}
|
|
if ( anchor & 0b001 ) {
|
|
reStr += '$';
|
|
}
|
|
return reStr;
|
|
};
|
|
rePatternFromUrlFilter.rePlainChars = /[.+?${}()|[\]\\]/g;
|
|
rePatternFromUrlFilter.reSeparators = /\^/g;
|
|
rePatternFromUrlFilter.reDanglingAsterisks = /^\*+|\*+$/g;
|
|
rePatternFromUrlFilter.reAsterisks = /\*+/g;
|
|
rePatternFromUrlFilter.restrHostnameAnchor1 = '^[a-z-]+://(?:[^/?#]+\\.)?';
|
|
rePatternFromUrlFilter.restrHostnameAnchor2 = '^[a-z-]+://(?:[^/?#]+)?';
|
|
|
|
/******************************************************************************/
|
|
|
|
async function fetchList(assetDetails) {
|
|
// Remember fetched URLs
|
|
const fetchedURLs = new Set();
|
|
|
|
// Fetch list and expand `!#include` directives
|
|
let parts = assetDetails.urls.map(url => ({ url }));
|
|
while ( parts.every(v => typeof v === 'string') === false ) {
|
|
const newParts = [];
|
|
for ( const part of parts ) {
|
|
if ( typeof part === 'string' ) {
|
|
newParts.push(part);
|
|
continue;
|
|
}
|
|
if ( fetchedURLs.has(part.url) ) {
|
|
newParts.push('');
|
|
continue;
|
|
}
|
|
fetchedURLs.add(part.url);
|
|
if (
|
|
assetDetails.trusted ||
|
|
part.url.startsWith('https://ublockorigin.github.io/uAssets/filters/')
|
|
) {
|
|
newParts.push(`!#trusted on ${secret}`);
|
|
}
|
|
newParts.push(
|
|
fetchText(part.url, cacheDir).then(details => {
|
|
const { url, error } = details;
|
|
if ( error !== undefined ) { return details; }
|
|
const content = details.content.trim();
|
|
if ( /* content === '' || */ /^<.*>$/.test(content) ) {
|
|
return { url, error: `Bad content: ${url}` };
|
|
}
|
|
return { url, content };
|
|
})
|
|
);
|
|
newParts.push(`!#trusted off ${secret}`);
|
|
}
|
|
if ( parts.some(v => typeof v === 'object' && v.error) ) { return; }
|
|
parts = await Promise.all(newParts);
|
|
parts = sfp.utils.preparser.expandIncludes(parts, env);
|
|
}
|
|
const text = parts.join('\n');
|
|
|
|
if ( text === '' ) {
|
|
log('No filterset found', false);
|
|
}
|
|
return text;
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
const isUnsupported = rule =>
|
|
rule._error !== undefined;
|
|
|
|
const isRegex = rule =>
|
|
rule.condition !== undefined &&
|
|
rule.condition.regexFilter !== undefined;
|
|
|
|
const isGood = rule =>
|
|
isUnsupported(rule) === false &&
|
|
/^(allow|block|redirect|modifyHeaders|allowAllRequests)$/.test(rule.action?.type);
|
|
|
|
const isURLSkip = rule =>
|
|
isUnsupported(rule) === false &&
|
|
rule.action !== undefined &&
|
|
rule.action.type === 'urlskip';
|
|
|
|
/******************************************************************************/
|
|
|
|
async function patchRuleset(ruleset) {
|
|
return import(`./${platform}/patch-ruleset.js`).then(module => {
|
|
return module.patchRuleset(ruleset)
|
|
}).catch(( ) => {
|
|
return ruleset;
|
|
});
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
// Two distinct hostnames:
|
|
// www.example.com
|
|
// example.com
|
|
// Can be reduced to a single one:
|
|
// example.com
|
|
// Since if example.com matches, then www.example.com (or any other subdomain
|
|
// of example.com) will always match.
|
|
|
|
function pruneHostnameArray(hostnames) {
|
|
const rootMap = new Map();
|
|
for ( const hostname of hostnames ) {
|
|
const labels = hostname.split('.');
|
|
let currentMap = rootMap;
|
|
let i = labels.length;
|
|
while ( i-- ) {
|
|
const label = labels[i];
|
|
let nextMap = currentMap.get(label);
|
|
if ( nextMap === null ) { break; }
|
|
if ( nextMap === undefined ) {
|
|
if ( i === 0 ) {
|
|
currentMap.set(label, (nextMap = null));
|
|
} else {
|
|
currentMap.set(label, (nextMap = new Map()));
|
|
}
|
|
} else if ( i === 0 ) {
|
|
currentMap.set(label, null);
|
|
}
|
|
currentMap = nextMap;
|
|
}
|
|
}
|
|
const assemble = (currentMap, currentHostname, out) => {
|
|
for ( const [ label, nextMap ] of currentMap ) {
|
|
const nextHostname = currentHostname === ''
|
|
? label
|
|
: `${label}.${currentHostname}`;
|
|
if ( nextMap === null ) {
|
|
out.push(nextHostname);
|
|
} else {
|
|
assemble(nextMap, nextHostname, out);
|
|
}
|
|
}
|
|
return out;
|
|
};
|
|
return assemble(rootMap, '', []);
|
|
}
|
|
|
|
/*******************************************************************************
|
|
*
|
|
* For large rulesets, one rule per line for compromise between size and
|
|
* readability. This also means that the number of lines in resulting file
|
|
* representative of the number of rules in the ruleset.
|
|
*
|
|
* */
|
|
|
|
function toJSONRuleset(ruleset) {
|
|
const nodupProps = [ 'domains', 'excludedDomains', 'requestDomains', 'excludedRequestDomains', 'initiatorDomains', 'excludedInitiatorDomains' ];
|
|
for ( const { condition } of ruleset ) {
|
|
if ( condition === undefined ) { continue; }
|
|
for ( const prop of nodupProps ) {
|
|
if ( condition[prop] === undefined ) { continue; }
|
|
condition[prop] = Array.from(new Set(condition[prop]));
|
|
}
|
|
}
|
|
const sortProps = [ 'requestDomains', 'initiatorDomains', 'domains' ];
|
|
ruleset.sort((a, b) => {
|
|
let aLen = 0, bLen = 0;
|
|
for ( const prop of sortProps ) {
|
|
aLen += a.condition[prop]?.length ?? 0;
|
|
bLen += b.condition[prop]?.length ?? 0;
|
|
}
|
|
return bLen - aLen;
|
|
});
|
|
const replacer = (k, v) => {
|
|
if ( k.startsWith('_') ) { return; }
|
|
if ( Array.isArray(v) ) {
|
|
return v.sort();
|
|
}
|
|
if ( v instanceof Object ) {
|
|
const sorted = {};
|
|
for ( const kk of Object.keys(v).sort() ) {
|
|
sorted[kk] = v[kk];
|
|
}
|
|
return sorted;
|
|
}
|
|
return v;
|
|
};
|
|
const indent = ruleset.length > 10 ? undefined : 1;
|
|
const out = [];
|
|
let id = 1;
|
|
for ( const rule of ruleset ) {
|
|
rule.id = id++;
|
|
out.push(JSON.stringify(rule, replacer, indent));
|
|
}
|
|
return `[\n${out.join(',\n')}\n]\n`;
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
function toStrictBlockRule(rule, out) {
|
|
if ( rule.action.type !== 'block' ) { return; }
|
|
const { condition } = rule;
|
|
if ( condition === undefined ) { return; }
|
|
if ( condition.domainType ) { return; }
|
|
if ( condition.excludedResourceTypes ) { return; }
|
|
if ( condition.requestMethods ) { return; }
|
|
if ( condition.excludedRequestMethods ) { return; }
|
|
if ( condition.responseHeaders ) { return; }
|
|
if ( condition.excludedResponseHeaders ) { return; }
|
|
if ( condition.initiatorDomains ) { return; }
|
|
if ( condition.excludedInitiatorDomains ) { return; }
|
|
const { resourceTypes } = condition;
|
|
if ( resourceTypes === undefined ) {
|
|
if ( condition.requestDomains === undefined ) { return; }
|
|
} else if ( resourceTypes.includes('main_frame') === false ) {
|
|
return;
|
|
}
|
|
let regexFilter;
|
|
if ( condition.urlFilter ) {
|
|
regexFilter = rePatternFromUrlFilter(condition.urlFilter);
|
|
} else if ( condition.regexFilter ) {
|
|
regexFilter = condition.regexFilter;
|
|
} else {
|
|
regexFilter = '^https?://.*';
|
|
}
|
|
if ( regexFilter.startsWith('^') === false ) {
|
|
regexFilter = `^.*${regexFilter}`;
|
|
}
|
|
if (
|
|
regexFilter.endsWith('$') === false &&
|
|
regexFilter.endsWith('.*') === false &&
|
|
regexFilter.endsWith('.+') === false
|
|
) {
|
|
regexFilter = `${regexFilter}.*`;
|
|
}
|
|
const strictBlockRule = out.get(regexFilter) || {
|
|
action: {
|
|
type: 'redirect',
|
|
redirect: {
|
|
regexSubstitution: `/strictblock.html#\\0`,
|
|
},
|
|
},
|
|
condition: {
|
|
regexFilter,
|
|
resourceTypes: [ 'main_frame' ],
|
|
},
|
|
priority: 29,
|
|
};
|
|
if ( condition.requestDomains ) {
|
|
strictBlockRule.condition.requestDomains ??= [];
|
|
strictBlockRule.condition.requestDomains = Array.from(
|
|
new Set([
|
|
...strictBlockRule.condition.requestDomains,
|
|
...condition.requestDomains,
|
|
])
|
|
);
|
|
}
|
|
if ( condition.excludedRequestDomains ) {
|
|
strictBlockRule.condition.excludedRequestDomains ??= [];
|
|
strictBlockRule.condition.excludedRequestDomains = Array.from(
|
|
new Set([
|
|
...strictBlockRule.condition.excludedRequestDomains,
|
|
...condition.excludedRequestDomains,
|
|
])
|
|
);
|
|
}
|
|
out.set(regexFilter, strictBlockRule);
|
|
}
|
|
toStrictBlockRule.ruleId = 1;
|
|
|
|
/******************************************************************************/
|
|
|
|
async function processNetworkFilters(assetDetails, network) {
|
|
const { ruleset: rules } = network;
|
|
log(`Input filter count: ${network.filterCount}`);
|
|
log(`\tAccepted filter count: ${network.acceptedFilterCount}`);
|
|
log(`\tRejected filter count: ${network.rejectedFilterCount}`);
|
|
log(`Output rule count: ${rules.length}`);
|
|
|
|
// Minimize requestDomains arrays
|
|
for ( const rule of rules ) {
|
|
const condition = rule.condition;
|
|
if ( condition === undefined ) { continue; }
|
|
const requestDomains = condition.requestDomains;
|
|
if ( requestDomains === undefined ) { continue; }
|
|
const beforeCount = requestDomains.length;
|
|
condition.requestDomains = pruneHostnameArray(requestDomains);
|
|
const afterCount = condition.requestDomains.length;
|
|
if ( afterCount !== beforeCount ) {
|
|
log(`\tPruning requestDomains: from ${beforeCount} to ${afterCount}`);
|
|
}
|
|
}
|
|
|
|
// Add native DNR ruleset if present
|
|
if ( assetDetails.dnrURL ) {
|
|
const result = await fetchText(assetDetails.dnrURL, cacheDir);
|
|
for ( const rule of JSON.parse(result.content) ) {
|
|
rules.push(rule);
|
|
}
|
|
}
|
|
|
|
const staticRules = await patchRuleset(
|
|
rules.filter(rule => isGood(rule) && isRegex(rule) === false)
|
|
);
|
|
log(`\tStatic rules: ${staticRules.length}`);
|
|
log(staticRules
|
|
.filter(rule => Array.isArray(rule._warning))
|
|
.map(rule => rule._warning.map(v => `\t\t${v}`))
|
|
.join('\n'), true
|
|
);
|
|
|
|
const regexRules = await patchRuleset(
|
|
rules.filter(rule => isGood(rule) && isRegex(rule))
|
|
);
|
|
log(`\tMaybe good (regexes): ${regexRules.length}`);
|
|
|
|
staticRules.forEach(rule => {
|
|
if ( rule.action.redirect?.extensionPath === undefined ) { return; }
|
|
requiredRedirectResources.add(
|
|
rule.action.redirect.extensionPath.replace(/^\/+/, '')
|
|
);
|
|
});
|
|
|
|
const urlskips = new Map();
|
|
for ( const rule of rules ) {
|
|
if ( isURLSkip(rule) === false ) { continue; }
|
|
if ( rule.__modifierAction !== 0 ) { continue; }
|
|
const { condition } = rule;
|
|
if ( condition.resourceTypes ) {
|
|
if ( condition.resourceTypes.includes('main_frame') === false ) {
|
|
continue;
|
|
}
|
|
}
|
|
const { urlFilter, regexFilter, requestDomains } = condition;
|
|
let re;
|
|
if ( urlFilter !== undefined ) {
|
|
re = rePatternFromUrlFilter(urlFilter);
|
|
} else if ( regexFilter !== undefined ) {
|
|
re = regexFilter;
|
|
} else {
|
|
re = '^';
|
|
}
|
|
const rawSteps = rule.__modifierValue;
|
|
const steps = rawSteps.includes(' ') && rawSteps.split(/ +/) || [ rawSteps ];
|
|
const keyEntry = {
|
|
re,
|
|
c: condition.isUrlFilterCaseSensitive,
|
|
steps,
|
|
}
|
|
const key = JSON.stringify(keyEntry);
|
|
let actualEntry = urlskips.get(key);
|
|
if ( actualEntry === undefined ) {
|
|
urlskips.set(key, keyEntry);
|
|
actualEntry = keyEntry;
|
|
}
|
|
if ( requestDomains !== undefined ) {
|
|
if ( actualEntry.hostnames === undefined ) {
|
|
actualEntry.hostnames = [];
|
|
}
|
|
actualEntry.hostnames.push(...requestDomains);
|
|
}
|
|
}
|
|
log(`\turlskip=: ${urlskips.size}`);
|
|
|
|
const bad = rules.filter(rule =>
|
|
isUnsupported(rule)
|
|
);
|
|
log(`\tUnsupported: ${bad.length}`);
|
|
log(bad.map(rule => rule._error.map(v => `\t\t${v}`)).join('\n'), true);
|
|
|
|
writeFile(`${rulesetDir}/main/${assetDetails.id}.json`,
|
|
toJSONRuleset(staticRules)
|
|
);
|
|
|
|
if ( regexRules.length !== 0 ) {
|
|
writeFile(`${rulesetDir}/regex/${assetDetails.id}.json`,
|
|
toJSONRuleset(regexRules)
|
|
);
|
|
}
|
|
|
|
const strictBlocked = new Map();
|
|
for ( const rule of staticRules ) {
|
|
toStrictBlockRule(rule, strictBlocked);
|
|
}
|
|
if ( strictBlocked.size !== 0 ) {
|
|
mergeRules(strictBlocked, 'requestDomains');
|
|
writeFile(`${rulesetDir}/strictblock/${assetDetails.id}.json`,
|
|
toJSONRuleset(Array.from(strictBlocked.values()))
|
|
);
|
|
}
|
|
|
|
if ( urlskips.size !== 0 ) {
|
|
writeFile(`${rulesetDir}/urlskip/${assetDetails.id}.json`,
|
|
JSON.stringify(Array.from(urlskips.values()), null, 1)
|
|
);
|
|
}
|
|
|
|
return {
|
|
total: rules.length,
|
|
plain: staticRules.length,
|
|
rejected: bad.length,
|
|
regex: regexRules.length,
|
|
strictblock: strictBlocked.size,
|
|
urlskip: urlskips.size,
|
|
};
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
// TODO: unify css/scriptlet processing code since now css styles are
|
|
// injected using scriptlet injection.
|
|
|
|
// Load all available scriptlets into a key-val map, where the key is the
|
|
// scriptlet token, and val is the whole content of the file.
|
|
|
|
let scriptletsMapPromise;
|
|
|
|
function loadAllSourceScriptlets() {
|
|
if ( scriptletsMapPromise !== undefined ) {
|
|
return scriptletsMapPromise;
|
|
}
|
|
|
|
scriptletsMapPromise = fs.readdir('./scriptlets').then(files => {
|
|
const readTemplateFile = file =>
|
|
fs.readFile(`./scriptlets/${file}`, { encoding: 'utf8' })
|
|
.then(text => ({ file, text }));
|
|
const readPromises = [];
|
|
for ( const file of files ) {
|
|
readPromises.push(readTemplateFile(file));
|
|
}
|
|
return Promise.all(readPromises).then(results => {
|
|
const originalScriptletMap = new Map();
|
|
for ( const details of results ) {
|
|
originalScriptletMap.set(
|
|
details.file.replace('.template.js', '')
|
|
.replace('.template.css', ''),
|
|
details.text
|
|
);
|
|
}
|
|
return originalScriptletMap;
|
|
});
|
|
});
|
|
|
|
return scriptletsMapPromise;
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
// http://www.cse.yorku.ca/~oz/hash.html#djb2
|
|
// Must mirror content script surveyor's version
|
|
|
|
async function processGenericCosmeticFilters(
|
|
assetDetails,
|
|
selectorList,
|
|
exceptionList,
|
|
declarativeMap
|
|
) {
|
|
const exceptionSet = new Set(
|
|
exceptionList &&
|
|
exceptionList.filter(a => a.key !== undefined).map(a => a.selector)
|
|
);
|
|
|
|
const genericSelectorMap = new Map();
|
|
if ( selectorList ) {
|
|
for ( const { key, selector } of selectorList ) {
|
|
if ( key === undefined ) { continue; }
|
|
if ( exceptionSet.has(selector) ) { continue; }
|
|
const type = key.charCodeAt(0);
|
|
const hash = hashFromStr(type, key.slice(1));
|
|
const selectors = genericSelectorMap.get(hash);
|
|
if ( selectors === undefined ) {
|
|
genericSelectorMap.set(hash, selector)
|
|
} else {
|
|
genericSelectorMap.set(hash, `${selectors},\n${selector}`)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Specific exceptions
|
|
const genericExceptionSieve = new Set();
|
|
const genericExceptionMap = new Map();
|
|
if ( declarativeMap ) {
|
|
for ( const [ exception, details ] of declarativeMap ) {
|
|
if ( details.rejected ) { continue; }
|
|
if ( details.key === undefined ) { continue; }
|
|
if ( details.matches !== undefined ) { continue; }
|
|
if ( details.excludeMatches === undefined ) { continue; }
|
|
const type = details.key.charCodeAt(0);
|
|
const hash = hashFromStr(type, details.key.slice(1));
|
|
genericExceptionSieve.add(hash);
|
|
for ( const hn of details.excludeMatches ) {
|
|
const exceptions = genericExceptionMap.get(hn);
|
|
if ( exceptions === undefined ) {
|
|
genericExceptionMap.set(hn, exception);
|
|
} else {
|
|
genericExceptionMap.set(hn, `${exceptions}\n${exception}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if ( genericSelectorMap.size === 0 ) {
|
|
if ( genericExceptionMap.size === 0 ) { return 0; }
|
|
}
|
|
|
|
const originalScriptletMap = await loadAllSourceScriptlets();
|
|
let patchedScriptlet = originalScriptletMap.get('css-generic').replace(
|
|
'$rulesetId$',
|
|
assetDetails.id
|
|
);
|
|
patchedScriptlet = safeReplace(patchedScriptlet,
|
|
/\bself\.\$genericSelectorMap\$/,
|
|
`${JSON.stringify(genericSelectorMap, scriptletJsonReplacer)}`
|
|
);
|
|
patchedScriptlet = safeReplace(patchedScriptlet,
|
|
/\bself\.\$genericExceptionSieve\$/,
|
|
`${JSON.stringify(genericExceptionSieve, scriptletJsonReplacer)}`
|
|
);
|
|
patchedScriptlet = safeReplace(patchedScriptlet,
|
|
/\bself\.\$genericExceptionMap\$/,
|
|
`${JSON.stringify(genericExceptionMap, scriptletJsonReplacer)}`
|
|
);
|
|
|
|
writeFile(`${scriptletDir}/generic/${assetDetails.id}.js`,
|
|
patchedScriptlet
|
|
);
|
|
|
|
log(`CSS-generic: ${genericExceptionSieve.size} specific CSS exceptions`);
|
|
log(`CSS-generic: ${genericSelectorMap.size} plain CSS selectors`);
|
|
|
|
return genericSelectorMap.size + genericExceptionSieve.size;
|
|
}
|
|
|
|
const hashFromStr = (type, s) => {
|
|
const len = s.length;
|
|
const step = len + 7 >>> 3;
|
|
let hash = (type << 5) + type ^ len;
|
|
for ( let i = 0; i < len; i += step ) {
|
|
hash = (hash << 5) + hash ^ s.charCodeAt(i);
|
|
}
|
|
return hash & 0xFFF;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
async function processGenericHighCosmeticFilters(
|
|
assetDetails,
|
|
genericSelectorList,
|
|
genericExceptionList
|
|
) {
|
|
if ( genericSelectorList === undefined ) { return 0; }
|
|
const genericSelectorSet = new Set(
|
|
genericSelectorList
|
|
.filter(a => a.key === undefined)
|
|
.map(a => a.selector)
|
|
);
|
|
// https://github.com/uBlockOrigin/uBOL-home/issues/365
|
|
if ( genericExceptionList ) {
|
|
for ( const entry of genericExceptionList ) {
|
|
if ( entry.key !== undefined ) { continue; }
|
|
globalHighlyGenericExceptionSet.add(entry.selector);
|
|
}
|
|
}
|
|
for ( const selector of globalHighlyGenericExceptionSet ) {
|
|
if ( genericSelectorSet.has(selector) === false ) { continue; }
|
|
genericSelectorSet.delete(selector);
|
|
log(`\tRemoving excepted highly generic filter ##${selector}`);
|
|
}
|
|
if ( genericSelectorSet.size === 0 ) { return 0; }
|
|
const selectorLists = Array.from(genericSelectorSet).sort().join(',\n');
|
|
const originalScriptletMap = await loadAllSourceScriptlets();
|
|
|
|
let patchedScriptlet = originalScriptletMap.get('css-generichigh').replace(
|
|
'$rulesetId$',
|
|
assetDetails.id
|
|
);
|
|
patchedScriptlet = safeReplace(patchedScriptlet,
|
|
/\$selectorList\$/,
|
|
selectorLists
|
|
);
|
|
|
|
writeFile(`${scriptletDir}/generichigh/${assetDetails.id}.css`,
|
|
patchedScriptlet
|
|
);
|
|
|
|
log(`CSS-generic-high: ${genericSelectorSet.size} plain CSS selectors`);
|
|
|
|
return genericSelectorSet.size;
|
|
}
|
|
|
|
const globalHighlyGenericExceptionSet = new Set();
|
|
|
|
/******************************************************************************/
|
|
|
|
const scriptletJsonReplacer = (k, v) => {
|
|
if ( k === 'n' ) {
|
|
if ( v === undefined || v.size === 0 ) { return; }
|
|
return Array.from(v);
|
|
}
|
|
if ( v instanceof Set || v instanceof Map ) {
|
|
if ( v.size === 0 ) { return; }
|
|
return Array.from(v);
|
|
}
|
|
return v;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
async function processCosmeticFilters(assetDetails, realm, mapin) {
|
|
if ( mapin === undefined ) { return 0; }
|
|
if ( mapin.size === 0 ) { return 0; }
|
|
|
|
// Collate all distinct selectors
|
|
const allSelectors = new Map();
|
|
const allHostnames = new Map();
|
|
let hasEntities = false;
|
|
for ( const [ selector, details ] of mapin ) {
|
|
if ( details.rejected ) { continue; }
|
|
if ( allSelectors.has(selector) === false ) {
|
|
allSelectors.set(selector, allSelectors.size);
|
|
}
|
|
const iSelector = allSelectors.get(selector);
|
|
if ( details.matches ) {
|
|
for ( const hn of details.matches ) {
|
|
if ( allHostnames.has(hn) === false ) {
|
|
allHostnames.set(hn, new Set());
|
|
}
|
|
allHostnames.get(hn).add(iSelector);
|
|
hasEntities ||= hn.endsWith('.*');
|
|
}
|
|
}
|
|
if ( details.excludeMatches ) {
|
|
for ( const hn of details.excludeMatches ) {
|
|
if ( allHostnames.has(hn) === false ) {
|
|
allHostnames.set(hn, new Set());
|
|
}
|
|
allHostnames.get(hn).add(~iSelector);
|
|
hasEntities ||= hn.endsWith('.*');
|
|
}
|
|
}
|
|
}
|
|
const allSelectorLists = new Map();
|
|
for ( const [ hn, selectorSet ] of allHostnames ) {
|
|
const list = JSON.stringify(Array.from(selectorSet).sort()).slice(1, -1);
|
|
if ( allSelectorLists.has(list) === false ) {
|
|
allSelectorLists.set(list, allSelectorLists.size);
|
|
}
|
|
allHostnames.set(hn, allSelectorLists.get(list));
|
|
}
|
|
|
|
// The cosmetic filters will be injected programmatically as content
|
|
// script and the decisions to activate the cosmetic filters will be
|
|
// done at injection time according to the document's hostname.
|
|
const originalScriptletMap = await loadAllSourceScriptlets();
|
|
let patchedScriptlet = originalScriptletMap.get(`css-${realm}`).replace(
|
|
'$rulesetId$',
|
|
assetDetails.id
|
|
);
|
|
patchedScriptlet = safeReplace(patchedScriptlet,
|
|
/\bself\.\$selectors\$/,
|
|
`/* ${allSelectors.size} */ ${JSON.stringify(Array.from(allSelectors.keys()))}`
|
|
);
|
|
patchedScriptlet = safeReplace(patchedScriptlet,
|
|
/\bself\.\$selectorLists\$/,
|
|
`/* ${allSelectorLists.size} */ ${JSON.stringify(Array.from(allSelectorLists.keys()).join(';'))}`
|
|
);
|
|
const sortedHostnames = Array.from(allHostnames.keys()).toSorted((a, b) => {
|
|
const d = a.length - b.length;
|
|
if ( d !== 0 ) { return d; }
|
|
return a < b ? -1 : 1;
|
|
});
|
|
patchedScriptlet = safeReplace(patchedScriptlet,
|
|
/\bself\.\$selectorListRefs\$/,
|
|
`/* ${sortedHostnames.length} */ "${JSON.stringify(sortedHostnames.map(a => allHostnames.get(a))).slice(1, -1)}"`
|
|
);
|
|
patchedScriptlet = safeReplace(patchedScriptlet,
|
|
/\bself\.\$hostnames\$/,
|
|
`/* ${sortedHostnames.length} */ ${JSON.stringify(sortedHostnames)}`
|
|
);
|
|
patchedScriptlet = safeReplace(patchedScriptlet,
|
|
'self.$hasEntities$',
|
|
JSON.stringify(hasEntities)
|
|
);
|
|
writeFile(`${scriptletDir}/${realm}/${assetDetails.id}.js`, patchedScriptlet);
|
|
|
|
log(`CSS-${realm}: ${allSelectors.size} distinct filters for ${allHostnames.size} distinct hostnames`);
|
|
|
|
return sortedHostnames.length;
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
async function processScriptletFilters(assetDetails, mapin) {
|
|
if ( mapin === undefined ) { return 0; }
|
|
if ( mapin.size === 0 ) { return 0; }
|
|
|
|
makeScriptlet.init();
|
|
|
|
for ( const details of mapin.values() ) {
|
|
makeScriptlet.compile(assetDetails, details);
|
|
}
|
|
const stats = await makeScriptlet.commit(
|
|
assetDetails.id,
|
|
`${scriptletDir}/scriptlet`,
|
|
writeFile
|
|
);
|
|
if ( stats.length !== 0 ) {
|
|
scriptletStats.set(assetDetails.id, stats);
|
|
}
|
|
makeScriptlet.reset();
|
|
return stats.length;
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
async function rulesetFromURLs(assetDetails) {
|
|
log('============================');
|
|
log(`Listset for '${assetDetails.id}':`);
|
|
|
|
if ( assetDetails.text === undefined && assetDetails.urls.length !== 0 ) {
|
|
const text = await fetchList(assetDetails);
|
|
if ( text === undefined ) {
|
|
process.exit(1);
|
|
}
|
|
assetDetails.text = text;
|
|
} else {
|
|
assetDetails.text = '';
|
|
}
|
|
|
|
if ( Array.isArray(assetDetails.filters) && assetDetails.filters.length ) {
|
|
const extra = [
|
|
`!#trusted on ${secret}`,
|
|
...assetDetails.filters,
|
|
`!#trusted off ${secret}`,
|
|
assetDetails.text,
|
|
];
|
|
assetDetails.text = extra.join('\n').trim();
|
|
}
|
|
|
|
if ( assetDetails.text === '' ) { return; }
|
|
|
|
const extensionPaths = [];
|
|
for ( const [ fname, details ] of redirectResourcesMap ) {
|
|
const path = `/web_accessible_resources/${fname}`;
|
|
extensionPaths.push([ fname, path ]);
|
|
if ( details.alias === undefined ) { continue; }
|
|
if ( typeof details.alias === 'string' ) {
|
|
extensionPaths.push([ details.alias, path ]);
|
|
continue;
|
|
}
|
|
if ( Array.isArray(details.alias) === false ) { continue; }
|
|
for ( const alias of details.alias ) {
|
|
extensionPaths.push([ alias, path ]);
|
|
}
|
|
}
|
|
|
|
const results = await dnrRulesetFromRawLists(
|
|
[ { name: assetDetails.id, text: assetDetails.text } ],
|
|
{ env, extensionPaths, secret, networkBad }
|
|
);
|
|
networkBad = results.networkBad;
|
|
|
|
// Release memory used by filter list content
|
|
assetDetails.text = undefined;
|
|
|
|
const netStats = await processNetworkFilters(
|
|
assetDetails,
|
|
results.network
|
|
);
|
|
|
|
// Split cosmetic filters into two groups: declarative and procedural
|
|
const declarativeCosmetic = new Map();
|
|
const proceduralCosmetic = new Map();
|
|
const rejectedCosmetic = [];
|
|
if ( results.specificCosmetic ) {
|
|
for ( const [ selector, details ] of results.specificCosmetic ) {
|
|
if ( details.rejected ) {
|
|
rejectedCosmetic.push(selector);
|
|
continue;
|
|
}
|
|
if ( selector.startsWith('{') === false ) {
|
|
declarativeCosmetic.set(selector, details);
|
|
continue;
|
|
}
|
|
const parsed = JSON.parse(selector);
|
|
parsed.raw = undefined;
|
|
proceduralCosmetic.set(JSON.stringify(parsed), details);
|
|
}
|
|
}
|
|
if ( rejectedCosmetic.length !== 0 ) {
|
|
log(`Rejected cosmetic filters: ${rejectedCosmetic.length}`);
|
|
log(rejectedCosmetic.map(line => `\t${line}`).join('\n'), true);
|
|
}
|
|
|
|
const genericDetailsForRuleset = {};
|
|
if (
|
|
Array.isArray(results.network.generichideExclusions) &&
|
|
results.network.generichideExclusions.length !== 0
|
|
) {
|
|
genericDetailsForRuleset.unhide = results.network.generichideExclusions
|
|
.filter(hn => hn.endsWith('.*') === false)
|
|
.sort();
|
|
}
|
|
if (
|
|
Array.isArray(results.network.generichideInclusions) &&
|
|
results.network.generichideInclusions.length !== 0
|
|
) {
|
|
genericDetailsForRuleset.hide = results.network.generichideInclusions
|
|
.filter(hn => hn.endsWith('.*') === false)
|
|
.sort();
|
|
}
|
|
if ( genericDetailsForRuleset.unhide || genericDetailsForRuleset.hide ) {
|
|
genericDetails.set(assetDetails.id, genericDetailsForRuleset);
|
|
}
|
|
|
|
const genericCosmeticStats = await processGenericCosmeticFilters(
|
|
assetDetails,
|
|
results.genericCosmeticFilters,
|
|
results.genericCosmeticExceptions,
|
|
declarativeCosmetic
|
|
);
|
|
const genericHighCosmeticStats = await processGenericHighCosmeticFilters(
|
|
assetDetails,
|
|
results.genericCosmeticFilters,
|
|
results.genericCosmeticExceptions,
|
|
);
|
|
const specificCosmeticStats = await processCosmeticFilters(
|
|
assetDetails,
|
|
'specific',
|
|
declarativeCosmetic
|
|
);
|
|
|
|
const proceduralStats = await processCosmeticFilters(
|
|
assetDetails,
|
|
'procedural',
|
|
proceduralCosmetic
|
|
);
|
|
const scriptletStats = await processScriptletFilters(
|
|
assetDetails,
|
|
results.scriptlet
|
|
);
|
|
|
|
rulesetDetails.push({
|
|
id: assetDetails.id,
|
|
name: assetDetails.name,
|
|
group: assetDetails.group,
|
|
parent: assetDetails.parent,
|
|
enabled: assetDetails.enabled,
|
|
lang: assetDetails.lang,
|
|
tags: assetDetails.tags,
|
|
homeURL: assetDetails.homeURL,
|
|
filters: {
|
|
total: results.network.filterCount,
|
|
accepted: results.network.acceptedFilterCount,
|
|
rejected: results.network.rejectedFilterCount,
|
|
},
|
|
rules: {
|
|
total: netStats.total,
|
|
plain: netStats.plain,
|
|
regex: netStats.regex,
|
|
removeparam: netStats.removeparam,
|
|
redirect: netStats.redirect,
|
|
modifyHeaders: netStats.modifyHeaders,
|
|
strictblock: netStats.strictblock,
|
|
urlskip: netStats.urlskip,
|
|
discarded: netStats.discarded,
|
|
rejected: netStats.rejected,
|
|
},
|
|
css: {
|
|
generic: genericCosmeticStats,
|
|
generichigh: genericHighCosmeticStats,
|
|
specific: specificCosmeticStats,
|
|
procedural: proceduralStats,
|
|
},
|
|
scriptlets: scriptletStats,
|
|
});
|
|
|
|
ruleResources.push({
|
|
id: assetDetails.id,
|
|
enabled: assetDetails.enabled,
|
|
path: `/rulesets/main/${assetDetails.id}.json`
|
|
});
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
async function main() {
|
|
|
|
let version = '';
|
|
{
|
|
const now = new Date();
|
|
const yearPart = now.getUTCFullYear();
|
|
const monthPart = now.getUTCMonth() + 1;
|
|
const dayPart = now.getUTCDate();
|
|
const hourPart = Math.floor(now.getUTCHours());
|
|
const minutePart = Math.floor(now.getUTCMinutes());
|
|
version = `${yearPart}.${monthPart*100+dayPart}.${hourPart*100+minutePart}`;
|
|
}
|
|
log(`Version: ${version}`, false);
|
|
|
|
// Get list of rulesets
|
|
const rulesets = await fs.readFile('rulesets.json', {
|
|
encoding: 'utf8'
|
|
}).then(text =>
|
|
JSON.parse(text)
|
|
);
|
|
|
|
for ( const ruleset of rulesets ) {
|
|
if ( ruleset.excludedPlatforms?.includes(platform) ) { continue; }
|
|
await rulesetFromURLs(ruleset);
|
|
}
|
|
|
|
logProgress('');
|
|
|
|
writeFile(`${rulesetDir}/ruleset-details.json`,
|
|
`${JSON.stringify(rulesetDetails, null, 1)}\n`
|
|
);
|
|
|
|
writeFile(`${rulesetDir}/scriptlet-details.json`,
|
|
`${JSON.stringify(scriptletStats, jsonSetMapReplacer, 1)}\n`
|
|
);
|
|
|
|
writeFile(`${rulesetDir}/generic-details.json`,
|
|
`${JSON.stringify(genericDetails, jsonSetMapReplacer, 1)}\n`
|
|
);
|
|
|
|
// Copy required redirect resources
|
|
for ( const path of requiredRedirectResources ) {
|
|
copyFile(`./${path}`, `${outputDir}/${path}`);
|
|
}
|
|
|
|
await Promise.all(writeOps);
|
|
|
|
// Patch manifest
|
|
// Get manifest content
|
|
const manifest = await fs.readFile(
|
|
`${outputDir}/manifest.json`,
|
|
{ encoding: 'utf8' }
|
|
).then(text =>
|
|
JSON.parse(text)
|
|
);
|
|
// Patch declarative_net_request key
|
|
manifest.declarative_net_request = { rule_resources: ruleResources };
|
|
// Patch web_accessible_resources key
|
|
manifest.web_accessible_resources = manifest.web_accessible_resources || [];
|
|
const web_accessible_resources = {
|
|
resources: Array.from(requiredRedirectResources).map(path => `${path}`),
|
|
matches: [ '<all_urls>' ],
|
|
};
|
|
if ( env.includes('chromium') && env.includes('safari') === false ) {
|
|
web_accessible_resources.use_dynamic_url = true;
|
|
}
|
|
manifest.web_accessible_resources.push(web_accessible_resources);
|
|
|
|
// Patch manifest version property
|
|
manifest.version = version;
|
|
// Commit changes
|
|
await fs.writeFile(`${outputDir}/manifest.json`,
|
|
JSON.stringify(manifest, null, 2) + '\n'
|
|
);
|
|
|
|
// Log results
|
|
const logContent = stdOutput.join('\n') + '\n';
|
|
await fs.writeFile(`${outputDir}/log.txt`, logContent);
|
|
}
|
|
|
|
main();
|
|
|
|
/******************************************************************************/
|