diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 571ad8e1e..41cac9e75 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -70,5 +70,8 @@ "short_name": "uBlock₀", "storage": { "managed_schema": "managed_storage.json" - } + }, + "web_accessible_resources": [ + "/web_accessible_resources/*" + ] } diff --git a/platform/chromium/vapi-background.js b/platform/chromium/vapi-background.js index 21c3dbcfb..1b9e48778 100644 --- a/platform/chromium/vapi-background.js +++ b/platform/chromium/vapi-background.js @@ -944,6 +944,38 @@ vAPI.messaging.broadcast = function(message) { /******************************************************************************/ /******************************************************************************/ +// https://github.com/gorhill/uBlock/issues/3474 +// https://github.com/gorhill/uBlock/issues/2823 +// - foil ability of web pages to identify uBO through +// its web accessible resources. +// https://github.com/gorhill/uBlock/issues/3497 +// - prevent web pages from interfering with uBO's element picker + +(function() { + vAPI.warSecret = + Math.floor(Math.random() * 982451653 + 982451653).toString(36) + + Math.floor(Math.random() * 982451653 + 982451653).toString(36); + + var key = 'secret=' + vAPI.warSecret; + var root = vAPI.getURL('/'); + var guard = function(details) { + if ( details.url.indexOf(key) === -1 ) { + return { redirectUrl: root }; + } + }; + + chrome.webRequest.onBeforeRequest.addListener( + guard, + { + urls: [ root + 'web_accessible_resources/*' ] + }, + [ 'blocking' ] + ); +})(); + +/******************************************************************************/ +/******************************************************************************/ + // https://developer.mozilla.org/en-US/Add-ons/WebExtensions/API/contextMenus#Browser_compatibility // Firefox for Android does no support browser.contextMenus. diff --git a/platform/webext/manifest.json b/platform/webext/manifest.json index 679ddbb1e..1e8cc2678 100644 --- a/platform/webext/manifest.json +++ b/platform/webext/manifest.json @@ -88,5 +88,8 @@ "48":"img/ublock.svg" } }, - "version":"1.9.15.101" + "version":"1.9.15.101", + "web_accessible_resources": [ + "/web_accessible_resources/*" + ] } diff --git a/src/js/assets.js b/src/js/assets.js index d0db48449..800848009 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2017 Raymond Hill + Copyright (C) 2014-2018 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -911,12 +911,6 @@ var updateFirst = function() { typeof vAPI.webextFlavor === 'string' && vAPI.webextFlavor.startsWith('Mozilla-Firefox-'); } - // This is to ensure the packaged version will always be used (in case - // there is a cache remnant from a pre-stable webext era). - // See https://github.com/uBlockOrigin/uAssets/commit/a6c77af4afb45800d4fd7c268a2a5eab5a64daf3#commitcomment-24642912 - if ( noRemoteResources ) { - api.remove('ublock-resources'); - } updaterStatus = 'updating'; updaterFetched.clear(); updaterUpdated = []; diff --git a/src/js/background.js b/src/js/background.js index 3d9a33018..b25c459bc 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2017 Raymond Hill + Copyright (C) 2014-2018 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -122,7 +122,7 @@ var µBlock = (function() { // jshint ignore:line // read-only systemSettings: { compiledMagic: 'puuijtkfpspv', - selfieMagic: 'puuijtkfpspv' + selfieMagic: 'tuqilngsxkwo' }, restoreBackupSettings: { diff --git a/src/js/messaging.js b/src/js/messaging.js index 56f4e0345..667d5572b 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -944,6 +944,7 @@ var onMessage = function(request, sender, callback) { // https://github.com/gorhill/uBlock/pull/2314#issuecomment-278716960 if ( request.assetKey === 'ublock-filters' ) { µb.assets.purge('ublock-resources'); + µb.redirectEngine.invalidateResourcesSelfie(); } break; diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js index 47efae2a1..6f7ea259f 100644 --- a/src/js/redirect-engine.js +++ b/src/js/redirect-engine.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2015-2017 Raymond Hill + Copyright (C) 2015-2018 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -28,14 +28,97 @@ /******************************************************************************/ /******************************************************************************/ +var warResolve = (function() { + var timer; + var toResolve = new Set(); + var toProcess = new Set(); + var reMimeParser = /^[^/]+\/([^\s;]+)/; + + var replacer = function(s) { + if ( s === '+' ) { return '-'; } + if ( s === '/' ) { return '_'; } + if ( s === '=' ) { return ''; } + return s; + }; + + var filenameFromToken = function(token, mime) { + var name = btoa(token).replace(/[+/=]/g, replacer); + var match = reMimeParser.exec(mime); + if ( match !== null ) { + name += '.' + match[1]; + } + return name; + }; + + var onResolved = function(success, token, url) { + var reng = µBlock.redirectEngine; + this.onload = this.onerror = null; + var resource = reng.resources.get(token); + if ( resource !== undefined && success ) { + resource.warURL = url; + } + toProcess.delete(token); + if ( toResolve.size === 0 && toProcess.size === 0 ) { + reng.selfieFromResources(); + } + }; + + var resolvePending = function() { + timer = undefined; + var reng = µBlock.redirectEngine, + resources = reng.resources, + n = 8; // max number of xhr at once + for ( var token of toResolve ) { + var resource = resources.get(token); + toResolve.delete(token); + if ( resource === undefined ) { continue; } + toProcess.add(token); + var url = vAPI.getURL( + '/web_accessible_resources/' + + filenameFromToken(token, resource.mime) + ); + var xhr = new XMLHttpRequest(); + xhr.timeout = 1000; + xhr.open('head', url + '?secret=' + vAPI.warSecret); + xhr.onload = onResolved.bind(this, true, token, url); + xhr.onerror = onResolved.bind(this, false, token, url); + xhr.responseType = 'text'; + xhr.send(); + n -= 1; + if ( n === 0 ) { break; } + } + if ( toResolve.size !== 0 ) { + timer = vAPI.setTimeout(resolvePending, 5); + } else if ( toProcess.size === 0 ) { + reng.selfieFromResources(); + } + }; + + return function(token) { + if ( vAPI.warSecret !== undefined ) { + toResolve.add(token); + } + if ( timer === undefined ) { + timer = vAPI.setTimeout(resolvePending, 1); + } + }; +})(); + +/******************************************************************************/ +/******************************************************************************/ + var RedirectEntry = function() { this.mime = ''; this.data = ''; + this.warURL = undefined; }; /******************************************************************************/ RedirectEntry.prototype.toURL = function() { + if ( this.warURL !== undefined ) { + return this.warURL + '?secret=' + vAPI.warSecret; + } if ( this.data.startsWith('data:') === false ) { if ( this.mime.indexOf(';') === -1 ) { this.data = 'data:' + this.mime + ';base64,' + btoa(this.data); @@ -75,6 +158,7 @@ RedirectEntry.fromSelfie = function(selfie) { var r = new RedirectEntry(); r.mime = selfie.mime; r.data = selfie.data; + r.warURL = selfie.warURL; return r; }; @@ -340,7 +424,6 @@ RedirectEngine.prototype.toSelfie = function() { } var µb = µBlock; return { - resources: µb.arrayFrom(this.resources), rules: rules, ruleTypes: µb.arrayFrom(this.ruleTypes), ruleSources: µb.arrayFrom(this.ruleSources), @@ -351,22 +434,11 @@ RedirectEngine.prototype.toSelfie = function() { /******************************************************************************/ RedirectEngine.prototype.fromSelfie = function(selfie) { - // Resources. - this.resources = new Map(); - var resources = selfie.resources, - item; - for ( var i = 0, n = resources.length; i < n; i++ ) { - item = resources[i]; - this.resources.set(item[0], RedirectEntry.fromSelfie(item[1])); - } - - // Rules. this.rules = new Map(selfie.rules); this.ruleTypes = new Set(selfie.ruleTypes); this.ruleSources = new Set(selfie.ruleSources); this.ruleDestinations = new Set(selfie.ruleDestinations); this.modifyTime = Date.now(); - return true; }; @@ -428,6 +500,7 @@ RedirectEngine.prototype.resourcesFromString = function(text) { // No more data, add the resource. this.resources.set(fields[0], RedirectEntry.fromFields(fields[1], fields.slice(2))); + warResolve(fields[0]); fields = undefined; } @@ -435,9 +508,52 @@ RedirectEngine.prototype.resourcesFromString = function(text) { // Process pending resource data. if ( fields !== undefined ) { this.resources.set(fields[0], RedirectEntry.fromFields(fields[1], fields.slice(2))); + warResolve(fields[0]); } }; +/******************************************************************************/ + +var resourcesSelfieVersion = 1; + +RedirectEngine.prototype.selfieFromResources = function() { + vAPI.cacheStorage.set({ + resourcesSelfie: { + version: resourcesSelfieVersion, + resources: µBlock.arrayFrom(this.resources) + } + }); +}; + +RedirectEngine.prototype.resourcesFromSelfie = function(callback) { + var me = this; + + var onSelfieReady = function(bin) { + if ( bin instanceof Object === false ) { + return callback(false); + } + var selfie = bin.resourcesSelfie; + if ( + selfie instanceof Object === false || + selfie.version !== resourcesSelfieVersion || + Array.isArray(selfie.resources) === false + ) { + return callback(false); + } + me.resources = new Map(); + for ( var entry of bin.resourcesSelfie.resources ) { + me.resources.set(entry[0], RedirectEntry.fromSelfie(entry[1])); + } + callback(true); + }; + + vAPI.cacheStorage.get('resourcesSelfie', onSelfieReady); +}; + +RedirectEngine.prototype.invalidateResourcesSelfie = function() { + vAPI.cacheStorage.remove('resourcesSelfie'); +}; + /******************************************************************************/ /******************************************************************************/ diff --git a/src/js/start.js b/src/js/start.js index 57f340812..3dc1b947b 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2017 Raymond Hill + Copyright (C) 2014-2018 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -103,18 +103,6 @@ var onPSLReady = function() { // To bring older versions up to date var onVersionReady = function(lastVersion) { - // Starting with 1.9.17, non-advanced users can have access to the dynamic - // filtering pane in read-only mode. Still, it should not be visible by - // default. - if ( lastVersion.localeCompare('1.9.17') < 0 ) { - if ( - µb.userSettings.advancedUserEnabled === false && - µb.userSettings.dynamicFilteringEnabled === true - ) { - µb.userSettings.dynamicFilteringEnabled = false; - µb.keyvalSetOne('dynamicFilteringEnabled', false); - } - } if ( lastVersion !== vAPI.app.version ) { vAPI.storage.set({ version: vAPI.app.version }); } @@ -140,6 +128,8 @@ var onSelfieReady = function(selfie) { µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine); µb.redirectEngine.fromSelfie(selfie.redirectEngine); µb.staticExtFilteringEngine.fromSelfie(selfie.staticExtFilteringEngine); + µb.loadRedirectResources(); + return true; }; diff --git a/src/js/storage.js b/src/js/storage.js index e0968e83f..8e2311775 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2017 Raymond Hill + Copyright (C) 2014-2018 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -795,17 +795,12 @@ /******************************************************************************/ -µBlock.loadRedirectResources = function(callback) { +µBlock.loadRedirectResources = function(updatedContent) { var µb = this, content = ''; - if ( typeof callback !== 'function' ) { - callback = this.noopFunc; - } - var onDone = function() { µb.redirectEngine.resourcesFromString(content); - callback(); }; var onUserResourcesLoaded = function(details) { @@ -825,7 +820,17 @@ µb.assets.fetchText(µb.hiddenSettings.userResourcesLocation, onUserResourcesLoaded); }; - this.assets.get('ublock-resources', onResourcesLoaded); + if ( typeof updatedContent === 'string' && updatedContent.length !== 0 ) { + return onResourcesLoaded({ content: updatedContent }); + } + + var onSelfieReady = function(success) { + if ( success !== true ) { + µb.assets.get('ublock-resources', onResourcesLoaded); + } + }; + + µb.redirectEngine.resourcesFromSelfie(onSelfieReady); }; /******************************************************************************/ @@ -1105,8 +1110,9 @@ this.compilePublicSuffixList(details.content); } } else if ( details.assetKey === 'ublock-resources' ) { + this.redirectEngine.invalidateResourcesSelfie(); if ( cached ) { - this.redirectEngine.resourcesFromString(details.content); + this.loadRedirectResources(details.content); } } vAPI.messaging.broadcast({ diff --git a/src/web_accessible_resources/to-import.txt b/src/web_accessible_resources/to-import.txt new file mode 100644 index 000000000..cbb74df59 --- /dev/null +++ b/src/web_accessible_resources/to-import.txt @@ -0,0 +1,56 @@ +# This is a list of resources (by token) which will be converted to +# "web accessible resources", such that uBO will be able to redirect +# to these through moz-extension: or chrome-extension: URLs. +# +# This addresses: +# - https://github.com/gorhill/uBlock/issues/3474 +# - https://github.com/gorhill/uBlock/issues/2823 +# +# uBO attaches a "secret" token internally when redirecting to any +# "web accessible resource", such that it is not possible for a web +# page to use one of these "web accessible resource" to directly +# detect the presence of uBO. +# +# To ensure valid filename on any platform OS, the resource tokens are +# converted to base64 (https://tools.ietf.org/html/rfc7515#appendix-C), +# and the result is used as filename. +# +# In case uBO redirects to a resource which has not been converted into +# a "web accessible resource", the redirection code will fall back to +# using a data: URI. +# +# The list below was gathered manually from scanning the use of the +# "redirect=" option in uBO's own filter lists. Eventually a script could +# be written to generate the list below. + +1x1-transparent.gif +2x2-transparent.png +32x32-transparent.png +3x2-transparent.png +addthis.com/addthis_widget.js +amazon-adsystem.com/aax2/amzn_ads.js +antiAdBlock.js +d3pkae9owd2lcf.cloudfront.net/mb105.js +disqus.com/embed.js +disqus.com/forums/*/embed.js +doubleclick.net/instream/ad_status.js +fuckadblock.js-3.2.0 +google-analytics.com/analytics.js +google-analytics.com/cx/api.js +google-analytics.com/ga.js +google-analytics.com/inpage_linkid.js +googlesyndication.com/adsbygoogle.js +googletagmanager.com/gtm.js +googletagservices.com/gpt.js +hd-main.js +ligatus.com/*/angular-tag.js +noopframe +noopjs +noopmp3-0.1s +nooptext +popads-dummy.js +popads.net.js +scorecardresearch.com/beacon.js +silent-noeval.js +static.chartbeat.com/chartbeat.js +widgets.outbrain.com/outbrain.js diff --git a/tools/import-war.py b/tools/import-war.py new file mode 100755 index 000000000..baf8339e6 --- /dev/null +++ b/tools/import-war.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +import base64 +import os +import re +import sys + +if len(sys.argv) == 1 or not sys.argv[1]: + raise SystemExit('Build dir missing.') + +# resource_dir = os.path.join(os.path.split(os.path.abspath(__file__))[0], '..') +build_dir = os.path.abspath(sys.argv[1]) + +# Read list of resource tokens to convert +to_import = set() +with open('./src/web_accessible_resources/to-import.txt', 'r') as f: + for line in f: + line = line.strip() + if len(line) != 0 and line[0] != '#': + to_import.add(line) + +# scan the file until a resource to import is found +def find_next_resource(f): + for line in f: + line = line.strip() + if len(line) == 0 or line[0] == '#': + continue + parts = line.partition(' ') + if parts[0] in to_import: + return (parts[0], parts[2].strip()) + return ('', '') + +def safe_filename_from_token(token, mime): + name = str(base64.b64encode(bytes(token, 'utf-8'), b'-_'), 'utf-8').strip('=') + # extract file extension from mime + match = re.search('^[^/]+/([^\s;]+)', mime) + if match: + name += '.' + match.group(1) + return name + +def import_resource(f, token, mime): + isBinary = mime.endswith(';base64') + lines = [] + for line in f: + if line.strip() == '': + break + if line.lstrip()[0] == '#': + continue + if isBinary: + line = line.strip() + lines.append(line) + filename = safe_filename_from_token(token, mime) + filepath = os.path.join(build_dir, 'web_accessible_resources', filename) + filedata = ''.join(lines) + if isBinary: + filedata = base64.b64decode(filedata) + else: + filedata = bytes(filedata, 'utf-8') + with open(filepath, 'wb') as fo: + fo.write(filedata) + +# Read content of the resources to convert +# - At this point, it is assumed resources.txt has been imported into the +# package. +resources_filename = os.path.join(build_dir, 'assets/ublock/resources.txt') +with open(resources_filename, 'r') as f: + while True: + token, mime = find_next_resource(f) + if token == '': + break + import_resource(f, token, mime) + diff --git a/tools/make-chromium.sh b/tools/make-chromium.sh index af36c29d1..2f3f06cab 100755 --- a/tools/make-chromium.sh +++ b/tools/make-chromium.sh @@ -31,7 +31,11 @@ mv /tmp/contentscript.js $DES/js/contentscript.js rm $DES/js/vapi-usercss.js # Chrome store-specific -cp -R $DES/_locales/nb $DES/_locales/no +cp -R $DES/_locales/nb $DES/_locales/no + +echo "*** uBlock0.chromium: Generating web accessible resources..." +cp -R src/web_accessible_resources $DES/ +python3 tools/import-war.py $DES/ echo "*** uBlock0.chromium: Generating meta..." python tools/make-chromium-meta.py $DES/ diff --git a/tools/make-webext-meta.py b/tools/make-webext-meta.py index b74b06e19..f12698d82 100644 --- a/tools/make-webext-meta.py +++ b/tools/make-webext-meta.py @@ -31,6 +31,9 @@ if match: else: builttype = 'rc' + str(buildtype - 100) webext_manifest['version'] = match.group(1) + builttype + gecko = webext_manifest['applications']['gecko'] + gecko['id'] = gecko['id'].replace('uBlock0', 'uBlock0.dev-build') + webext_manifest['name'] = webext_manifest['name'] + ' dev build' else: webext_manifest['version'] = chromium_manifest['version'] diff --git a/tools/make-webext.sh b/tools/make-webext.sh index 930dd2184..37a23b3a9 100755 --- a/tools/make-webext.sh +++ b/tools/make-webext.sh @@ -42,6 +42,10 @@ rm $DES/img/icon_128.png rm $DES/options_ui.html rm $DES/js/options_ui.js +echo "*** uBlock0.chromium: Generating web accessible resources..." +cp -R src/web_accessible_resources $DES/ +python3 tools/import-war.py $DES/ + echo "*** uBlock0.webext: Generating meta..." python tools/make-webext-meta.py $DES/