From 6a8c27b6dfb81faa37a878a58b5241feaeea5747 Mon Sep 17 00:00:00 2001
From: Raymond Hill <rhill@raymondhill.net>
Date: Fri, 15 Dec 2017 07:39:21 -0500
Subject: [PATCH] fix #3331: ability to fetch sublists using `!# include`
 directives

---
 src/js/assets.js   | 78 ++++++++++++++++++++++++++++++++++++++++++++--
 src/js/uritools.js |  2 +-
 2 files changed, 77 insertions(+), 3 deletions(-)

diff --git a/src/js/assets.js b/src/js/assets.js
index 5ee4a92c5..c2fb47ff7 100644
--- a/src/js/assets.js
+++ b/src/js/assets.js
@@ -167,6 +167,72 @@ api.fetchText = function(url, onLoad, onError) {
     }
 };
 
+/******************************************************************************/
+
+// https://github.com/gorhill/uBlock/issues/3331
+//   Support the seamless loading of sublists.
+
+api.fetchFilterList = function(mainlistURL, onLoad, onError) {
+    var µburi = µBlock.URI,
+        content = [],
+        errored = false,
+        pendingSublistURLs = new Set([ mainlistURL ]),
+        loadedSublistURLs = new Set(),
+        mainOriginURL = µburi.originFromURI(mainlistURL);
+
+    var onLocalLoadSuccess = function(details) {
+        if ( errored ) { return; }
+
+        var isSublist = details.url !== mainlistURL,
+            sublistURL;
+
+        pendingSublistURLs.delete(details.url);
+        loadedSublistURLs.add(details.url);
+        if ( isSublist ) { content.push('\n! ' + '>>>>>>>> ' + details.url); }
+        content.push(details.content.trim());
+        if ( isSublist ) { content.push('! <<<<<<<< ' + details.url); }
+
+        if ( mainOriginURL !== '' ) {
+            var subOriginURL,
+                reInclude = /^!# include (\S+)/gm,
+                match = reInclude.exec(details.content);
+            while ( match !== null ) {
+                sublistURL = match[1];
+                subOriginURL = µburi.originFromURI(sublistURL);
+                if ( subOriginURL !== '' && subOriginURL !== mainOriginURL ) {
+                    continue;
+                }
+                if ( subOriginURL === '' ) {
+                    sublistURL = mainOriginURL + '/' + sublistURL;
+                }
+                if ( loadedSublistURLs.has(sublistURL) ) { continue; }
+                pendingSublistURLs.add(sublistURL);
+                match = reInclude.exec(details.content);
+            }
+        }
+
+        if ( pendingSublistURLs.size !== 0 ) {
+            for ( sublistURL of pendingSublistURLs ) {
+                api.fetchText(sublistURL, onLocalLoadSuccess, onLocalLoadError);
+            }
+            return;
+        }
+
+        details.url = mainlistURL;
+        details.content = content.join('\n').trim();
+        onLoad(details);
+    };
+
+    var onLocalLoadError = function(details) {
+        errored = true;
+        details.url = mainlistURL;
+        details.content = '';
+        onError(details);
+    };
+
+    this.fetchText(mainlistURL, onLocalLoadSuccess, onLocalLoadError);
+};
+
 /*******************************************************************************
 
     The purpose of the asset source registry is to keep key detail information
@@ -651,7 +717,11 @@ api.get = function(assetKey, options, callback) {
         if ( !contentURL ) {
             return reportBack('', 'E_NOTFOUND');
         }
-        api.fetchText(contentURL, onContentLoaded, onContentNotLoaded);
+        if ( assetDetails.content === 'filters' ) {
+            api.fetchFilterList(contentURL, onContentLoaded, onContentNotLoaded);
+        } else {
+            api.fetchText(contentURL, onContentLoaded, onContentNotLoaded);
+        }
     };
 
     var onContentLoaded = function(details) {
@@ -735,7 +805,11 @@ var getRemote = function(assetKey, callback) {
         if ( !contentURL ) {
             return reportBack('', 'E_NOTFOUND');
         }
-        api.fetchText(contentURL, onRemoteContentLoaded, onRemoteContentError);
+        if ( assetDetails.content === 'filters' ) {
+            api.fetchFilterList(contentURL, onRemoteContentLoaded, onRemoteContentError);
+        } else {
+            api.fetchText(contentURL, onRemoteContentLoaded, onRemoteContentError);
+        }
     };
 
     getAssetSourceRegistry(function(registry) {
diff --git a/src/js/uritools.js b/src/js/uritools.js
index f624d891e..eaca7731d 100644
--- a/src/js/uritools.js
+++ b/src/js/uritools.js
@@ -51,7 +51,7 @@ var reRFC3986 = /^([^:\/?#]+:)?(\/\/[^\/?#]*)?([^?#]*)(\?[^#]*)?(#.*)?/;
 // Derived
 var reSchemeFromURI          = /^[^:\/?#]+:/;
 var reAuthorityFromURI       = /^(?:[^:\/?#]+:)?(\/\/[^\/?#]+)/;
-var reOriginFromURI          = /^(?:[^:\/?#]+:)?(?:\/\/[^\/?#]+)/;
+var reOriginFromURI          = /^(?:[^:\/?#]+:)\/\/(?:[^\/?#]+)?/;
 var reCommonHostnameFromURL  = /^https?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])\//;
 var rePathFromURI            = /^(?:[^:\/?#]+:)?(?:\/\/[^\/?#]*)?([^?#]*)/;
 var reMustNormalizeHostname  = /[^0-9a-z._-]/;