patch for an issue where it would fail again if the encoding is still messed up, also patch for an issue with the protocol, will now auto assign http if it isn't present (issue #225 and issue #226)

etc/checksum/md5sum.md5

@@ -62,7 +62,7 @@ bbd8b4c6100070d420d48dc7dfc297eb  ./lib/firewall/webknight.py
 81a29a14d72980a306fbaec0dc772048  ./lib/firewall/fortigate.py
 6ea65a0160c21e144e92334acc2e3667  ./lib/firewall/anquanbao.py
 22a0ad8f2fa1a16b651cb5ae37ca9b0d  ./lib/firewall/generic.py
-c3aa1db3ee34b841d91ad850c2cfd7b2  ./lib/attacks/gist_lookup/__init__.py
+b5ff3286060c0bbc0fe1f0f591131c9c  ./lib/attacks/gist_lookup/__init__.py
 86224bd899c2a2438042cbdc077dc4cc  ./lib/attacks/clickjacking_scan/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/attacks/__init__.py
 4c644b0e3a62b6c1528d34a04837aa35  ./lib/attacks/sqlmap_scan/__init__.py
@@ -70,7 +70,7 @@ d41d8cd98f00b204e9800998ecf8427e  ./lib/attacks/__init__.py
 7bc7a6f3e85651aab3588f087563eded  ./lib/attacks/whois_lookup/__init__.py
 8ec72bb33df998e32b925e3060d9d17a  ./lib/attacks/whois_lookup/whois.py
 f65411ebaf58392e6e2b617dba99ff90  ./lib/attacks/admin_panel_finder/__init__.py
-0e3fbd55788c040bb96faa1ef0cf1394  ./lib/attacks/xss_scan/__init__.py
+9353f5d6188963dc5c2d5cee69439688  ./lib/attacks/xss_scan/__init__.py
 7642d078fe304a7ca8bfaaa070a0cb31  ./lib/attacks/nmap_scan/__init__.py
 216999fa0e84866d5c1d96d5676034e4  ./lib/attacks/nmap_scan/nmap_opts.py
 ac942d36f7d78c249e587417736e88e6  ./lib/header_check/__init__.py
@@ -78,7 +78,7 @@ cd8e35cfd995d0a93892cfc83f01dea7  ./lib/core/common.py
 4433353fb5c55578391d8b4006191ee8  ./lib/core/errors.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/core/__init__.py
 e5cba9d37cf54d14255a81da1fa83f90  ./lib/core/settings.py
-801a4f7ac892b74676c649bd4844ccdb  ./lib/core/decorators.py
+de4254c5e40f7aa4fb81e0608f758a2c  ./lib/core/decorators.py
 9a02e5b913d210350545ac26510a63c9  ./var/search/__init__.py
 1ed3c450e620ff1edd8b0864179fdea7  ./var/search/selenium_search.py
 cfcce04aac694eee7a6c73969861ce43  ./var/search/pgp_search.py

lib/attacks/gist_lookup/__init__.py

@@ -1,9 +1,10 @@
 import re
+import sys
 
 from bs4 import BeautifulSoup
 
-import lib.core.settings
 import lib.core.common
+import lib.core.settings
 
 
 def __create_url(redirect, template="https://gist.github.com{}"):
@@ -20,18 +21,22 @@ def get_raw_html(redirect, verbose=False):
     tag, descriptor = "a", "href"
     raw_gist_regex = re.compile(r".raw.[a-z0-9]{40}", re.I)
     _, status, html, _ = lib.core.common.get_page(redirect)
-    soup = BeautifulSoup(html, "html.parser")
-    for link in soup.findAll(tag):
-        raw_gist_redirect = link.get(descriptor)
-        if raw_gist_regex.search(str(raw_gist_redirect)) is not None:
-            url = __create_url(raw_gist_redirect)
-            if verbose:
-                lib.core.settings.logger.debug(lib.core.settings.set_color(
-                    "found raw Gist URL '{}'...".format(url), level=10
-                ))
-            _, _, html, _ = lib.core.common.get_page(url)
-            raw_soup = BeautifulSoup(html, "html.parser")
-            return raw_soup, url
+
+    if status == 200:
+        soup = BeautifulSoup(html, "html.parser")
+        for link in soup.findAll(tag):
+            raw_gist_redirect = link.get(descriptor)
+            if raw_gist_regex.search(str(raw_gist_redirect)) is not None:
+                url = __create_url(raw_gist_redirect)
+                if verbose:
+                    lib.core.settings.logger.debug(lib.core.settings.set_color(
+                        "found raw Gist URL '{}'...".format(url), level=10
+                    ))
+                _, _, html, _ = lib.core.common.get_page(url)
+                raw_soup = BeautifulSoup(html, "html.parser")
+                return raw_soup, url
+    else:
+        return None, None
 
 
 def get_links(page_set, proxy=None, agent=None):
@@ -43,6 +48,7 @@ def get_links(page_set, proxy=None, agent=None):
     tag, descriptor = "a", "href"
     gist_regex = re.compile(r"[a-f0-9]{32}", re.I)
     gist_skip_schema = ("stargazers", "forks", "#comments")
+
     for i in range(page_set):
         lib.core.settings.logger.info(lib.core.settings.set_color(
             "fetching all Gists on page #{}...".format(i+1)
@@ -50,19 +56,27 @@ def get_links(page_set, proxy=None, agent=None):
         _, status, html, _ = lib.core.common.get_page(
             gist_search_url.format(i+1), proxy=proxy, agent=agent
         )
-        soup = BeautifulSoup(html, "html.parser")
-        for link in soup.findAll(tag):
-            redirect = link.get(descriptor)
-            if not any(s in redirect for s in gist_skip_schema):
-                if gist_regex.search(redirect) is not None:
-                    if not any(protocol in redirect for protocol in ["https://", "http://"]):
-                        redirects.add(__create_url(redirect))
-                    else:
-                        redirects.add(redirect)
+        if status == 200:
+            soup = BeautifulSoup(html, "html.parser")
+            for link in soup.findAll(tag):
+                redirect = link.get(descriptor)
+                if not any(s in redirect for s in gist_skip_schema):
+                    if gist_regex.search(redirect) is not None:
+                        if not any(protocol in redirect for protocol in ["https://", "http://"]):
+                            redirects.add(__create_url(redirect))
+                        else:
+                            redirects.add(redirect)
+        else:
+            lib.core.settings.logger.warning(lib.core.settings.set_color(
+                "page #{} failed to load with status code {} (reason '{}')...".format(
+                    i+1, status, lib.core.common.STATUS_CODES[int(status)]
+                ), level=30
+            ))
+            continue
     return redirects
 
 
-def check_files_for_information(url, data_to_search, query):
+def check_files_for_information(data_to_search, query):
     """
     check the files to see if they contain any of the information that was specified
     """
@@ -99,6 +113,7 @@ def check_files_for_information(url, data_to_search, query):
             )
 
 
+# @lib.core.decorators.tail_call_optimized
 def github_gist_search_main(query, **kwargs):
     """
     main function for searching Gists
@@ -108,6 +123,11 @@ def github_gist_search_main(query, **kwargs):
     verbose = kwargs.get("verbose", False)
     page_set = kwargs.get("page_set", 10)
 
+    # there seems to be a recursion issue in this function,
+    # so until I get this figured out, we're going to change
+    # the maximum recursion of the system when we get here
+    sys.setrecursionlimit(1500)
+
     try:
         lib.core.settings.logger.info(lib.core.settings.set_color(
             "searching a total of {} pages of Gists for '{}'...".format(
@@ -126,8 +146,9 @@ def github_gist_search_main(query, **kwargs):
                 ), level=15
             ))
         for link in list(links):
-            gist, gist_link = get_raw_html(link, verbose=verbose)
-            check_files_for_information(gist_link, gist, query)
+            if link is not None:
+                gist, gist_link = get_raw_html(link, verbose=verbose)
+                check_files_for_information(gist, query)
     except KeyboardInterrupt:
         if not lib.core.common.pause():
             lib.core.common.shutdown()

lib/attacks/xss_scan/__init__.py

@@ -11,6 +11,7 @@ import requests
 
 import lib.core.common
 import lib.core.settings
+import lib.core.decorators
 from lib.core.errors import InvalidTamperProvided
 
 
@@ -109,22 +110,34 @@ def scan_xss(url, agent=None, proxy=None):
     be tampered or encoded if the site is not vulnerable
     """
 
-    try:
-        _, status, html_data, _ = lib.core.common.get_page(url, agent=agent, proxy=proxy)
-    except requests.exceptions.ChunkedEncodingError:
-        lib.core.settings.logger.warning(lib.core.settings.set_color(
-            "encoding seems to be messed up, trying the request again...", level=30
-        ))
-        _, status, html_data, _ = lib.core.common.get_page(url, agent=agent, proxy=proxy)
+    retry_flags = 3
+    auto_assign = "http://{}"
+    url_verification = re.compile(r"http(s)?", re.I)
+
+    if url_verification.search(url) is None:
+        lib.core.settings.logger.warning(lib.core.settings.set_color(
+            "protocol missing from URL, automatically assigning protocol...", level=30
+        ))
+        url = auto_assign.format(url)
+
+    while retry_flags > 0:
+        try:
+            _, status, html_data, _ = lib.core.common.get_page(url, agent=agent, proxy=proxy)
+            query = find_xss_script(url)
+            for db in lib.core.settings.DBMS_ERRORS.keys():
+                for item in lib.core.settings.DBMS_ERRORS[db]:
+                    if re.findall(item, html_data):
+                        return "sqli", db
+            if status != 404:
+                if query in html_data:
+                    return True, None
+            retry_flags -= 1
+        except requests.exceptions.ChunkedEncodingError:
+            lib.core.settings.logger.warning(lib.core.settings.set_color(
+                "encoding seems to be messed up, retrying request...", level=30
+            ))
+            retry_flags -= 1
 
-    query = find_xss_script(url)
-    for db in lib.core.settings.DBMS_ERRORS.keys():
-        for item in lib.core.settings.DBMS_ERRORS[db]:
-            if re.findall(item, html_data):
-                return "sqli", db
-    if status != 404:
-        if query in html_data:
-            return True, None
     return False, None
 
 

lib/core/decorators.py

@@ -6,6 +6,7 @@ import lib.core.settings
 
 
 class TimeOut:
+
     def __init__(self, seconds=1, error_message='Timeout'):
         self.seconds = seconds
         self.error_message = error_message