created a get_page function that does the requests for me, this way I'm not calling requests.get over and over again

etc/checksum/md5sum.md5

@@ -62,22 +62,22 @@ bbd8b4c6100070d420d48dc7dfc297eb  ./lib/firewall/webknight.py
 81a29a14d72980a306fbaec0dc772048  ./lib/firewall/fortigate.py
 6ea65a0160c21e144e92334acc2e3667  ./lib/firewall/anquanbao.py
 22a0ad8f2fa1a16b651cb5ae37ca9b0d  ./lib/firewall/generic.py
-ddf45537bdcea15d3b428495985bad09  ./lib/attacks/gist_lookup/__init__.py
+cf23d0f522b183bea9dc0fcf4aa64115  ./lib/attacks/gist_lookup/__init__.py
-7183dbd7106ecb436176cebcca4e499f  ./lib/attacks/clickjacking_scan/__init__.py
+86224bd899c2a2438042cbdc077dc4cc  ./lib/attacks/clickjacking_scan/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/attacks/__init__.py
 4c644b0e3a62b6c1528d34a04837aa35  ./lib/attacks/sqlmap_scan/__init__.py
 5e5bb575014ebe613db6bf671d008cf8  ./lib/attacks/sqlmap_scan/sqlmap_opts.py
 7bc7a6f3e85651aab3588f087563eded  ./lib/attacks/whois_lookup/__init__.py
 8ec72bb33df998e32b925e3060d9d17a  ./lib/attacks/whois_lookup/whois.py
-67ac65be1b4394ad80ec0fd3472c5309  ./lib/attacks/admin_panel_finder/__init__.py
+8c6f696f6a983e6309d7d09e409d97e6  ./lib/attacks/admin_panel_finder/__init__.py
-acd30079085bd6be6e24fef80bdc669c  ./lib/attacks/xss_scan/__init__.py
+48ed675099dc8549a7d80bd2d9da7d17  ./lib/attacks/xss_scan/__init__.py
 7642d078fe304a7ca8bfaaa070a0cb31  ./lib/attacks/nmap_scan/__init__.py
 216999fa0e84866d5c1d96d5676034e4  ./lib/attacks/nmap_scan/nmap_opts.py
 888686098a0850750f2435d0e1645944  ./lib/header_check/__init__.py
-0721c655d433dbc170756f3e40a786cf  ./lib/core/common.py
+1c0535bf9fc6138d317bd1c31a2d7fe1  ./lib/core/common.py
 4433353fb5c55578391d8b4006191ee8  ./lib/core/errors.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/core/__init__.py
-6057ea051ca9ed35fc5aa5811cdfca3d  ./lib/core/settings.py
+c31e5ed269fe4ef888bf7eb4425539e4  ./lib/core/settings.py
 801a4f7ac892b74676c649bd4844ccdb  ./lib/core/decorators.py
 9a02e5b913d210350545ac26510a63c9  ./var/search/__init__.py
 0545ee54ade186681b25d157fb32f350  ./var/search/selenium_search.py
@@ -85,4 +85,4 @@ cfcce04aac694eee7a6c73969861ce43  ./var/search/pgp_search.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/auto_issue/__init__.py
 0c11c16126baf789388a661bbbefb149  ./var/auto_issue/github.py
-7608a7a8195d6d66ebae7664f25639fd  ./var/blackwidow/__init__.py
+222102e56b3f4b9facda7ed28690d3f7  ./var/blackwidow/__init__.py

lib/attacks/admin_panel_finder/__init__.py

@@ -7,7 +7,6 @@ try:                 # Python 2
 except ImportError:  # Python 3
     from urllib2 import urlopen, HTTPError
 
-import requests
 from requests.exceptions import (
     ConnectionError,
     TooManyRedirects
@@ -41,9 +40,7 @@ def check_for_externals(url, data_sep="-" * 30, **kwargs):
     try:
         url = lib.core.settings.replace_http(url)
         full_url = "{}{}{}".format("http://", url, currently_searching)
-        conn = requests.get(full_url)
+        _, data, code, _ = lib.core.common.get_page(full_url)
-        data = conn.content
-        code = conn.status_code
     except (TooManyRedirects, ConnectionError):
         lib.core.settings.logger.error(lib.core.settings.set_color(
             "connection to '{}' failed, assuming does not exist and continuing...".format(full_url), level=40

lib/attacks/clickjacking_scan/__init__.py

@@ -1,5 +1,3 @@
-import requests
-
 import lib.core.common
 import lib.core.settings
 import var.auto_issue.github
@@ -41,7 +39,7 @@ class ClickJackingScanner(object):
                 lib.core.common.HTTP_HEADER.USER_AGENT: agent,
                 lib.core.common.HTTP_HEADER.CONNECTION: "close"
             }
-        req = requests.get(self.url, headers=headers, proxies=lib.core.settings.proxy_string_to_dict(proxy))
+        req, _, _, headers = lib.core.common.get_page(self.url, headers=headers, proxy=proxy)
         headers = req.headers
         if self.safe in headers:
             return False

lib/attacks/gist_lookup/__init__.py

@@ -12,10 +12,10 @@ def __check_remaining_rate_limit():
     check how many requests you have left to run
     """
     url = lib.core.settings.GITHUB_GIST_SEARCH_URLS["check_rate"]
-    data = requests.get(url, params={lib.core.common.HTTP_HEADER.AUTHORIZATION: "token {}".format(
+    _, _, data, headers = lib.core.common.get_page(url, auth="token {}".format(
         lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)
-    )})
+    ))
-    remaining = data.headers["X-RateLimit-Remaining"]
+    remaining = headers["X-RateLimit-Remaining"]
     if int(remaining) == 0:
         lib.core.settings.logger.error(lib.core.settings.set_color(
             "Github only allows 60 unauthenticated requests per hour, you have hit that limit "
@@ -45,7 +45,7 @@ def get_raw_data(page_set, proxy=None, agent=None, verbose=False):
     if proxy is not None:
         proxy = lib.core.settings.proxy_string_to_dict(proxy)
     for page in list(page_set):
-        data = requests.get(url.format(page), params=headers, proxies=proxy)
+        _, _, data, _ = lib.core.common.get_page(url.format(page), agent=agent, proxy=proxy)
         # load the found info into JSON format
         # so we can pull using keys
         data = json.loads(data.content)
@@ -94,13 +94,13 @@ def check_files_for_information(found_url, data_to_search):
     )
     total_found = set()
     try:
-        data = requests.get(found_url)
+        _, _, data, _ = lib.core.common.get_page(found_url)
     except requests.exceptions.ConnectionError:
         lib.core.settings.logger.warning(lib.core.settings.set_color(
             "to many requests are being sent to quickly, adding sleep time...", level=30
         ))
         time.sleep(3)
-        data = requests.get(found_url)
+        _, _, data, _ = lib.core.common.get_page(found_url)
     for data_regex in data_regex_schema:
         if data_regex.search(data.content) is not None:
             lib.core.settings.logger.info(lib.core.settings.set_color(

lib/attacks/xss_scan/__init__.py

@@ -108,23 +108,15 @@ def scan_xss(url, agent=None, proxy=None):
     chance that the URL is vulnerable to XSS attacks. Usually what will happen is the payload will
     be tampered or encoded if the site is not vulnerable
     """
-    user_agent = agent or lib.core.settings.DEFAULT_USER_AGENT
-    config_proxy = lib.core.settings.proxy_string_to_dict(proxy)
-    config_headers = {
-        lib.core.common.HTTP_HEADER.CONNECTION: "close",
-        lib.core.common.HTTP_HEADER.USER_AGENT: user_agent
-    }
 
     try:
-        xss_request = requests.get(url, proxies=config_proxy, headers=config_headers)
+        _, status, html_data, _ = lib.core.common.get_page(url, agent=agent, proxy=proxy)
     except requests.exceptions.ChunkedEncodingError:
         lib.core.settings.logger.warning(lib.core.settings.set_color(
             "encoding seems to be messed up, trying the request again...", level=30
         ))
-        xss_request = requests.get(url, proxies=config_proxy, headers=config_headers)
+        _, status, html_data, _ = lib.core.common.get_page(url, agent=agent, proxy=proxy)
 
-    status = xss_request.status_code
-    html_data = xss_request.content
     query = find_xss_script(url)
     for db in lib.core.settings.DBMS_ERRORS.keys():
         for item in lib.core.settings.DBMS_ERRORS[db]:
@@ -149,10 +141,9 @@ def main_xss(start_url, proxy=None, agent=None, **kwargs):
         "candidate to perform XSS tests on, would you like to continue anyways"
     )
     if not batch:
-        if not lib.core.settings.URL_QUERY_REGEX.match(start_url):
+        question = lib.core.common.prompt(
-            question = lib.core.common.prompt(
+            question_msg, opts="yN"
-                question_msg, opts="yN"
+        ) if not lib.core.settings.URL_QUERY_REGEX.match(start_url) else "y"
-            )
     else:
         question = lib.core.common.prompt(
             question_msg, opts="yN", default="y"

lib/core/common.py

@@ -13,6 +13,7 @@ except ImportError:
         unquote
     )
 
+import requests
 from lxml import etree
 
 import lib.core.settings
@@ -281,3 +282,55 @@ def run_fix(message, command, fail_message, exit_process=False):
         lib.core.settings.logger.fatal(lib.core.settings.set_color(
             fail_message, level=50
         ))
+
+
+def get_page(url, **kwargs):
+    agent = kwargs.get("agent", None)
+    proxy = kwargs.get("proxy", None)
+    xforward = kwargs.get("xforward", False)
+    auth = kwargs.get("auth", None)
+    skip_verf = kwargs.get("skip_verf", False)
+
+    if agent is None:
+        agent = lib.core.settings.DEFAULT_USER_AGENT
+
+    if xforward:
+        ip_list = (
+            lib.core.settings.create_random_ip(),
+            lib.core.settings.create_random_ip(),
+            lib.core.settings.create_random_ip()
+        )
+        headers = {
+            HTTP_HEADER.CONNECTION: "close",
+            HTTP_HEADER.USER_AGENT: agent,
+            HTTP_HEADER.X_FORWARDED_FOR: "{}, {}, {}".format(
+                ip_list[0], ip_list[1], ip_list[2]
+            )
+        }
+    elif auth:
+        headers = {
+            HTTP_HEADER.CONNECTION: "close",
+            HTTP_HEADER.USER_AGENT: agent,
+            HTTP_HEADER.AUTHORIZATION: "{}".format(
+                auth
+            )
+        }
+    else:
+        headers = {
+            HTTP_HEADER.CONNECTION: "close",
+            HTTP_HEADER.USER_AGENT: agent
+        }
+
+    if proxy is not None:
+        proxies = {
+            "https": proxy,
+            "http": proxy
+        }
+    else:
+        proxies = {}
+
+    req = requests.get(url, params=headers, proxies=proxies, verify=False if skip_verf else True)
+    status = req.status_code
+    html = req.content
+    headers = req.headers
+    return req, status, html, headers

lib/core/settings.py

@@ -46,7 +46,7 @@ CLONE = "https://github.com/ekultek/zeus-scanner.git"
 ISSUE_LINK = "https://github.com/ekultek/zeus-scanner/issues"
 
 # current version <major.minor.commit.patch ID>
-VERSION = "1.3.14.{}".format(PATCH_ID)
+VERSION = "1.3.15".format(PATCH_ID)
 
 # colors to output depending on the version
 VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30}

var/blackwidow/__init__.py

@@ -3,7 +3,6 @@ import sys
 reload(sys)
 sys.setdefaultencoding("utf-8")  # this will take care of most of the Unicode errors.
 
-import requests
 from bs4 import BeautifulSoup
 
 import lib.core.errors
@@ -23,17 +22,6 @@ class Blackwidow(object):
         self.forward = forward or None
         self.proxy = lib.core.settings.proxy_string_to_dict(proxy) or None
         self.user_agent = user_agent or lib.core.settings.DEFAULT_USER_AGENT
-        if self.forward is not None:
-            self.headers = {
-                "user-agent": self.user_agent,
-                "X-Forwarded-For": "{}, {}, {}".format(
-                    self.forward[0], self.forward[1], self.forward[2]
-                )
-            }
-        else:
-            self.headers = {
-                "user-agent": self.user_agent
-            }
 
     @staticmethod
     def get_url_ext(url):
@@ -52,10 +40,13 @@ class Blackwidow(object):
         """
         try:
             # verify=False will take care of SSLErrors
-            attempt = requests.get(self.url, params=self.headers, proxies=self.proxy, verify=False)
+            attempt, status, _, _ = lib.core.common.get_page(
-            if attempt.status_code == 200:
+                self.url, agent=self.user_agent, xforward=self.forward, skip_verf=True,
-                return ("ok", None)
+                proxy=self.proxy
-            return ("fail", attempt.status_code)
+            )
+            if status == 200:
+                return "ok", None
+            return "fail", attempt.status_code
         except Exception as e:
             if "Max retries exceeded with url" in str(e):
                 info_msg = ""
@@ -84,8 +75,9 @@ class Blackwidow(object):
         """
         unique_links = set()
         true_url = lib.core.settings.replace_http(given_url)
-        req = requests.get(given_url, params=self.headers, proxies=self.proxy)
+        _, status, html_page, _ = lib.core.common.get_page(
-        html_page = req.content
+            given_url, agent=self.user_agent, proxy=self.proxy
+        )
         soup = BeautifulSoup(html_page, "html.parser")
         for link in soup.findAll(attribute):
             found_redirect = str(link.get(descriptor)).decode("unicode_escape")