created a common.py file and drafted write_to_log_file into it, also created an HTTP_HEADER class that will be used instead of repeating myself, fixed everything accordingly

etc/checksum/md5sum.md5

@@ -23,7 +23,7 @@ ca6935a72fd0527d15a78a17a35e56e8  ./bin/drivers/geckodriver-v0.19.0-linux64.tar.
 4ccb56fb3700005c9f9188f84152f21a  ./bin/drivers/geckodriver-v0.18.0-linux64.tar.gz
 07cd383c8aef8ea5ef194a506141afd6  ./bin/drivers/geckodriver-v0.19.0-linux32.tar.gz
 6ea65a0160c21e144e92334acc2e3667  ./lib/firewall/anquanbao.py
-1ebfc01f84201fb723ff727957d18b26  ./lib/firewall/cloudflare.py
+627fd88a80a9ee049e08b113a232f065  ./lib/firewall/cloudflare.py
 54815706261c32b57fbbdc99244b5cdd  ./lib/firewall/modsecurity.py
 6b370050b40d8c1d2221424f756c7842  ./lib/firewall/paloalto.py
 24342e7de0c51595d593cef74d80d3a0  ./lib/firewall/sucuri.py
@@ -52,23 +52,24 @@ a0fedc86cfb4a370e6c1a606010812ed  ./lib/tamper_scripts/space2null_encode.pyc
 694231b4c2f99406481c34ced85ddfe1  ./lib/tamper_scripts/base64_encode.py
 6ac38bec8c32eab57efa01f7a06dff14  ./lib/tamper_scripts/space2null_encode.py
 3b8c95a6a3b7cecce5118f2fb1ccc6b8  ./lib/tamper_scripts/appendnull_encode.py
-3308a53435cd255107a9301723844d6e  ./lib/attacks/clickjacking_scan/__init__.py
+a39c2376b66ff3a2fa239c97a69e891f  ./lib/attacks/clickjacking_scan/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/attacks/__init__.py
 7272a7fd0b0c2e9192bc4adb6154d2f0  ./lib/attacks/sqlmap_scan/__init__.py
 aa7268a8f085734a6c577c86440f7a1b  ./lib/attacks/sqlmap_scan/sqlmap_opts.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/attacks/whois_lookup/__init__.py
-28bb20770353ed5684615822e2048811  ./lib/attacks/whois_lookup/whois.py
+ef5dc31f38028aa0d73258b36f0e36f9  ./lib/attacks/whois_lookup/whois.py
-6b9ba948ca5ba51ef3d8423e738c41ba  ./lib/attacks/admin_panel_finder/__init__.py
+4c8cc6050796510f242539267292cfb2  ./lib/attacks/admin_panel_finder/__init__.py
-ceb1b278b0861c976dfecc91cb64e53d  ./lib/attacks/xss_scan/__init__.py
+edb971a085a2c3be9277def5a0f14d49  ./lib/attacks/xss_scan/__init__.py
 27358f26bda30d7356143c3ea1fa99c5  ./lib/attacks/nmap_scan/__init__.py
 21faf4679cdeaa731029a48f8963d6e7  ./lib/attacks/nmap_scan/nmap_opts.py
-94fb7c32f7db112f14a14297311d0aa3  ./lib/attacks/gist_lookup/__init__.py
+7eff376f860e72ff32094cd96d86a6a2  ./lib/attacks/gist_lookup/__init__.py
 1faa2b5dfad6eb538bbfe42942d2a9da  ./lib/core/errors.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/core/__init__.py
-f14e59a7d0b25368fc6ef7efb7793d0b  ./lib/core/settings.py
+c631f2b8aec464b0a771f71bb6d1d137  ./lib/core/settings.py
-f8dca2fa45acb95f7081546bf6aec025  ./lib/header_check/__init__.py
+aa541bfde844b009227f664749a39ea8  ./lib/core/common.py
+7198de0a126cca4b06c31b205e2a5e23  ./lib/header_check/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/google_search/__init__.py
-a53336c939aa2756b0cf35563880bdd8  ./var/google_search/search.py
+180a9deb0c4937bc22ec1d18c7b540d5  ./var/google_search/search.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/auto_issue/__init__.py
 4c7c008b28eaac5afb57a4bea46f1b19  ./var/auto_issue/github.py

lib/attacks/admin_panel_finder/__init__.py

@@ -9,6 +9,7 @@ except ImportError:  # Python 3
 
 import requests
 
+import lib.core.common
 import lib.core.settings
 from var.auto_issue.github import request_issue_creation
 
@@ -71,12 +72,12 @@ def check_for_externals(url, data_sep="-" * 30, **kwargs):
         lib.core.settings.logger.info(lib.core.settings.set_color(
             "robots.txt page will be saved into a file...", level=25
         ))
-        return lib.core.settings.write_to_log_file(data, lib.core.settings.ROBOTS_PAGE_PATH, "{}-robots_text.log".format(url))
+        return lib.core.common.write_to_log_file(data, lib.core.settings.ROBOTS_PAGE_PATH, "{}-robots_text.log".format(url))
     elif sitemap:
         lib.core.settings.logger.info(lib.core.settings.set_color(
             "found a sitemap, saving to file...", level=25
         ))
-        return lib.core.settings.write_to_log_file(
+        return lib.core.common.write_to_log_file(
             data, lib.core.settings.SITEMAP_FILE_LOG_PATH, "{}-sitemap.xml".format(
                                                        lib.core.settings.replace_http(url)
             )
@@ -166,7 +167,7 @@ def check_for_admin_page(url, exts, protocol="http://", **kwargs):
         "only writing successful connections to log file...", level=30
     ))
     if len(connections) > 0:
-        lib.core.settings.write_to_log_file(list(connections), lib.core.settings.ADMIN_PAGE_FILE_PATH, "{}-admin-page.log".format(
+        lib.core.common.write_to_log_file(list(connections), lib.core.settings.ADMIN_PAGE_FILE_PATH, "{}-admin-page.log".format(
             lib.core.settings.replace_http(url)
         ))
 

lib/attacks/clickjacking_scan/__init__.py

@@ -1,5 +1,6 @@
 import requests
 
+import lib.core.common
 import lib.core.settings
 import var.auto_issue.github
 
@@ -8,7 +9,7 @@ class ClickJackingScanner(object):
 
     def __init__(self, url):
         self.url = url
-        self.safe = "X-Frame-Options"
+        self.safe = lib.core.common.HTTP_HEADER.X_FRAME_OPT
         self.html = open(lib.core.settings.CLICKJACKING_TEST_PAGE_PATH).read()
 
     def generate_html(self):
@@ -29,16 +30,16 @@ class ClickJackingScanner(object):
         if forward is not None:
             ip_addrs = lib.core.settings.create_random_ip()
             headers = {
-                "user-agent": agent,
+                lib.core.common.HTTP_HEADER.USER_AGENT: agent,
-                "X-Forwarded-From": "{}, {}, {}".format(
+                lib.core.common.HTTP_HEADER.X_FORWARDED_FROM: "{}, {}, {}".format(
                     ip_addrs[0], ip_addrs[1], ip_addrs[2]
                 ),
-                "Connection": "close"
+                lib.core.common.HTTP_HEADER.CONNECTION: "close"
             }
         else:
             headers = {
-                "user-agent": agent,
+                lib.core.common.HTTP_HEADER.USER_AGENT: agent,
-                "Connection": "close"
+                lib.core.common.HTTP_HEADER.CONNECTION: "close"
             }
         req = requests.get(self.url, headers=headers, proxies=lib.core.settings.proxy_string_to_dict(proxy))
         headers = req.headers
@@ -93,7 +94,7 @@ def clickjacking_main(url, **kwargs):
                 "it appears that provided URL '{}' is vulnerable to clickjacking, writing "
                 "to HTML file...".format(url), level=25
             ))
-            lib.core.settings.write_to_log_file(
+            lib.core.common.write_to_log_file(
                 data,
                 lib.core.settings.CLICKJACKING_RESULTS_PATH,
                 "{}-clickjacking.html".format(lib.core.settings.replace_http(url))

lib/attacks/gist_lookup/__init__.py

@@ -4,6 +4,7 @@ import time
 import requests
 
 import lib.core.settings
+import lib.core.common
 
 
 def __check_remaining_rate_limit():
@@ -11,7 +12,7 @@ def __check_remaining_rate_limit():
     check how many requests you have left to run
     """
     url = lib.core.settings.GITHUB_GIST_SEARCH_URLS["check_rate"]
-    data = requests.get(url, params={"Authorization": "token {}".format(
+    data = requests.get(url, params={lib.core.common.HTTP_HEADER.AUTHORIZATION: "token {}".format(
         lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)
     )})
     remaining = data.headers["X-RateLimit-Remaining"]
@@ -31,8 +32,8 @@ def get_raw_data(page_set, proxy=None, agent=None, verbose=False):
     retval = set()
     url = lib.core.settings.GITHUB_GIST_SEARCH_URLS["search"]
     headers = {
-        "User-Agent": agent,
+        lib.core.common.HTTP_HEADER.USER_AGENT: agent,
-        "Authorization": "token {}".format(lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)),
+        lib.core.common.HTTP_HEADER.AUTHORIZATION: "token {}".format(lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)),
     }
     lib.core.settings.logger.info(lib.core.settings.set_color(
         "searching a total of {} pages of Gists...".format(page_set[-1])
@@ -81,7 +82,7 @@ def check_files_for_information(found_url, data_to_search):
             "found a match with given specifics, saving full Gist to log file..."
         ))
         total_found.add(found_url)
-        lib.core.settings.write_to_log_file(
+        lib.core.common.write_to_log_file(
             data.content, lib.core.settings.GIST_MATCH_LOG, "gist-match-{}.log"
         )
     return len(total_found)

lib/attacks/whois_lookup/whois.py

@@ -2,6 +2,7 @@ import json
 import time
 import urllib2
 
+import lib.core.common
 import lib.core.settings
 
 
@@ -10,8 +11,8 @@ def gather_raw_whois_info(domain):
     get the raw JSON data for from the whois API
     """
     auth_headers = {
-        "Content-Type": "application/json",
+        lib.core.common.HTTP_HEADER.CONTENT_TYPE: "application/json",
-        "Authorization": "Token {}".format(lib.core.settings.get_token(lib.core.settings.WHOIS_AUTH_PATH)),
+        lib.core.common.HTTP_HEADER.AUTHORIZATION: "Token {}".format(lib.core.settings.get_token(lib.core.settings.WHOIS_AUTH_PATH)),
     }
     request = urllib2.Request(
         lib.core.settings.WHOIS_JSON_LINK.format(domain), headers=auth_headers
@@ -77,7 +78,8 @@ def whois_lookup_main(domain, **kwargs):
         time.sleep(timeout)
     try:
         raw_information = gather_raw_whois_info(domain)
-    except Exception:
+    except Exception as e:
+        lib.core.settings.logger.exception(str(e))
         lib.core.settings.logger.error(lib.core.settings.set_color(
             "unable to produce information from WhoIs lookup...", level=40
         ))
@@ -97,7 +99,7 @@ def whois_lookup_main(domain, **kwargs):
                 "unable to display any information from WhoIs lookup on domain '{}'...".format(domain), level=50
             ))
             return None
-    lib.core.settings.write_to_log_file(
+    lib.core.common.write_to_log_file(
         raw_information, lib.core.settings.WHOIS_RESULTS_LOG_PATH,
         "{}-whois.json".format(domain)
     )

lib/attacks/xss_scan/__init__.py

@@ -9,6 +9,7 @@ except ImportError:
 
 import requests
 
+import lib.core.common
 import lib.core.settings
 from lib.core.errors import InvalidTamperProvided
 
@@ -109,7 +110,10 @@ def scan_xss(url, agent=None, proxy=None):
     """
     user_agent = agent or lib.core.settings.DEFAULT_USER_AGENT
     config_proxy = lib.core.settings.proxy_string_to_dict(proxy)
-    config_headers = {"connection": "close", "user-agent": user_agent}
+    config_headers = {
+        lib.core.common.HTTP_HEADER.CONNECTION: "close",
+        lib.core.common.HTTP_HEADER.USER_AGENT: user_agent
+    }
     xss_request = requests.get(url, proxies=config_proxy, headers=config_headers)
     status = xss_request.status_code
     html_data = xss_request.content

lib/core/common.py

@@ -0,0 +1,96 @@
+import os
+import re
+import json
+
+from lxml import etree
+
+import lib.core.settings
+
+
+class HTTP_HEADER:
+    ACCEPT = "Accept"
+    ACCEPT_CHARSET = "Accept-Charset"
+    ACCEPT_ENCODING = "Accept-Encoding"
+    ACCEPT_LANGUAGE = "Accept-Language"
+    AUTHORIZATION = "Authorization"
+    CACHE_CONTROL = "Cache-Control"
+    CONNECTION = "Connection"
+    CONTENT_ENCODING = "Content-Encoding"
+    CONTENT_LENGTH = "Content-Length"
+    CONTENT_RANGE = "Content-Range"
+    CONTENT_TYPE = "Content-Type"
+    COOKIE = "Cookie"
+    EXPIRES = "Expires"
+    HOST = "Host"
+    IF_MODIFIED_SINCE = "If-Modified-Since"
+    LAST_MODIFIED = "Last-Modified"
+    LOCATION = "Location"
+    PRAGMA = "Pragma"
+    PROXY_AUTHORIZATION = "Proxy-Authorization"
+    PROXY_CONNECTION = "Proxy-Connection"
+    RANGE = "Range"
+    REFERER = "Referer"
+    REFRESH = "Refresh"
+    SERVER = "Server"
+    SET_COOKIE = "Set-Cookie"
+    TRANSFER_ENCODING = "Transfer-Encoding"
+    URI = "URI"
+    USER_AGENT = "User-Agent"
+    VIA = "Via"
+    X_POWERED_BY = "X-Powered-By"
+    X_DATA_ORIGIN = "X-Data-Origin"
+    X_FRAME_OPT = "X-Frame-Options"
+    X_FORWARDED_FROM = "X-Forwarded-From"
+
+
+def write_to_log_file(data_to_write, path, filename, blacklist=False):
+    """
+    write all found data to a log file
+    """
+    lib.core.settings.create_dir(path.format(os.getcwd()))
+    full_file_path = "{}/{}".format(
+        path.format(os.getcwd()), filename.format(len(os.listdir(path.format(
+            os.getcwd()
+        ))) + 1)
+    )
+    skip_log_schema = ("url-log", "blackwidow-log", "zeus-log", "extracted", ".blacklist", "gist-match")
+    to_search = filename.split("-")[0]
+    amount = len([f for f in os.listdir(path) if to_search in f])
+    new_filename = "{}({}).{}".format(
+                    filename.split("-")[0], amount, filename.split(".")[-1]
+                )
+    with open(full_file_path, "a+") as log:
+        data = re.sub(r'\s+', '', log.read())
+        if re.match(r'^<.+>$', data):  # matches HTML and XML
+            try:
+                log.write(etree.tostring(data_to_write, pretty_print=True))
+            except TypeError:
+                return write_to_log_file(data_to_write, path, new_filename)
+        elif amount > 0 and not any(_ in filename for _ in list(skip_log_schema)):
+            return write_to_log_file(data_to_write, path, new_filename)
+        elif blacklist:
+            items = log.readlines()
+            if any(d.strip() == data_to_write for d in items):
+                lib.core.settings.logger.info(lib.core.settings.set_color(
+                    "query already in blacklist..."
+                ))
+                return full_file_path
+            else:
+                log.write(data_to_write + "\n")
+        else:
+            if isinstance(data_to_write, list):
+                for item in data_to_write:
+                    item = item.strip()
+                    log.write(str(item) + "\n")
+            elif isinstance(data_to_write, (tuple, set)):
+                for item in list(data_to_write):
+                    item = item.strip()
+                    log.write(str(item) + "\n")
+            elif isinstance(data_to_write, dict):
+                json.dump(data_to_write, log, sort_keys=True, indent=4)
+            else:
+                log.write(data_to_write + "\n")
+    lib.core.settings.logger.info(lib.core.settings.set_color(
+        "successfully wrote found items to '{}'...".format(full_file_path)
+    ))
+    return full_file_path

lib/core/settings.py

@@ -3,7 +3,6 @@ import io
 import re
 import sys
 import glob
-import json
 import time
 import shlex
 import difflib
@@ -24,7 +23,6 @@ except ImportError:
 import psutil
 import requests
 import whichcraft
-from lxml import etree
 
 import bin.unzip_gecko
 import lib.core.errors
@@ -582,59 +580,6 @@ def fix_log_file(logfile=get_latest_log_file(CURRENT_LOG_FILE_PATH)):
             fixed.write(line + "\n")  # rewrite everything back to normal
 
 
-def write_to_log_file(data_to_write, path, filename, blacklist=False):
-    """
-    write all found data to a log file
-    """
-    create_dir(path.format(os.getcwd()))
-    full_file_path = "{}/{}".format(
-        path.format(os.getcwd()), filename.format(len(os.listdir(path.format(
-            os.getcwd()
-        ))) + 1)
-    )
-    skip_log_schema = ("url-log", "blackwidow-log", "zeus-log", "extracted", ".blacklist", "gist-match")
-    to_search = filename.split("-")[0]
-    amount = len([f for f in os.listdir(path) if to_search in f])
-    new_filename = "{}({}).{}".format(
-                    filename.split("-")[0], amount, filename.split(".")[-1]
-                )
-    with open(full_file_path, "a+") as log:
-        data = re.sub(r'\s+', '', log.read())
-        if re.match(r'^<.+>$', data):  # matches HTML and XML
-            try:
-                log.write(etree.tostring(data_to_write, pretty_print=True))
-            except TypeError:
-                return write_to_log_file(data_to_write, path, new_filename)
-        elif amount > 0 and not any(_ in filename for _ in list(skip_log_schema)):
-            return write_to_log_file(data_to_write, path, new_filename)
-        elif blacklist:
-            items = log.readlines()
-            if any(d.strip() == data_to_write for d in items):
-                logger.info(set_color(
-                    "query already in blacklist..."
-                ))
-                return full_file_path
-            else:
-                log.write(data_to_write + "\n")
-        else:
-            if isinstance(data_to_write, list):
-                for item in data_to_write:
-                    item = item.strip()
-                    log.write(str(item) + "\n")
-            elif isinstance(data_to_write, (tuple, set)):
-                for item in list(data_to_write):
-                    item = item.strip()
-                    log.write(str(item) + "\n")
-            elif isinstance(data_to_write, dict):
-                json.dump(data_to_write, log, sort_keys=True, indent=4)
-            else:
-                log.write(data_to_write + "\n")
-    logger.info(set_color(
-        "successfully wrote found items to '{}'...".format(full_file_path)
-    ))
-    return full_file_path
-
-
 def search_for_process(name):
     """
     search for a given process to see if it's started or not

lib/firewall/cloudflare.py

@@ -1,5 +1,7 @@
 import re
 
+from lib.core.common import HTTP_HEADER
+
 
 __item__ = "CloudFlare Web Application Firewall (CloudFlare)"
 
@@ -12,9 +14,9 @@ def detect(content, **kwargs):
         if detection.search(content) is not None:
             return True
     try:
-        if re.compile(r"cloudflare-nginx", re.I).search(headers.get("Server")) is not None:
+        if re.compile(r"cloudflare-nginx", re.I).search(headers.get(HTTP_HEADER.SERVER)) is not None:
             return True
-        if re.compile(r"\A__cfduid=", re.I).search(headers.get("Cookie")) is not None:
+        if re.compile(r"\A__cfduid=", re.I).search(headers.get(HTTP_HEADER.COOKIE)) is not None:
             return True
         if re.compile(r"CF_RAY", re.I).search(str(headers)) is not None:
             return True

lib/header_check/__init__.py

@@ -7,12 +7,15 @@ from xml.dom import minidom
 from requests.exceptions import ConnectionError
 
 from var.auto_issue.github import request_issue_creation
+from lib.core.common import (
+    write_to_log_file,
+    HTTP_HEADER
+)
 from lib.core.settings import (
     logger, set_color,
     HEADER_XML_DATA,
     proxy_string_to_dict,
     create_random_ip,
-    write_to_log_file,
     replace_http,
     HEADER_RESULT_PATH,
     COOKIE_LOG_PATH,
@@ -37,14 +40,14 @@ def detect_protection(url, **kwargs):
             create_random_ip()
         )
         headers = {
-            "Connection": "close",
+            HTTP_HEADER.CONNECTION: "close",
-            "user-agent": agent,
+            HTTP_HEADER.USER_AGENT: agent,
-            "X-Forward-From": "{}, {}, {}".format(ip_list[0], ip_list[1], ip_list[2])
+            HTTP_HEADER.X_FORWARDED_FROM: "{}, {}, {}".format(ip_list[0], ip_list[1], ip_list[2])
         }
     else:
         headers = {
-            "Connection": "close",
+            HTTP_HEADER.CONNECTION: "close",
-            "user-agent": agent
+            HTTP_HEADER.USER_AGENT: agent
         }
 
     url = "{} {}".format(url.strip(), PROTECTION_CHECK_PAYLOAD)
@@ -137,15 +140,15 @@ def load_headers(url, **kwargs):
         proxy = proxy_string_to_dict(proxy)
     if not xforward:
         header_value = {
-            "connection": "close",
+            HTTP_HEADER.CONNECTION: "close",
-            "user-agent": agent
+            HTTP_HEADER.USER_AGENT: agent
         }
     else:
         ip_list = create_random_ip(), create_random_ip(), create_random_ip()
         header_value = {
-            "connection": "close",
+            HTTP_HEADER.CONNECTION: "close",
-            "user-agent": agent,
+            HTTP_HEADER.USER_AGENT: agent,
-            "X-Forwarded-For": "{}, {}, {}".format(
+            HTTP_HEADER.X_FORWARDED_FROM: "{}, {}, {}".format(
                 ip_list[0], ip_list[1], ip_list[2]
             )
         }

var/google_search/search.py

@@ -27,6 +27,10 @@ from selenium.webdriver.remote.errorhandler import (
 )
 
 from var.auto_issue.github import request_issue_creation
+from lib.core.common import (
+    write_to_log_file,
+    HTTP_HEADER
+)
 from lib.core.settings import (
     logger,
     set_color,
@@ -36,7 +40,6 @@ from lib.core.settings import (
     URL_REGEX,
     shutdown,
     URL_LOG_PATH,
-    write_to_log_file,
     prompt,
     EXTRACTED_URL_LOG,
     URL_EXCLUDES,
@@ -397,14 +400,14 @@ def parse_search_results(query, url_to_search, verbose=False, **kwargs):
             ))
 
         headers = {
-            "Connection": "close",
+            HTTP_HEADER.CONNECTION: "close",
-            "user-agent": user_agent,
+            HTTP_HEADER.USER_AGENT: user_agent,
-            "X-Forwarded-For": "{}, {}, {}".format(ip_to_use[0], ip_to_use[1], ip_to_use[2])
+            HTTP_HEADER.X_FORWARDED_FROM: "{}, {}, {}".format(ip_to_use[0], ip_to_use[1], ip_to_use[2])
         }
     else:
         headers = {
-            "Connection": "close",
+            HTTP_HEADER.CONNECTION: "close",
-            "user-agent": user_agent
+            HTTP_HEADER.USER_AGENT: user_agent
         }
     logger.info(set_color(
         "attempting to gather query URL..."