diff --git a/etc/checksum/md5sum.md5 b/etc/checksum/md5sum.md5 index bda5a13..c11ef74 100644 --- a/etc/checksum/md5sum.md5 +++ b/etc/checksum/md5sum.md5 @@ -23,7 +23,7 @@ ca6935a72fd0527d15a78a17a35e56e8 ./bin/drivers/geckodriver-v0.19.0-linux64.tar. 4ccb56fb3700005c9f9188f84152f21a ./bin/drivers/geckodriver-v0.18.0-linux64.tar.gz 07cd383c8aef8ea5ef194a506141afd6 ./bin/drivers/geckodriver-v0.19.0-linux32.tar.gz 6ea65a0160c21e144e92334acc2e3667 ./lib/firewall/anquanbao.py -1ebfc01f84201fb723ff727957d18b26 ./lib/firewall/cloudflare.py +627fd88a80a9ee049e08b113a232f065 ./lib/firewall/cloudflare.py 54815706261c32b57fbbdc99244b5cdd ./lib/firewall/modsecurity.py 6b370050b40d8c1d2221424f756c7842 ./lib/firewall/paloalto.py 24342e7de0c51595d593cef74d80d3a0 ./lib/firewall/sucuri.py @@ -52,23 +52,24 @@ a0fedc86cfb4a370e6c1a606010812ed ./lib/tamper_scripts/space2null_encode.pyc 694231b4c2f99406481c34ced85ddfe1 ./lib/tamper_scripts/base64_encode.py 6ac38bec8c32eab57efa01f7a06dff14 ./lib/tamper_scripts/space2null_encode.py 3b8c95a6a3b7cecce5118f2fb1ccc6b8 ./lib/tamper_scripts/appendnull_encode.py -3308a53435cd255107a9301723844d6e ./lib/attacks/clickjacking_scan/__init__.py +a39c2376b66ff3a2fa239c97a69e891f ./lib/attacks/clickjacking_scan/__init__.py d41d8cd98f00b204e9800998ecf8427e ./lib/attacks/__init__.py 7272a7fd0b0c2e9192bc4adb6154d2f0 ./lib/attacks/sqlmap_scan/__init__.py aa7268a8f085734a6c577c86440f7a1b ./lib/attacks/sqlmap_scan/sqlmap_opts.py d41d8cd98f00b204e9800998ecf8427e ./lib/attacks/whois_lookup/__init__.py -28bb20770353ed5684615822e2048811 ./lib/attacks/whois_lookup/whois.py -6b9ba948ca5ba51ef3d8423e738c41ba ./lib/attacks/admin_panel_finder/__init__.py -ceb1b278b0861c976dfecc91cb64e53d ./lib/attacks/xss_scan/__init__.py +ef5dc31f38028aa0d73258b36f0e36f9 ./lib/attacks/whois_lookup/whois.py +4c8cc6050796510f242539267292cfb2 ./lib/attacks/admin_panel_finder/__init__.py +edb971a085a2c3be9277def5a0f14d49 ./lib/attacks/xss_scan/__init__.py 27358f26bda30d7356143c3ea1fa99c5 ./lib/attacks/nmap_scan/__init__.py 21faf4679cdeaa731029a48f8963d6e7 ./lib/attacks/nmap_scan/nmap_opts.py -94fb7c32f7db112f14a14297311d0aa3 ./lib/attacks/gist_lookup/__init__.py +7eff376f860e72ff32094cd96d86a6a2 ./lib/attacks/gist_lookup/__init__.py 1faa2b5dfad6eb538bbfe42942d2a9da ./lib/core/errors.py d41d8cd98f00b204e9800998ecf8427e ./lib/core/__init__.py -f14e59a7d0b25368fc6ef7efb7793d0b ./lib/core/settings.py -f8dca2fa45acb95f7081546bf6aec025 ./lib/header_check/__init__.py +c631f2b8aec464b0a771f71bb6d1d137 ./lib/core/settings.py +aa541bfde844b009227f664749a39ea8 ./lib/core/common.py +7198de0a126cca4b06c31b205e2a5e23 ./lib/header_check/__init__.py d41d8cd98f00b204e9800998ecf8427e ./var/google_search/__init__.py -a53336c939aa2756b0cf35563880bdd8 ./var/google_search/search.py +180a9deb0c4937bc22ec1d18c7b540d5 ./var/google_search/search.py d41d8cd98f00b204e9800998ecf8427e ./var/__init__.py d41d8cd98f00b204e9800998ecf8427e ./var/auto_issue/__init__.py 4c7c008b28eaac5afb57a4bea46f1b19 ./var/auto_issue/github.py diff --git a/lib/attacks/admin_panel_finder/__init__.py b/lib/attacks/admin_panel_finder/__init__.py index 62f5ef2..180545b 100644 --- a/lib/attacks/admin_panel_finder/__init__.py +++ b/lib/attacks/admin_panel_finder/__init__.py @@ -9,6 +9,7 @@ except ImportError: # Python 3 import requests +import lib.core.common import lib.core.settings from var.auto_issue.github import request_issue_creation @@ -71,12 +72,12 @@ def check_for_externals(url, data_sep="-" * 30, **kwargs): lib.core.settings.logger.info(lib.core.settings.set_color( "robots.txt page will be saved into a file...", level=25 )) - return lib.core.settings.write_to_log_file(data, lib.core.settings.ROBOTS_PAGE_PATH, "{}-robots_text.log".format(url)) + return lib.core.common.write_to_log_file(data, lib.core.settings.ROBOTS_PAGE_PATH, "{}-robots_text.log".format(url)) elif sitemap: lib.core.settings.logger.info(lib.core.settings.set_color( "found a sitemap, saving to file...", level=25 )) - return lib.core.settings.write_to_log_file( + return lib.core.common.write_to_log_file( data, lib.core.settings.SITEMAP_FILE_LOG_PATH, "{}-sitemap.xml".format( lib.core.settings.replace_http(url) ) @@ -166,7 +167,7 @@ def check_for_admin_page(url, exts, protocol="http://", **kwargs): "only writing successful connections to log file...", level=30 )) if len(connections) > 0: - lib.core.settings.write_to_log_file(list(connections), lib.core.settings.ADMIN_PAGE_FILE_PATH, "{}-admin-page.log".format( + lib.core.common.write_to_log_file(list(connections), lib.core.settings.ADMIN_PAGE_FILE_PATH, "{}-admin-page.log".format( lib.core.settings.replace_http(url) )) diff --git a/lib/attacks/clickjacking_scan/__init__.py b/lib/attacks/clickjacking_scan/__init__.py index 3bdfc5e..8e1d1fc 100644 --- a/lib/attacks/clickjacking_scan/__init__.py +++ b/lib/attacks/clickjacking_scan/__init__.py @@ -1,5 +1,6 @@ import requests +import lib.core.common import lib.core.settings import var.auto_issue.github @@ -8,7 +9,7 @@ class ClickJackingScanner(object): def __init__(self, url): self.url = url - self.safe = "X-Frame-Options" + self.safe = lib.core.common.HTTP_HEADER.X_FRAME_OPT self.html = open(lib.core.settings.CLICKJACKING_TEST_PAGE_PATH).read() def generate_html(self): @@ -29,16 +30,16 @@ class ClickJackingScanner(object): if forward is not None: ip_addrs = lib.core.settings.create_random_ip() headers = { - "user-agent": agent, - "X-Forwarded-From": "{}, {}, {}".format( + lib.core.common.HTTP_HEADER.USER_AGENT: agent, + lib.core.common.HTTP_HEADER.X_FORWARDED_FROM: "{}, {}, {}".format( ip_addrs[0], ip_addrs[1], ip_addrs[2] ), - "Connection": "close" + lib.core.common.HTTP_HEADER.CONNECTION: "close" } else: headers = { - "user-agent": agent, - "Connection": "close" + lib.core.common.HTTP_HEADER.USER_AGENT: agent, + lib.core.common.HTTP_HEADER.CONNECTION: "close" } req = requests.get(self.url, headers=headers, proxies=lib.core.settings.proxy_string_to_dict(proxy)) headers = req.headers @@ -93,7 +94,7 @@ def clickjacking_main(url, **kwargs): "it appears that provided URL '{}' is vulnerable to clickjacking, writing " "to HTML file...".format(url), level=25 )) - lib.core.settings.write_to_log_file( + lib.core.common.write_to_log_file( data, lib.core.settings.CLICKJACKING_RESULTS_PATH, "{}-clickjacking.html".format(lib.core.settings.replace_http(url)) diff --git a/lib/attacks/gist_lookup/__init__.py b/lib/attacks/gist_lookup/__init__.py index 7b35901..3229f98 100644 --- a/lib/attacks/gist_lookup/__init__.py +++ b/lib/attacks/gist_lookup/__init__.py @@ -4,6 +4,7 @@ import time import requests import lib.core.settings +import lib.core.common def __check_remaining_rate_limit(): @@ -11,7 +12,7 @@ def __check_remaining_rate_limit(): check how many requests you have left to run """ url = lib.core.settings.GITHUB_GIST_SEARCH_URLS["check_rate"] - data = requests.get(url, params={"Authorization": "token {}".format( + data = requests.get(url, params={lib.core.common.HTTP_HEADER.AUTHORIZATION: "token {}".format( lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH) )}) remaining = data.headers["X-RateLimit-Remaining"] @@ -31,8 +32,8 @@ def get_raw_data(page_set, proxy=None, agent=None, verbose=False): retval = set() url = lib.core.settings.GITHUB_GIST_SEARCH_URLS["search"] headers = { - "User-Agent": agent, - "Authorization": "token {}".format(lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)), + lib.core.common.HTTP_HEADER.USER_AGENT: agent, + lib.core.common.HTTP_HEADER.AUTHORIZATION: "token {}".format(lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)), } lib.core.settings.logger.info(lib.core.settings.set_color( "searching a total of {} pages of Gists...".format(page_set[-1]) @@ -81,7 +82,7 @@ def check_files_for_information(found_url, data_to_search): "found a match with given specifics, saving full Gist to log file..." )) total_found.add(found_url) - lib.core.settings.write_to_log_file( + lib.core.common.write_to_log_file( data.content, lib.core.settings.GIST_MATCH_LOG, "gist-match-{}.log" ) return len(total_found) diff --git a/lib/attacks/whois_lookup/whois.py b/lib/attacks/whois_lookup/whois.py index 33b9ba0..b7031d5 100644 --- a/lib/attacks/whois_lookup/whois.py +++ b/lib/attacks/whois_lookup/whois.py @@ -2,6 +2,7 @@ import json import time import urllib2 +import lib.core.common import lib.core.settings @@ -10,8 +11,8 @@ def gather_raw_whois_info(domain): get the raw JSON data for from the whois API """ auth_headers = { - "Content-Type": "application/json", - "Authorization": "Token {}".format(lib.core.settings.get_token(lib.core.settings.WHOIS_AUTH_PATH)), + lib.core.common.HTTP_HEADER.CONTENT_TYPE: "application/json", + lib.core.common.HTTP_HEADER.AUTHORIZATION: "Token {}".format(lib.core.settings.get_token(lib.core.settings.WHOIS_AUTH_PATH)), } request = urllib2.Request( lib.core.settings.WHOIS_JSON_LINK.format(domain), headers=auth_headers @@ -77,7 +78,8 @@ def whois_lookup_main(domain, **kwargs): time.sleep(timeout) try: raw_information = gather_raw_whois_info(domain) - except Exception: + except Exception as e: + lib.core.settings.logger.exception(str(e)) lib.core.settings.logger.error(lib.core.settings.set_color( "unable to produce information from WhoIs lookup...", level=40 )) @@ -97,7 +99,7 @@ def whois_lookup_main(domain, **kwargs): "unable to display any information from WhoIs lookup on domain '{}'...".format(domain), level=50 )) return None - lib.core.settings.write_to_log_file( + lib.core.common.write_to_log_file( raw_information, lib.core.settings.WHOIS_RESULTS_LOG_PATH, "{}-whois.json".format(domain) ) \ No newline at end of file diff --git a/lib/attacks/xss_scan/__init__.py b/lib/attacks/xss_scan/__init__.py index cbc6d87..aff8ce7 100644 --- a/lib/attacks/xss_scan/__init__.py +++ b/lib/attacks/xss_scan/__init__.py @@ -9,6 +9,7 @@ except ImportError: import requests +import lib.core.common import lib.core.settings from lib.core.errors import InvalidTamperProvided @@ -109,7 +110,10 @@ def scan_xss(url, agent=None, proxy=None): """ user_agent = agent or lib.core.settings.DEFAULT_USER_AGENT config_proxy = lib.core.settings.proxy_string_to_dict(proxy) - config_headers = {"connection": "close", "user-agent": user_agent} + config_headers = { + lib.core.common.HTTP_HEADER.CONNECTION: "close", + lib.core.common.HTTP_HEADER.USER_AGENT: user_agent + } xss_request = requests.get(url, proxies=config_proxy, headers=config_headers) status = xss_request.status_code html_data = xss_request.content diff --git a/lib/core/common.py b/lib/core/common.py new file mode 100644 index 0000000..c88c5d7 --- /dev/null +++ b/lib/core/common.py @@ -0,0 +1,96 @@ +import os +import re +import json + +from lxml import etree + +import lib.core.settings + + +class HTTP_HEADER: + ACCEPT = "Accept" + ACCEPT_CHARSET = "Accept-Charset" + ACCEPT_ENCODING = "Accept-Encoding" + ACCEPT_LANGUAGE = "Accept-Language" + AUTHORIZATION = "Authorization" + CACHE_CONTROL = "Cache-Control" + CONNECTION = "Connection" + CONTENT_ENCODING = "Content-Encoding" + CONTENT_LENGTH = "Content-Length" + CONTENT_RANGE = "Content-Range" + CONTENT_TYPE = "Content-Type" + COOKIE = "Cookie" + EXPIRES = "Expires" + HOST = "Host" + IF_MODIFIED_SINCE = "If-Modified-Since" + LAST_MODIFIED = "Last-Modified" + LOCATION = "Location" + PRAGMA = "Pragma" + PROXY_AUTHORIZATION = "Proxy-Authorization" + PROXY_CONNECTION = "Proxy-Connection" + RANGE = "Range" + REFERER = "Referer" + REFRESH = "Refresh" + SERVER = "Server" + SET_COOKIE = "Set-Cookie" + TRANSFER_ENCODING = "Transfer-Encoding" + URI = "URI" + USER_AGENT = "User-Agent" + VIA = "Via" + X_POWERED_BY = "X-Powered-By" + X_DATA_ORIGIN = "X-Data-Origin" + X_FRAME_OPT = "X-Frame-Options" + X_FORWARDED_FROM = "X-Forwarded-From" + + +def write_to_log_file(data_to_write, path, filename, blacklist=False): + """ + write all found data to a log file + """ + lib.core.settings.create_dir(path.format(os.getcwd())) + full_file_path = "{}/{}".format( + path.format(os.getcwd()), filename.format(len(os.listdir(path.format( + os.getcwd() + ))) + 1) + ) + skip_log_schema = ("url-log", "blackwidow-log", "zeus-log", "extracted", ".blacklist", "gist-match") + to_search = filename.split("-")[0] + amount = len([f for f in os.listdir(path) if to_search in f]) + new_filename = "{}({}).{}".format( + filename.split("-")[0], amount, filename.split(".")[-1] + ) + with open(full_file_path, "a+") as log: + data = re.sub(r'\s+', '', log.read()) + if re.match(r'^<.+>$', data): # matches HTML and XML + try: + log.write(etree.tostring(data_to_write, pretty_print=True)) + except TypeError: + return write_to_log_file(data_to_write, path, new_filename) + elif amount > 0 and not any(_ in filename for _ in list(skip_log_schema)): + return write_to_log_file(data_to_write, path, new_filename) + elif blacklist: + items = log.readlines() + if any(d.strip() == data_to_write for d in items): + lib.core.settings.logger.info(lib.core.settings.set_color( + "query already in blacklist..." + )) + return full_file_path + else: + log.write(data_to_write + "\n") + else: + if isinstance(data_to_write, list): + for item in data_to_write: + item = item.strip() + log.write(str(item) + "\n") + elif isinstance(data_to_write, (tuple, set)): + for item in list(data_to_write): + item = item.strip() + log.write(str(item) + "\n") + elif isinstance(data_to_write, dict): + json.dump(data_to_write, log, sort_keys=True, indent=4) + else: + log.write(data_to_write + "\n") + lib.core.settings.logger.info(lib.core.settings.set_color( + "successfully wrote found items to '{}'...".format(full_file_path) + )) + return full_file_path diff --git a/lib/core/settings.py b/lib/core/settings.py index 1a1489d..a349d3b 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -3,7 +3,6 @@ import io import re import sys import glob -import json import time import shlex import difflib @@ -24,7 +23,6 @@ except ImportError: import psutil import requests import whichcraft -from lxml import etree import bin.unzip_gecko import lib.core.errors @@ -582,59 +580,6 @@ def fix_log_file(logfile=get_latest_log_file(CURRENT_LOG_FILE_PATH)): fixed.write(line + "\n") # rewrite everything back to normal -def write_to_log_file(data_to_write, path, filename, blacklist=False): - """ - write all found data to a log file - """ - create_dir(path.format(os.getcwd())) - full_file_path = "{}/{}".format( - path.format(os.getcwd()), filename.format(len(os.listdir(path.format( - os.getcwd() - ))) + 1) - ) - skip_log_schema = ("url-log", "blackwidow-log", "zeus-log", "extracted", ".blacklist", "gist-match") - to_search = filename.split("-")[0] - amount = len([f for f in os.listdir(path) if to_search in f]) - new_filename = "{}({}).{}".format( - filename.split("-")[0], amount, filename.split(".")[-1] - ) - with open(full_file_path, "a+") as log: - data = re.sub(r'\s+', '', log.read()) - if re.match(r'^<.+>$', data): # matches HTML and XML - try: - log.write(etree.tostring(data_to_write, pretty_print=True)) - except TypeError: - return write_to_log_file(data_to_write, path, new_filename) - elif amount > 0 and not any(_ in filename for _ in list(skip_log_schema)): - return write_to_log_file(data_to_write, path, new_filename) - elif blacklist: - items = log.readlines() - if any(d.strip() == data_to_write for d in items): - logger.info(set_color( - "query already in blacklist..." - )) - return full_file_path - else: - log.write(data_to_write + "\n") - else: - if isinstance(data_to_write, list): - for item in data_to_write: - item = item.strip() - log.write(str(item) + "\n") - elif isinstance(data_to_write, (tuple, set)): - for item in list(data_to_write): - item = item.strip() - log.write(str(item) + "\n") - elif isinstance(data_to_write, dict): - json.dump(data_to_write, log, sort_keys=True, indent=4) - else: - log.write(data_to_write + "\n") - logger.info(set_color( - "successfully wrote found items to '{}'...".format(full_file_path) - )) - return full_file_path - - def search_for_process(name): """ search for a given process to see if it's started or not diff --git a/lib/firewall/cloudflare.py b/lib/firewall/cloudflare.py index e853fa2..9e9a3f2 100644 --- a/lib/firewall/cloudflare.py +++ b/lib/firewall/cloudflare.py @@ -1,5 +1,7 @@ import re +from lib.core.common import HTTP_HEADER + __item__ = "CloudFlare Web Application Firewall (CloudFlare)" @@ -12,9 +14,9 @@ def detect(content, **kwargs): if detection.search(content) is not None: return True try: - if re.compile(r"cloudflare-nginx", re.I).search(headers.get("Server")) is not None: + if re.compile(r"cloudflare-nginx", re.I).search(headers.get(HTTP_HEADER.SERVER)) is not None: return True - if re.compile(r"\A__cfduid=", re.I).search(headers.get("Cookie")) is not None: + if re.compile(r"\A__cfduid=", re.I).search(headers.get(HTTP_HEADER.COOKIE)) is not None: return True if re.compile(r"CF_RAY", re.I).search(str(headers)) is not None: return True diff --git a/lib/header_check/__init__.py b/lib/header_check/__init__.py index b907e29..d80a765 100644 --- a/lib/header_check/__init__.py +++ b/lib/header_check/__init__.py @@ -7,12 +7,15 @@ from xml.dom import minidom from requests.exceptions import ConnectionError from var.auto_issue.github import request_issue_creation +from lib.core.common import ( + write_to_log_file, + HTTP_HEADER +) from lib.core.settings import ( logger, set_color, HEADER_XML_DATA, proxy_string_to_dict, create_random_ip, - write_to_log_file, replace_http, HEADER_RESULT_PATH, COOKIE_LOG_PATH, @@ -37,14 +40,14 @@ def detect_protection(url, **kwargs): create_random_ip() ) headers = { - "Connection": "close", - "user-agent": agent, - "X-Forward-From": "{}, {}, {}".format(ip_list[0], ip_list[1], ip_list[2]) + HTTP_HEADER.CONNECTION: "close", + HTTP_HEADER.USER_AGENT: agent, + HTTP_HEADER.X_FORWARDED_FROM: "{}, {}, {}".format(ip_list[0], ip_list[1], ip_list[2]) } else: headers = { - "Connection": "close", - "user-agent": agent + HTTP_HEADER.CONNECTION: "close", + HTTP_HEADER.USER_AGENT: agent } url = "{} {}".format(url.strip(), PROTECTION_CHECK_PAYLOAD) @@ -137,15 +140,15 @@ def load_headers(url, **kwargs): proxy = proxy_string_to_dict(proxy) if not xforward: header_value = { - "connection": "close", - "user-agent": agent + HTTP_HEADER.CONNECTION: "close", + HTTP_HEADER.USER_AGENT: agent } else: ip_list = create_random_ip(), create_random_ip(), create_random_ip() header_value = { - "connection": "close", - "user-agent": agent, - "X-Forwarded-For": "{}, {}, {}".format( + HTTP_HEADER.CONNECTION: "close", + HTTP_HEADER.USER_AGENT: agent, + HTTP_HEADER.X_FORWARDED_FROM: "{}, {}, {}".format( ip_list[0], ip_list[1], ip_list[2] ) } diff --git a/var/google_search/search.py b/var/google_search/search.py index 6f81b04..2421d72 100644 --- a/var/google_search/search.py +++ b/var/google_search/search.py @@ -27,6 +27,10 @@ from selenium.webdriver.remote.errorhandler import ( ) from var.auto_issue.github import request_issue_creation +from lib.core.common import ( + write_to_log_file, + HTTP_HEADER +) from lib.core.settings import ( logger, set_color, @@ -36,7 +40,6 @@ from lib.core.settings import ( URL_REGEX, shutdown, URL_LOG_PATH, - write_to_log_file, prompt, EXTRACTED_URL_LOG, URL_EXCLUDES, @@ -397,14 +400,14 @@ def parse_search_results(query, url_to_search, verbose=False, **kwargs): )) headers = { - "Connection": "close", - "user-agent": user_agent, - "X-Forwarded-For": "{}, {}, {}".format(ip_to_use[0], ip_to_use[1], ip_to_use[2]) + HTTP_HEADER.CONNECTION: "close", + HTTP_HEADER.USER_AGENT: user_agent, + HTTP_HEADER.X_FORWARDED_FROM: "{}, {}, {}".format(ip_to_use[0], ip_to_use[1], ip_to_use[2]) } else: headers = { - "Connection": "close", - "user-agent": user_agent + HTTP_HEADER.CONNECTION: "close", + HTTP_HEADER.USER_AGENT: user_agent } logger.info(set_color( "attempting to gather query URL..."