From 4651fcd2ac424104cf4af204faaf75acf7c36853 Mon Sep 17 00:00:00 2001 From: ekultek Date: Mon, 27 Nov 2017 13:09:53 -0600 Subject: [PATCH] major patch for an issue with the firewall identification (private report) it was super unreliable, should work better now --- etc/checksum/md5sum.md5 | 25 +++++++------- lib/core/settings.py | 2 +- lib/firewall/aws.py | 9 +++-- lib/firewall/cloudflare.py | 16 +++++---- lib/firewall/cloudfront.py | 7 ++-- lib/firewall/sonicwall.py | 2 +- lib/firewall/squid.py | 18 +++++----- lib/firewall/sucuri.py | 5 ++- lib/firewall/urlscan.py | 19 +++++++++++ lib/firewall/varnish.py | 18 +++++----- lib/firewall/webknight.py | 7 ++-- lib/firewall/yundun.py | 4 +-- lib/firewall/yunsuo.py | 2 +- lib/header_check/__init__.py | 65 ++++++++++++++++++++---------------- 14 files changed, 115 insertions(+), 84 deletions(-) create mode 100644 lib/firewall/urlscan.py diff --git a/etc/checksum/md5sum.md5 b/etc/checksum/md5sum.md5 index 730fdfa..a4e387d 100644 --- a/etc/checksum/md5sum.md5 +++ b/etc/checksum/md5sum.md5 @@ -34,25 +34,26 @@ d41d8cd98f00b204e9800998ecf8427e ./lib/tamper_scripts/__init__.py 6ac38bec8c32eab57efa01f7a06dff14 ./lib/tamper_scripts/space2null_encode.py 3b8c95a6a3b7cecce5118f2fb1ccc6b8 ./lib/tamper_scripts/appendnull_encode.py d41d8cd98f00b204e9800998ecf8427e ./lib/__init__.py -2bfc3884ae96cc2443ebf94359b380c0 ./lib/firewall/cloudfront.py +6299b188a730844954044887f528435a ./lib/firewall/cloudfront.py d41d8cd98f00b204e9800998ecf8427e ./lib/firewall/__init__.py -24342e7de0c51595d593cef74d80d3a0 ./lib/firewall/sucuri.py -e903d6aa36ca4d0eff8de3056eaa59c7 ./lib/firewall/aws.py +eb3a3066efbcf87dbc10a49be445cb8f ./lib/firewall/urlscan.py +d4986f3d95a773d7c3585b07bcd6310e ./lib/firewall/sucuri.py +e4514021485dbb94c3f0023b04af01ad ./lib/firewall/aws.py 1f303641d59686d544f2986ff74c6b31 ./lib/firewall/webseal.py e4eef006dd909c222b1b9f48826c3ef5 ./lib/firewall/pk.py 6b370050b40d8c1d2221424f756c7842 ./lib/firewall/paloalto.py 6bbe2f6f6a2a1ddf0e416e94ec1f0763 ./lib/firewall/siteguard.py -627fd88a80a9ee049e08b113a232f065 ./lib/firewall/cloudflare.py +787e21ed577ff05b095aa0f0e5e5e9bf ./lib/firewall/cloudflare.py c3f01fc8ff7dfe7759f63bf16b00f127 ./lib/firewall/wordfence.py 6ccf3a1df5aa6429cd3365b7b8ded8f4 ./lib/firewall/powerful.py 54815706261c32b57fbbdc99244b5cdd ./lib/firewall/modsecurity.py -fde5445df5d77d245656adea96673cfa ./lib/firewall/squid.py -1105bb1a4236f884548c6a013a51945e ./lib/firewall/sonicwall.py +5ee20e2c158d0734b4dd5a8eb47f8ea5 ./lib/firewall/squid.py +71744d7a95f42063a8fb6e720932cd3d ./lib/firewall/sonicwall.py 9070b43428bd17fd5faf86995cb559a2 ./lib/firewall/stringray.py -0c9cae75883a59d1a224e27339803dbe ./lib/firewall/varnish.py -8c4a95660ca3c142f3658a2df1146171 ./lib/firewall/yundun.py -608ea8be34f9ec4c92c4e22eb0c5b39c ./lib/firewall/yunsuo.py -75da205112928437d8b3f451af16e7b8 ./lib/firewall/webknight.py +bf5285dc059c761e1719bc734ae8504f ./lib/firewall/varnish.py +78e6b01feb9bad68c2fc8a79e75930fd ./lib/firewall/yundun.py +cb45428e92485b759ff5cb46a0be9c73 ./lib/firewall/yunsuo.py +bbd8b4c6100070d420d48dc7dfc297eb ./lib/firewall/webknight.py 95b908a21c0ff456ae59df4c6c189c54 ./lib/firewall/wallarm.py 6ea65a0160c21e144e92334acc2e3667 ./lib/firewall/anquanbao.py ad238be8225c2791d82d4c7582afe820 ./lib/firewall/generic.py @@ -67,11 +68,11 @@ d2846e039fefee741db24dd64f7bd50e ./lib/attacks/admin_panel_finder/__init__.py b5cd5e913cc62112776153bdf0f60fa4 ./lib/attacks/xss_scan/__init__.py 63c45495ec1ed2e98946bef514d8805e ./lib/attacks/nmap_scan/__init__.py 216999fa0e84866d5c1d96d5676034e4 ./lib/attacks/nmap_scan/nmap_opts.py -e11185d7dd3d6c94d7f4ddbcfd1d26cc ./lib/header_check/__init__.py +545de148cbb330e3e1aa91de05d6e16c ./lib/header_check/__init__.py 9a2bb0d52f64e12d5a63ce83874ea74a ./lib/core/common.py 1faa2b5dfad6eb538bbfe42942d2a9da ./lib/core/errors.py d41d8cd98f00b204e9800998ecf8427e ./lib/core/__init__.py -5318e70bd8aac1f24ce9735b3a5faf7c ./lib/core/settings.py +4126cf6962354db415835b42d054b85b ./lib/core/settings.py 9a02e5b913d210350545ac26510a63c9 ./var/search/__init__.py 8402f23a2586b6f684fb1c3c04c4386f ./var/search/selenium_search.py 63ba132381a0cc2d7629852bd5e4aa17 ./var/search/pgp_search.py diff --git a/lib/core/settings.py b/lib/core/settings.py index b88cd17..a09de6c 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -46,7 +46,7 @@ CLONE = "https://github.com/ekultek/zeus-scanner.git" ISSUE_LINK = "https://github.com/ekultek/zeus-scanner/issues" # current version -VERSION = "1.2.39.{}".format(PATCH_ID) +VERSION = "1.2.40.{}".format(PATCH_ID) # colors to output depending on the version VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30} diff --git a/lib/firewall/aws.py b/lib/firewall/aws.py index c53c174..c40f2f7 100644 --- a/lib/firewall/aws.py +++ b/lib/firewall/aws.py @@ -1,5 +1,7 @@ import re +from lib.core.common import HTTP_HEADER + __item__ = "Amazon Web Services Web Application Firewall (Amazon)" @@ -12,10 +14,13 @@ def detect(content, **kwargs): re.compile(r"AccessDenied<.Code>", re.I), re.compile(r"\bAWS", re.I), re.compile(r"x.amz.id.\d+", re.I), - re.compile(r"x.amz.request.id", re.I) + re.compile(r"x.amz.request.id", re.I), + re.compile(r"amazon.\d+", re.I) ) for detection in detection_schema: if detection.search(content) is not None: return True - elif detection.search(str(headers)) is not None: + if detection.search(headers.get(HTTP_HEADER.SERVER, "")) is not None: + return True + if detection.search(headers.get(HTTP_HEADER.X_POWERED_BY, "")) is not None: return True diff --git a/lib/firewall/cloudflare.py b/lib/firewall/cloudflare.py index 9e9a3f2..2e631a8 100644 --- a/lib/firewall/cloudflare.py +++ b/lib/firewall/cloudflare.py @@ -9,16 +9,18 @@ __item__ = "CloudFlare Web Application Firewall (CloudFlare)" def detect(content, **kwargs): headers = kwargs.get("headers", None) content = str(content) - detection_schemas = (re.compile(r"CloudFlare Ray ID:|var CloudFlare=", re.I),) + detection_schemas = ( + re.compile(r"CloudFlare Ray ID:|var CloudFlare=", re.I), + re.compile(r"cloudflare-nginx", re.I), + re.compile(r"\A__cfduid=", re.I), + re.compile(r"CF_RAY", re.I) + ) for detection in detection_schemas: if detection.search(content) is not None: return True - try: - if re.compile(r"cloudflare-nginx", re.I).search(headers.get(HTTP_HEADER.SERVER)) is not None: + elif detection.search(headers.get(HTTP_HEADER.SERVER, "")) is not None: return True - if re.compile(r"\A__cfduid=", re.I).search(headers.get(HTTP_HEADER.COOKIE)) is not None: + elif detection.search(headers.get(HTTP_HEADER.COOKIE, "")) is not None: return True - if re.compile(r"CF_RAY", re.I).search(str(headers)) is not None: + elif detection.search(str(headers)) is not None: return True - except Exception: - pass diff --git a/lib/firewall/cloudfront.py b/lib/firewall/cloudfront.py index fda4a8b..db988a1 100644 --- a/lib/firewall/cloudfront.py +++ b/lib/firewall/cloudfront.py @@ -11,7 +11,6 @@ def detect(content, **kwargs): re.compile(r"cloudfront", re.I), re.compile(r"X-Amz-Cf-Id", re.I) ) - if headers is not None: - for detection in detection_schema: - if detection.search(str(headers)) is not None: - return True + for detection in detection_schema: + if detection.search(str(headers)) is not None: + return True diff --git a/lib/firewall/sonicwall.py b/lib/firewall/sonicwall.py index 38f679a..0047508 100644 --- a/lib/firewall/sonicwall.py +++ b/lib/firewall/sonicwall.py @@ -20,5 +20,5 @@ def detect(content, **kwargs): for detection in detection_schema: if detection.search(content) is not None: return True - elif detection.search(headers.get(HTTP_HEADER.SERVER)) is not None: + if detection.search(headers.get(HTTP_HEADER.SERVER, "")) is not None: return True diff --git a/lib/firewall/squid.py b/lib/firewall/squid.py index e2dd10d..7e98371 100644 --- a/lib/firewall/squid.py +++ b/lib/firewall/squid.py @@ -1,5 +1,7 @@ import re +from lib.core.common import HTTP_HEADER + __item__ = "Squid Proxy (IDS)" @@ -9,17 +11,13 @@ def detect(content, **kwargs): headers = kwargs.get("headers", None) detection_schema = ( re.compile(r"squid", re.I), - re.compile(r"Access control configuration prevents", re.I) + re.compile(r"Access control configuration prevents", re.I), + re.compile(r"X.Squid.Error", re.I), ) for detection in detection_schema: if detection.search(content) is not None: return True - if headers is not None: - headers = str(headers) - detection_schema = ( - re.compile(r"X.Squid.Error", re.I), - re.compile(r"squid", re.I) - ) - for detection in detection_schema: - if detection.search(headers) is not None: - return True + if detection.search(headers.get(HTTP_HEADER.SERVER, "")) is not None: + return True + if detection.search(str(headers)) is not None: + return True diff --git a/lib/firewall/sucuri.py b/lib/firewall/sucuri.py index 9d27ecf..2ac303c 100644 --- a/lib/firewall/sucuri.py +++ b/lib/firewall/sucuri.py @@ -1,5 +1,6 @@ import re +from lib.core.common import HTTP_HEADER __item__ = "Sucuri Firewall (Sucuri Cloudproxy)" @@ -15,7 +16,5 @@ def detect(content, **kwargs): for detection in detection_schema: if detection.search(content) is not None: return True - if headers is not None: - headers = str(headers) - if re.compile(r"X-Sucuri-ID", re.I).search(headers) is not None: + if re.compile(r"X-Sucuri-ID", re.I).search(headers.get(HTTP_HEADER.SERVER, "")) is not None: return True diff --git a/lib/firewall/urlscan.py b/lib/firewall/urlscan.py new file mode 100644 index 0000000..9f70d01 --- /dev/null +++ b/lib/firewall/urlscan.py @@ -0,0 +1,19 @@ +import re + +from lib.core.common import HTTP_HEADER + + +__item__ = "UrlScan (Microsoft)" + + +def detect(content, **kwargs): + headers = kwargs.get("headers", None) + detection_schema = ( + re.compile(r"rejected.by.url.scan", re.I), + re.compile(r"/rejected.by.url.scan", re.I) + ) + for detection in detection_schema: + if detection.search(content) is not None: + return True + if detection.search(headers.get(HTTP_HEADER.LOCATION, "")) is not None: + return True diff --git a/lib/firewall/varnish.py b/lib/firewall/varnish.py index 692b874..0ad2cda 100644 --- a/lib/firewall/varnish.py +++ b/lib/firewall/varnish.py @@ -9,16 +9,18 @@ __item__ = "Varnish FireWall (OWASP)" def detect(content, **kwargs): content = str(content) headers = kwargs.get("headers", None) - status = kwargs.get("status", None) detection_schema = ( re.compile(r"\bXID: \d+", re.I), re.compile(r"varnish\Z", re.I), re.compile(r"varnish"), re.I ) - for detection in detection_schema: - if detection.search(content) is not None and status == 404: - return True - elif detection.search(headers.get(HTTP_HEADER.VIA)) is not None: - return True - elif detection.search(headers.get(HTTP_HEADER.SERVER)) is not None: - return True + try: + for detection in detection_schema: + if detection.search(content) is not None: + return True + if detection.search(headers.get(HTTP_HEADER.VIA, "")) is not None: + return True + if detection.search(headers.get(HTTP_HEADER.SERVER, "")) is not None: + return True + except: + pass diff --git a/lib/firewall/webknight.py b/lib/firewall/webknight.py index 2259b1f..62b51f0 100644 --- a/lib/firewall/webknight.py +++ b/lib/firewall/webknight.py @@ -16,7 +16,6 @@ def detect(content, **kwargs): if status is not None: if status == 999: return True - if headers is not None: - for detection in detection_schema: - if detection.search(headers.get(HTTP_HEADER.SERVER)) is not None: - return True + for detection in detection_schema: + if detection.search(headers.get(HTTP_HEADER.SERVER, "")) is not None: + return True diff --git a/lib/firewall/yundun.py b/lib/firewall/yundun.py index 798e089..c0519ed 100644 --- a/lib/firewall/yundun.py +++ b/lib/firewall/yundun.py @@ -13,7 +13,7 @@ def detect(content, **kwargs): ) if headers is not None: for detection in detection_schema: - if detection.search(headers.get(HTTP_HEADER.X_CACHE)) is not None: + if detection.search(headers.get(HTTP_HEADER.X_CACHE, "")) is not None: return True - if detection.search(headers.get(HTTP_HEADER.SERVER)) is not None: + if detection.search(headers.get(HTTP_HEADER.SERVER, "")) is not None: return True diff --git a/lib/firewall/yunsuo.py b/lib/firewall/yunsuo.py index 988c6ce..3bdf49d 100644 --- a/lib/firewall/yunsuo.py +++ b/lib/firewall/yunsuo.py @@ -15,5 +15,5 @@ def detect(content, **kwargs): for detection in detection_schema: if detection.search(content) is not None: return True - if re.search(r"yunsuo_session", headers.get(HTTP_HEADER.SET_COOKIE), re.I) is not None: + if re.search(r"yunsuo_session", headers.get(HTTP_HEADER.SET_COOKIE, ""), re.I) is not None: return True diff --git a/lib/header_check/__init__.py b/lib/header_check/__init__.py index 40759bf..1ba0ad7 100644 --- a/lib/header_check/__init__.py +++ b/lib/header_check/__init__.py @@ -78,41 +78,48 @@ def detect_protection(url, **kwargs): return None retval = [] - if status != 200 and "not found" not in html.lower(): - file_list = [f for f in os.listdir(DETECT_FIREWALL_PATH) if not any(ex in f for ex in ["__init__", ".pyc"])] - for item in file_list: - item = item[:-3] - detection_name = "lib.firewall.{}" - detection_name = detection_name.format(item) - detection_name = importlib.import_module(detection_name) - if detection_name.detect(html, headers=headers, status=status): - retval.append(detection_name.__item__) - if len(retval) > 1: - if "Generic (Unknown)" in retval: - item = retval.index("Generic (Unknown)") - del retval[item] - else: - if retval[0] == "Generic (Unknown)": + file_list = [f for f in os.listdir(DETECT_FIREWALL_PATH) if not any(ex in f for ex in ["__init__", ".pyc"])] + for item in file_list: + item = item[:-3] + if verbose: + logger.debug(set_color( + "loading script '{}'...".format(item), level=10 + )) + detection_name = "lib.firewall.{}" + detection_name = detection_name.format(item) + detection_name = importlib.import_module(detection_name) + if detection_name.detect(html, headers=headers, status=status) is True: + retval.append(detection_name.__item__) + if len(retval) != 0: + if len(retval) >= 2: + try: + del retval[retval.index("Generic (Unknown)")] + except: logger.warning(set_color( - "identified WAF/IDS/IPS is unknown to Zeus, if you know the firewall and the context " - "of the firewall, please create an issue ({}), fingerprint of the firewall will be " - "written to a log file...".format(ISSUE_LINK), level=30 + "multiple firewalls identified ({}), displaying most likely...".format( + ", ".join(retval) + ), level=30 )) - full_finger_print = "HTTP/1.1 {}\n{}\n{}".format(status, headers, html) - write_to_log_file( - full_finger_print, UNKNOWN_FIREWALL_FINGERPRINT_PATH, UNKNOWN_FIREWALL_FILENAME.format( - replace_http(url) - ) - ) + del retval[retval.index(retval[1])] + if retval[0] == "Generic (Unknown)": + logger.warning(set_color( + "discovered firewall is unknown to Zeus, saving fingerprint to file. " + "if you know the details or the context of the firewall please create " + "an issue with the fingerprint, or a pull request with the script...", level=30 + )) + fingerprint = "\n{}".format( + status, headers, html + ) + write_to_log_file(fingerprint, UNKNOWN_FIREWALL_FINGERPRINT_PATH, UNKNOWN_FIREWALL_FILENAME) + return "".join(retval) if isinstance(retval, list) else retval else: - retval = None - - return ''.join(retval) if isinstance(retval, list) else retval + return None except Exception as e: + print logger.exception(e) if "Read timed out." or "Connection reset by peer" in str(e): logger.warning(set_color( - "detection request timed out, assuming no protection and continuing...", level=30 + "detection request failed, assuming no protection and continuing...", level=30 )) return None else: @@ -222,7 +229,7 @@ def main_header_check(url, **kwargs): )) else: logger.warning(set_color( - "the target URL WAF/IDS/IPS has been identified as '{}'...".format(identified), level=30 + "the target URL WAF/IDS/IPS has been identified as '{}'...".format(identified), level=35 )) if verbose: