From c382c010fc51450dc3dce166fa4ee5b0febb86c3 Mon Sep 17 00:00:00 2001 From: ekultek Date: Mon, 4 Dec 2017 14:37:07 -0600 Subject: [PATCH] patch for an issue where it would fail again if the encoding is still messed up, also patch for an issue with the protocol, will now auto assign http if it isn't present (issue #225 and issue #226) --- etc/checksum/md5sum.md5 | 6 +-- lib/attacks/gist_lookup/__init__.py | 71 +++++++++++++++++++---------- lib/attacks/xss_scan/__init__.py | 43 +++++++++++------ lib/core/decorators.py | 1 + 4 files changed, 78 insertions(+), 43 deletions(-) diff --git a/etc/checksum/md5sum.md5 b/etc/checksum/md5sum.md5 index f2d44c6..1ebaac5 100644 --- a/etc/checksum/md5sum.md5 +++ b/etc/checksum/md5sum.md5 @@ -62,7 +62,7 @@ bbd8b4c6100070d420d48dc7dfc297eb ./lib/firewall/webknight.py 81a29a14d72980a306fbaec0dc772048 ./lib/firewall/fortigate.py 6ea65a0160c21e144e92334acc2e3667 ./lib/firewall/anquanbao.py 22a0ad8f2fa1a16b651cb5ae37ca9b0d ./lib/firewall/generic.py -c3aa1db3ee34b841d91ad850c2cfd7b2 ./lib/attacks/gist_lookup/__init__.py +b5ff3286060c0bbc0fe1f0f591131c9c ./lib/attacks/gist_lookup/__init__.py 86224bd899c2a2438042cbdc077dc4cc ./lib/attacks/clickjacking_scan/__init__.py d41d8cd98f00b204e9800998ecf8427e ./lib/attacks/__init__.py 4c644b0e3a62b6c1528d34a04837aa35 ./lib/attacks/sqlmap_scan/__init__.py @@ -70,7 +70,7 @@ d41d8cd98f00b204e9800998ecf8427e ./lib/attacks/__init__.py 7bc7a6f3e85651aab3588f087563eded ./lib/attacks/whois_lookup/__init__.py 8ec72bb33df998e32b925e3060d9d17a ./lib/attacks/whois_lookup/whois.py f65411ebaf58392e6e2b617dba99ff90 ./lib/attacks/admin_panel_finder/__init__.py -0e3fbd55788c040bb96faa1ef0cf1394 ./lib/attacks/xss_scan/__init__.py +9353f5d6188963dc5c2d5cee69439688 ./lib/attacks/xss_scan/__init__.py 7642d078fe304a7ca8bfaaa070a0cb31 ./lib/attacks/nmap_scan/__init__.py 216999fa0e84866d5c1d96d5676034e4 ./lib/attacks/nmap_scan/nmap_opts.py ac942d36f7d78c249e587417736e88e6 ./lib/header_check/__init__.py @@ -78,7 +78,7 @@ cd8e35cfd995d0a93892cfc83f01dea7 ./lib/core/common.py 4433353fb5c55578391d8b4006191ee8 ./lib/core/errors.py d41d8cd98f00b204e9800998ecf8427e ./lib/core/__init__.py e5cba9d37cf54d14255a81da1fa83f90 ./lib/core/settings.py -801a4f7ac892b74676c649bd4844ccdb ./lib/core/decorators.py +de4254c5e40f7aa4fb81e0608f758a2c ./lib/core/decorators.py 9a02e5b913d210350545ac26510a63c9 ./var/search/__init__.py 1ed3c450e620ff1edd8b0864179fdea7 ./var/search/selenium_search.py cfcce04aac694eee7a6c73969861ce43 ./var/search/pgp_search.py diff --git a/lib/attacks/gist_lookup/__init__.py b/lib/attacks/gist_lookup/__init__.py index f7f1251..9b13276 100644 --- a/lib/attacks/gist_lookup/__init__.py +++ b/lib/attacks/gist_lookup/__init__.py @@ -1,9 +1,10 @@ import re +import sys from bs4 import BeautifulSoup -import lib.core.settings import lib.core.common +import lib.core.settings def __create_url(redirect, template="https://gist.github.com{}"): @@ -20,18 +21,22 @@ def get_raw_html(redirect, verbose=False): tag, descriptor = "a", "href" raw_gist_regex = re.compile(r".raw.[a-z0-9]{40}", re.I) _, status, html, _ = lib.core.common.get_page(redirect) - soup = BeautifulSoup(html, "html.parser") - for link in soup.findAll(tag): - raw_gist_redirect = link.get(descriptor) - if raw_gist_regex.search(str(raw_gist_redirect)) is not None: - url = __create_url(raw_gist_redirect) - if verbose: - lib.core.settings.logger.debug(lib.core.settings.set_color( - "found raw Gist URL '{}'...".format(url), level=10 - )) - _, _, html, _ = lib.core.common.get_page(url) - raw_soup = BeautifulSoup(html, "html.parser") - return raw_soup, url + + if status == 200: + soup = BeautifulSoup(html, "html.parser") + for link in soup.findAll(tag): + raw_gist_redirect = link.get(descriptor) + if raw_gist_regex.search(str(raw_gist_redirect)) is not None: + url = __create_url(raw_gist_redirect) + if verbose: + lib.core.settings.logger.debug(lib.core.settings.set_color( + "found raw Gist URL '{}'...".format(url), level=10 + )) + _, _, html, _ = lib.core.common.get_page(url) + raw_soup = BeautifulSoup(html, "html.parser") + return raw_soup, url + else: + return None, None def get_links(page_set, proxy=None, agent=None): @@ -43,6 +48,7 @@ def get_links(page_set, proxy=None, agent=None): tag, descriptor = "a", "href" gist_regex = re.compile(r"[a-f0-9]{32}", re.I) gist_skip_schema = ("stargazers", "forks", "#comments") + for i in range(page_set): lib.core.settings.logger.info(lib.core.settings.set_color( "fetching all Gists on page #{}...".format(i+1) @@ -50,19 +56,27 @@ def get_links(page_set, proxy=None, agent=None): _, status, html, _ = lib.core.common.get_page( gist_search_url.format(i+1), proxy=proxy, agent=agent ) - soup = BeautifulSoup(html, "html.parser") - for link in soup.findAll(tag): - redirect = link.get(descriptor) - if not any(s in redirect for s in gist_skip_schema): - if gist_regex.search(redirect) is not None: - if not any(protocol in redirect for protocol in ["https://", "http://"]): - redirects.add(__create_url(redirect)) - else: - redirects.add(redirect) + if status == 200: + soup = BeautifulSoup(html, "html.parser") + for link in soup.findAll(tag): + redirect = link.get(descriptor) + if not any(s in redirect for s in gist_skip_schema): + if gist_regex.search(redirect) is not None: + if not any(protocol in redirect for protocol in ["https://", "http://"]): + redirects.add(__create_url(redirect)) + else: + redirects.add(redirect) + else: + lib.core.settings.logger.warning(lib.core.settings.set_color( + "page #{} failed to load with status code {} (reason '{}')...".format( + i+1, status, lib.core.common.STATUS_CODES[int(status)] + ), level=30 + )) + continue return redirects -def check_files_for_information(url, data_to_search, query): +def check_files_for_information(data_to_search, query): """ check the files to see if they contain any of the information that was specified """ @@ -99,6 +113,7 @@ def check_files_for_information(url, data_to_search, query): ) +# @lib.core.decorators.tail_call_optimized def github_gist_search_main(query, **kwargs): """ main function for searching Gists @@ -108,6 +123,11 @@ def github_gist_search_main(query, **kwargs): verbose = kwargs.get("verbose", False) page_set = kwargs.get("page_set", 10) + # there seems to be a recursion issue in this function, + # so until I get this figured out, we're going to change + # the maximum recursion of the system when we get here + sys.setrecursionlimit(1500) + try: lib.core.settings.logger.info(lib.core.settings.set_color( "searching a total of {} pages of Gists for '{}'...".format( @@ -126,8 +146,9 @@ def github_gist_search_main(query, **kwargs): ), level=15 )) for link in list(links): - gist, gist_link = get_raw_html(link, verbose=verbose) - check_files_for_information(gist_link, gist, query) + if link is not None: + gist, gist_link = get_raw_html(link, verbose=verbose) + check_files_for_information(gist, query) except KeyboardInterrupt: if not lib.core.common.pause(): lib.core.common.shutdown() diff --git a/lib/attacks/xss_scan/__init__.py b/lib/attacks/xss_scan/__init__.py index 481d087..6247a80 100644 --- a/lib/attacks/xss_scan/__init__.py +++ b/lib/attacks/xss_scan/__init__.py @@ -11,6 +11,7 @@ import requests import lib.core.common import lib.core.settings +import lib.core.decorators from lib.core.errors import InvalidTamperProvided @@ -109,22 +110,34 @@ def scan_xss(url, agent=None, proxy=None): be tampered or encoded if the site is not vulnerable """ - try: - _, status, html_data, _ = lib.core.common.get_page(url, agent=agent, proxy=proxy) - except requests.exceptions.ChunkedEncodingError: - lib.core.settings.logger.warning(lib.core.settings.set_color( - "encoding seems to be messed up, trying the request again...", level=30 - )) - _, status, html_data, _ = lib.core.common.get_page(url, agent=agent, proxy=proxy) + retry_flags = 3 + auto_assign = "http://{}" + url_verification = re.compile(r"http(s)?", re.I) + + if url_verification.search(url) is None: + lib.core.settings.logger.warning(lib.core.settings.set_color( + "protocol missing from URL, automatically assigning protocol...", level=30 + )) + url = auto_assign.format(url) + + while retry_flags > 0: + try: + _, status, html_data, _ = lib.core.common.get_page(url, agent=agent, proxy=proxy) + query = find_xss_script(url) + for db in lib.core.settings.DBMS_ERRORS.keys(): + for item in lib.core.settings.DBMS_ERRORS[db]: + if re.findall(item, html_data): + return "sqli", db + if status != 404: + if query in html_data: + return True, None + retry_flags -= 1 + except requests.exceptions.ChunkedEncodingError: + lib.core.settings.logger.warning(lib.core.settings.set_color( + "encoding seems to be messed up, retrying request...", level=30 + )) + retry_flags -= 1 - query = find_xss_script(url) - for db in lib.core.settings.DBMS_ERRORS.keys(): - for item in lib.core.settings.DBMS_ERRORS[db]: - if re.findall(item, html_data): - return "sqli", db - if status != 404: - if query in html_data: - return True, None return False, None diff --git a/lib/core/decorators.py b/lib/core/decorators.py index 49a6810..50c4c28 100644 --- a/lib/core/decorators.py +++ b/lib/core/decorators.py @@ -6,6 +6,7 @@ import lib.core.settings class TimeOut: + def __init__(self, seconds=1, error_message='Timeout'): self.seconds = seconds self.error_message = error_message