diff --git a/etc/checksum/md5sum.md5 b/etc/checksum/md5sum.md5 index f2d44c6..1ebaac5 100644 --- a/etc/checksum/md5sum.md5 +++ b/etc/checksum/md5sum.md5 @@ -62,7 +62,7 @@ bbd8b4c6100070d420d48dc7dfc297eb ./lib/firewall/webknight.py 81a29a14d72980a306fbaec0dc772048 ./lib/firewall/fortigate.py 6ea65a0160c21e144e92334acc2e3667 ./lib/firewall/anquanbao.py 22a0ad8f2fa1a16b651cb5ae37ca9b0d ./lib/firewall/generic.py -c3aa1db3ee34b841d91ad850c2cfd7b2 ./lib/attacks/gist_lookup/__init__.py +b5ff3286060c0bbc0fe1f0f591131c9c ./lib/attacks/gist_lookup/__init__.py 86224bd899c2a2438042cbdc077dc4cc ./lib/attacks/clickjacking_scan/__init__.py d41d8cd98f00b204e9800998ecf8427e ./lib/attacks/__init__.py 4c644b0e3a62b6c1528d34a04837aa35 ./lib/attacks/sqlmap_scan/__init__.py @@ -70,7 +70,7 @@ d41d8cd98f00b204e9800998ecf8427e ./lib/attacks/__init__.py 7bc7a6f3e85651aab3588f087563eded ./lib/attacks/whois_lookup/__init__.py 8ec72bb33df998e32b925e3060d9d17a ./lib/attacks/whois_lookup/whois.py f65411ebaf58392e6e2b617dba99ff90 ./lib/attacks/admin_panel_finder/__init__.py -0e3fbd55788c040bb96faa1ef0cf1394 ./lib/attacks/xss_scan/__init__.py +9353f5d6188963dc5c2d5cee69439688 ./lib/attacks/xss_scan/__init__.py 7642d078fe304a7ca8bfaaa070a0cb31 ./lib/attacks/nmap_scan/__init__.py 216999fa0e84866d5c1d96d5676034e4 ./lib/attacks/nmap_scan/nmap_opts.py ac942d36f7d78c249e587417736e88e6 ./lib/header_check/__init__.py @@ -78,7 +78,7 @@ cd8e35cfd995d0a93892cfc83f01dea7 ./lib/core/common.py 4433353fb5c55578391d8b4006191ee8 ./lib/core/errors.py d41d8cd98f00b204e9800998ecf8427e ./lib/core/__init__.py e5cba9d37cf54d14255a81da1fa83f90 ./lib/core/settings.py -801a4f7ac892b74676c649bd4844ccdb ./lib/core/decorators.py +de4254c5e40f7aa4fb81e0608f758a2c ./lib/core/decorators.py 9a02e5b913d210350545ac26510a63c9 ./var/search/__init__.py 1ed3c450e620ff1edd8b0864179fdea7 ./var/search/selenium_search.py cfcce04aac694eee7a6c73969861ce43 ./var/search/pgp_search.py diff --git a/lib/attacks/gist_lookup/__init__.py b/lib/attacks/gist_lookup/__init__.py index f7f1251..9b13276 100644 --- a/lib/attacks/gist_lookup/__init__.py +++ b/lib/attacks/gist_lookup/__init__.py @@ -1,9 +1,10 @@ import re +import sys from bs4 import BeautifulSoup -import lib.core.settings import lib.core.common +import lib.core.settings def __create_url(redirect, template="https://gist.github.com{}"): @@ -20,18 +21,22 @@ def get_raw_html(redirect, verbose=False): tag, descriptor = "a", "href" raw_gist_regex = re.compile(r".raw.[a-z0-9]{40}", re.I) _, status, html, _ = lib.core.common.get_page(redirect) - soup = BeautifulSoup(html, "html.parser") - for link in soup.findAll(tag): - raw_gist_redirect = link.get(descriptor) - if raw_gist_regex.search(str(raw_gist_redirect)) is not None: - url = __create_url(raw_gist_redirect) - if verbose: - lib.core.settings.logger.debug(lib.core.settings.set_color( - "found raw Gist URL '{}'...".format(url), level=10 - )) - _, _, html, _ = lib.core.common.get_page(url) - raw_soup = BeautifulSoup(html, "html.parser") - return raw_soup, url + + if status == 200: + soup = BeautifulSoup(html, "html.parser") + for link in soup.findAll(tag): + raw_gist_redirect = link.get(descriptor) + if raw_gist_regex.search(str(raw_gist_redirect)) is not None: + url = __create_url(raw_gist_redirect) + if verbose: + lib.core.settings.logger.debug(lib.core.settings.set_color( + "found raw Gist URL '{}'...".format(url), level=10 + )) + _, _, html, _ = lib.core.common.get_page(url) + raw_soup = BeautifulSoup(html, "html.parser") + return raw_soup, url + else: + return None, None def get_links(page_set, proxy=None, agent=None): @@ -43,6 +48,7 @@ def get_links(page_set, proxy=None, agent=None): tag, descriptor = "a", "href" gist_regex = re.compile(r"[a-f0-9]{32}", re.I) gist_skip_schema = ("stargazers", "forks", "#comments") + for i in range(page_set): lib.core.settings.logger.info(lib.core.settings.set_color( "fetching all Gists on page #{}...".format(i+1) @@ -50,19 +56,27 @@ def get_links(page_set, proxy=None, agent=None): _, status, html, _ = lib.core.common.get_page( gist_search_url.format(i+1), proxy=proxy, agent=agent ) - soup = BeautifulSoup(html, "html.parser") - for link in soup.findAll(tag): - redirect = link.get(descriptor) - if not any(s in redirect for s in gist_skip_schema): - if gist_regex.search(redirect) is not None: - if not any(protocol in redirect for protocol in ["https://", "http://"]): - redirects.add(__create_url(redirect)) - else: - redirects.add(redirect) + if status == 200: + soup = BeautifulSoup(html, "html.parser") + for link in soup.findAll(tag): + redirect = link.get(descriptor) + if not any(s in redirect for s in gist_skip_schema): + if gist_regex.search(redirect) is not None: + if not any(protocol in redirect for protocol in ["https://", "http://"]): + redirects.add(__create_url(redirect)) + else: + redirects.add(redirect) + else: + lib.core.settings.logger.warning(lib.core.settings.set_color( + "page #{} failed to load with status code {} (reason '{}')...".format( + i+1, status, lib.core.common.STATUS_CODES[int(status)] + ), level=30 + )) + continue return redirects -def check_files_for_information(url, data_to_search, query): +def check_files_for_information(data_to_search, query): """ check the files to see if they contain any of the information that was specified """ @@ -99,6 +113,7 @@ def check_files_for_information(url, data_to_search, query): ) +# @lib.core.decorators.tail_call_optimized def github_gist_search_main(query, **kwargs): """ main function for searching Gists @@ -108,6 +123,11 @@ def github_gist_search_main(query, **kwargs): verbose = kwargs.get("verbose", False) page_set = kwargs.get("page_set", 10) + # there seems to be a recursion issue in this function, + # so until I get this figured out, we're going to change + # the maximum recursion of the system when we get here + sys.setrecursionlimit(1500) + try: lib.core.settings.logger.info(lib.core.settings.set_color( "searching a total of {} pages of Gists for '{}'...".format( @@ -126,8 +146,9 @@ def github_gist_search_main(query, **kwargs): ), level=15 )) for link in list(links): - gist, gist_link = get_raw_html(link, verbose=verbose) - check_files_for_information(gist_link, gist, query) + if link is not None: + gist, gist_link = get_raw_html(link, verbose=verbose) + check_files_for_information(gist, query) except KeyboardInterrupt: if not lib.core.common.pause(): lib.core.common.shutdown() diff --git a/lib/attacks/xss_scan/__init__.py b/lib/attacks/xss_scan/__init__.py index 481d087..6247a80 100644 --- a/lib/attacks/xss_scan/__init__.py +++ b/lib/attacks/xss_scan/__init__.py @@ -11,6 +11,7 @@ import requests import lib.core.common import lib.core.settings +import lib.core.decorators from lib.core.errors import InvalidTamperProvided @@ -109,22 +110,34 @@ def scan_xss(url, agent=None, proxy=None): be tampered or encoded if the site is not vulnerable """ - try: - _, status, html_data, _ = lib.core.common.get_page(url, agent=agent, proxy=proxy) - except requests.exceptions.ChunkedEncodingError: - lib.core.settings.logger.warning(lib.core.settings.set_color( - "encoding seems to be messed up, trying the request again...", level=30 - )) - _, status, html_data, _ = lib.core.common.get_page(url, agent=agent, proxy=proxy) + retry_flags = 3 + auto_assign = "http://{}" + url_verification = re.compile(r"http(s)?", re.I) + + if url_verification.search(url) is None: + lib.core.settings.logger.warning(lib.core.settings.set_color( + "protocol missing from URL, automatically assigning protocol...", level=30 + )) + url = auto_assign.format(url) + + while retry_flags > 0: + try: + _, status, html_data, _ = lib.core.common.get_page(url, agent=agent, proxy=proxy) + query = find_xss_script(url) + for db in lib.core.settings.DBMS_ERRORS.keys(): + for item in lib.core.settings.DBMS_ERRORS[db]: + if re.findall(item, html_data): + return "sqli", db + if status != 404: + if query in html_data: + return True, None + retry_flags -= 1 + except requests.exceptions.ChunkedEncodingError: + lib.core.settings.logger.warning(lib.core.settings.set_color( + "encoding seems to be messed up, retrying request...", level=30 + )) + retry_flags -= 1 - query = find_xss_script(url) - for db in lib.core.settings.DBMS_ERRORS.keys(): - for item in lib.core.settings.DBMS_ERRORS[db]: - if re.findall(item, html_data): - return "sqli", db - if status != 404: - if query in html_data: - return True, None return False, None diff --git a/lib/core/decorators.py b/lib/core/decorators.py index 49a6810..50c4c28 100644 --- a/lib/core/decorators.py +++ b/lib/core/decorators.py @@ -6,6 +6,7 @@ import lib.core.settings class TimeOut: + def __init__(self, seconds=1, error_message='Timeout'): self.seconds = seconds self.error_message = error_message