From bedcde92708233dcfe2db8d8e7e1a104e710d35a Mon Sep 17 00:00:00 2001 From: ekultek Date: Sun, 3 Dec 2017 13:30:50 -0600 Subject: [PATCH] complete rework of how the Gist search works, you no longer have a certain amount you can search, this is also a fix for issue #221 where the dict would not load into JSON format, that is no longer required --- etc/checksum/md5sum.md5 | 4 +- lib/attacks/gist_lookup/__init__.py | 212 ++++++++++++---------------- lib/core/settings.py | 7 +- 3 files changed, 96 insertions(+), 127 deletions(-) diff --git a/etc/checksum/md5sum.md5 b/etc/checksum/md5sum.md5 index c80e56d..f2d44c6 100644 --- a/etc/checksum/md5sum.md5 +++ b/etc/checksum/md5sum.md5 @@ -62,7 +62,7 @@ bbd8b4c6100070d420d48dc7dfc297eb ./lib/firewall/webknight.py 81a29a14d72980a306fbaec0dc772048 ./lib/firewall/fortigate.py 6ea65a0160c21e144e92334acc2e3667 ./lib/firewall/anquanbao.py 22a0ad8f2fa1a16b651cb5ae37ca9b0d ./lib/firewall/generic.py -cf23d0f522b183bea9dc0fcf4aa64115 ./lib/attacks/gist_lookup/__init__.py +c3aa1db3ee34b841d91ad850c2cfd7b2 ./lib/attacks/gist_lookup/__init__.py 86224bd899c2a2438042cbdc077dc4cc ./lib/attacks/clickjacking_scan/__init__.py d41d8cd98f00b204e9800998ecf8427e ./lib/attacks/__init__.py 4c644b0e3a62b6c1528d34a04837aa35 ./lib/attacks/sqlmap_scan/__init__.py @@ -77,7 +77,7 @@ ac942d36f7d78c249e587417736e88e6 ./lib/header_check/__init__.py cd8e35cfd995d0a93892cfc83f01dea7 ./lib/core/common.py 4433353fb5c55578391d8b4006191ee8 ./lib/core/errors.py d41d8cd98f00b204e9800998ecf8427e ./lib/core/__init__.py -f723499d996a3bbbd3022cc372dffdd9 ./lib/core/settings.py +e5cba9d37cf54d14255a81da1fa83f90 ./lib/core/settings.py 801a4f7ac892b74676c649bd4844ccdb ./lib/core/decorators.py 9a02e5b913d210350545ac26510a63c9 ./var/search/__init__.py 1ed3c450e620ff1edd8b0864179fdea7 ./var/search/selenium_search.py diff --git a/lib/attacks/gist_lookup/__init__.py b/lib/attacks/gist_lookup/__init__.py index 07d9d90..f7f1251 100644 --- a/lib/attacks/gist_lookup/__init__.py +++ b/lib/attacks/gist_lookup/__init__.py @@ -1,73 +1,68 @@ import re -import json -import time -import requests + +from bs4 import BeautifulSoup import lib.core.settings import lib.core.common -def __check_remaining_rate_limit(): +def __create_url(redirect, template="https://gist.github.com{}"): """ - check how many requests you have left to run + create the URL for the Gists """ - url = lib.core.settings.GITHUB_GIST_SEARCH_URLS["check_rate"] - _, _, data, headers = lib.core.common.get_page(url, auth="token {}".format( - lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH) - )) - remaining = headers["X-RateLimit-Remaining"] - if int(remaining) == 0: - lib.core.settings.logger.error(lib.core.settings.set_color( - "Github only allows 60 unauthenticated requests per hour, you have hit that limit " - "if you need to do more requests it is recommended to run behind a proxy with a different " - "user-agent (IE --proxy socks5://127.0.0.1:9050 --random-agent)...", level=40 - )) - lib.core.common.shutdown() - else: - lib.core.settings.logger.warning(lib.core.settings.set_color( - "you have {} unauthenticated requests remaining...".format(remaining), level=30 - )) + return template.format(redirect) -def get_raw_data(page_set, proxy=None, agent=None, verbose=False): +def get_raw_html(redirect, verbose=False): + """ + get the raw HTML of the Gist plus the URL for it + """ + tag, descriptor = "a", "href" + raw_gist_regex = re.compile(r".raw.[a-z0-9]{40}", re.I) + _, status, html, _ = lib.core.common.get_page(redirect) + soup = BeautifulSoup(html, "html.parser") + for link in soup.findAll(tag): + raw_gist_redirect = link.get(descriptor) + if raw_gist_regex.search(str(raw_gist_redirect)) is not None: + url = __create_url(raw_gist_redirect) + if verbose: + lib.core.settings.logger.debug(lib.core.settings.set_color( + "found raw Gist URL '{}'...".format(url), level=10 + )) + _, _, html, _ = lib.core.common.get_page(url) + raw_soup = BeautifulSoup(html, "html.parser") + return raw_soup, url + + +def get_links(page_set, proxy=None, agent=None): """ parse 10 pages of Github gists and use them """ - retval = set() - url = lib.core.settings.GITHUB_GIST_SEARCH_URLS["search"] - headers = { - lib.core.common.HTTP_HEADER.USER_AGENT: agent, - lib.core.common.HTTP_HEADER.AUTHORIZATION: "token {}".format(lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)), - } - lib.core.settings.logger.info(lib.core.settings.set_color( - "searching a total of {} pages of Gists...".format(page_set[-1]) - )) - if proxy is not None: - proxy = lib.core.settings.proxy_string_to_dict(proxy) - for page in list(page_set): - _, _, data, _ = lib.core.common.get_page(url.format(page), agent=agent, proxy=proxy) - # load the found info into JSON format - # so we can pull using keys - data = json.loads(data.content) - for item in data: - # get the URL to the raw data so we can search it - gist_file = item["files"] - gist_filename = gist_file.keys() - try: - skip_schema = ("-", " ", "") - if not any(s == gist_filename for s in skip_schema): - if verbose: - lib.core.settings.logger.debug(lib.core.settings.set_color( - "found filename '{}'...".format(''.join(gist_filename)), level=10 - )) - retval.add(gist_file[''.join(gist_filename)]["raw_url"]) - # sometimes the URL doesn't like being pulled, so we'll just skip those ones - except Exception: - pass - return retval + redirects, retval = set(), set() + gist_search_url = "https://gist.github.com/discover?page={}" + tag, descriptor = "a", "href" + gist_regex = re.compile(r"[a-f0-9]{32}", re.I) + gist_skip_schema = ("stargazers", "forks", "#comments") + for i in range(page_set): + lib.core.settings.logger.info(lib.core.settings.set_color( + "fetching all Gists on page #{}...".format(i+1) + )) + _, status, html, _ = lib.core.common.get_page( + gist_search_url.format(i+1), proxy=proxy, agent=agent + ) + soup = BeautifulSoup(html, "html.parser") + for link in soup.findAll(tag): + redirect = link.get(descriptor) + if not any(s in redirect for s in gist_skip_schema): + if gist_regex.search(redirect) is not None: + if not any(protocol in redirect for protocol in ["https://", "http://"]): + redirects.add(__create_url(redirect)) + else: + redirects.add(redirect) + return redirects -def check_files_for_information(found_url, data_to_search): +def check_files_for_information(url, data_to_search, query): """ check the files to see if they contain any of the information that was specified """ @@ -75,95 +70,68 @@ def check_files_for_information(found_url, data_to_search): # bases while we do the searching. # this will make it so that if there is a match anywhere # in anything, we'll find it. - if "www" in data_to_search: - # we need to create the query to be just the domain name - data_to_search = data_to_search.split(".")[1] + data_to_search = str(data_to_search) data_regex_schema = ( - # a normal regex with our string - re.compile(r"{}".format(data_to_search), re.I), # match a URL with or without www - re.compile(r"(http(s)?)?(.//)?(www.)?{}".format(data_to_search), re.I), + re.compile(r"(http(s)?)?(.//)?(www.)?{}".format(query), re.I), # match our string and any random character around it (I like to call it the tittyex) - re.compile(r"(.)?{}(.)?".format(data_to_search), re.I), + re.compile(r"(.)?{}(.)?".format(query), re.I), # single boundary match, checks if it's inside of something else - re.compile(r"\b{}".format(data_to_search), re.I), + re.compile(r"\b{}".format(query), re.I), # double boundary, same as above but with another boundary - re.compile(r"\b{}\b".format(data_to_search), re.I), + re.compile(r"\b{}\b".format(query), re.I), # wildcard match - re.compile(r"{}*".format(data_to_search), re.I) + re.compile(r"{}*".format(query), re.I), + # normal match + re.compile(r"{}".format(query), re.I) ) - total_found = set() - try: - _, _, data, _ = lib.core.common.get_page(found_url) - except requests.exceptions.ConnectionError: - lib.core.settings.logger.warning(lib.core.settings.set_color( - "to many requests are being sent to quickly, adding sleep time...", level=30 - )) - time.sleep(3) - _, _, data, _ = lib.core.common.get_page(found_url) - for data_regex in data_regex_schema: - if data_regex.search(data.content) is not None: + for regex in list(data_regex_schema): + if regex.search(data_to_search) is not None: lib.core.settings.logger.info(lib.core.settings.set_color( - "found a match with given specifics ('{}'), saving full Gist to log file...".format( - data_regex.pattern + "found match with given specifics ('{}'), saving Gist to file...".format( + regex.pattern ), level=25 )) - total_found.add(found_url) lib.core.common.write_to_log_file( - data.content, lib.core.settings.GIST_MATCH_LOG, lib.core.settings.GIST_FILENAME.format( - lib.core.settings.replace_http(data_to_search) - ) + data_to_search, + lib.core.settings.GIST_MATCH_LOG, + lib.core.settings.GIST_FILENAME.format(query) ) - return len(total_found) def github_gist_search_main(query, **kwargs): + """ + main function for searching Gists + """ proxy = kwargs.get("proxy", None) agent = kwargs.get("agent", None) verbose = kwargs.get("verbose", False) - thread = kwargs.get("do_threading", False) - # proc_num = kwargs.get("proc_num", 5) # TODO:/ - page_set = kwargs.get("page_set", (1, 2, 3, 4, 5)) - total_found = 0 + page_set = kwargs.get("page_set", 10) try: + lib.core.settings.logger.info(lib.core.settings.set_color( + "searching a total of {} pages of Gists for '{}'...".format( + page_set, query + ) + )) + + if "www." in query: + query = query.split(".")[1] + + links = get_links(page_set, proxy=proxy, agent=agent) if verbose: lib.core.settings.logger.debug(lib.core.settings.set_color( - "checking if you have exceeded your search limit...", level=10 - )) - __check_remaining_rate_limit() - lib.core.settings.logger.info(lib.core.settings.set_color( - "searching Github Gists for '{}'...".format(query) - )) - gathered_links = get_raw_data(page_set, proxy=proxy, agent=agent, verbose=verbose) - lib.core.settings.logger.info(lib.core.settings.set_color( - "pulled a total of {} URL's to search...".format(len(gathered_links)), level=25 - )) - if not thread: - lib.core.settings.logger.info(lib.core.settings.set_color( - "performing Github Gist search, this will probably take awhile..." - )) - for url in gathered_links: - total = check_files_for_information(url, query) - total_found += total - else: - lib.core.settings.logger.warning(lib.core.settings.set_color( - "multi-threading is not implemented yet...", level=35 - )) - lib.core.settings.logger.info(lib.core.settings.set_color( - "performing Github Gist search, this will probably take awhile..." - )) - for url in gathered_links: - total = check_files_for_information(url, query) - total_found += total - if total_found > 0: - lib.core.settings.logger.info(lib.core.settings.set_color( - "found a total of {} interesting Gists...".format(total_found) - )) - else: - lib.core.settings.logger.warning(lib.core.settings.set_color( - "did not find any interesting Gists...", level=30 + "found a total of {} links to search...".format( + len(links) + ), level=15 )) + for link in list(links): + gist, gist_link = get_raw_html(link, verbose=verbose) + check_files_for_information(gist_link, gist, query) except KeyboardInterrupt: if not lib.core.common.pause(): - lib.core.common.shutdown() \ No newline at end of file + lib.core.common.shutdown() + except Exception as e: + lib.core.settings.logger.exception(lib.core.settings.set_color( + "Gist search has failed with error '{}'...".format(str(e)), level=50 + )) \ No newline at end of file diff --git a/lib/core/settings.py b/lib/core/settings.py index 53d0b51..211aaa3 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -46,7 +46,7 @@ CLONE = "https://github.com/ekultek/zeus-scanner.git" ISSUE_LINK = "https://github.com/ekultek/zeus-scanner/issues" # current version -VERSION = "1.3.16.{}".format(PATCH_ID) +VERSION = "1.3.17.{}".format(PATCH_ID) # colors to output depending on the version VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30} @@ -315,10 +315,11 @@ URL_EXCLUDES = ( "drive.google", "books.google", "news.google", "www.google", "mail.google", "accounts.google", "schema.org", "www.