From 77bc6dc956789ba872233739b709d6e907892089 Mon Sep 17 00:00:00 2001 From: ekultek Date: Sat, 11 Nov 2017 15:11:06 -0600 Subject: [PATCH] multiple page searching no longer requires Google's API, you can now perform as many requests as you want and get as many links as you want as long as it does not go over 500 pages. Removed a couple dependencies that needed to be there for Google's API --- etc/checksum/md5sum.md5 | 10 +-- lib/core/settings.py | 10 ++- requirements.txt | 2 - var/google_search/search.py | 139 ++++++++++++++++++++---------------- zeus.py | 18 +++-- 5 files changed, 99 insertions(+), 80 deletions(-) diff --git a/etc/checksum/md5sum.md5 b/etc/checksum/md5sum.md5 index cfa5738..ac066e1 100644 --- a/etc/checksum/md5sum.md5 +++ b/etc/checksum/md5sum.md5 @@ -1,4 +1,4 @@ -7ad57e1cf9dfca20650b7eb7f2f7686c ./zeus.py +6b7b3eac4bcb0bf320973a2e5cf5078e ./zeus.py 4b32db388e8acda35570c734d27c950c ./etc/scripts/launch_sqlmap.sh 6ad5f22ec4a6f8324bfb1b01ab6d51ec ./etc/scripts/cleanup.sh 155c9482f690f1482f324a7ffd8b8098 ./etc/scripts/fix_pie.sh @@ -41,19 +41,19 @@ d41d8cd98f00b204e9800998ecf8427e ./lib/__init__.py 3308a53435cd255107a9301723844d6e ./lib/attacks/clickjacking_scan/__init__.py d41d8cd98f00b204e9800998ecf8427e ./lib/attacks/__init__.py 7272a7fd0b0c2e9192bc4adb6154d2f0 ./lib/attacks/sqlmap_scan/__init__.py -5e5bb575014ebe613db6bf671d008cf8 ./lib/attacks/sqlmap_scan/sqlmap_opts.py +aa7268a8f085734a6c577c86440f7a1b ./lib/attacks/sqlmap_scan/sqlmap_opts.py d41d8cd98f00b204e9800998ecf8427e ./lib/attacks/whois_lookup/__init__.py d8fab18b15d1546f6585fe926c27868f ./lib/attacks/whois_lookup/whois.py -c96c7f39e6636e3a0986470212a46314 ./lib/attacks/admin_panel_finder/__init__.py +3917cdce61918f3992fc682105367ce8 ./lib/attacks/admin_panel_finder/__init__.py 77a8c4b040680cc00fcfe7924b1a4b1b ./lib/attacks/xss_scan/__init__.py 27358f26bda30d7356143c3ea1fa99c5 ./lib/attacks/nmap_scan/__init__.py 21faf4679cdeaa731029a48f8963d6e7 ./lib/attacks/nmap_scan/nmap_opts.py 1faa2b5dfad6eb538bbfe42942d2a9da ./lib/core/errors.py d41d8cd98f00b204e9800998ecf8427e ./lib/core/__init__.py -f07a174b8894114a44ce86fa47e96d3d ./lib/core/settings.py +d7882d1d6002eef2667d10ced8e5e4f4 ./lib/core/settings.py 60d6f072cc80f08e9214131dc33fe63a ./lib/header_check/__init__.py d41d8cd98f00b204e9800998ecf8427e ./var/google_search/__init__.py -a576adcbf0c7c4e7feca1016d632e53b ./var/google_search/search.py +049b8a1cc93dc44b12dd93efdf785412 ./var/google_search/search.py d41d8cd98f00b204e9800998ecf8427e ./var/__init__.py d41d8cd98f00b204e9800998ecf8427e ./var/auto_issue/__init__.py dadca85c232153021ba9ff253d8ee1d9 ./var/auto_issue/github.py diff --git a/lib/core/settings.py b/lib/core/settings.py index 3eac3fe..9dcf95c 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -52,7 +52,7 @@ PATCH_ID = str(subprocess.check_output(["git", "rev-parse", "origin/master"]))[: CLONE = "https://github.com/ekultek/zeus-scanner.git" # current version -VERSION = "1.2.4".format(PATCH_ID) +VERSION = "1.2.5".format(PATCH_ID) # colors to output depending on the version VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30} @@ -87,6 +87,9 @@ DEFAULT_USER_AGENT = "Zeus-Scanner/{} (Language=Python/{}; Platform={})".format( # max number of threads allowed MAX_THREADS = 10 +# max amount of pages to search +MAX_PAGE_NUMBER = 500 + # path to the checksum CHECKSUM_PATH = "{}/etc/checksum/md5sum.md5".format(os.getcwd()) @@ -176,7 +179,8 @@ AUTHORIZED_SEARCH_ENGINES = { "aol": "http://aol.com", "bing": "http://bing.com", "duckduckgo": "http://duckduckgo.com/html", - "google": "http://google.com" + "google": "http://google.com", + "search-results": "http://www1.search-results.com/web?tpr={}&q={}&page={}" } # extensions to exclude from the spider @@ -205,7 +209,7 @@ URL_EXCLUDES = ( "www.google", "mail.google", "accounts.google", "schema.org", "www. 0: - results.add(next(index)) - limit -= 1 - found += 1 - except Exception as e: - if "Error 503" in str(e): - logger.fatal(set_color( - "Google is blocking the current IP address, dumping already found URL's...", level=50 - )) - results = results - pass - retval = set() - for url in results: - if URL_REGEX.match(url) and URL_QUERY_REGEX.match(url): + if not xforward: + params = { + "Connection": "close", + "user-agent": agent + } + else: + ip_list = (create_random_ip(), create_random_ip(), create_random_ip()) + params = { + "Connection": "close", + "user-agent": agent, + "X-Forwarded-For": "{}, {}, {}".format(ip_list[0], ip_list[1], ip_list[2]) + } + + page_number = 1 + try: + while len(retval) <= link_amount: if verbose: logger.debug(set_color( - "found '{}'...".format(url), level=10 + "searching page number {}...".format(page_number), level=10 )) - retval.add(url) - - if len(retval) != 0: - logger.info(set_color( - "a total of {} links found out of requested {}...".format( - len(retval), link_amount + if page_number % 10 == 0: + logger.info(set_color( + "currently on page {} of search results...".format( + page_number + ) + )) + page_request = requests.get( + search_engine.format(page_number, query, page_number), params=params, + proxies=proxy_string_to_dict(proxy) ) - )) - write_to_log_file(list(retval), URL_LOG_PATH, "url-log-{}.log") - else: + if page_request.status_code == 200: + html_page = page_request.content + soup = BeautifulSoup(html_page, "html.parser") + for link in soup.findAll(attrib): + redirect = link.get(desc) + if redirect is not None: + if not any(ex in redirect for ex in URL_EXCLUDES): + if URL_REGEX.match(redirect): + retval.add(redirect) + if page_number < MAX_PAGE_NUMBER: + page_number += 1 + else: + logger.warning(set_color( + "hit max page number {}...".format(MAX_PAGE_NUMBER), level=30 + )) + break + except KeyboardInterrupt: logger.error(set_color( - "unable to extract URL's from results...", level=40 + "user aborted, dumping already found URL(s)...", level=40 )) + write_to_log_file(retval, URL_LOG_PATH, "url-log-{}.log") + logger.info(set_color( + "found a total of {} URL(s)...".format(len(retval)), level=25 + )) + shutdown() + except Exception as e: + logger.exception(set_color( + "Zeus ran into an unexpected error '{}'...".format(e), level=50 + )) + request_issue_creation() + shutdown() + + logger.info(set_color( + "a total of {} URL(s) found out of the requested {}...".format(len(retval), link_amount), level=25 + )) + write_to_log_file(retval, URL_LOG_PATH, "url-log-{}.log") + return list(retval) if len(retval) != 0 else None diff --git a/zeus.py b/zeus.py index f1acac7..4984c30 100755 --- a/zeus.py +++ b/zeus.py @@ -340,17 +340,21 @@ if __name__ == "__main__": # search multiple pages of Google elif opt.dorkToUse is not None and opt.searchMultiplePages: if opt.amountToSearch is None: - logger.fatal(set_color( - "did not specify amount of links to find...", level=50 + logger.warning(set_color( + "did not specify amount of links to find defaulting to 40...", level=30 )) - shutdown() - link_amount_to_search = opt.amountToSearch + link_amount_to_search = 40 + else: + link_amount_to_search = opt.amountToSearch + logger.info(set_color( - "searching Google using dork '{}' for a total of {} links...".format(opt.dorkToUse, opt.amountToSearch) + "searching Google using dork '{}' for a total of {} links...".format(opt.dorkToUse, link_amount_to_search) )) try: - search.search_multiple_pages(opt.dorkToUse, link_amount_to_search, proxy=proxy_to_use, - agent=agent_to_use, verbose=opt.runInVerbose) + search.search_multiple_pages( + opt.dorkToUse, link_amount_to_search, proxy=proxy_to_use, + agent=agent_to_use, verbose=opt.runInVerbose, xforward=opt.forwardedForRandomIP + ) except Exception as e: if "Error 400" in str(e): logger.fatal(set_color(