From 94dac1c2bc43641d40d904c76e7becbbbb2ae4f4 Mon Sep 17 00:00:00 2001 From: ekultek Date: Tue, 3 Oct 2017 11:19:16 -0500 Subject: [PATCH] added the ability to search multiple pages of Google (issue #4) --- lib/settings.py | 23 +++++++++++- requirements.txt | 1 + var/google_search/search.py | 74 ++++++++++++++++++++++++++++--------- zeus.py | 56 +++++++++++++++++++++++++++- 4 files changed, 132 insertions(+), 22 deletions(-) diff --git a/lib/settings.py b/lib/settings.py index e3f6237..caf6ecb 100644 --- a/lib/settings.py +++ b/lib/settings.py @@ -22,7 +22,7 @@ except NameError: # clone link CLONE = "https://github.com/ekultek/zeus-scanner.git" # current version -VERSION = "1.0.26" +VERSION = "1.0.27" # colors to output depending on the version VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30} # version string formatting @@ -380,4 +380,23 @@ def fix_log_file(logfile=get_latest_log_file(CURRENT_LOG_FILE_PATH)): open(logfile, "w").close() with open(logfile, "a+") as fixed: for line in retval.split("\n"): - fixed.write(line + "\n") \ No newline at end of file + fixed.write(line + "\n") + + +def write_to_log_file(data_to_write, path, filename): + create_dir(path.format(os.getcwd())) + full_file_path = "{}/{}".format( + path.format(os.getcwd()), filename.format(len(os.listdir(path.format( + os.getcwd() + ))) + 1) + ) + with open(full_file_path, "a+") as log: + if isinstance(data_to_write, list): + for item in data_to_write: + item = item.strip() + log.write(str(item) + "\n") + logger.info(set_color( + "successfully wrote found items to '{}'...".format(full_file_path) + )) + else: + log.write(data_to_write + "\n") diff --git a/requirements.txt b/requirements.txt index 8570146..83f1f29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ python-nmap==0.6.1 whichcraft==0.4.1 pyvirtualdisplay==0.2.1 lxml==3.7.3 +google-api-python-client==1.6.4 diff --git a/var/google_search/search.py b/var/google_search/search.py index 5c395ea..f1068a4 100644 --- a/var/google_search/search.py +++ b/var/google_search/search.py @@ -2,11 +2,23 @@ import os import re import time try: - from urllib import unquote + import urllib2 except ImportError: - from urllib.parse import unquote + import urllib as urllib2 + +try: + from urllib import ( + unquote, + quote + ) +except ImportError: + from urllib.parse import ( + unquote, + quote + ) import requests +import google as google_api from selenium import webdriver from pyvirtualdisplay import Display from selenium.webdriver.common.keys import Keys @@ -20,7 +32,8 @@ from lib.settings import ( URL_QUERY_REGEX, URL_REGEX, shutdown, - create_dir, + URL_LOG_PATH, + write_to_log_file, ) try: @@ -160,21 +173,17 @@ def get_urls(query, url, verbose=False, warning=True, user_agent=None, proxy=Non def parse_search_results( - query, url, verbose=False, dirname="{}/log/url-log", filename="url-log-{}.log", **kwargs): + query, url_to_search, verbose=False, **kwargs): """ Parse a webpage from Google for URL's with a GET(query) parameter """ exclude = "google" or "webcache" or "youtube" splitter = "&" - - create_dir(dirname.format(os.getcwd())) - full_file_path = "{}/{}".format( - dirname.format(os.getcwd()), filename.format(len(os.listdir(dirname.format( - os.getcwd() - ))) + 1) - ) + retval = set() + query_url = None def __get_headers(): + proxy_string, user_agent = None, None try: proxy_string = kwargs.get("proxy") except: @@ -223,7 +232,7 @@ def parse_search_results( "attempting to gather query URL..." )) try: - query_url = get_urls(query, url, verbose=verbose, user_agent=user_agent, proxy=proxy_string) + query_url = get_urls(query, url_to_search, verbose=verbose, user_agent=user_agent, proxy=proxy_string) except Exception as e: if "WebDriverException" in str(e): logger.exception(set_color( @@ -241,12 +250,12 @@ def parse_search_results( logger.info(set_color( "URL successfully gathered, searching for GET parameters..." )) + logger.info(set_color(proxy_string_info)) req = requests.get(query_url, proxies=proxy_string) logger.info(set_color(user_agent_info)) req.headers.update(headers) found_urls = URL_REGEX.findall(req.text) - retval = set() for urls in list(found_urls): for url in list(urls): url = unquote(url) @@ -267,16 +276,45 @@ def parse_search_results( "found a total of {} URL's with a GET parameter...".format(len(retval)) )) if len(retval) != 0: + file_path = write_to_log_file(retval, URL_LOG_PATH, "url-log-{}.log") logger.info(set_color( - "saving found URL's under '{}'...".format(full_file_path) + "found URL's have been saved to '{}'...".format(file_path) )) - with open(full_file_path, "a+") as log: - for url in list(retval): - log.write(url + "\n") else: logger.critical(set_color( "did not find any usable URL's with the given query '{}' " - "using search engine '{}'...".format(query, url), level=50 + "using search engine '{}'...".format(query, url_to_search), level=50 )) shutdown() return list(retval) if len(retval) != 0 else None + + +def search_multiple_pages(query, link_amount, verbose=False): + logger.warning(set_color( + "multiple pages will be searched using Google's API client, searches may be blocked after a certain " + "amount of time...", level=30 + )) + results, limit, found, index = set(), link_amount, 0, google_api.search(query) + while limit > 0: + results.add(next(index)) + limit -= 1 + found += 1 + + retval = set() + for url in results: + if URL_REGEX.match(url) and URL_QUERY_REGEX.match(url): + if verbose: + logger.debug(set_color( + "found '{}'...".format(url), level=10 + )) + retval.add(url) + + logger.info(set_color( + "a total of {} links found out of requested {}...".format( + len(retval), link_amount + ) + )) + + write_to_log_file(list(retval), URL_LOG_PATH, "url-log-{}.log") + + diff --git a/zeus.py b/zeus.py index 0eaeac0..f71c89f 100755 --- a/zeus.py +++ b/zeus.py @@ -46,7 +46,7 @@ from lib.settings import ( NMAP_MAN_PAGE_URL, SQLMAP_MAN_PAGE_URL, get_true_url, - fix_log_file + fix_log_file, ) if __name__ == "__main__": @@ -111,6 +111,13 @@ if __name__ == "__main__": engines.add_option("-A", "--search-engine-aol", dest="useAOL", action="store_true", help="Use AOL as the search engine") + search_items = optparse.OptionGroup(parser, "Search options", + "Arguments that will control the search criteria") + search_items.add_option("-L", "--links", dest="amountToSearch", type=int, metavar="HOW-MANY-LINKS", + help="Specify how many links to try and search on Google") + search_items.add_option("-M", "--multi", dest="searchMultiplePages", action="store_true", + help="Search multiple pages of Google") + # obfuscation options anon = optparse.OptionGroup(parser, "Anonymity arguments", "Arguments that help with anonymity and hiding identity") @@ -141,6 +148,7 @@ if __name__ == "__main__": parser.add_option_group(mandatory) parser.add_option_group(attacks) + parser.add_option_group(search_items) parser.add_option_group(anon) parser.add_option_group(engines) parser.add_option_group(misc) @@ -391,7 +399,7 @@ if __name__ == "__main__": try: # use a personal dork as the query - if opt.dorkToUse is not None: + if opt.dorkToUse is not None and not opt.searchMultiplePages: logger.info(set_color( "starting dork scan with query '{}'...".format(opt.dorkToUse) )) @@ -423,6 +431,50 @@ if __name__ == "__main__": auto=opt.autoStartSqlmap, verbose=opt.runInVerbose, batch=opt.runInBatch ) + # search multiple pages of Google + elif opt.dorkToUse is not None and opt.searchMultiplePages: + if opt.amountToSearch is None: + logger.fatal(set_color( + "did not specify amount of links to find...", level=50 + )) + shutdown() + link_amount_to_search = opt.amountToSearch + logger.info(set_color( + "searching Google using dork '{}' for a total of {} links...".format(opt.dorkToUse, opt.amountToSearch) + )) + if proxy_to_use: + logger.warning(set_color( + "proxy not implemented for multiple page searching...", level=30 + )) + try: + search.search_multiple_pages(opt.dorkToUse, link_amount_to_search, verbose=opt.runInVerbose) + except Exception as e: + if "Error 400" in str(e): + logger.fatal(set_color( + "failed to connect to search engine...".format(e), level=50 + )) + elif "Error 503" in str(e): + logger.fatal(set_color( + "Google has blocked your IP address from doing anymore searches via API, " + "you can still search using headless browsers (-d )...", level=50 + )) + else: + logger.exception(set_color( + "failed with unexpected error '{}'...".format(e), level=50 + )) + shutdown() + + urls_to_use = get_latest_log_file(URL_LOG_PATH) + if opt.runSqliScan or opt.runPortScan or opt.intelCheck or opt.adminPanelFinder or opt.runXssScan: + with open(urls_to_use) as urls: + for url in urls.readlines(): + __run_attacks( + url.strip(), + sqlmap=opt.runSqliScan, nmap=opt.runPortScan, intel=opt.intelCheck, xss=opt.runXssScan, + admin=opt.adminPanelFinder, given_path=opt.givenSearchPath, + auto=opt.autoStartSqlmap, verbose=opt.runInVerbose, batch=opt.runInBatch + ) + # use a file full of dorks as the queries elif opt.dorkFileToUse is not None: with open(opt.dorkFileToUse) as dorks: