prehistoric-systems/Zeus-Scanner
1
0
Fork 0
mirror of https://github.com/Ekultek/Zeus-Scanner.git synced 2026-03-11 08:55:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

created a website spider, will pull all useable URL's from a given webpage

Browse source
This commit is contained in:
ekultek 2017-09-11 11:43:41 -05:00
parent 74bf299e27
commit 03498f3d9f
2 changed files with 189 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
var/blackwidow/__init__.py Normal file
View file

@ -0,0 +1,113 @@
import os
import requests
import lib.settings
import lib.errors
class Blackwidow(object):
"""
spider to scrape a webpage for all available URL's
"""
def __init__(self, url, user_agent=None, proxy=None):
self.url = url
if proxy is not None:
self.proxy = proxy
else:
self.proxy = None
if user_agent is None:
self.user_agent = lib.settings.DEFAULT_USER_AGENT
else:
self.user_agent = user_agent
@staticmethod
def get_url_ext(url):
"""
get the extenstion of the URL
"""
try:
data = url.split(".")
if data[-1] in lib.settings.SPIDER_EXT_EXCLUDE:
return True
return False
except Exception:
pass
def test_connection(self):
"""
make sure the connection is good before you continue
"""
try:
attempt = requests.get(self.url, params={"user-agent": self.user_agent}, proxies=self.proxy)
if attempt.status_code == 200:
return "ok"
else:
raise lib.errors.SpiderTestFailure(
"failed to connect to '{}', received status code: {}".format(
self.url, attempt.status_code
)
)
except Exception as e:
lib.settings.logger.exception(lib.settings.set_color(
"failed to connect to '{}' received error '{}'...".format(
self.url, e
)
))
def scrape_page_for_links(self, given_url):
"""
scrape the webpage's HTML for usable GET links
"""
unique_links = set()
while True:
req = requests.get(given_url, params={"user-agent": self.user_agent}, proxies=self.proxy)
html_page = req.content
found_links = lib.settings.URL_REGEX.findall(html_page)
for link in list(found_links):
if lib.settings.URL_QUERY_REGEX.match(link[0]) and not Blackwidow.get_url_ext(link[0]):
unique_links.add(link)
else:
pass
break
return list(unique_links)
@staticmethod
def write_to_file(urls, dir_path="{}/log/blackwidow-log", log_file="blackwidow-log-{}.log"):
"""
write to the log file
"""
full_path = "{}/{}".format(dir_path.format(os.getcwd()),
log_file.format(len(os.listdir(dir_path.format(os.getcwd())))))
lib.settings.create_dir(dir_path.format(os.getcwd()))
with open(full_path, "a+") as log:
for url in urls:
log.write(url + "\n")
lib.settings.logger.info(lib.settings.set_color(
"found a total of {} unique usable URL's, saved to '{}'...".format(len(urls), full_path)
))
def blackwidow_main(url, proxy=None, agent=None, verbose=False):
"""
scrape a given URL for all available links
"""
lib.settings.create_dir("{}/{}".format(os.getcwd(), "log/blackwidow-log"))
lib.settings.logger.info(lib.settings.set_color(
"starting blackwidow on '{}'...".format(url)
))
crawler = Blackwidow(url, user_agent=agent, proxy=proxy)
if verbose:
lib.settings.logger.debug(lib.settings.set_color(
"testing connection to the URL...", level=10
))
crawler.test_connection()
if verbose:
lib.settings.logger.debug(lib.settings.set_color(
"connection satisfied, continuing process...", level=10
))
found = crawler.scrape_page_for_links(url)
to_use = [data[0] for data in found]
crawler.write_to_file(to_use)

99
zeus.py
View file

@ -1,5 +1,6 @@
#!/usr/bin/env python
import os
import time
import optparse
import subprocess
@ -7,7 +8,9 @@ import random
import httplib as http_client
from var.google_search import search
from var import blackwidow
from lib.attacks.sqlmap_scan.sqlmap_opts import SQLMAP_API_OPTIONS
from lib.errors import InvalidInputProvided
from lib.attacks import (
nmap_scan,
sqlmap_scan
@ -24,15 +27,21 @@ from lib.settings import (
CURRENT_LOG_FILE_PATH,
AUTHORIZED_SEARCH_ENGINES,
URL_LOG_PATH,
SPIDER_LOG_PATH,
replace_http,
prompt,
get_random_dork,
update_zeus
update_zeus,
VERSION_STRING,
URL_REGEX, URL_QUERY_REGEX
)
if __name__ == "__main__":
parser = optparse.OptionParser(usage="zeus.py -[d|l] [OPTIONS]")
parser = optparse.OptionParser(usage="{} -d|l|r DORK|FILE [-s|p [--OPTS]] [-D|B|A] [--OPTS]".format(
os.path.basename(__file__)
))
# mandatory options
mandatory = optparse.OptionGroup(parser, "Mandatory Options",
@ -43,6 +52,8 @@ if __name__ == "__main__":
help="Specify a file full of dorks to run through"),
mandatory.add_option("-r", "--rand-dork", dest="useRandomDork", action="store_true",
help="Use a random dork from the etc/dorks.txt file to perform the scan")
mandatory.add_option("-b", "--blackwidow", dest="spiderWebSite", metavar="URL DEPTH",
help="Spider a single webpage for all available URL's")
# attack options
attacks = optparse.OptionGroup(parser, "Attack arguments",
@ -51,6 +62,15 @@ if __name__ == "__main__":
help="Run a Sqlmap SQLi scan on the discovered URL's")
attacks.add_option("-p", "--port-scan", dest="runPortScan", action="store_true",
help="Run a Nmap port scan on the discovered URL's")
attacks.add_option("--sqlmap-args", dest="sqlmapArguments", metavar="SQLMAP-ARGS",
help="Pass the arguments to send to the sqlmap API within quotes & "
"separated by a comma. IE 'dbms mysql, verbose 3, level 5'")
attacks.add_option("--auto-start", dest="autoStartSqlmap", action="store_true",
help="Attempt to automatically find sqlmap on your system")
attacks.add_option("--search-here", dest="givenSearchPath", metavar="PATH-TO-START",
help="Start searching for sqlmap in this given path")
attacks.add_option("--show", dest="showSqlmapArguments", action="store_true",
help="Show the arguments that the sqlmap API understands")
# search engine options
engines = optparse.OptionGroup(parser, "Search engine arguments",
@ -62,48 +82,51 @@ if __name__ == "__main__":
engines.add_option("-A", "--search-engine-aol", dest="useAOL", action="store_true",
help="Use AOL as the search engine")
# obfuscation options
anon = optparse.OptionGroup(parser, "Anonymity arguments",
"Arguments that help with anonymity and hiding identity")
anon.add_option("--proxy", dest="proxyConfig", metavar="PROXY-STRING",
help="Use a proxy to do the scraping, will not auto configure to the API's")
anon.add_option("--proxy-file", dest="proxyFileRand", metavar="FILE-PATH",
help="Grab a random proxy from a given file of proxies")
anon.add_option("--random-agent", dest="useRandomAgent", action="store_true",
help="Use a random user-agent from the etc/agents.txt file")
anon.add_option("--agent", dest="usePersonalAgent", metavar="USER-AGENT",
help="Use your own personal user-agent")
# miscellaneous options
misc = optparse.OptionGroup(parser, "Misc Options",
"These options affect how the program will run")
misc.add_option("--verbose", dest="runInVerbose", action="store_true",
help="Run the application in verbose mode (more output)")
misc.add_option("--proxy", dest="proxyConfig", metavar="PROXY-STRING",
help="Use a proxy to do the scraping, will not auto configure "
"to the API's")
misc.add_option("--proxy-file", dest="proxyFileRand", metavar="FILE-PATH",
help="Grab a random proxy from a given file of proxies")
misc.add_option("--random-agent", dest="useRandomAgent", action="store_true",
help="Use a random user-agent from the etc/agents.txt file")
misc.add_option("--agent", dest="usePersonalAgent", metavar="USER-AGENT",
help="Use your own personal user-agent")
misc.add_option("--show-requests", dest="showRequestInfo", action="store_true",
help="Show your request information (more verbose output) this "
"will also show all requests made to the API's used")
misc.add_option("--sqlmap-args", dest="sqlmapArguments", metavar="SQLMAP-ARGS",
help="Pass the arguments to send to the sqlmap API within quotes & "
"separated by a comma. IE 'dbms mysql, verbose 3, level 5'")
misc.add_option("--auto-start", dest="autoStartSqlmap", action="store_true",
help="Attempt to automatically find sqlmap on your system")
misc.add_option("--search-here", dest="givenSearchPath", metavar="PATH-TO-START",
help="Start searching for sqlmap in this given path")
misc.add_option("--show", dest="showSqlmapArguments", action="store_true",
help="Show the arguments that the sqlmap API understands")
help="Show all HTTP requests made by the application")
misc.add_option("--batch", dest="runInBatch", action="store_true",
help="Skip the questions and run in default batch mode")
misc.add_option("--update", dest="updateZeus", action="store_true",
help="Update to the latest development version")
misc.add_option("--hide", dest="hideBanner", action="store_true",
help="Hide the banner during running")
misc.add_option("--version", dest="showCurrentVersion", action="store_true",
help="Show the current version and exit")
parser.add_option_group(mandatory)
parser.add_option_group(attacks)
parser.add_option_group(anon)
parser.add_option_group(engines)
parser.add_option_group(misc)
opt, _ = parser.parse_args()
if opt.showCurrentVersion:
print(VERSION_STRING)
exit(0)
# run the setup on the program
setup(verbose=opt.runInVerbose)
print(BANNER)
if not opt.hideBanner:
print(BANNER)
start_up()
@ -138,6 +161,7 @@ if __name__ == "__main__":
opts_being_used.append((o, v))
return dict(opts_being_used)
if opt.runInVerbose:
being_run = __find_running_opts()
logger.debug(set_color(
@ -154,6 +178,7 @@ if __name__ == "__main__":
))
http_client.HTTPConnection.debuglevel = 1
def __config_headers():
"""
configure the request headers, this will configure user agents and proxies
@ -178,6 +203,7 @@ if __name__ == "__main__":
agent = None
return proxy, agent
def __config_search_engine(verbose=False):
"""
configure the search engine if a one different from google is given
@ -212,6 +238,7 @@ if __name__ == "__main__":
se = AUTHORIZED_SEARCH_ENGINES["google"]
return se
def __create_sqlmap_arguments():
"""
create the sqlmap arguments (a list of tuples) that will be passed to the API
@ -230,6 +257,7 @@ if __name__ == "__main__":
))
return retval
def __run_attacks(url, sqlmap=False, verbose=False, nmap=False, given_path=None, auto=False, batch=False):
"""
run the attacks if any are requested
@ -339,6 +367,31 @@ if __name__ == "__main__":
level=50
))
elif opt.spiderWebSite:
if not URL_REGEX.match(opt.spiderWebSite):
raise InvalidInputProvided(
"provided URL did not match to a true URL, check the URL and try again..."
)
else:
if URL_QUERY_REGEX.match(opt.spiderWebSite):
is_sure = prompt(
"it is recomened to not use a URL that has a GET(query) parameter in it, "
"would you like to continue", "yN"
)
if is_sure.lower().startswith("y"):
pass
else:
shutdown()
blackwidow.blackwidow_main(opt.spiderWebSite, agent=agent_to_use, proxy=proxy_to_use, verbose=opt.runInVerbose)
urls_to_use = get_latest_log_file(SPIDER_LOG_PATH)
if opt.runSqliScan or opt.runPortScan:
with open(urls_to_use) as urls:
for url in urls.readlines():
__run_attacks(url.strip(), sqlmap=opt.runSqliScan, nmap=opt.runPortScan,
given_path=opt.givenSearchPath, auto=opt.autoStartSqlmap,
batch=opt.runInBatch)
else:
logger.critical(set_color(