diff --git a/.gitignore b/.gitignore index 82e091c..ca92001 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,4 @@ lib/core/__pycache__/ var/__pycache__/ var/auto_issue/__pycache__/ var/blackwidow/__pycache__/ -var/google_search/__pycache__/ \ No newline at end of file +var/search/__pycache__/ \ No newline at end of file diff --git a/etc/checksum/md5sum.md5 b/etc/checksum/md5sum.md5 index 79ae8c3..0bed143 100644 --- a/etc/checksum/md5sum.md5 +++ b/etc/checksum/md5sum.md5 @@ -1,4 +1,4 @@ -6287156b539d0cd67fdd050715f6ecd3 ./zeus.py +79f00e0d6fbc7e8da033205394f2f0e5 ./zeus.py 4b32db388e8acda35570c734d27c950c ./etc/scripts/launch_sqlmap.sh 6ad5f22ec4a6f8324bfb1b01ab6d51ec ./etc/scripts/cleanup.sh 74d7bee13890a9dd279bb857591647ce ./etc/scripts/reinstall.sh @@ -57,7 +57,7 @@ fde5445df5d77d245656adea96673cfa ./lib/firewall/squid.py 6ea65a0160c21e144e92334acc2e3667 ./lib/firewall/anquanbao.py ad238be8225c2791d82d4c7582afe820 ./lib/firewall/generic.py 746f201907050843889975ace2dc66db ./lib/attacks/gist_lookup/__init__.py -566dc051ce3ac8b877a530fa53f47c66 ./lib/attacks/clickjacking_scan/__init__.py +b981a5f1fddc740ec42351b59f16040b ./lib/attacks/clickjacking_scan/__init__.py d41d8cd98f00b204e9800998ecf8427e ./lib/attacks/__init__.py 4fd64046da32edb2cefa6ba5f3ff5848 ./lib/attacks/sqlmap_scan/__init__.py 5e5bb575014ebe613db6bf671d008cf8 ./lib/attacks/sqlmap_scan/sqlmap_opts.py @@ -67,14 +67,14 @@ d41d8cd98f00b204e9800998ecf8427e ./lib/attacks/whois_lookup/__init__.py a27929dffdf979f59675ad21ff5f77a6 ./lib/attacks/xss_scan/__init__.py fa40d9681c7c3024bd50ad12ef231d6d ./lib/attacks/nmap_scan/__init__.py 216999fa0e84866d5c1d96d5676034e4 ./lib/attacks/nmap_scan/nmap_opts.py -1477efbe270c386057b0170faee4335f ./lib/header_check/__init__.py -833a08e289a14935e476742ec3a27e81 ./lib/core/common.py +6c8768d487af7bc0b4bc0d4d0882691c ./lib/header_check/__init__.py +097ddbb1444de0c09535d02265152cf1 ./lib/core/common.py 1faa2b5dfad6eb538bbfe42942d2a9da ./lib/core/errors.py d41d8cd98f00b204e9800998ecf8427e ./lib/core/__init__.py -922da57ee1f9d9ba492484bcdde4eff7 ./lib/core/settings.py -d41d8cd98f00b204e9800998ecf8427e ./var/google_search/__init__.py -ebe79f9647d78d519a0307aaaab1e425 ./var/google_search/search.py +b4b487abccf2e7837329f5cec7b5081d ./lib/core/settings.py +10950ee27df3186646eeead8cb7f53a2 ./var/search/selenium_search.py +7a1d9976db3e7d2923898c3065b9ca33 ./var/search/pgp_search.py d41d8cd98f00b204e9800998ecf8427e ./var/__init__.py d41d8cd98f00b204e9800998ecf8427e ./var/auto_issue/__init__.py a12d5ffa2d45347c86e5fab154a89943 ./var/auto_issue/github.py -40228c36886e5908c4c78c945759bd50 ./var/blackwidow/__init__.py \ No newline at end of file +7f605eefd38c7404660e8266a7a52192 ./var/blackwidow/__init__.py \ No newline at end of file diff --git a/lib/attacks/clickjacking_scan/__init__.py b/lib/attacks/clickjacking_scan/__init__.py index 72d780c..3db75f3 100644 --- a/lib/attacks/clickjacking_scan/__init__.py +++ b/lib/attacks/clickjacking_scan/__init__.py @@ -31,7 +31,7 @@ class ClickJackingScanner(object): ip_addrs = lib.core.settings.create_random_ip() headers = { lib.core.common.HTTP_HEADER.USER_AGENT: agent, - lib.core.common.HTTP_HEADER.X_FORWARDED_FROM: "{}, {}, {}".format( + lib.core.common.HTTP_HEADER.X_FORWARDED_FOR: "{}, {}, {}".format( ip_addrs[0], ip_addrs[1], ip_addrs[2] ), lib.core.common.HTTP_HEADER.CONNECTION: "close" diff --git a/lib/core/common.py b/lib/core/common.py index fdcffda..f65e929 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -42,7 +42,7 @@ class HTTP_HEADER: X_POWERED_BY = "X-Powered-By" X_DATA_ORIGIN = "X-Data-Origin" X_FRAME_OPT = "X-Frame-Options" - X_FORWARDED_FROM = "X-Forwarded-From" + X_FORWARDED_FOR = "X-Forwarded-For" def write_to_log_file(data_to_write, path, filename, blacklist=False): @@ -55,7 +55,10 @@ def write_to_log_file(data_to_write, path, filename, blacklist=False): os.getcwd() ))) + 1) ) - skip_log_schema = ("url-log", "blackwidow-log", "zeus-log", "extracted", ".blacklist", "gist-match") + skip_log_schema = ( + "url-log", "blackwidow-log", "zeus-log", + "extracted", ".blacklist", "gist-match" + ) to_search = filename.split("-")[0] amount = len([f for f in os.listdir(path) if to_search in f]) new_filename = "{}({}).{}".format( diff --git a/lib/core/settings.py b/lib/core/settings.py index baab01d..9ee93b9 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -28,17 +28,8 @@ import bin.unzip_gecko import lib.core.errors import lib.core.common -from lib.attacks.admin_panel_finder import main -from lib.attacks.xss_scan import main_xss -from lib.attacks.whois_lookup.whois import whois_lookup_main -from lib.attacks.clickjacking_scan import clickjacking_main -from lib.attacks.gist_lookup import github_gist_search_main from lib.attacks.sqlmap_scan.sqlmap_opts import SQLMAP_API_OPTIONS from lib.attacks.nmap_scan.nmap_opts import NMAP_API_OPTS -from lib.attacks import ( - nmap_scan, - sqlmap_scan, -) try: raw_input # Python 2 @@ -46,7 +37,6 @@ except NameError: raw_input = input # Python 3 # get the master patch ID when a patch is pushed to the program - PATCH_ID = str(subprocess.check_output(["git", "rev-parse", "origin/master"]))[:6] # clone link @@ -56,7 +46,7 @@ CLONE = "https://github.com/ekultek/zeus-scanner.git" ISSUE_LINK = "https://github.com/ekultek/zeus-scanner/issues" # current version -VERSION = "1.2.28".format(PATCH_ID) +VERSION = "1.2.29".format(PATCH_ID) # colors to output depending on the version VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30} @@ -172,9 +162,12 @@ GIST_MATCH_LOG = "{}/log/gists".format(os.getcwd()) # unknown firewall log path UNKNOWN_FIREWALL_FINGERPRINT_PATH = "{}/log/unknown-firewall".format(os.getcwd()) -# blacklisted forks, if your dork doesn't pull any URL's it'll be sent here +# blacklisted dorks, if your dork doesn't pull any URL's it'll be sent here BLACKLIST_FILE_PATH = "{}/log/blacklist".format(os.getcwd()) +# found PGP keys file path +PGP_KEYS_FILE_PATH = "{}/log/pgp_keys".format(os.getcwd()) + # the current log file being used CURRENT_LOG_FILE_PATH = "{}/log".format(os.getcwd()) @@ -210,6 +203,9 @@ EXTRACTED_URL_FILENAME = "extracted-url-{}.log" # filename for the URL log URL_FILENAME = "url-log-{}.log" +# filename to save the PGP keys +PGP_KEY_FILENAME = "{}-{}.pgp" + # filename for the blacklist log BLACKLIST_FILENAME = ".blacklist" @@ -231,6 +227,9 @@ SQLMAP_MAN_PAGE_URL = "https://github.com/sqlmapproject/sqlmap/wiki/Usage" # whois API link WHOIS_JSON_LINK = "https://jsonwhoisapi.com/api/v1/whois?identifier={}" +# PGP key identifier to ensure that the link we find is a PGP key +PGP_IDENTIFIER_REGEX = re.compile(r"(0x)?[a-z0-9]{16}", re.I) + # regex to find GET params in a URL, IE php?id= URL_QUERY_REGEX = re.compile(r"(.*)[?|#](.*){1}\=(.*)") @@ -238,7 +237,7 @@ URL_QUERY_REGEX = re.compile(r"(.*)[?|#](.*){1}\=(.*)") URL_REGEX = re.compile(r"((https?):((//)|(\\\\))+([\w\d:#@%/;$()~_?\+-=\\\.&](#!)?)*)") # regex to discover if there are any results on the page -NO_RESULTS_REGEX = re.compile("did not match with any results.", re.IGNORECASE) +NO_RESULTS_REGEX = re.compile("did not match with any results.", re.I) # WAF/IDS/IPS checking payload PROTECTION_CHECK_PAYLOAD = ( @@ -256,9 +255,11 @@ AUTHORIZED_SEARCH_ENGINES = { "bing": "http://bing.com", "duckduckgo": "http://duckduckgo.com/html", "google": "http://google.com", - "search-results": "http://www1.search-results.com/web?tpr={}&q={}&page={}" + "search-results": "http://www1.search-results.com/web?tpr={}&q={}&page={}", + "pgp": "https://pgp.mit.edu/pks/lookup?search={}&op=index" } +# search page for Gists and rate checking URL GITHUB_GIST_SEARCH_URLS = { "search": "https://api.github.com/gists/public?page={}&per_page=100", "check_rate": "https://api.github.com/users/ZeusIssueReporter" @@ -790,6 +791,9 @@ def create_random_ip(): generated = __get_nodes() if generated == "0.0.0.0" or "255.255.255.255": generated = __get_nodes() # if it isn't a real IP regenerate it + logger.info(set_color( + "random IP address generated for header '{}'...".format(generated) + )) return generated @@ -894,6 +898,7 @@ def run_attacks(url, **kwargs): whois = kwargs.get("whois", False) clickjacking = kwargs.get("clickjacking", False) github = kwargs.get("github", False) + pgp = kwargs.get("pgp", False) auto_start = kwargs.get("auto_start", False) sqlmap_arguments = kwargs.get("sqlmap_args", None) nmap_arguments = kwargs.get("nmap_args", None) @@ -946,37 +951,47 @@ def run_attacks(url, **kwargs): if question.lower().startswith("y"): if sqlmap: + from lib.attacks import sqlmap_scan return sqlmap_scan.sqlmap_scan_main( url.strip(), verbose=verbose, opts=create_arguments(sqlmap=True, sqlmap_args=sqlmap_arguments, conf=conf_file), auto_start=auto_start) elif nmap: + from lib.attacks import nmap_scan url_ip_address = replace_http(url.strip()) return nmap_scan.perform_port_scan( url_ip_address, verbose=verbose, opts=create_arguments(nmap=True, nmap_args=nmap_arguments) ) elif admin: + from lib.attacks.admin_panel_finder import main main( url, show=show_all, proc_num=threads, verbose=verbose, do_threading=do_threading, batch=batch ) elif xss: + from lib.attacks.xss_scan import main_xss if check_for_protection(PROTECTED, "xss"): main_xss( url, verbose=verbose, proxy=proxy, agent=agent, tamper=tamper_script, batch=batch, ) elif whois: + from lib.attacks.whois_lookup.whois import whois_lookup_main whois_lookup_main( url, verbose=verbose, timeout=timeout ) elif clickjacking: + from lib.attacks.clickjacking_scan import clickjacking_main if check_for_protection(PROTECTED, "clickjacking"): clickjacking_main(url, agent=agent, proxy=proxy, forward=forwarded, batch=batch) elif github: + from lib.attacks.gist_lookup import github_gist_search_main query = replace_http(url) github_gist_search_main(query, agent=agent, proxy=proxy, verbose=verbose) + elif pgp: + from var.search.pgp_search import pgp_main + pgp_main(url, verbose=verbose) else: pass else: diff --git a/lib/header_check/__init__.py b/lib/header_check/__init__.py index a855369..54a02e9 100644 --- a/lib/header_check/__init__.py +++ b/lib/header_check/__init__.py @@ -45,7 +45,7 @@ def detect_protection(url, **kwargs): headers = { HTTP_HEADER.CONNECTION: "close", HTTP_HEADER.USER_AGENT: agent, - HTTP_HEADER.X_FORWARDED_FROM: "{}, {}, {}".format(ip_list[0], ip_list[1], ip_list[2]) + HTTP_HEADER.X_FORWARDED_FOR: "{}, {}, {}".format(ip_list[0], ip_list[1], ip_list[2]) } else: headers = { diff --git a/var/blackwidow/__init__.py b/var/blackwidow/__init__.py index 26c8977..f842313 100644 --- a/var/blackwidow/__init__.py +++ b/var/blackwidow/__init__.py @@ -114,10 +114,6 @@ def blackwidow_main(url, **kwargs): lib.core.settings.create_random_ip(), lib.core.settings.create_random_ip() ) - if verbose: - lib.core.settings.logger.debug(lib.core.settings.set_color( - "random IP addresses generated for header '{}'...".format(forward), level=10 - )) if verbose: lib.core.settings.logger.debug(lib.core.settings.set_color( diff --git a/var/google_search/__init__.py b/var/search/__init__.py similarity index 100% rename from var/google_search/__init__.py rename to var/search/__init__.py diff --git a/var/search/pgp_search.py b/var/search/pgp_search.py new file mode 100644 index 0000000..cf9f11c --- /dev/null +++ b/var/search/pgp_search.py @@ -0,0 +1,174 @@ +import re + +import requests +from bs4 import BeautifulSoup +from requests.exceptions import ReadTimeout + +import lib.core.common +import lib.core.settings + + +def __create_url(ext): + """ + create the URL with the identifier, usually a hash + """ + url = lib.core.settings.AUTHORIZED_SEARCH_ENGINES["pgp"] + items = url.split("/") + # make sure that there's a `/` in the extension + if "/" in ext[0]: + retval = "{}//{}{}".format(items[0], items[2], ext) + else: + # otherwise we'll just add it + retval = "{}//{}/{}".format(items[0], items[2], ext) + return retval + + +def __set_headers(**kwargs): + """ + set the HTTP headers + """ + agent = kwargs.get("agent", None) + xforward = kwargs.get("xforward", False) + if not xforward: + headers = { + lib.core.common.HTTP_HEADER.CONNECTION: "close", + lib.core.common.HTTP_HEADER.USER_AGENT: agent + } + else: + ip_list = ( + lib.core.settings.create_random_ip(), + lib.core.settings.create_random_ip(), + lib.core.settings.create_random_ip() + ) + headers = { + lib.core.common.HTTP_HEADER.CONNECTION: "close", + lib.core.common.HTTP_HEADER.USER_AGENT: agent, + lib.core.common.HTTP_HEADER.X_FORWARDED_FOR: "{}, {}, {}".format( + ip_list[0], ip_list[1], ip_list[2] + ) + } + return headers + + +def obtain_html(url, query, **kwargs): + """ + obtain the HTML containing the URL redirects to the public PGP keys + """ + agent = kwargs.get("agent", None) + xforward = kwargs.get("xforwad", False) + proxy = kwargs.get("proxy", None) + url = url.format(query) + # regular expression to match if no results are given + result_regex = re.compile("<.+>no.results.found<.+.>", re.I) + req = requests.get( + url, + params=__set_headers(agent=agent, xforward=xforward), # set the headers + proxies=lib.core.settings.proxy_string_to_dict(proxy), + timeout=10 + ) + status, html = req.status_code, req.content + if status == 200: + # check against the regex + if result_regex.search(str(html)) is not None: + return None + else: + return html + return None + + +def gather_urls(html, attribute="a", descriptor="href"): + """ + get the URLs within the HTML + """ + redirects, retval = set(), set() + soup = BeautifulSoup(html, "html.parser") + for link in soup.findAll(attribute): + found_redirect = str(link.get(descriptor)).decode("unicode_escape") + if lib.core.settings.PGP_IDENTIFIER_REGEX.search(found_redirect) is not None: + redirects.add(found_redirect) + for link in redirects: + url = __create_url(link) + if lib.core.settings.URL_REGEX.match(url): + retval.add(url) + return list(retval) + + +def get_pgp_keys(url_list, query, attribute="pre", **kwargs): + """ + get the PGP keys by connecting to the URLs and pulling the information from the HTML + """ + agent = kwargs.get("agent", None) + proxy = kwargs.get("proxy", None) + xforward = kwargs.get("xforward", None) + verbose = kwargs.get("verbose", False) + data_sep = "-" * 30 + extracted_keys, identifiers = set(), [] + # regex to match the beginning of a PGP key + identity_matcher = re.compile(r"\bbegin.pgp.public.key.block", re.I) + for url in url_list: + if verbose: + lib.core.settings.logger.debug(lib.core.settings.set_color( + "checking '{}'...".format(url), level=10 + )) + identifiers.append(lib.core.settings.PGP_IDENTIFIER_REGEX.search(str(url)).group()) + req = requests.get( + url, + params=__set_headers(agent=agent, xforward=xforward), + proxies=lib.core.settings.proxy_string_to_dict(proxy), + timeout=10 + ) + status, html = req.status_code, req.content + if status == 200: + soup = BeautifulSoup(html, "html.parser") + context = soup.findAll(attribute)[0] + if identity_matcher.search(str(context)) is not None: + extracted_keys.add(context) + for i, k in enumerate(extracted_keys): + pgp_key = str(k).split("<{}>".format(attribute)) # split the string by the tag + pgp_key = pgp_key[1].split("".format(attribute))[0] # split it against by an end tag + if verbose: + lib.core.settings.logger.debug(lib.core.settings.set_color( + "found PGP:", level=10 + )) + # output the found PGP key if you run in verbose + print("{}\n{}\n{}".format(data_sep, pgp_key, data_sep)) + lib.core.common.write_to_log_file( + pgp_key, lib.core.settings.PGP_KEYS_FILE_PATH, lib.core.settings.PGP_KEY_FILENAME.format(identifiers[i], query) + ) + + +def pgp_main(query, verbose=False): + try: + query = lib.core.settings.replace_http(query, queries=False, complete=True).split(".")[0] + # make sure the query isn't going to fail + except Exception: + query = query + lib.core.settings.logger.info(lib.core.settings.set_color( + "searching public PGP files with given query '{}'...".format(query) + )) + try: + html = obtain_html( + lib.core.settings.AUTHORIZED_SEARCH_ENGINES["pgp"], query, agent=lib.core.settings.DEFAULT_USER_AGENT + ) + except ReadTimeout: + lib.core.settings.logger.warning(lib.core.settings.set_color( + "connection timed out, assuming no PGP keys...", level=30 + )) + html = None + if html is not None: + urls = gather_urls(html) + lib.core.settings.logger.info(lib.core.settings.set_color( + "found a total of {} URLs...".format(len(urls)) + )) + if verbose: + lib.core.settings.logger.debug(lib.core.settings.set_color( + "found a '{}'...".format(urls), level=10 + )) + lib.core.settings.logger.info(lib.core.settings.set_color( + "gathering PGP key(s) and writing to a file...", level=25 + )) + return get_pgp_keys(urls, query, verbose=verbose) + else: + lib.core.settings.logger.warning(lib.core.settings.set_color( + "did not find anything using query '{}'...".format(query), level=30 + )) diff --git a/var/google_search/search.py b/var/search/selenium_search.py similarity index 99% rename from var/google_search/search.py rename to var/search/selenium_search.py index 8fe225f..c47a6b0 100644 --- a/var/google_search/search.py +++ b/var/search/selenium_search.py @@ -405,7 +405,7 @@ def parse_search_results(query, url_to_search, verbose=False, **kwargs): headers = { HTTP_HEADER.CONNECTION: "close", HTTP_HEADER.USER_AGENT: user_agent, - HTTP_HEADER.X_FORWARDED_FROM: "{}, {}, {}".format(ip_to_use[0], ip_to_use[1], ip_to_use[2]) + HTTP_HEADER.X_FORWARDED_FOR: "{}, {}, {}".format(ip_to_use[0], ip_to_use[1], ip_to_use[2]) } else: headers = { diff --git a/zeus.py b/zeus.py index b6ee0f3..38c5143 100755 --- a/zeus.py +++ b/zeus.py @@ -9,7 +9,7 @@ import warnings import subprocess from var import blackwidow -from var.google_search import search +from var.search import selenium_search from var.auto_issue.github import request_issue_creation from lib.header_check import main_header_check from lib.attacks.nmap_scan.nmap_opts import NMAP_API_OPTS @@ -85,6 +85,8 @@ if __name__ == "__main__": help="Perform a clickjacking scan on a provided URL") attacks.add_option("-g", "--github-search", dest="searchGithub", action="store_true", help="Perform a Github Gist search for any information on the found websites") + attacks.add_option("-P", "--pgp", dest="pgpLookup", action="store_true", + help="Perform a PGP public key lookup on the found URLs") attacks.add_option("--sqlmap-args", dest="sqlmapArguments", metavar="SQLMAP-ARGS", help="Pass the arguments to send to the sqlmap API within quotes & " "separated by a comma. IE 'dbms mysql, verbose 3, level 5'") @@ -97,7 +99,7 @@ if __name__ == "__main__": help="Show the arguments that the sqlmap API understands") attacks.add_option("--show-nmap", dest="showNmapArgs", action="store_true", help="Show the arguments that nmap understands") - attacks.add_option("-P", "--show-possibles", dest="showAllConnections", action="store_true", + attacks.add_option("--show-possibles", dest="showAllConnections", action="store_true", help="Show all connections made during the admin panel search") attacks.add_option("--tamper", dest="tamperXssPayloads", metavar="TAMPER-SCRIPT", help="Send the XSS payloads through tampering before sending to the target") @@ -265,7 +267,7 @@ if __name__ == "__main__": opt.runSqliScan, opt.runPortScan, opt.adminPanelFinder, opt.runXssScan, opt.performWhoisLookup, opt.performClickjackingScan, - opt.searchGithub + opt.searchGithub, opt.pgpLookup ] if any(options): with open(urls_to_use) as urls: @@ -291,7 +293,7 @@ if __name__ == "__main__": ) run_attacks( url.strip(), - sqlmap=opt.runSqliScan, nmap=opt.runPortScan, + sqlmap=opt.runSqliScan, nmap=opt.runPortScan, pgp=opt.pgpLookup, xss=opt.runXssScan, whois=opt.performWhoisLookup, admin=opt.adminPanelFinder, clickjacking=opt.performClickjackingScan, github=opt.searchGithub, verbose=opt.runInVerbose, batch=opt.runInBatch, @@ -322,7 +324,7 @@ if __name__ == "__main__": "starting dork scan with query '{}'...".format(opt.dorkToUse) )) try: - search.parse_search_results( + selenium_search.parse_search_results( opt.dorkToUse, search_engine, verbose=opt.runInVerbose, proxy=proxy_to_use, agent=agent_to_use, pull_all=opt.noExclude, parse_webcache=opt.parseWebcache, forward_for=opt.forwardedForRandomIP, tor=opt.useTor, batch=opt.runInBatch, @@ -381,7 +383,7 @@ if __name__ == "__main__": ) )) try: - search.search_multiple_pages( + selenium_search.search_multiple_pages( dork_to_use, link_amount_to_search, proxy=proxy_to_use, agent=agent_to_use, verbose=opt.runInVerbose, xforward=opt.forwardedForRandomIP, batch=opt.runInBatch, @@ -414,7 +416,7 @@ if __name__ == "__main__": "starting dork scan with query '{}'...".format(dork) )) try: - search.parse_search_results( + selenium_search.parse_search_results( dork, search_engine, verbose=opt.runInVerbose, proxy=proxy_to_use, agent=agent_to_use, pull_all=opt.noExclude, parse_webcache=opt.parseWebcache, tor=opt.useTor, batch=opt.runInBatch @@ -439,7 +441,7 @@ if __name__ == "__main__": "using random dork '{}' as the search query...".format(random_dork) )) try: - search.parse_search_results( + selenium_search.parse_search_results( random_dork, search_engine, verbose=opt.runInVerbose, proxy=proxy_to_use, agent=agent_to_use, pull_all=opt.noExclude, parse_webcache=opt.parseWebcache, tor=opt.useTor, batch=opt.runInBatch