diff --git a/README.md b/README.md index 220bdaf..bb6de34 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![GitHub license](https://img.shields.io/badge/license-GPL-blue.svg?style=flat-square)](https://raw.githubusercontent.com/Ekultek/Zeus-Scanner/master/.github/LICENSE.md) [![Twitter](https://img.shields.io/twitter/url/https/github.com/ekultek/zeus-scanner.svg?style=social)](https://twitter.com/Zeus_Scanner) [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://github.com/Ekultek/Zeus-Scanner#donations) + # Helpful links directory - [Overview](https://github.com/Ekultek/Zeus-Scanner#zeus-scanner) diff --git a/etc/checksum/md5sum.md5 b/etc/checksum/md5sum.md5 index 536b854..58ff900 100644 --- a/etc/checksum/md5sum.md5 +++ b/etc/checksum/md5sum.md5 @@ -43,10 +43,10 @@ f27322b9716e1a2b0b0b0487f3149474 ./lib/attacks/whois_lookup/whois.py f746d2867f493104a78d0540cf50c03f ./lib/attacks/intel_me/__init__.py 1faa2b5dfad6eb538bbfe42942d2a9da ./lib/core/errors.py d41d8cd98f00b204e9800998ecf8427e ./lib/core/__init__.py -8255b986984e7b069907548ec7bc2478 ./lib/core/settings.py +819b60912d654bdd81b9d96bd757989f ./lib/core/settings.py d41d8cd98f00b204e9800998ecf8427e ./var/google_search/__init__.py b92ee17da90b17a0abb4e07e24fca3e1 ./var/google_search/search.py d41d8cd98f00b204e9800998ecf8427e ./var/__init__.py d41d8cd98f00b204e9800998ecf8427e ./var/auto_issue/__init__.py 4506850a02aa18e12bef4efeb760ad9e ./var/auto_issue/github.py -c9aff08ed524687c3de90393a5a49f64 ./var/blackwidow/__init__.py \ No newline at end of file +3f8ab5a50094d0aa9b95cf7797f3326e ./var/blackwidow/__init__.py \ No newline at end of file diff --git a/lib/core/settings.py b/lib/core/settings.py index 52873f2..b09efbb 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -38,7 +38,7 @@ PATCH_ID = str(subprocess.check_output(["git", "rev-parse", "origin/master"]))[: # clone link CLONE = "https://github.com/ekultek/zeus-scanner.git" # current version -VERSION = "1.1.5".format(PATCH_ID) +VERSION = "1.1.6.{}".format(PATCH_ID) # colors to output depending on the version VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30} # version string formatting diff --git a/var/blackwidow/__init__.py b/var/blackwidow/__init__.py index 8f1850e..01118b1 100644 --- a/var/blackwidow/__init__.py +++ b/var/blackwidow/__init__.py @@ -21,12 +21,12 @@ class Blackwidow(object): @staticmethod def get_url_ext(url): """ - get the extenstion of the URL + get the extension of the URL """ try: data = url.split(".") return data[-1] in lib.core.settings.SPIDER_EXT_EXCLUDE - except Exception: + except (IndexError, Exception): pass def test_connection(self): @@ -34,7 +34,8 @@ class Blackwidow(object): make sure the connection is good before you continue """ try: - attempt = requests.get(self.url, params={"user-agent": self.user_agent}, proxies=self.proxy) + attempt = requests.get(self.url, params={"user-agent": self.user_agent}, + proxies=lib.core.settings.proxy_string_to_dict(self.proxy)) if attempt.status_code == 200: return "ok" raise lib.core.errors.SpiderTestFailure( @@ -55,7 +56,8 @@ class Blackwidow(object): """ unique_links = set() true_url = lib.core.settings.replace_http(given_url) - req = requests.get(given_url, params={"user-agent": self.user_agent}, proxies=self.proxy) + req = requests.get(given_url, params={"user-agent": self.user_agent}, + proxies=lib.core.settings.proxy_string_to_dict(self.proxy)) html_page = req.content soup = BeautifulSoup(html_page, "html.parser") for link in soup.findAll(attribute): @@ -67,10 +69,22 @@ class Blackwidow(object): return list(unique_links) -def blackwidow_main(url, proxy=None, agent=None, verbose=False): +def blackwidow_main(url, **kwargs): """ scrape a given URL for all available links """ + verbose = kwargs.get("verbose", False) + proxy = kwargs.get("proxy", None) + agent = kwargs.get("agent", None) + if verbose: + lib.core.settings.logger.debug(lib.core.settings.set_color( + "settings user-agent to '{}'...".format(agent) + )) + if proxy is not None: + if verbose: + lib.core.settings.logger.debug(lib.core.settings.set_color( + "running behind proxy '{}'...".format(proxy) + )) lib.core.settings.create_dir("{}/{}".format(os.getcwd(), "log/blackwidow-log")) lib.core.settings.logger.info(lib.core.settings.set_color( "starting blackwidow on '{}'...".format(url) @@ -85,17 +99,19 @@ def blackwidow_main(url, proxy=None, agent=None, verbose=False): lib.core.settings.logger.debug(lib.core.settings.set_color( "connection satisfied, continuing process...", level=10 )) + lib.core.settings.logger.info(lib.core.settings.set_color( + "crawling given URL '{}' for links...".format(url) + )) found = crawler.scrape_page_for_links(url) - file_path = lib.core.settings.write_to_log_file(found, path=lib.core.settings.SPIDER_LOG_PATH, filename="blackwidow-log-{}.log") - with open(file_path) as data: - found = data.readlines() - if len(found) > 0: - lib.core.settings.logger.info(lib.core.settings.set_color( - "found a total of {} links from '{}'...".format( - len(found), url - ) - )) - else: - lib.core.settings.logger.fatal(lib.core.settings.set_color( - "did not find any usable links from '{}'...".format(url), level=50 - )) \ No newline at end of file + if len(found) > 0: + lib.core.settings.logger.info(lib.core.settings.set_color( + "found a total of {} links from given URL '{}'...".format( + len(found), url + ) + )) + lib.core.settings.write_to_log_file(found, path=lib.core.settings.SPIDER_LOG_PATH, + filename="blackwidow-log-{}.log") + else: + lib.core.settings.logger.fatal(lib.core.settings.set_color( + "did not find any usable links from '{}'...".format(url), level=50 + )) \ No newline at end of file