patched a privately reported issue that would not allow you to run the blackwidow crawler behind a proxy, forgot a space in the README file

README.md

@@ -4,6 +4,7 @@
 [![GitHub license](https://img.shields.io/badge/license-GPL-blue.svg?style=flat-square)](https://raw.githubusercontent.com/Ekultek/Zeus-Scanner/master/.github/LICENSE.md)
 [![Twitter](https://img.shields.io/twitter/url/https/github.com/ekultek/zeus-scanner.svg?style=social)](https://twitter.com/Zeus_Scanner)
 [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://github.com/Ekultek/Zeus-Scanner#donations)
+
 # Helpful links directory
 
 - [Overview](https://github.com/Ekultek/Zeus-Scanner#zeus-scanner)

etc/checksum/md5sum.md5

@@ -43,10 +43,10 @@ f27322b9716e1a2b0b0b0487f3149474  ./lib/attacks/whois_lookup/whois.py
 f746d2867f493104a78d0540cf50c03f  ./lib/attacks/intel_me/__init__.py
 1faa2b5dfad6eb538bbfe42942d2a9da  ./lib/core/errors.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/core/__init__.py
-8255b986984e7b069907548ec7bc2478  ./lib/core/settings.py
+819b60912d654bdd81b9d96bd757989f  ./lib/core/settings.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/google_search/__init__.py
 b92ee17da90b17a0abb4e07e24fca3e1  ./var/google_search/search.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/auto_issue/__init__.py
 4506850a02aa18e12bef4efeb760ad9e  ./var/auto_issue/github.py
-c9aff08ed524687c3de90393a5a49f64  ./var/blackwidow/__init__.py
+3f8ab5a50094d0aa9b95cf7797f3326e  ./var/blackwidow/__init__.py

lib/core/settings.py

@@ -38,7 +38,7 @@ PATCH_ID = str(subprocess.check_output(["git", "rev-parse", "origin/master"]))[:
 # clone link
 CLONE = "https://github.com/ekultek/zeus-scanner.git"
 # current version <major.minor.commit.patch ID>
-VERSION = "1.1.5".format(PATCH_ID)
+VERSION = "1.1.6.{}".format(PATCH_ID)
 # colors to output depending on the version
 VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30}
 # version string formatting

var/blackwidow/__init__.py

@@ -21,12 +21,12 @@ class Blackwidow(object):
     @staticmethod
     def get_url_ext(url):
         """
-        get the extenstion of the URL
+        get the extension of the URL
         """
         try:
             data = url.split(".")
             return data[-1] in lib.core.settings.SPIDER_EXT_EXCLUDE
-        except Exception:
+        except (IndexError, Exception):
             pass
 
     def test_connection(self):
@@ -34,7 +34,8 @@ class Blackwidow(object):
         make sure the connection is good before you continue
         """
         try:
-            attempt = requests.get(self.url, params={"user-agent": self.user_agent}, proxies=self.proxy)
+            attempt = requests.get(self.url, params={"user-agent": self.user_agent},
+                                   proxies=lib.core.settings.proxy_string_to_dict(self.proxy))
             if attempt.status_code == 200:
                 return "ok"
             raise lib.core.errors.SpiderTestFailure(
@@ -55,7 +56,8 @@ class Blackwidow(object):
         """
         unique_links = set()
         true_url = lib.core.settings.replace_http(given_url)
-        req = requests.get(given_url, params={"user-agent": self.user_agent}, proxies=self.proxy)
+        req = requests.get(given_url, params={"user-agent": self.user_agent},
+                           proxies=lib.core.settings.proxy_string_to_dict(self.proxy))
         html_page = req.content
         soup = BeautifulSoup(html_page, "html.parser")
         for link in soup.findAll(attribute):
@@ -67,10 +69,22 @@ class Blackwidow(object):
         return list(unique_links)
 
 
-def blackwidow_main(url, proxy=None, agent=None, verbose=False):
+def blackwidow_main(url, **kwargs):
     """
     scrape a given URL for all available links
     """
+    verbose = kwargs.get("verbose", False)
+    proxy = kwargs.get("proxy", None)
+    agent = kwargs.get("agent", None)
+    if verbose:
+        lib.core.settings.logger.debug(lib.core.settings.set_color(
+            "settings user-agent to '{}'...".format(agent)
+        ))
+    if proxy is not None:
+        if verbose:
+            lib.core.settings.logger.debug(lib.core.settings.set_color(
+                "running behind proxy '{}'...".format(proxy)
+            ))
     lib.core.settings.create_dir("{}/{}".format(os.getcwd(), "log/blackwidow-log"))
     lib.core.settings.logger.info(lib.core.settings.set_color(
         "starting blackwidow on '{}'...".format(url)
@@ -85,17 +99,19 @@ def blackwidow_main(url, proxy=None, agent=None, verbose=False):
         lib.core.settings.logger.debug(lib.core.settings.set_color(
             "connection satisfied, continuing process...", level=10
         ))
+    lib.core.settings.logger.info(lib.core.settings.set_color(
+        "crawling given URL '{}' for links...".format(url)
+    ))
     found = crawler.scrape_page_for_links(url)
-    file_path = lib.core.settings.write_to_log_file(found, path=lib.core.settings.SPIDER_LOG_PATH, filename="blackwidow-log-{}.log")
-    with open(file_path) as data:
-        found = data.readlines()
-        if len(found) > 0:
-            lib.core.settings.logger.info(lib.core.settings.set_color(
-                "found a total of {} links from '{}'...".format(
-                    len(found), url
-                )
-            ))
-        else:
-            lib.core.settings.logger.fatal(lib.core.settings.set_color(
-                "did not find any usable links from '{}'...".format(url), level=50
-            ))
+    if len(found) > 0:
+        lib.core.settings.logger.info(lib.core.settings.set_color(
+            "found a total of {} links from given URL '{}'...".format(
+                len(found), url
+            )
+        ))
+        lib.core.settings.write_to_log_file(found, path=lib.core.settings.SPIDER_LOG_PATH,
+                                            filename="blackwidow-log-{}.log")
+    else:
+        lib.core.settings.logger.fatal(lib.core.settings.set_color(
+            "did not find any usable links from '{}'...".format(url), level=50
+        ))