added a new requirement BeautifulSoup, fixed the crawler will pull all links with an 'a' tag and descriptor of 'href', bumped version number

etc/checksum/md5sum.md5

@@ -1,8 +1,8 @@
 4910b563b0f2403dbe4a89f10001de0b  ./zeus.py
 6ad5f22ec4a6f8324bfb1b01ab6d51ec  ./etc/scripts/cleanup.sh
 155c9482f690f1482f324a7ffd8b8098  ./etc/scripts/fix_pie.sh
-0e435c641bc636ac0b3d54e032d9cf6a  .etc/scripts/install_nmap.sh
-4b32db388e8acda35570c734d27c950c  .etc/scripts/launch_sqlmap.sh
+0e435c641bc636ac0b3d54e032d9cf6a  ./etc/scripts/install_nmap.sh
+4b32db388e8acda35570c734d27c950c  ./etc/scripts/launch_sqlmap.sh
 642a77905d8bb4e5533e0e9c2137c0fa  ./etc/agents.txt
 66b11aa388ea909de7b212341259a318  ./etc/auths/git_auth
 8f686b05c5c5dfc02f0fcaa7ebc8677c  ./etc/auths/whois_auth
@@ -49,4 +49,4 @@ b92ee17da90b17a0abb4e07e24fca3e1  ./var/google_search/search.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/auto_issue/__init__.py
 4506850a02aa18e12bef4efeb760ad9e  ./var/auto_issue/github.py
-a83bf6c450035a733fa81a46c16fdd2b  ./var/blackwidow/__init__.py
+c9aff08ed524687c3de90393a5a49f64  ./var/blackwidow/__init__.py

lib/core/settings.py

@@ -38,7 +38,7 @@ PATCH_ID = str(subprocess.check_output(["git", "rev-parse", "origin/master"]))[:
 # clone link
 CLONE = "https://github.com/ekultek/zeus-scanner.git"
 # current version <major.minor.commit.patch ID>
-VERSION = "1.1.4".format(PATCH_ID)
+VERSION = "1.1.5".format(PATCH_ID)
 # colors to output depending on the version
 VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30}
 # version string formatting

requirements.txt

@@ -6,4 +6,5 @@ pyvirtualdisplay==0.2.1
 lxml==3.7.3
 google-api-python-client==1.6.4
 httplib2==0.10.3
-psutil==5.0.1
+psutil==5.0.1
+beautifulsoup4==4.6.0

var/blackwidow/__init__.py

@@ -1,6 +1,7 @@
 import os
 
 import requests
+from bs4 import BeautifulSoup
 
 import lib.core.errors
 import lib.core.settings
@@ -48,19 +49,21 @@ class Blackwidow(object):
                 )
             ))
 
-    def scrape_page_for_links(self, given_url):
+    def scrape_page_for_links(self, given_url, attribute="a", descriptor="href"):
         """
         scrape the webpage's HTML for usable GET links
         """
         unique_links = set()
-        while True:
-            req = requests.get(given_url, params={"user-agent": self.user_agent}, proxies=self.proxy)
-            html_page = req.content
-            found_links = lib.core.settings.URL_REGEX.findall(html_page)
-            for link in list(found_links):
-                if lib.core.settings.URL_QUERY_REGEX.match(link[0]) and not Blackwidow.get_url_ext(link[0]):
-                    unique_links.add(link)
-            break
+        true_url = lib.core.settings.replace_http(given_url)
+        req = requests.get(given_url, params={"user-agent": self.user_agent}, proxies=self.proxy)
+        html_page = req.content
+        soup = BeautifulSoup(html_page, "html.parser")
+        for link in soup.findAll(attribute):
+            found_redirect = link.get(descriptor)
+            if found_redirect is not None and lib.core.settings.URL_REGEX.match(found_redirect):
+                unique_links.add(found_redirect)
+            else:
+                unique_links.add("http://{}/{}".format(true_url, found_redirect))
         return list(unique_links)
 
 
@@ -83,5 +86,16 @@ def blackwidow_main(url, proxy=None, agent=None, verbose=False):
             "connection satisfied, continuing process...", level=10
         ))
     found = crawler.scrape_page_for_links(url)
-    to_use = [data[0] for data in found]
-    lib.core.settings.write_to_log_file(to_use, path=lib.core.settings.SPIDER_LOG_PATH, filename="blackwidow-log-{}.log")
+    file_path = lib.core.settings.write_to_log_file(found, path=lib.core.settings.SPIDER_LOG_PATH, filename="blackwidow-log-{}.log")
+    with open(file_path) as data:
+        found = data.readlines()
+        if len(found) > 0:
+            lib.core.settings.logger.info(lib.core.settings.set_color(
+                "found a total of {} links from '{}'...".format(
+                    len(found), url
+                )
+            ))
+        else:
+            lib.core.settings.logger.fatal(lib.core.settings.set_color(
+                "did not find any usable links from '{}'...".format(url), level=50
+            ))