diff --git a/etc/checksum/md5sum.md5 b/etc/checksum/md5sum.md5 index f9bb307..536b854 100644 --- a/etc/checksum/md5sum.md5 +++ b/etc/checksum/md5sum.md5 @@ -1,8 +1,8 @@ 4910b563b0f2403dbe4a89f10001de0b ./zeus.py 6ad5f22ec4a6f8324bfb1b01ab6d51ec ./etc/scripts/cleanup.sh 155c9482f690f1482f324a7ffd8b8098 ./etc/scripts/fix_pie.sh -0e435c641bc636ac0b3d54e032d9cf6a .etc/scripts/install_nmap.sh -4b32db388e8acda35570c734d27c950c .etc/scripts/launch_sqlmap.sh +0e435c641bc636ac0b3d54e032d9cf6a ./etc/scripts/install_nmap.sh +4b32db388e8acda35570c734d27c950c ./etc/scripts/launch_sqlmap.sh 642a77905d8bb4e5533e0e9c2137c0fa ./etc/agents.txt 66b11aa388ea909de7b212341259a318 ./etc/auths/git_auth 8f686b05c5c5dfc02f0fcaa7ebc8677c ./etc/auths/whois_auth @@ -49,4 +49,4 @@ b92ee17da90b17a0abb4e07e24fca3e1 ./var/google_search/search.py d41d8cd98f00b204e9800998ecf8427e ./var/__init__.py d41d8cd98f00b204e9800998ecf8427e ./var/auto_issue/__init__.py 4506850a02aa18e12bef4efeb760ad9e ./var/auto_issue/github.py -a83bf6c450035a733fa81a46c16fdd2b ./var/blackwidow/__init__.py \ No newline at end of file +c9aff08ed524687c3de90393a5a49f64 ./var/blackwidow/__init__.py \ No newline at end of file diff --git a/lib/core/settings.py b/lib/core/settings.py index 94920a1..52873f2 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -38,7 +38,7 @@ PATCH_ID = str(subprocess.check_output(["git", "rev-parse", "origin/master"]))[: # clone link CLONE = "https://github.com/ekultek/zeus-scanner.git" # current version -VERSION = "1.1.4".format(PATCH_ID) +VERSION = "1.1.5".format(PATCH_ID) # colors to output depending on the version VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30} # version string formatting diff --git a/requirements.txt b/requirements.txt index 01fdf63..ea4a74e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ pyvirtualdisplay==0.2.1 lxml==3.7.3 google-api-python-client==1.6.4 httplib2==0.10.3 -psutil==5.0.1 \ No newline at end of file +psutil==5.0.1 +beautifulsoup4==4.6.0 \ No newline at end of file diff --git a/var/blackwidow/__init__.py b/var/blackwidow/__init__.py index 2fccdd2..8f1850e 100644 --- a/var/blackwidow/__init__.py +++ b/var/blackwidow/__init__.py @@ -1,6 +1,7 @@ import os import requests +from bs4 import BeautifulSoup import lib.core.errors import lib.core.settings @@ -48,19 +49,21 @@ class Blackwidow(object): ) )) - def scrape_page_for_links(self, given_url): + def scrape_page_for_links(self, given_url, attribute="a", descriptor="href"): """ scrape the webpage's HTML for usable GET links """ unique_links = set() - while True: - req = requests.get(given_url, params={"user-agent": self.user_agent}, proxies=self.proxy) - html_page = req.content - found_links = lib.core.settings.URL_REGEX.findall(html_page) - for link in list(found_links): - if lib.core.settings.URL_QUERY_REGEX.match(link[0]) and not Blackwidow.get_url_ext(link[0]): - unique_links.add(link) - break + true_url = lib.core.settings.replace_http(given_url) + req = requests.get(given_url, params={"user-agent": self.user_agent}, proxies=self.proxy) + html_page = req.content + soup = BeautifulSoup(html_page, "html.parser") + for link in soup.findAll(attribute): + found_redirect = link.get(descriptor) + if found_redirect is not None and lib.core.settings.URL_REGEX.match(found_redirect): + unique_links.add(found_redirect) + else: + unique_links.add("http://{}/{}".format(true_url, found_redirect)) return list(unique_links) @@ -83,5 +86,16 @@ def blackwidow_main(url, proxy=None, agent=None, verbose=False): "connection satisfied, continuing process...", level=10 )) found = crawler.scrape_page_for_links(url) - to_use = [data[0] for data in found] - lib.core.settings.write_to_log_file(to_use, path=lib.core.settings.SPIDER_LOG_PATH, filename="blackwidow-log-{}.log") + file_path = lib.core.settings.write_to_log_file(found, path=lib.core.settings.SPIDER_LOG_PATH, filename="blackwidow-log-{}.log") + with open(file_path) as data: + found = data.readlines() + if len(found) > 0: + lib.core.settings.logger.info(lib.core.settings.set_color( + "found a total of {} links from '{}'...".format( + len(found), url + ) + )) + else: + lib.core.settings.logger.fatal(lib.core.settings.set_color( + "did not find any usable links from '{}'...".format(url), level=50 + )) \ No newline at end of file