mirror of
https://github.com/Ekultek/Zeus-Scanner.git
synced 2026-03-11 08:55:51 +00:00
87 lines
2.9 KiB
Python
87 lines
2.9 KiB
Python
import os
|
|
|
|
import requests
|
|
|
|
import lib.core.errors
|
|
import lib.core.settings
|
|
|
|
|
|
class Blackwidow(object):
|
|
|
|
"""
|
|
spider to scrape a webpage for all available URL's
|
|
"""
|
|
|
|
def __init__(self, url, user_agent=None, proxy=None):
|
|
self.url = url
|
|
self.proxy = proxy or None
|
|
self.user_agent = user_agent or lib.core.settings.DEFAULT_USER_AGENT
|
|
|
|
@staticmethod
|
|
def get_url_ext(url):
|
|
"""
|
|
get the extenstion of the URL
|
|
"""
|
|
try:
|
|
data = url.split(".")
|
|
return data[-1] in lib.core.settings.SPIDER_EXT_EXCLUDE
|
|
except Exception:
|
|
pass
|
|
|
|
def test_connection(self):
|
|
"""
|
|
make sure the connection is good before you continue
|
|
"""
|
|
try:
|
|
attempt = requests.get(self.url, params={"user-agent": self.user_agent}, proxies=self.proxy)
|
|
if attempt.status_code == 200:
|
|
return "ok"
|
|
raise lib.core.errors.SpiderTestFailure(
|
|
"failed to connect to '{}', received status code: {}".format(
|
|
self.url, attempt.status_code
|
|
)
|
|
)
|
|
except Exception as e:
|
|
lib.core.settings.logger.exception(lib.core.settings.set_color(
|
|
"failed to connect to '{}' received error '{}'...".format(
|
|
self.url, e
|
|
)
|
|
))
|
|
|
|
def scrape_page_for_links(self, given_url):
|
|
"""
|
|
scrape the webpage's HTML for usable GET links
|
|
"""
|
|
unique_links = set()
|
|
while True:
|
|
req = requests.get(given_url, params={"user-agent": self.user_agent}, proxies=self.proxy)
|
|
html_page = req.content
|
|
found_links = lib.core.settings.URL_REGEX.findall(html_page)
|
|
for link in list(found_links):
|
|
if lib.core.settings.URL_QUERY_REGEX.match(link[0]) and not Blackwidow.get_url_ext(link[0]):
|
|
unique_links.add(link)
|
|
break
|
|
return list(unique_links)
|
|
|
|
|
|
def blackwidow_main(url, proxy=None, agent=None, verbose=False):
|
|
"""
|
|
scrape a given URL for all available links
|
|
"""
|
|
lib.core.settings.create_dir("{}/{}".format(os.getcwd(), "log/blackwidow-log"))
|
|
lib.core.settings.logger.info(lib.core.settings.set_color(
|
|
"starting blackwidow on '{}'...".format(url)
|
|
))
|
|
crawler = Blackwidow(url, user_agent=agent, proxy=proxy)
|
|
if verbose:
|
|
lib.core.settings.logger.debug(lib.core.settings.set_color(
|
|
"testing connection to the URL...", level=10
|
|
))
|
|
crawler.test_connection()
|
|
if verbose:
|
|
lib.core.settings.logger.debug(lib.core.settings.set_color(
|
|
"connection satisfied, continuing process...", level=10
|
|
))
|
|
found = crawler.scrape_page_for_links(url)
|
|
to_use = [data[0] for data in found]
|
|
lib.core.settings.write_to_log_file(to_use, path=lib.core.settings.SPIDER_LOG_PATH, filename="blackwidow-log-{}.log")
|