diff --git a/lib/core/settings.py b/lib/core/settings.py index 0ea60c9..8bef38f 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -10,6 +10,8 @@ import difflib import logging import string import random +import socket +import struct import platform import subprocess @@ -38,7 +40,7 @@ PATCH_ID = str(subprocess.check_output(["git", "rev-parse", "origin/master"]))[: # clone link CLONE = "https://github.com/ekultek/zeus-scanner.git" # current version -VERSION = "1.1.9.{}".format(PATCH_ID) +VERSION = "1.1.10.{}".format(PATCH_ID) # colors to output depending on the version VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30} # version string formatting @@ -685,3 +687,17 @@ def create_arguments(**kwargs): warning_msg.format(line), level=30 )) return retval + + +def create_random_ip(): + """ + create a random IP address, no testing if it is valid or not + """ + + def __get_nodes(): + return str(socket.inet_ntoa(struct.pack(">I", random.randint(1, 0xffffffff)))) + + generated = __get_nodes() + if generated == "0.0.0.0" or "255.255.255.255": + generated = __get_nodes() + return generated diff --git a/var/blackwidow/__init__.py b/var/blackwidow/__init__.py index 01118b1..4c50f8f 100644 --- a/var/blackwidow/__init__.py +++ b/var/blackwidow/__init__.py @@ -1,4 +1,7 @@ import os +import sys +reload(sys) +sys.setdefaultencoding("utf-8") # this will take care of most of the Unicode errors. import requests from bs4 import BeautifulSoup @@ -13,10 +16,22 @@ class Blackwidow(object): spider to scrape a webpage for all available URL's """ - def __init__(self, url, user_agent=None, proxy=None): + def __init__(self, url, user_agent=None, proxy=None, forward=None): self.url = url - self.proxy = proxy or None + self.forward = forward or None + self.proxy = lib.core.settings.proxy_string_to_dict(proxy) or None self.user_agent = user_agent or lib.core.settings.DEFAULT_USER_AGENT + if self.forward is not None: + self.headers = { + "user-agent": self.user_agent, + "X-Forwarded-For": "{}, {}, {}".format( + self.forward[0], self.forward[1], self.forward[2] + ) + } + else: + self.headers = { + "user-agent": self.user_agent + } @staticmethod def get_url_ext(url): @@ -34,8 +49,7 @@ class Blackwidow(object): make sure the connection is good before you continue """ try: - attempt = requests.get(self.url, params={"user-agent": self.user_agent}, - proxies=lib.core.settings.proxy_string_to_dict(self.proxy)) + attempt = requests.get(self.url, params=self.headers, proxies=self.proxy) if attempt.status_code == 200: return "ok" raise lib.core.errors.SpiderTestFailure( @@ -56,12 +70,11 @@ class Blackwidow(object): """ unique_links = set() true_url = lib.core.settings.replace_http(given_url) - req = requests.get(given_url, params={"user-agent": self.user_agent}, - proxies=lib.core.settings.proxy_string_to_dict(self.proxy)) + req = requests.get(given_url, params=self.headers, proxies=self.proxy) html_page = req.content soup = BeautifulSoup(html_page, "html.parser") for link in soup.findAll(attribute): - found_redirect = link.get(descriptor) + found_redirect = str(link.get(descriptor)).decode("unicode_escape") if found_redirect is not None and lib.core.settings.URL_REGEX.match(found_redirect): unique_links.add(found_redirect) else: @@ -76,20 +89,33 @@ def blackwidow_main(url, **kwargs): verbose = kwargs.get("verbose", False) proxy = kwargs.get("proxy", None) agent = kwargs.get("agent", None) + forward = kwargs.get("forward", None) + + if forward is not None: + forward = ( + lib.core.settings.create_random_ip(), + lib.core.settings.create_random_ip(), + lib.core.settings.create_random_ip() + ) + if verbose: + lib.core.settings.logger.debug(lib.core.settings.set_color( + "random IP addresses generated for header '{}'...".format(forward), level=10 + )) + if verbose: lib.core.settings.logger.debug(lib.core.settings.set_color( - "settings user-agent to '{}'...".format(agent) + "settings user-agent to '{}'...".format(agent), level=10 )) if proxy is not None: if verbose: lib.core.settings.logger.debug(lib.core.settings.set_color( - "running behind proxy '{}'...".format(proxy) + "running behind proxy '{}'...".format(proxy), level=10 )) lib.core.settings.create_dir("{}/{}".format(os.getcwd(), "log/blackwidow-log")) lib.core.settings.logger.info(lib.core.settings.set_color( "starting blackwidow on '{}'...".format(url) )) - crawler = Blackwidow(url, user_agent=agent, proxy=proxy) + crawler = Blackwidow(url, user_agent=agent, proxy=proxy, forward=forward) if verbose: lib.core.settings.logger.debug(lib.core.settings.set_color( "testing connection to the URL...", level=10