From b58a6b48d46831a4e12f35b25f9d51b63b163dd5 Mon Sep 17 00:00:00 2001 From: ekultek Date: Sun, 26 Nov 2017 16:44:04 -0600 Subject: [PATCH] enabled X-Forwarded-For headers in selenium searches --- etc/checksum/md5sum.md5 | 6 ++--- lib/core/settings.py | 2 +- var/search/__init__.py | 49 ++++++++++++++++++++++++++++++++--- var/search/selenium_search.py | 5 ++-- 4 files changed, 53 insertions(+), 9 deletions(-) diff --git a/etc/checksum/md5sum.md5 b/etc/checksum/md5sum.md5 index 7ecd14d..ffcc29b 100644 --- a/etc/checksum/md5sum.md5 +++ b/etc/checksum/md5sum.md5 @@ -71,9 +71,9 @@ e11185d7dd3d6c94d7f4ddbcfd1d26cc ./lib/header_check/__init__.py 45fc5f5847a3a9909cbd5e0add7b4586 ./lib/core/common.py 1faa2b5dfad6eb538bbfe42942d2a9da ./lib/core/errors.py d41d8cd98f00b204e9800998ecf8427e ./lib/core/__init__.py -086bf2b15a72771d87ba4bc8c52e2827 ./lib/core/settings.py -a2ff723e08c865330249c484cdf91e66 ./var/search/__init__.py -cd16c7e4eb550b2c8531dee993eb4c2a ./var/search/selenium_search.py +299e7ca9950a04fca62f958bd31a2abb ./lib/core/settings.py +9a02e5b913d210350545ac26510a63c9 ./var/search/__init__.py +8402f23a2586b6f684fb1c3c04c4386f ./var/search/selenium_search.py 63ba132381a0cc2d7629852bd5e4aa17 ./var/search/pgp_search.py d41d8cd98f00b204e9800998ecf8427e ./var/__init__.py d41d8cd98f00b204e9800998ecf8427e ./var/auto_issue/__init__.py diff --git a/lib/core/settings.py b/lib/core/settings.py index b93aaed..8969ecb 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -46,7 +46,7 @@ CLONE = "https://github.com/ekultek/zeus-scanner.git" ISSUE_LINK = "https://github.com/ekultek/zeus-scanner/issues" # current version -VERSION = "1.2.37.{}".format(PATCH_ID) +VERSION = "1.2.38".format(PATCH_ID) # colors to output depending on the version VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30} diff --git a/var/search/__init__.py b/var/search/__init__.py index 3ab26c4..09c46a2 100644 --- a/var/search/__init__.py +++ b/var/search/__init__.py @@ -3,9 +3,11 @@ from selenium import webdriver from selenium.webdriver.common.proxy import * from selenium.webdriver.remote.errorhandler import WebDriverException +from lib.core.common import HTTP_HEADER from lib.core.settings import ( logger, set_color, + create_random_ip, DEFAULT_USER_AGENT ) @@ -19,7 +21,7 @@ class SetBrowser(object): def __init__(self, **kwargs): self.agent = kwargs.get("agent", DEFAULT_USER_AGENT) self.proxy = kwargs.get("proxy", None) - # self.xforward = kwargs.get("xforward", False) # TODO:/ + self.xforward = kwargs.get("xforward", False) self.tor = kwargs.get("tor", False) self.tor_port = kwargs.get("port", 9050) @@ -83,6 +85,41 @@ class SetBrowser(object): ff_browser.set_preference(setting[0], setting[1]) return ff_browser + def __set_x_forward(self, profile): + """ + set the X-Forwarded-For headers for selenium, this can only be done + if you are using a profile for Firefox, and ONLY IN FIREFOX. + """ + ip_list = ( + create_random_ip(), + create_random_ip(), + create_random_ip() + ) + # references: + # https://eveningsamurai.wordpress.com/2013/11/21/changing-http-headers-for-a-selenium-webdriver-request/ + # https://stackoverflow.com/questions/6478672/how-to-send-an-http-requestheader-using-selenium-2/22238398#22238398 + # https://blog.giantgeek.com/?p=1455 + + # amount of headers to modify + profile.set_preference("modifyheaders.headers.count", 1) + # action to take on the headers + profile.set_preference("modifyheaders.headers.action0", "Add") + # header name, in this case it's `X-Forwarded-For` + profile.set_preference("modifyheaders.headers.name0", HTTP_HEADER.X_FORWARDED_FOR) + # header value, in this case, it's 3 random IP addresses + profile.set_preference("modifyheaders.headers.value0", "{}, {}, {}".format( + ip_list[0], ip_list[1], ip_list[2] + )) + # enable the header modification + profile.set_preference("modifyheaders.headers.enabled0", True) + # send it through the configuration + profile.set_preference("modifyheaders.config.active", True) + # turn it on from the new configuration + profile.set_preference("modifyheaders.config.alwaysOn", True) + # as always, change the user agent + profile.set_preference("general.useragent.override", self.agent) + return profile + def set_browser(self): """ set the browser settings @@ -93,7 +130,10 @@ class SetBrowser(object): logger.info(set_color( "setting the browser..." )) - profile = profile.set_preference("general.useragent.override", self.agent) + profile.set_preference("general.useragent.override", self.agent) + browser = webdriver.Firefox(profile, proxy=self.__set_proxy()) + elif self.xforward: + profile = self.__set_x_forward(profile) browser = webdriver.Firefox(profile, proxy=self.__set_proxy()) else: logger.info(set_color( @@ -103,9 +143,12 @@ class SetBrowser(object): browser = webdriver.Firefox(profile) except (OSError, WebDriverException): if not self.tor: - profile = profile.set_preference("general.useragent.override", self.agent) + profile.set_preference("general.useragent.override", self.agent) browser = webdriver.Firefox(profile, proxy=self.__set_proxy(), executable_path=whichcraft.which("geckodriver")) + elif self.xforward: + profile = self.__set_x_forward(profile) + browser = webdriver.Firefox(profile, proxy=self.__set_proxy()) else: profile = self.__tor_browser_emulation(profile) browser = webdriver.Firefox(profile, executable_path=whichcraft.which("geckodriver")) diff --git a/var/search/selenium_search.py b/var/search/selenium_search.py index 89fda9f..6a16947 100644 --- a/var/search/selenium_search.py +++ b/var/search/selenium_search.py @@ -73,12 +73,13 @@ def get_urls(query, url, verbose=False, **kwargs): proxy, user_agent = kwargs.get("proxy", None), kwargs.get("user_agent", None) tor, tor_port = kwargs.get("tor", False), kwargs.get("tor_port", None) batch = kwargs.get("batch", False) + xforward = kwargs.get("xforward", False) logger.info(set_color( "setting up virtual display to hide the browser..." )) ff_display = Display(visible=0, size=(800, 600)) ff_display.start() - browser = var.search.SetBrowser(agent=user_agent, proxy=proxy, tor=tor).set_browser() + browser = var.search.SetBrowser(agent=user_agent, proxy=proxy, tor=tor, xforward=xforward).set_browser() logger.info(set_color("browser will open shortly...", level=25)) browser.get(url) if verbose: @@ -260,7 +261,7 @@ def parse_search_results(query, url_to_search, verbose=False, **kwargs): try: query_url = get_urls( query, url_to_search, verbose=verbose, user_agent=user_agent, proxy=proxy_string, - tor=tor, batch=batch + tor=tor, batch=batch, xforward=forward_for ) except Exception as e: if "'/usr/lib/firefoxdriver/webdriver.xpi'" in str(e):