moved the URL parsing to it's own class, saves some space in the searching file and is a little quicker now

etc/checksum/md5sum.md5

@@ -68,12 +68,12 @@ a27929dffdf979f59675ad21ff5f77a6  ./lib/attacks/xss_scan/__init__.py
 fa40d9681c7c3024bd50ad12ef231d6d  ./lib/attacks/nmap_scan/__init__.py
 216999fa0e84866d5c1d96d5676034e4  ./lib/attacks/nmap_scan/nmap_opts.py
 81ec8da1193e93d1302bfd5a05b33730  ./lib/header_check/__init__.py
-097ddbb1444de0c09535d02265152cf1  ./lib/core/common.py
+aa2857204de4427c5da5d5a94700500f  ./lib/core/common.py
 1faa2b5dfad6eb538bbfe42942d2a9da  ./lib/core/errors.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/core/__init__.py
-08b9809eec3689f0d4661366fc08b454  ./lib/core/settings.py
-9ba84ab8e80011d7d7f7e017ef103f43  ./var/search/__init__.py
-617570f36f8525ba3b2768cceae87951  ./var/search/selenium_search.py
+195a949241e56dae2215dc7b825cea69  ./lib/core/settings.py
+9989b7122c79a4cd38be1e00adab5187  ./var/search/__init__.py
+cd16c7e4eb550b2c8531dee993eb4c2a  ./var/search/selenium_search.py
 7a1d9976db3e7d2923898c3065b9ca33  ./var/search/pgp_search.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/auto_issue/__init__.py

lib/core/common.py

@@ -2,6 +2,14 @@ import os
 import re
 import json
 import time
+try:
+    from urllib import (  # python 2
+        unquote
+    )
+except ImportError:
+    from urllib.parse import (  # python 3
+        unquote
+    )
 
 from lxml import etree
 
@@ -45,6 +53,55 @@ class HTTP_HEADER:
     X_FORWARDED_FOR = "X-Forwarded-For"
 
 
+class URLParser(object):
+
+    def __init__(self, url):
+        self.url = url
+        self.url_match_regex = re.compile(r"((https?):((//)|(\\\\))+([\w\d:#@%/;$()~_?\+-=\\\.&](#!)?)*)")
+        self.webcache_regex = re.compile(r"cache:(.{,16})?:")
+        self.possible_leftovers = ("<", ">", ";", ",")
+        self.webcache_schema = "webcache"
+        self.constant_ip_ban_splitter = "continue="
+        self.content_ip_ban_seperator = ("Fid", "&gs_")
+
+    def extract_webcache_url(self, splitter="+"):
+        """
+        extract the URL from Google's webcache URL
+        """
+        url = self.url
+        data = self.webcache_regex.split(url)
+        to_extract = data[2].split(splitter)
+        extracted = to_extract[0]
+        if self.url_match_regex.match(extracted):
+            return extracted
+        return None
+
+    def extract_ip_ban_url(self):
+        """
+        extract the true URL from Google's IP ban URL
+        """
+        url = unquote(self.url)
+        to_use_separator = None
+        retval_url = None
+        url_data_list = url.split(self.constant_ip_ban_splitter)
+        for item in url_data_list:
+            for sep in list(self.content_ip_ban_seperator):
+                if sep in item:
+                    to_use_separator = sep
+            retval_url = item.split(to_use_separator)
+        return unquote(retval_url[0])
+
+    def strip_url_leftovers(self):
+        """
+        strip any leftovers that come up with the URL every now and then
+        """
+        url = self.url
+        for possible in self.possible_leftovers:
+            if possible in url:
+                url = url.split(possible)[0]
+        return url
+
+
 def write_to_log_file(data_to_write, path, filename, blacklist=False):
     """
     write all found data to a log file

lib/core/settings.py

@@ -46,7 +46,7 @@ CLONE = "https://github.com/ekultek/zeus-scanner.git"
 ISSUE_LINK = "https://github.com/ekultek/zeus-scanner/issues"
 
 # current version <major.minor.commit.patch ID>
-VERSION = "1.2.33".format(PATCH_ID)
+VERSION = "1.2.34.{}".format(PATCH_ID)
 
 # colors to output depending on the version
 VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30}
@@ -245,6 +245,9 @@ URL_QUERY_REGEX = re.compile(r"(.*)[?|#](.*){1}\=(.*)")
 # regex to recognize a URL
 URL_REGEX = re.compile(r"((https?):((//)|(\\\\))+([\w\d:#@%/;$()~_?\+-=\\\.&](#!)?)*)")
 
+# regex to match Google's IP ban URL
+IP_BAN_REGEX = re.compile(r"http(s)?.//ipv\d{1}.google.[a-z]{1,5}.", re.I)
+
 # regex to discover if there are any results on the page
 NO_RESULTS_REGEX = re.compile("did not match with any results.", re.I)
 
@@ -399,6 +402,7 @@ def set_color(org_string, level=None):
     """
     color_levels = {
         10: "\033[36m{}\033[0m",         # DEBUG
+        15: "\033[1m\033[36m{}\033[0m",  # GOOD DEBUG INFO
         20: "\033[32m{}\033[0m",         # INFO *default
         25: "\033[1m\033[32m{}\033[0m",  # GOOD INFO
         30: "\033[33m{}\033[0m",         # WARNING

var/search/__init__.py

@@ -84,7 +84,7 @@ class SetBrowser(object):
 
     def set_browser(self):
         """
-        set up the browser
+        set the browser settings
         """
         profile = webdriver.FirefoxProfile()
         try:
@@ -92,7 +92,6 @@ class SetBrowser(object):
                 logger.info(set_color(
                     "setting the browser..."
                 ))
-                # override the user-agent to be our person one
                 profile = profile.set_preference("general.useragent.override", self.agent)
                 browser = webdriver.Firefox(profile, proxy=self.__set_proxy())
             else: