From 04cd49e722cc3c3d39c2634af7c0e61b460db3a9 Mon Sep 17 00:00:00 2001
From: ekultek <thomas.perkins23@icloud.com>
Date: Sun, 29 Oct 2017 11:46:36 -0500
Subject: [PATCH] added some new URL's to the skip schema, edited the DDG link
 to the HTML link, made sure that the URL is not just a protocol before adding
 to the file

---
 etc/checksum/md5sum.md5     |  4 ++--
 lib/core/settings.py        |  6 ++---
 var/google_search/search.py | 45 +++++++++++++++++++------------------
 3 files changed, 28 insertions(+), 27 deletions(-)

diff --git a/etc/checksum/md5sum.md5 b/etc/checksum/md5sum.md5
index 60753a2..b91314d 100644
--- a/etc/checksum/md5sum.md5
+++ b/etc/checksum/md5sum.md5
@@ -43,9 +43,9 @@ f27322b9716e1a2b0b0b0487f3149474  ./lib/attacks/whois_lookup/whois.py
 f746d2867f493104a78d0540cf50c03f  ./lib/attacks/intel_me/__init__.py
 1faa2b5dfad6eb538bbfe42942d2a9da  ./lib/core/errors.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/core/__init__.py
-9dd8b8617f2b465f29d3e82c040eeb9e  ./lib/core/settings.py
+f5e9b36328f3e144451738b617caa0ef  ./lib/core/settings.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/google_search/__init__.py
-b92ee17da90b17a0abb4e07e24fca3e1  ./var/google_search/search.py
+2043c53d8dfd45a375059cf02740cf9d  ./var/google_search/search.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/auto_issue/__init__.py
 4506850a02aa18e12bef4efeb760ad9e  ./var/auto_issue/github.py
diff --git a/lib/core/settings.py b/lib/core/settings.py
index f9c1234..9cb8601 100644
--- a/lib/core/settings.py
+++ b/lib/core/settings.py
@@ -38,7 +38,7 @@ PATCH_ID = str(subprocess.check_output(["git", "rev-parse", "origin/master"]))[:
 # clone link
 CLONE = "https://github.com/ekultek/zeus-scanner.git"
 # current version <major.minor.commit.patch ID>
-VERSION = "1.1.7.{}".format(PATCH_ID)
+VERSION = "1.1.8".format(PATCH_ID)
 # colors to output depending on the version
 VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30}
 # version string formatting
@@ -112,7 +112,7 @@ ALREADY_USED = set()
 AUTHORIZED_SEARCH_ENGINES = {
     "aol": "http://aol.com",
     "bing": "http://bing.com",
-    "duckduckgo": "http://duckduckgo.com",
+    "duckduckgo": "http://duckduckgo.com/html",
     "google": "http://google.com"
 }
 # extensions to exclude from the spider
@@ -153,7 +153,7 @@ URL_EXCLUDES = (
     "drive.google", "books.google", "news.google",
     "www.google", "mail.google", "accounts.google",
     "schema.org", "www.<b", "https://cid-", "https://<strong",  # these are some weird things that get pulled up?
-    "plus.google"
+    "plus.google", "www.w3.org", "schemas.live.com"
 )
 # regular expressions used for DBMS recognition based on error message response
 DBMS_ERRORS = {
diff --git a/var/google_search/search.py b/var/google_search/search.py
index 71cca29..4065942 100644
--- a/var/google_search/search.py
+++ b/var/google_search/search.py
@@ -349,29 +349,30 @@ def parse_search_results(
         for url in list(urls):
             url = unquote(url)
             if not any(u in url for u in URL_EXCLUDES):
-                if URL_REGEX.match(url):
-                    if isinstance(url, unicode):
-                        url = str(url).encode("utf-8")
-                    if pull_all:
-                        retval.add(url.split(splitter)[0])
-                    else:
-                        if URL_QUERY_REGEX.match(url.split(splitter)[0]):
+                if not url == "http://" and not url == "https://":
+                    if URL_REGEX.match(url):
+                        if isinstance(url, unicode):
+                            url = str(url).encode("utf-8")
+                        if pull_all:
+                            retval.add(url.split(splitter)[0])
+                        else:
+                            if URL_QUERY_REGEX.match(url.split(splitter)[0]):
+                                retval.add(url.split(splitter)[0])
+                        if verbose:
+                            try:
+                                logger.debug(set_color(
+                                    "found '{}'...".format(url.split(splitter)[0]), level=10
+                                ))
+                            except TypeError:
+                                logger.debug(set_color(
+                                    "found '{}'...".format(str(url).split(splitter)[0]), level=10
+                                ))
+                            except AttributeError:
+                                logger.debug(set_color(
+                                    "found '{}...".format(str(url)), level=10
+                                ))
+                        if url is not None:
                             retval.add(url.split(splitter)[0])
-                    if verbose:
-                        try:
-                            logger.debug(set_color(
-                                "found '{}'...".format(url.split(splitter)[0]), level=10
-                            ))
-                        except TypeError:
-                            logger.debug(set_color(
-                                "found '{}'...".format(str(url).split(splitter)[0]), level=10
-                            ))
-                        except AttributeError:
-                            logger.debug(set_color(
-                                "found '{}...".format(str(url)), level=10
-                            ))
-                    if url is not None:
-                        retval.add(url.split(splitter)[0])
     true_retval = set()
     for url in list(retval):
         if any(l in url for l in possible_leftovers):