complete rework of how the Gist search works, you no longer have a certain amount you can search, this is also a fix for issue #221 where the dict would not load into JSON format, that is no longer required

etc/checksum/md5sum.md5

@@ -62,7 +62,7 @@ bbd8b4c6100070d420d48dc7dfc297eb  ./lib/firewall/webknight.py
 81a29a14d72980a306fbaec0dc772048  ./lib/firewall/fortigate.py
 6ea65a0160c21e144e92334acc2e3667  ./lib/firewall/anquanbao.py
 22a0ad8f2fa1a16b651cb5ae37ca9b0d  ./lib/firewall/generic.py
-cf23d0f522b183bea9dc0fcf4aa64115  ./lib/attacks/gist_lookup/__init__.py
+c3aa1db3ee34b841d91ad850c2cfd7b2  ./lib/attacks/gist_lookup/__init__.py
 86224bd899c2a2438042cbdc077dc4cc  ./lib/attacks/clickjacking_scan/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/attacks/__init__.py
 4c644b0e3a62b6c1528d34a04837aa35  ./lib/attacks/sqlmap_scan/__init__.py
@@ -77,7 +77,7 @@ ac942d36f7d78c249e587417736e88e6  ./lib/header_check/__init__.py
 cd8e35cfd995d0a93892cfc83f01dea7  ./lib/core/common.py
 4433353fb5c55578391d8b4006191ee8  ./lib/core/errors.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/core/__init__.py
-f723499d996a3bbbd3022cc372dffdd9  ./lib/core/settings.py
+e5cba9d37cf54d14255a81da1fa83f90  ./lib/core/settings.py
 801a4f7ac892b74676c649bd4844ccdb  ./lib/core/decorators.py
 9a02e5b913d210350545ac26510a63c9  ./var/search/__init__.py
 1ed3c450e620ff1edd8b0864179fdea7  ./var/search/selenium_search.py

lib/attacks/gist_lookup/__init__.py

@@ -1,73 +1,68 @@
 import re
-import json
+
-import time
+from bs4 import BeautifulSoup
-import requests
 
 import lib.core.settings
 import lib.core.common
 
 
-def __check_remaining_rate_limit():
+def __create_url(redirect, template="https://gist.github.com{}"):
     """
-    check how many requests you have left to run
+    create the URL for the Gists
     """
-    url = lib.core.settings.GITHUB_GIST_SEARCH_URLS["check_rate"]
+    return template.format(redirect)
-    _, _, data, headers = lib.core.common.get_page(url, auth="token {}".format(
-        lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)
-    ))
-    remaining = headers["X-RateLimit-Remaining"]
-    if int(remaining) == 0:
-        lib.core.settings.logger.error(lib.core.settings.set_color(
-            "Github only allows 60 unauthenticated requests per hour, you have hit that limit "
-            "if you need to do more requests it is recommended to run behind a proxy with a different "
-            "user-agent (IE --proxy socks5://127.0.0.1:9050 --random-agent)...", level=40
-        ))
-        lib.core.common.shutdown()
-    else:
-        lib.core.settings.logger.warning(lib.core.settings.set_color(
-            "you have {} unauthenticated requests remaining...".format(remaining), level=30
-        ))
 
 
-def get_raw_data(page_set, proxy=None, agent=None, verbose=False):
+def get_raw_html(redirect, verbose=False):
+    """
+    get the raw HTML of the Gist plus the URL for it
+    """
+    tag, descriptor = "a", "href"
+    raw_gist_regex = re.compile(r".raw.[a-z0-9]{40}", re.I)
+    _, status, html, _ = lib.core.common.get_page(redirect)
+    soup = BeautifulSoup(html, "html.parser")
+    for link in soup.findAll(tag):
+        raw_gist_redirect = link.get(descriptor)
+        if raw_gist_regex.search(str(raw_gist_redirect)) is not None:
+            url = __create_url(raw_gist_redirect)
+            if verbose:
+                lib.core.settings.logger.debug(lib.core.settings.set_color(
+                    "found raw Gist URL '{}'...".format(url), level=10
+                ))
+            _, _, html, _ = lib.core.common.get_page(url)
+            raw_soup = BeautifulSoup(html, "html.parser")
+            return raw_soup, url
+
+
+def get_links(page_set, proxy=None, agent=None):
     """
     parse 10 pages of Github gists and use them
     """
-    retval = set()
+    redirects, retval = set(), set()
-    url = lib.core.settings.GITHUB_GIST_SEARCH_URLS["search"]
+    gist_search_url = "https://gist.github.com/discover?page={}"
-    headers = {
+    tag, descriptor = "a", "href"
-        lib.core.common.HTTP_HEADER.USER_AGENT: agent,
+    gist_regex = re.compile(r"[a-f0-9]{32}", re.I)
-        lib.core.common.HTTP_HEADER.AUTHORIZATION: "token {}".format(lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)),
+    gist_skip_schema = ("stargazers", "forks", "#comments")
-    }
+    for i in range(page_set):
-    lib.core.settings.logger.info(lib.core.settings.set_color(
+        lib.core.settings.logger.info(lib.core.settings.set_color(
-        "searching a total of {} pages of Gists...".format(page_set[-1])
+            "fetching all Gists on page #{}...".format(i+1)
-    ))
+        ))
-    if proxy is not None:
+        _, status, html, _ = lib.core.common.get_page(
-        proxy = lib.core.settings.proxy_string_to_dict(proxy)
+            gist_search_url.format(i+1), proxy=proxy, agent=agent
-    for page in list(page_set):
+        )
-        _, _, data, _ = lib.core.common.get_page(url.format(page), agent=agent, proxy=proxy)
+        soup = BeautifulSoup(html, "html.parser")
-        # load the found info into JSON format
+        for link in soup.findAll(tag):
-        # so we can pull using keys
+            redirect = link.get(descriptor)
-        data = json.loads(data.content)
+            if not any(s in redirect for s in gist_skip_schema):
-        for item in data:
+                if gist_regex.search(redirect) is not None:
-            # get the URL to the raw data so we can search it
+                    if not any(protocol in redirect for protocol in ["https://", "http://"]):
-            gist_file = item["files"]
+                        redirects.add(__create_url(redirect))
-            gist_filename = gist_file.keys()
+                    else:
-            try:
+                        redirects.add(redirect)
-                skip_schema = ("-", " ", "")
+    return redirects
-                if not any(s == gist_filename for s in skip_schema):
-                    if verbose:
-                        lib.core.settings.logger.debug(lib.core.settings.set_color(
-                            "found filename '{}'...".format(''.join(gist_filename)), level=10
-                        ))
-                    retval.add(gist_file[''.join(gist_filename)]["raw_url"])
-            # sometimes the URL doesn't like being pulled, so we'll just skip those ones
-            except Exception:
-                pass
-    return retval
 
 
-def check_files_for_information(found_url, data_to_search):
+def check_files_for_information(url, data_to_search, query):
     """
     check the files to see if they contain any of the information that was specified
     """
@@ -75,95 +70,68 @@ def check_files_for_information(found_url, data_to_search):
     # bases while we do the searching.
     # this will make it so that if there is a match anywhere
     # in anything, we'll find it.
-    if "www" in data_to_search:
+    data_to_search = str(data_to_search)
-        # we need to create the query to be just the domain name
-        data_to_search = data_to_search.split(".")[1]
     data_regex_schema = (
-        # a normal regex with our string
-        re.compile(r"{}".format(data_to_search), re.I),
         # match a URL with or without www
-        re.compile(r"(http(s)?)?(.//)?(www.)?{}".format(data_to_search), re.I),
+        re.compile(r"(http(s)?)?(.//)?(www.)?{}".format(query), re.I),
         # match our string and any random character around it (I like to call it the tittyex)
-        re.compile(r"(.)?{}(.)?".format(data_to_search), re.I),
+        re.compile(r"(.)?{}(.)?".format(query), re.I),
         # single boundary match, checks if it's inside of something else
-        re.compile(r"\b{}".format(data_to_search), re.I),
+        re.compile(r"\b{}".format(query), re.I),
         # double boundary, same as above but with another boundary
-        re.compile(r"\b{}\b".format(data_to_search), re.I),
+        re.compile(r"\b{}\b".format(query), re.I),
         # wildcard match
-        re.compile(r"{}*".format(data_to_search), re.I)
+        re.compile(r"{}*".format(query), re.I),
+        # normal match
+        re.compile(r"{}".format(query), re.I)
     )
-    total_found = set()
+    for regex in list(data_regex_schema):
-    try:
+        if regex.search(data_to_search) is not None:
-        _, _, data, _ = lib.core.common.get_page(found_url)
-    except requests.exceptions.ConnectionError:
-        lib.core.settings.logger.warning(lib.core.settings.set_color(
-            "to many requests are being sent to quickly, adding sleep time...", level=30
-        ))
-        time.sleep(3)
-        _, _, data, _ = lib.core.common.get_page(found_url)
-    for data_regex in data_regex_schema:
-        if data_regex.search(data.content) is not None:
             lib.core.settings.logger.info(lib.core.settings.set_color(
-                "found a match with given specifics ('{}'), saving full Gist to log file...".format(
+                "found match with given specifics ('{}'), saving Gist to file...".format(
-                    data_regex.pattern
+                    regex.pattern
                 ), level=25
             ))
-            total_found.add(found_url)
             lib.core.common.write_to_log_file(
-                data.content, lib.core.settings.GIST_MATCH_LOG, lib.core.settings.GIST_FILENAME.format(
+                data_to_search,
-                    lib.core.settings.replace_http(data_to_search)
+                lib.core.settings.GIST_MATCH_LOG,
-                )
+                lib.core.settings.GIST_FILENAME.format(query)
             )
-    return len(total_found)
 
 
 def github_gist_search_main(query, **kwargs):
+    """
+    main function for searching Gists
+    """
     proxy = kwargs.get("proxy", None)
     agent = kwargs.get("agent", None)
     verbose = kwargs.get("verbose", False)
-    thread = kwargs.get("do_threading", False)
+    page_set = kwargs.get("page_set", 10)
-    # proc_num = kwargs.get("proc_num", 5)  # TODO:/
-    page_set = kwargs.get("page_set", (1, 2, 3, 4, 5))
-    total_found = 0
 
     try:
+        lib.core.settings.logger.info(lib.core.settings.set_color(
+            "searching a total of {} pages of Gists for '{}'...".format(
+                page_set, query
+            )
+        ))
+
+        if "www." in query:
+            query = query.split(".")[1]
+
+        links = get_links(page_set, proxy=proxy, agent=agent)
         if verbose:
             lib.core.settings.logger.debug(lib.core.settings.set_color(
-                "checking if you have exceeded your search limit...", level=10
+                "found a total of {} links to search...".format(
-            ))
+                    len(links)
-        __check_remaining_rate_limit()
+                ), level=15
-        lib.core.settings.logger.info(lib.core.settings.set_color(
-            "searching Github Gists for '{}'...".format(query)
-        ))
-        gathered_links = get_raw_data(page_set, proxy=proxy, agent=agent, verbose=verbose)
-        lib.core.settings.logger.info(lib.core.settings.set_color(
-            "pulled a total of {} URL's to search...".format(len(gathered_links)), level=25
-        ))
-        if not thread:
-            lib.core.settings.logger.info(lib.core.settings.set_color(
-                "performing Github Gist search, this will probably take awhile..."
-            ))
-            for url in gathered_links:
-                total = check_files_for_information(url, query)
-                total_found += total
-        else:
-            lib.core.settings.logger.warning(lib.core.settings.set_color(
-                "multi-threading is not implemented yet...", level=35
-            ))
-            lib.core.settings.logger.info(lib.core.settings.set_color(
-                "performing Github Gist search, this will probably take awhile..."
-            ))
-            for url in gathered_links:
-                total = check_files_for_information(url, query)
-                total_found += total
-        if total_found > 0:
-            lib.core.settings.logger.info(lib.core.settings.set_color(
-                "found a total of {} interesting Gists...".format(total_found)
-            ))
-        else:
-            lib.core.settings.logger.warning(lib.core.settings.set_color(
-                "did not find any interesting Gists...", level=30
             ))
+        for link in list(links):
+            gist, gist_link = get_raw_html(link, verbose=verbose)
+            check_files_for_information(gist_link, gist, query)
     except KeyboardInterrupt:
         if not lib.core.common.pause():
             lib.core.common.shutdown()
+    except Exception as e:
+        lib.core.settings.logger.exception(lib.core.settings.set_color(
+            "Gist search has failed with error '{}'...".format(str(e)), level=50
+        ))

lib/core/settings.py

@@ -46,7 +46,7 @@ CLONE = "https://github.com/ekultek/zeus-scanner.git"
 ISSUE_LINK = "https://github.com/ekultek/zeus-scanner/issues"
 
 # current version <major.minor.commit.patch ID>
-VERSION = "1.3.16.{}".format(PATCH_ID)
+VERSION = "1.3.17.{}".format(PATCH_ID)
 
 # colors to output depending on the version
 VERSION_TYPE_COLORS = {"dev": 33, "stable": 92, "other": 30}
@@ -315,10 +315,11 @@ URL_EXCLUDES = (
     "drive.google", "books.google", "news.google",
     "www.google", "mail.google", "accounts.google",
     "schema.org", "www.<b", "https://cid-", "https://<strong",  # these are some weird things that get pulled up?
-    "plus.google", "www.w3.org", "schemas.live.com",
+    "plus.google", "www.w3.org", "schemas.live.com", "https://my."
     "torproject.org", "search-results.com", "index.com",
     "gov", ".gov", "facebook.com", "instagram.com", "snapchat",
-    "stackoverflow", "stackexchange", "github.com"
+    "stackoverflow", "stackexchange", "github.com", "apple.com",
+    "http://my."
 )
 
 # regular expressions used for DBMS recognition based on error message response