created a Github Gist searching tool that will search 3 pages of Gists (usually around 500 links) and create a regex for the found URL, from there it will search through all the Gists and save any of them that have a match (issue #153)

etc/checksum/md5sum.md5

@@ -1,4 +1,4 @@
-01ae751f79ec95fe5792a56d62f6a9be  ./zeus.py
+dab1f0143e2755b02401dece68a161ed  ./zeus.py
 4b32db388e8acda35570c734d27c950c  ./etc/scripts/launch_sqlmap.sh
 6ad5f22ec4a6f8324bfb1b01ab6d51ec  ./etc/scripts/cleanup.sh
 155c9482f690f1482f324a7ffd8b8098  ./etc/scripts/fix_pie.sh
@@ -57,18 +57,19 @@ d41d8cd98f00b204e9800998ecf8427e  ./lib/attacks/__init__.py
 7272a7fd0b0c2e9192bc4adb6154d2f0  ./lib/attacks/sqlmap_scan/__init__.py
 aa7268a8f085734a6c577c86440f7a1b  ./lib/attacks/sqlmap_scan/sqlmap_opts.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/attacks/whois_lookup/__init__.py
-9b2bd4904ec2385eb38a3b6cd59dbc99  ./lib/attacks/whois_lookup/whois.py
+28bb20770353ed5684615822e2048811  ./lib/attacks/whois_lookup/whois.py
 6b9ba948ca5ba51ef3d8423e738c41ba  ./lib/attacks/admin_panel_finder/__init__.py
 ceb1b278b0861c976dfecc91cb64e53d  ./lib/attacks/xss_scan/__init__.py
 27358f26bda30d7356143c3ea1fa99c5  ./lib/attacks/nmap_scan/__init__.py
 21faf4679cdeaa731029a48f8963d6e7  ./lib/attacks/nmap_scan/nmap_opts.py
+94fb7c32f7db112f14a14297311d0aa3  ./lib/attacks/gist_lookup/__init__.py
 1faa2b5dfad6eb538bbfe42942d2a9da  ./lib/core/errors.py
 d41d8cd98f00b204e9800998ecf8427e  ./lib/core/__init__.py
-da9f08a209175067a56a557da804ac00  ./lib/core/settings.py
+d00ee11f8294c2121bbf94f150bb346b  ./lib/core/settings.py
 f8dca2fa45acb95f7081546bf6aec025  ./lib/header_check/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/google_search/__init__.py
 1c21d355668d4f503a1c4bc41b9f5124  ./var/google_search/search.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/__init__.py
 d41d8cd98f00b204e9800998ecf8427e  ./var/auto_issue/__init__.py
-dadca85c232153021ba9ff253d8ee1d9  ./var/auto_issue/github.py
+4c7c008b28eaac5afb57a4bea46f1b19  ./var/auto_issue/github.py
 059765fe1ae084ad267d4b7aa7a34032  ./var/blackwidow/__init__.py

lib/attacks/gist_lookup/__init__.py

@@ -0,0 +1,135 @@
+import re
+import json
+import time
+import requests
+
+import lib.core.settings
+
+
+def __check_remaining_rate_limit():
+    """
+    check how many requests you have left to run
+    """
+    url = lib.core.settings.GITHUB_GIST_SEARCH_URLS["check_rate"]
+    data = requests.get(url, params={"Authorization": "token {}".format(
+        lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)
+    )})
+    remaining = data.headers["X-RateLimit-Remaining"]
+    if int(remaining) == 0:
+        lib.core.settings.logger.error(lib.core.settings.set_color(
+            "Github only allows 60 unauthenticated requests per hour, you have hit that limit "
+            "if you need to do more requests it is recommended to run behind a proxy with a different "
+            "user-agent (IE --proxy socks5://127.0.0.1:9050 --random-agent)...", level=40
+        ))
+        lib.core.settings.shutdown()
+
+
+def get_raw_data(page_set, proxy=None, agent=None, verbose=False):
+    """
+    parse 10 pages of Github gists and use them
+    """
+    retval = set()
+    url = lib.core.settings.GITHUB_GIST_SEARCH_URLS["search"]
+    headers = {
+        "User-Agent": agent,
+        "Authorization": "token {}".format(lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)),
+    }
+    lib.core.settings.logger.info(lib.core.settings.set_color(
+        "searching a total of {} pages of Gists...".format(page_set[-1])
+    ))
+    if proxy is not None:
+        proxy = lib.core.settings.proxy_string_to_dict(proxy)
+    for page in list(page_set):
+        data = requests.get(url.format(page), params=headers, proxies=proxy)
+        # load the found info into JSON format
+        # so we can pull using keys
+        data = json.loads(data.content)
+        for item in data:
+            # get the URL to the raw data so we can search it
+            gist_file = item["files"]
+            gist_filename = gist_file.keys()
+            try:
+                if verbose:
+                    lib.core.settings.logger.debug(lib.core.settings.set_color(
+                        "found filename '{}'...".format(''.join(gist_filename)), level=10
+                    ))
+                retval.add(gist_file[''.join(gist_filename)]["raw_url"])
+            # sometimes the URL doesn't like being pulled, so we'll just skip those ones
+            except Exception:
+                pass
+    return retval
+
+
+def check_files_for_information(found_url, data_to_search):
+    """
+    check the files to see if they contain any of the information you specified
+    """
+    # create a regex to search the data
+    data_regex = re.compile(data_to_search, re.I)
+    total_found = set()
+    try:
+        data = requests.get(found_url)
+    except requests.exceptions.ConnectionError:
+        lib.core.settings.logger.warning(lib.core.settings.set_color(
+            "to many requests are being sent to quickly, adding sleep time...", level=30
+        ))
+        time.sleep(3)
+        data = requests.get(found_url)
+
+    if data_regex.search(data.content) is not None:
+        lib.core.settings.logger.info(lib.core.settings.set_color(
+            "found a match with given specifics, saving full Gist to log file..."
+        ))
+        total_found.add(found_url)
+        lib.core.settings.write_to_log_file(
+            data.content, lib.core.settings.GIST_MATCH_LOG, "gist-match-{}.log"
+        )
+    return len(total_found)
+
+
+def github_gist_search_main(query, **kwargs):
+    proxy = kwargs.get("proxy", None)
+    agent = kwargs.get("agent", None)
+    verbose = kwargs.get("verbose", False)
+    thread = kwargs.get("do_threading", False)
+    proc_num = kwargs.get("proc_num", 5)
+    page_set = kwargs.get("page_set", (1, 2, 3))
+    total_found = 0
+
+    if verbose:
+        lib.core.settings.logger.debug(lib.core.settings.set_color(
+            "checking if you have exceeded your search limit...", level=10
+        ))
+    __check_remaining_rate_limit()
+    lib.core.settings.logger.info(lib.core.settings.set_color(
+        "searching Github Gists for '{}'...".format(query)
+    ))
+    gathered_links = get_raw_data(page_set, proxy=proxy, agent=agent, verbose=verbose)
+    lib.core.settings.logger.info(lib.core.settings.set_color(
+        "pulled a total of {} URL's to search...".format(len(gathered_links)), level=25
+    ))
+    if not thread:
+        lib.core.settings.logger.info(lib.core.settings.set_color(
+            "performing Github Gist search, this will probably take awhile..."
+        ))
+        for url in gathered_links:
+            total = check_files_for_information(url, query)
+            total_found += total
+    else:
+        lib.core.settings.logger.warning(lib.core.settings.set_color(
+            "multi-threading is not implemented yet...", level=35
+        ))
+        lib.core.settings.logger.info(lib.core.settings.set_color(
+            "performing Github Gist search, this will probably take awhile..."
+        ))
+        for url in gathered_links:
+            total = check_files_for_information(url, query)
+            total_found += total
+    if total_found > 0:
+        lib.core.settings.logger.info(lib.core.settings.set_color(
+            "found a total of {} interesting Gists...".format(total_found)
+        ))
+    else:
+        lib.core.settings.logger.warning(lib.core.settings.set_color(
+            "did not find any interesting Gists...", level=30
+        ))

lib/attacks/whois_lookup/whois.py

@@ -1,43 +1,17 @@
-import os
 import json
 import time
 import urllib2
 
-from base64 import b64decode
-
 import lib.core.settings
 
 
-def __get_encoded_string(path="{}/etc/auths/whois_auth"):
-    with open(path.format(os.getcwd())) as log:
-        return log.read()
-
-
-def __get_n(encoded):
-    return encoded.split(":")[-1]
-
-
-def __decode(encoded, n):
-    token = encoded.split(":")[0]
-    for _ in range(0, n):
-        token = b64decode(token)
-    return token
-
-
-def __get_token():
-    encoded = __get_encoded_string()
-    n = __get_n(encoded)
-    token = __decode(encoded, int(n))
-    return token
-
-
 def gather_raw_whois_info(domain):
     """
     get the raw JSON data for from the whois API
     """
     auth_headers = {
         "Content-Type": "application/json",
-        "Authorization": "Token {}".format(__get_token()),
+        "Authorization": "Token {}".format(lib.core.settings.get_token(lib.core.settings.WHOIS_AUTH_PATH)),
     }
     request = urllib2.Request(
         lib.core.settings.WHOIS_JSON_LINK.format(domain), headers=auth_headers

lib/core/settings.py

@@ -8,6 +8,7 @@ import time
 import shlex
 import difflib
 import logging
+import base64
 import string
 import random
 import socket
@@ -32,6 +33,7 @@ from lib.attacks.admin_panel_finder import main
 from lib.attacks.xss_scan import main_xss
 from lib.attacks.whois_lookup.whois import whois_lookup_main
 from lib.attacks.clickjacking_scan import clickjacking_main
+from lib.attacks.gist_lookup import github_gist_search_main
 from lib.attacks.sqlmap_scan.sqlmap_opts import SQLMAP_API_OPTIONS
 from lib.attacks.nmap_scan.nmap_opts import NMAP_API_OPTS
 from lib.attacks import (
@@ -165,6 +167,9 @@ SPIDER_LOG_PATH = "{}/log/blackwidow-log".format(os.getcwd())
 # cookies log path
 COOKIE_LOG_PATH = "{}/log/cookies".format(os.getcwd())
 
+# log to write to for gist searching
+GIST_MATCH_LOG = "{}/log/gists".format(os.getcwd())
+
 # unknown firewall log path
 UNKNOWN_FIREWALL_FINGERPRINT_PATH = "{}/log/unknown-firewall".format(os.getcwd())
 
@@ -174,6 +179,12 @@ BLACKLIST_FILE_PATH = "{}/log/blacklist".format(os.getcwd())
 # the current log file being used
 CURRENT_LOG_FILE_PATH = "{}/log".format(os.getcwd())
 
+# github autohorization token path
+GITHUB_AUTH_PATH = "{}/etc/auths/git_auth".format(os.getcwd())
+
+# whois authorization token path
+WHOIS_AUTH_PATH = "{}/etc/auths/whois_auth".format(os.getcwd())
+
 # nmap's manual page for their options
 NMAP_MAN_PAGE_URL = "https://nmap.org/book/man-briefoptions.html"
 
@@ -211,6 +222,12 @@ AUTHORIZED_SEARCH_ENGINES = {
     "search-results": "http://www1.search-results.com/web?tpr={}&q={}&page={}"
 }
 
+GITHUB_GIST_SEARCH_URLS = {
+    "search": "https://api.github.com/gists/public?page={}&per_page=100",
+    "check_rate": "https://api.github.com/users/ZeusIssueReporter"
+
+}
+
 # extensions to exclude from the spider
 SPIDER_EXT_EXCLUDE = (
     "3ds", "3g2", "3gp", "7z", "DS_Store", "a", "aac", "adp", "ai", "aif", "aiff",
@@ -575,7 +592,7 @@ def write_to_log_file(data_to_write, path, filename, blacklist=False):
             os.getcwd()
         ))) + 1)
     )
-    skip_log_schema = ("url-log", "blackwidow-log", "zeus-log", "extracted", ".blacklist")
+    skip_log_schema = ("url-log", "blackwidow-log", "zeus-log", "extracted", ".blacklist", "gist-match")
     to_search = filename.split("-")[0]
     amount = len([f for f in os.listdir(path) if to_search in f])
     new_filename = "{}({}).{}".format(
@@ -949,6 +966,7 @@ def run_attacks(url, **kwargs):
     verbose = kwargs.get("verbose", False)
     whois = kwargs.get("whois", False)
     clickjacking = kwargs.get("clickjacking", False)
+    github = kwargs.get("github", False)
     auto_start = kwargs.get("auto_start", False)
     sqlmap_arguments = kwargs.get("sqlmap_args", None)
     nmap_arguments = kwargs.get("nmap_args", None)
@@ -1029,6 +1047,9 @@ def run_attacks(url, **kwargs):
             if check_for_protection(PROTECTED, "clickjacking"):
                 clickjacking_main(url, agent=agent, proxy=proxy,
                                   forward=forwarded, batch=batch)
+        elif github:
+            query = replace_http(url)
+            github_gist_search_main(query, agent=agent, proxy=proxy, verbose=verbose)
         else:
             pass
     else:
@@ -1079,3 +1100,38 @@ def calculate_success(amount_of_urls):
     else:
         success_rate = "outstanding"
     return success_rate
+
+
+def __get_encoded_string(path):
+    """
+    get the encoded authorization string
+    """
+    with open(path.format(os.getcwd())) as log:
+        return log.read()
+
+
+def __get_n(encoded):
+    """
+    get the n'th number for decoding
+    """
+    return encoded.split(":")[-1]
+
+
+def __decode(encoded, n):
+    """
+    decode the string
+    """
+    token = encoded.split(":")[0]
+    for _ in range(0, n):
+        token = base64.b64decode(token)
+    return token
+
+
+def get_token(path):
+    """
+    get the authorization token
+    """
+    encoded = __get_encoded_string(path)
+    n = __get_n(encoded)
+    token = __decode(encoded, int(n))
+    return token

var/auto_issue/github.py

@@ -1,4 +1,3 @@
-import os
 import sys
 try:
     import urllib2  # python 2
@@ -7,27 +6,9 @@ except ImportError:
 import json
 import platform
 
-from base64 import b64decode
-
 import lib.core.settings
 
 
-def __get_encoded_string(filename="{}/etc/auths/git_auth"):
-    with open(filename.format(os.getcwd())) as data:
-        return data.read()
-
-
-def get_decode_num(data):
-    return data.split(":")[-1]
-
-
-def decode(n, token):
-    token = token.split(":")[0]
-    for _ in range(int(n)):
-        token = b64decode(token)
-    return token
-
-
 def request_issue_creation():
     if not lib.core.settings.get_md5sum():
         lib.core.settings.logger.fatal(lib.core.settings.set_color(
@@ -73,9 +54,7 @@ def request_issue_creation():
         "getting authorization..."
     ))
 
-    encoded = __get_encoded_string()
-    n = get_decode_num(encoded)
-    token = decode(n, encoded)
+    token = lib.core.settings.get_token(lib.core.settings.GITHUB_AUTH_PATH)
 
     current_log_file = lib.core.settings.get_latest_log_file(lib.core.settings.CURRENT_LOG_FILE_PATH)
     stacktrace = __extract_stacktrace(current_log_file)

zeus.py

@@ -85,6 +85,8 @@ if __name__ == "__main__":
                        help="Perform a WhoIs lookup on the provided domain")
     attacks.add_option("-c", "--clickjacking", dest="performClickjackingScan", action="store_true",
                        help="Perform a clickjacking scan on a provided URL")
+    attacks.add_option("-g", "--github-search", dest="searchGithub", action="store_true",
+                       help="Perform a Github Gist search for any information on the found websites")
     attacks.add_option("--sqlmap-args", dest="sqlmapArguments", metavar="SQLMAP-ARGS",
                        help="Pass the arguments to send to the sqlmap API within quotes & "
                             "separated by a comma. IE 'dbms mysql, verbose 3, level 5'")
@@ -272,7 +274,8 @@ if __name__ == "__main__":
         options = [
             opt.runSqliScan, opt.runPortScan,
             opt.adminPanelFinder, opt.runXssScan,
-            opt.performWhoisLookup, opt.performClickjackingScan
+            opt.performWhoisLookup, opt.performClickjackingScan,
+            opt.searchGithub
         ]
         if any(options):
             with open(urls_to_use) as urls:
@@ -300,7 +303,7 @@ if __name__ == "__main__":
                             url.strip(),
                             sqlmap=opt.runSqliScan, nmap=opt.runPortScan,
                             xss=opt.runXssScan, whois=opt.performWhoisLookup, admin=opt.adminPanelFinder,
-                            clickjacking=opt.performClickjackingScan,
+                            clickjacking=opt.performClickjackingScan, github=opt.searchGithub,
                             verbose=opt.runInVerbose, batch=opt.runInBatch,
                             auto_start=opt.autoStartSqlmap, xforward=opt.forwardedForRandomIP,
                             sqlmap_args=opt.sqlmapArguments, nmap_args=opt.nmapArguments,