prehistoric-systems/Zeus-Scanner
1
0
Fork 0
mirror of https://github.com/Ekultek/Zeus-Scanner.git synced 2026-03-11 08:55:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

will now cleanup the leftover junk from being parsed, it seems that sometimes random tags get pulled with the URL

Browse source
This commit is contained in:
ekultek 2017-10-23 22:27:01 -05:00
parent f6888b59af
commit 0c291d6d10
1 changed files with 13 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
var/google_search/search.py
View file

@ -49,6 +49,16 @@ except NameError:
unicode = str unicode = str
def strip_leftovers(url, possibles):
"""
strip leftover HTML tags and random garbage data that is sometimes found in the URL's
"""
for p in possibles:
if p in url:
url = url.split(p)[0]
return url
def bypass_ip_block(url): def bypass_ip_block(url):
""" """
bypass Google's IP blocking by extracting the true URL from the ban URL. bypass Google's IP blocking by extracting the true URL from the ban URL.
@ -214,6 +224,7 @@ def parse_search_results(
""" """
Parse a webpage from Google for URL's with a GET(query) parameter Parse a webpage from Google for URL's with a GET(query) parameter
""" """
possible_leftovers = ("<", ">", ";", ",")
splitter = "&amp;" splitter = "&amp;"
retval = set() retval = set()
query_url = None query_url = None
@ -352,6 +363,8 @@ def parse_search_results(
retval.add(url.split(splitter)[0]) retval.add(url.split(splitter)[0])
true_retval = set() true_retval = set()
for url in list(retval): for url in list(retval):
if any(l in url for l in possible_leftovers):
url = strip_leftovers(url, list(possible_leftovers))
if parse_webcache: if parse_webcache:
if "webcache" in url: if "webcache" in url:
logger.info(set_color( logger.info(set_color(