From fa43d499c5bb2c31b425146fbdc2832a9f6beb17 Mon Sep 17 00:00:00 2001 From: ekultek Date: Tue, 3 Oct 2017 14:53:45 -0500 Subject: [PATCH] edited for the new write_to_file function --- var/blackwidow/__init__.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/var/blackwidow/__init__.py b/var/blackwidow/__init__.py index 039cdcc..0f072e9 100644 --- a/var/blackwidow/__init__.py +++ b/var/blackwidow/__init__.py @@ -63,21 +63,6 @@ class Blackwidow(object): break return list(unique_links) - @staticmethod - def write_to_file(urls, dir_path="{}/log/blackwidow-log", log_file="blackwidow-log-{}.log"): - """ - write to the log file - """ - full_path = "{}/{}".format(dir_path.format(os.getcwd()), - log_file.format(len(os.listdir(dir_path.format(os.getcwd()))) + 1)) - lib.settings.create_dir(dir_path.format(os.getcwd())) - with open(full_path, "a+") as log: - for url in urls: - log.write(url + "\n") - lib.settings.logger.info(lib.settings.set_color( - "found a total of {} unique usable URL's, saved to '{}'...".format(len(urls), full_path) - )) - def blackwidow_main(url, proxy=None, agent=None, verbose=False): """ @@ -99,4 +84,4 @@ def blackwidow_main(url, proxy=None, agent=None, verbose=False): )) found = crawler.scrape_page_for_links(url) to_use = [data[0] for data in found] - crawler.write_to_file(to_use) + lib.settings.write_to_log_file(to_use, path=lib.settings.SPIDER_LOG_PATH, filename="blackwidow-log-{}.log")