From 2a3790f8e8c45a6867e1f68be68272ddaa4dc9e8 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 4 Nov 2014 13:24:54 -0500 Subject: [PATCH 001/195] use iptables-allports for recidive --- ChangeLog | 3 +++ config/jail.conf | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0e107735..c4fc1a44 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,9 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 * grep'ing for IP in *mail-whois-lines.conf should now match also at the begginning and EOL. Thanks Dean Lee + * recidive uses iptables-allports banaction by default now. + Avoids problems with iptables versions not understanding 'all' for + protocols and ports - New Features: diff --git a/config/jail.conf b/config/jail.conf index d119d229..d2949023 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -655,8 +655,7 @@ maxretry = 5 [recidive] logpath = /var/log/fail2ban.log -port = all -protocol = all +banaction = iptables-allports bantime = 604800 ; 1 week findtime = 86400 ; 1 day maxretry = 5 From fe72a5585c3a9970036b15d7957e8d064d9bdefa Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Tue, 30 Dec 2014 19:06:17 -0500 Subject: [PATCH 002/195] Create Jail for Postfix based on RBL Use RBL blocks to ban addresses, unique Jail so maxretry can be set to 1 (vs postfix.conf) --- ChangeLog | 2 ++ config/filter.d/postfix-rbl.conf | 19 +++++++++++++++++++ config/jail.conf | 7 +++++++ fail2ban/tests/files/logs/postfix-rbl | 2 ++ 4 files changed, 30 insertions(+) create mode 100644 config/filter.d/postfix-rbl.conf create mode 100644 fail2ban/tests/files/logs/postfix-rbl diff --git a/ChangeLog b/ChangeLog index c80dac5a..b9d60f75 100644 --- a/ChangeLog +++ b/ChangeLog @@ -23,6 +23,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * filter.d/postfix-sasl.conf - failregex is now case insensitive - New Features: + - New filter: + - postfix-rbl Thanks Lee Clemens - New interpolation feature for config readers - `%(known/parameter)s`. (means last known option with name `parameter`). This interpolation makes possible to extend a stock filter or jail regexp in .local file diff --git a/config/filter.d/postfix-rbl.conf b/config/filter.d/postfix-rbl.conf new file mode 100644 index 00000000..4b572241 --- /dev/null +++ b/config/filter.d/postfix-rbl.conf @@ -0,0 +1,19 @@ +# Fail2Ban filter for Postfix's RBL based Blocked hosts +# +# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + +[Definition] + +_daemon = postfix/smtpd + +failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using \S+; Blocked.* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ + +ignoreregex = + +# Author: Lee Clemens diff --git a/config/jail.conf b/config/jail.conf index 6a95aa12..5544ba6d 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -465,6 +465,13 @@ port = smtp,465,submission logpath = %(postfix_log)s +[postfix-rbl] + +port = smtp,465,submission +logpath = %(syslog_mail)s +maxretry = 1 + + [sendmail-auth] port = submission,465,smtp diff --git a/fail2ban/tests/files/logs/postfix-rbl b/fail2ban/tests/files/logs/postfix-rbl new file mode 100644 index 00000000..fd420fa7 --- /dev/null +++ b/fail2ban/tests/files/logs/postfix-rbl @@ -0,0 +1,2 @@ +# failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" } +Dec 30 18:19:15 xxx postfix/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from= to= proto=ESMTP helo= From 0f48cf42844c92120854a05dc47dba6ba3ae8d82 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Tue, 30 Dec 2014 19:14:39 -0500 Subject: [PATCH 003/195] loosen up regex for spamhaus (spamcop says "Blocked" as part of url) --- config/filter.d/postfix-rbl.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/postfix-rbl.conf b/config/filter.d/postfix-rbl.conf index 4b572241..05a8bbc7 100644 --- a/config/filter.d/postfix-rbl.conf +++ b/config/filter.d/postfix-rbl.conf @@ -12,7 +12,7 @@ before = common.conf _daemon = postfix/smtpd -failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using \S+; Blocked.* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ +failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using .* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ ignoreregex = From d0bcacd97be8d2eac24c048d0f58b5b93ee90340 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 2 Jan 2015 02:55:40 -0500 Subject: [PATCH 004/195] Fix strptime thread safety issue strptime thread safety hack-around - http://bugs.python.org/issue7980 --- bin/fail2ban-client | 3 +++ bin/fail2ban-regex | 3 +++ bin/fail2ban-server | 3 +++ 3 files changed, 9 insertions(+) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 866a5287..b91c339c 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -430,6 +430,9 @@ class ServerExecutionException(Exception): pass if __name__ == "__main__": # pragma: no cover - can't test main + from time import strptime + # strptime thread safety hack-around - http://bugs.python.org/issue7980 + strptime("2012", "%Y") client = Fail2banClient() # Exit with correct return value if client.start(sys.argv): diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 5644dd37..96aca926 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -474,6 +474,9 @@ class Fail2banRegex(object): if __name__ == "__main__": + from time import strptime + # strptime thread safety hack-around - http://bugs.python.org/issue7980 + strptime("2012", "%Y") parser = get_opt_parser() (opts, args) = parser.parse_args() diff --git a/bin/fail2ban-server b/bin/fail2ban-server index ec0c0dbe..dc8d81b4 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -131,6 +131,9 @@ class Fail2banServer: return False if __name__ == "__main__": + from time import strptime + # strptime thread safety hack-around - http://bugs.python.org/issue7980 + strptime("2012", "%Y") server = Fail2banServer() if server.start(sys.argv): sys.exit(0) From b26725f654f2c58a219367f5221da0c148a5cd55 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 3 Jan 2015 13:45:06 -0500 Subject: [PATCH 005/195] Move strptime workaround to fail2ban/__init__.py --- bin/fail2ban-client | 3 --- bin/fail2ban-regex | 3 --- bin/fail2ban-server | 3 --- fail2ban/__init__.py | 4 ++++ 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index b91c339c..866a5287 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -430,9 +430,6 @@ class ServerExecutionException(Exception): pass if __name__ == "__main__": # pragma: no cover - can't test main - from time import strptime - # strptime thread safety hack-around - http://bugs.python.org/issue7980 - strptime("2012", "%Y") client = Fail2banClient() # Exit with correct return value if client.start(sys.argv): diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 96aca926..5644dd37 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -474,9 +474,6 @@ class Fail2banRegex(object): if __name__ == "__main__": - from time import strptime - # strptime thread safety hack-around - http://bugs.python.org/issue7980 - strptime("2012", "%Y") parser = get_opt_parser() (opts, args) = parser.parse_args() diff --git a/bin/fail2ban-server b/bin/fail2ban-server index dc8d81b4..ec0c0dbe 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -131,9 +131,6 @@ class Fail2banServer: return False if __name__ == "__main__": - from time import strptime - # strptime thread safety hack-around - http://bugs.python.org/issue7980 - strptime("2012", "%Y") server = Fail2banServer() if server.start(sys.argv): sys.exit(0) diff --git a/fail2ban/__init__.py b/fail2ban/__init__.py index cbaaf757..b7906099 100644 --- a/fail2ban/__init__.py +++ b/fail2ban/__init__.py @@ -68,3 +68,7 @@ logging.notice = _root_notice # add NOTICE to the priority map of all the levels logging.handlers.SysLogHandler.priority_map['NOTICE'] = 'notice' + +from time import strptime +# strptime thread safety hack-around - http://bugs.python.org/issue7980 +strptime("2012", "%Y") From 541a747d7949e42844c2637d8e94cf92aedf8075 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 3 Jan 2015 15:19:58 -0500 Subject: [PATCH 006/195] Update Changelog with strptime fix --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index c80dac5a..5bf5a084 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * filters.d/exim.conf - cover different settings of exim logs details. Thanks bes.internal * filter.d/postfix-sasl.conf - failregex is now case insensitive + * fail2ban/__init__.py - Added strptime thread safety hack-around - New Features: - New interpolation feature for config readers - `%(known/parameter)s`. From 2f360ce44724a570f18e20b79f528e1ea947606e Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 3 Jan 2015 15:26:34 -0500 Subject: [PATCH 007/195] Update Changelog with strptime fix --- ChangeLog | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7e686adb..040ff48c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,11 +21,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * filters.d/exim.conf - cover different settings of exim logs details. Thanks bes.internal * filter.d/postfix-sasl.conf - failregex is now case insensitive -<<<<<<< HEAD - * fail2ban/__init__.py - Added strptime thread safety hack-around -======= * filters.d/postfix.conf - add 'Client host rejected error message' failregex ->>>>>>> 8fc32bb33f6f8c43fa107f1a6fc3a71bf000a767 + * fail2ban/__init__.py - Added strptime thread safety hack-around - New Features: - New interpolation feature for config readers - `%(known/parameter)s`. From 4714028c69412018498f469837238f0f7b793f53 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 3 Jan 2015 16:16:23 -0500 Subject: [PATCH 008/195] Change case and tense for consistency --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 040ff48c..73a3693b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,7 +22,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released details. Thanks bes.internal * filter.d/postfix-sasl.conf - failregex is now case insensitive * filters.d/postfix.conf - add 'Client host rejected error message' failregex - * fail2ban/__init__.py - Added strptime thread safety hack-around + * fail2ban/__init__.py - add strptime thread safety hack-around - New Features: - New interpolation feature for config readers - `%(known/parameter)s`. From d7b7f4bc91be34e80b48c05c86913fc0d4d168d4 Mon Sep 17 00:00:00 2001 From: TorontoMedia Date: Thu, 8 Jan 2015 21:06:43 -0500 Subject: [PATCH 009/195] Update firewallcmd-allports.conf --- config/action.d/firewallcmd-allports.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/action.d/firewallcmd-allports.conf b/config/action.d/firewallcmd-allports.conf index c0c378a4..ce83164b 100644 --- a/config/action.d/firewallcmd-allports.conf +++ b/config/action.d/firewallcmd-allports.conf @@ -8,6 +8,8 @@ before = iptables-blocktype.conf +[Definition] + actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b- firewall-cmd --direct --add-rule ipv4 filter f2b- 1000 -j RETURN firewall-cmd --direct --add-rule ipv4 filter 0 -j f2b- From 083031524d9005fc5a39e9837aaaa545b2a2a2fe Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 8 Jan 2015 21:14:37 -0500 Subject: [PATCH 010/195] BF: adding missing Definition section header to firewallcmd-allports --- config/action.d/firewallcmd-allports.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/action.d/firewallcmd-allports.conf b/config/action.d/firewallcmd-allports.conf index c0c378a4..ce83164b 100644 --- a/config/action.d/firewallcmd-allports.conf +++ b/config/action.d/firewallcmd-allports.conf @@ -8,6 +8,8 @@ before = iptables-blocktype.conf +[Definition] + actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b- firewall-cmd --direct --add-rule ipv4 filter f2b- 1000 -j RETURN firewall-cmd --direct --add-rule ipv4 filter 0 -j f2b- From fdd93d1475d595f7af8614307208dac60ee9bb40 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 8 Jan 2015 21:48:50 -0500 Subject: [PATCH 011/195] ENH: unittest to catch actions without Init or Definition section and all must have actionban at least --- fail2ban/tests/clientreadertestcase.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 4f77a698..a94fad4e 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -459,6 +459,22 @@ class JailsReaderTest(LogCaptureTestCase): self.assertTrue(self._is_logged("No file(s) found for glob /weapons/of/mass/destruction")) if STOCK: + def testReadStockActionConf(self): + for actionConfig in glob.glob(os.path.join(CONFIG_DIR, 'action.d', '*.conf')): + actionName = os.path.basename(actionConfig).replace('.conf', '') + actionReader = ActionReader(actionName, "TEST", {}, basedir=CONFIG_DIR) + self.assertTrue(actionReader.read()) + actionReader.getOptions({}) # populate _opts + if not actionName.endswith('-common'): + self.assertTrue('Definition' in actionReader.sections(), + msg="Action file %r is lacking [Definition] section" % actionConfig) + # all must have some actionban defined + self.assertTrue(actionReader._opts.get('actionban', '').strip(), + msg="Action file %r is lacking actionban" % actionConfig) + self.assertTrue('Init' in actionReader.sections(), + msg="Action file %r is lacking [Init] section" % actionConfig) + + def testReadStockJailConf(self): jails = JailsReader(basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm self.assertTrue(jails.read()) # opens fine From 12e3cca3f2e1b28228463f59a65e1b5a38114974 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 19 Jan 2015 10:28:53 +0100 Subject: [PATCH 012/195] port[s] typo fixed in jail.conf/nginx-http-auth, issue gh-913 --- ChangeLog | 1 + config/jail.conf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b4cacb35..0a901547 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released ----------- - Fixes: + * port[s] typo in jail.conf/nginx-http-auth gh-913. Thanks Frederik Wagner (fnerdwq) * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 * grep'ing for IP in *mail-whois-lines.conf should now match also at the begginning and EOL. Thanks Dean Lee diff --git a/config/jail.conf b/config/jail.conf index 5544ba6d..70104878 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -291,7 +291,7 @@ maxretry = 1 [nginx-http-auth] -ports = http,https +port = http,https logpath = %(nginx_error_log)s From b04a51246f8fa4d1c87d34a3bc6b31369c968e1f Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Jan 2015 11:32:15 +0100 Subject: [PATCH 013/195] infinite busy loop on _escapedTags match in substituteRecursiveTags gh-907 --- ChangeLog | 2 ++ fail2ban/server/action.py | 22 +++++++++------------- fail2ban/tests/actiontestcase.py | 2 ++ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0a901547..c24c9dc8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released ----------- - Fixes: + * infinite busy loop on _escapedTags match in substituteRecursiveTags gh-907. + Thanks TonyThompson * port[s] typo in jail.conf/nginx-http-auth gh-913. Thanks Frederik Wagner (fnerdwq) * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 * grep'ing for IP in *mail-whois-lines.conf should now match also diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index da7517f6..c69ba88f 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -394,20 +394,16 @@ class CommandAction(ActionBase): # recursive definitions are bad #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) return False - elif found_tag in cls._escapedTags: - # Escaped so won't match + if found_tag in cls._escapedTags or not tags.has_key(found_tag): + # Escaped or missing tags - just continue on searching after end of match + # Missing tags are ok - cInfo can contain aInfo elements like and valid shell + # constructs like . + m = t.search(value, m.end()) continue - else: - if tags.has_key(found_tag): - value = value.replace('<%s>' % found_tag , tags[found_tag]) - #logSys.log(5, 'value now: %s' % value) - done.append(found_tag) - m = t.search(value, m.start()) - else: - # Missing tags are ok so we just continue on searching. - # cInfo can contain aInfo elements like and valid shell - # constructs like . - m = t.search(value, m.start() + 1) + value = value.replace('<%s>' % found_tag , tags[found_tag]) + #logSys.log(5, 'value now: %s' % value) + done.append(found_tag) + m = t.search(value, m.start()) #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) tags[tag] = value return tags diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 5a58149f..36e0ddb8 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -59,6 +59,8 @@ class CommandActionTest(LogCaptureTestCase): self.assertEqual(CommandAction.substituteRecursiveTags({'A': ''}), {'A': ''}) self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) + # Escaped tags should be ignored + self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) # Multiple stuff on same line is ok self.assertEqual(CommandAction.substituteRecursiveTags({'failregex': 'to= fromip= evilperson=', 'honeypot': 'pokie', 'ignoreregex': ''}), { 'failregex': "to=pokie fromip= evilperson=pokie", From 607af36ad3ac9716d8820572538e4071babe1a5a Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Jan 2015 14:08:30 +0100 Subject: [PATCH 014/195] workaround for the "Bad file descriptor" issue on Python 2.7, gh-161 : asyncore.loop() using poll by the way, prevents to write "'build/bdist.linux-x86_64' does not exist -- can't clean it" into stderr; --- fail2ban/server/asyncserver.py | 7 +++++-- fail2ban/tests/misctestcase.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index a54d41a1..c99cdffe 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -149,8 +149,11 @@ class AsyncServer(asyncore.dispatcher): self.__init = True # TODO Add try..catch # There's a bug report for Python 2.6/3.0 that use_poll=True yields some 2.5 incompatibilities: - logSys.debug("Detected Python 2.6 or greater. asyncore.loop() not using poll") - asyncore.loop(use_poll=False) # fixes the "Unexpected communication problem" issue on Python 2.6 and 3.0 + if sys.version_info >= (2, 7) and sys.version_info < (2, 8): # if python 2.7 ... + logSys.debug("Detected Python 2.7. asyncore.loop() using poll") + asyncore.loop(use_poll=True) # workaround for the "Bad file descriptor" issue on Python 2.7, gh-161 + else: + asyncore.loop(use_poll=False) # fixes the "Unexpected communication problem" issue on Python 2.6 and 3.0 ## # Stops the communication server. diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 2f69aef0..a682b63f 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -113,7 +113,7 @@ class SetupTest(unittest.TestCase): # clean up shutil.rmtree(tmp) # remove build directory - os.system("%s %s clean --all >/dev/null" + os.system("%s %s clean --all >/dev/null 2>&1" % (sys.executable, self.setup)) class TestsUtilsTest(unittest.TestCase): From 33e9e2174ad98466c6f33dd78ba7baf30f3a8340 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Jan 2015 17:09:13 +0100 Subject: [PATCH 015/195] recursive/embedded version of issue/907; test cases merged from remote-tracking branch 'yarikoptic:enh/embedded_tags' into issue/907 infinite busy loop on _escapedTags match in substituteRecursiveTags gh-907 --- ChangeLog | 3 ++ fail2ban/server/action.py | 61 ++++++++++++++++++-------------- fail2ban/tests/actiontestcase.py | 5 +++ 3 files changed, 42 insertions(+), 27 deletions(-) diff --git a/ChangeLog b/ChangeLog index c24c9dc8..f01ed930 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,9 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - New Features: - New filter: - postfix-rbl Thanks Lee Clemens + - New recursive embedded substitution feature added: + - `<HOST>` becomes `` for PREF=`IPV4`; + - `<HOST>` becomes `1.2.3.4` for PREF=`IPV4` and IPV4HOST=`1.2.3.4`; - New interpolation feature for config readers - `%(known/parameter)s`. (means last known option with name `parameter`). This interpolation makes possible to extend a stock filter or jail regexp in .local file diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index c69ba88f..bfd2adc2 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -378,34 +378,41 @@ class CommandAction(ActionBase): Dictionary of tags(keys) and their values, with tags within the values recursively replaced. """ - t = re.compile(r'<([^ >]+)>') - for tag in tags.iterkeys(): - if tag in cls._escapedTags: - # Escaped so won't match - continue - value = str(tags[tag]) - m = t.search(value) - done = [] - #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) - while m: - found_tag = m.group(1) - #logSys.log(5, 'found: %s' % found_tag) - if found_tag == tag or found_tag in done: - # recursive definitions are bad - #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) - return False - if found_tag in cls._escapedTags or not tags.has_key(found_tag): - # Escaped or missing tags - just continue on searching after end of match - # Missing tags are ok - cInfo can contain aInfo elements like and valid shell - # constructs like . - m = t.search(value, m.end()) + t = re.compile(r'<([^ <>]+)>') + while True: + repFlag = False + for tag in tags.iterkeys(): + if tag in cls._escapedTags: + # Escaped so won't match continue - value = value.replace('<%s>' % found_tag , tags[found_tag]) - #logSys.log(5, 'value now: %s' % value) - done.append(found_tag) - m = t.search(value, m.start()) - #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) - tags[tag] = value + value = str(tags[tag]) + m = t.search(value) + done = [] + #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) + while m: + found_tag = m.group(1) + #logSys.log(5, 'found: %s' % found_tag) + if found_tag == tag or found_tag in done: + # recursive definitions are bad + #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) + return False + if found_tag in cls._escapedTags or not tags.has_key(found_tag): + # Escaped or missing tags - just continue on searching after end of match + # Missing tags are ok - cInfo can contain aInfo elements like and valid shell + # constructs like . + m = t.search(value, m.end()) + continue + value = value.replace('<%s>' % found_tag , tags[found_tag]) + #logSys.log(5, 'value now: %s' % value) + done.append(found_tag) + m = t.search(value, m.start()) + #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) + # if was substituted, check again later: + if tags[tag] != value: + repFlag = True + tags[tag] = value + if not repFlag: + break return tags @staticmethod diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 36e0ddb8..7cd0cd4d 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -73,6 +73,11 @@ class CommandActionTest(LogCaptureTestCase): 'ABC': '123 192.0.2.0', 'xyz': '890 123 192.0.2.0', }) + # obscure embedded case + self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4'}), + {'A': '', 'PREF': 'IPV4'}) + self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}), + {'A': '1.2.3.4', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}) def testReplaceTag(self): aInfo = { From 6b42878b8cea7a3d3ea8ad261a0e51f480d051e4 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Jan 2015 17:31:17 +0100 Subject: [PATCH 016/195] better recognition of embedded-recursive substitution to repeat interpolation --- fail2ban/server/action.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index bfd2adc2..70b3eb7e 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -407,9 +407,11 @@ class CommandAction(ActionBase): done.append(found_tag) m = t.search(value, m.start()) #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) - # if was substituted, check again later: + # was substituted? if tags[tag] != value: - repFlag = True + # check again later if embedded-recursive substitution: + if value.startswith('<') and value.endswith('>'): + repFlag = True tags[tag] = value if not repFlag: break From d0b932aaca91cb044861a3e61496c6c06b03c263 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 21 Jan 2015 09:44:55 +0100 Subject: [PATCH 017/195] code review + more test cases (embedded replace in a string) --- fail2ban/server/action.py | 8 ++++++-- fail2ban/tests/actiontestcase.py | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 70b3eb7e..8e0cd97f 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -362,6 +362,7 @@ class CommandAction(ActionBase): @classmethod def substituteRecursiveTags(cls, tags): """Sort out tag definitions within other tags. + Since v.0.9.2 supports embedded interpolation (see test cases for examples). so: becomes: a = 3 a = 3 @@ -379,13 +380,16 @@ class CommandAction(ActionBase): within the values recursively replaced. """ t = re.compile(r'<([^ <>]+)>') + # repeat substitution while embedded-recursive (repFlag is True) while True: repFlag = False + # substitute each value: for tag in tags.iterkeys(): if tag in cls._escapedTags: # Escaped so won't match continue value = str(tags[tag]) + # search and replace all tags within value, that can be interpolated using other tags: m = t.search(value) done = [] #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) @@ -409,8 +413,8 @@ class CommandAction(ActionBase): #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) # was substituted? if tags[tag] != value: - # check again later if embedded-recursive substitution: - if value.startswith('<') and value.endswith('>'): + # check still contains any tag - should be repeated (possible embedded-recursive substitution): + if t.search(value): repFlag = True tags[tag] = value if not repFlag: diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 7cd0cd4d..2e826a8e 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -78,6 +78,9 @@ class CommandActionTest(LogCaptureTestCase): {'A': '', 'PREF': 'IPV4'}) self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}), {'A': '1.2.3.4', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}) + # more embedded within a string and two interpolations + self.assertEqual(CommandAction.substituteRecursiveTags({'A': 'A HOST> B IP C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}), + {'A': 'A 1.2.3.4 B IPV4 C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}) def testReplaceTag(self): aInfo = { From 40eb7a40f73ff98aca4ddc227045fa7c54b7d74e Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 13:54:00 -0500 Subject: [PATCH 018/195] Add pypy3 Travis Support --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index bd2d294c..0a4486a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ python: - "3.3" - "3.4" - "pypy" + - "pypy3" before_install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi install: From d01099f3478e330ccab076c9f2cbb52d66baf599 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 14:43:00 -0500 Subject: [PATCH 019/195] Use travis_retry for commands which use the internet --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index bd2d294c..ead9ef46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,11 @@ python: - "3.4" - "pypy" before_install: - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get update -qq; fi install: - - pip install pyinotify - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; pip install -q coveralls; cd -; fi + - travis_retry pip install pyinotify + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; travis_retry pip install -q coveralls; cd -; fi script: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi # test installation From 625bcef8a1be6bbcdd4d4522aafe46d917d8e472 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:12:04 -0500 Subject: [PATCH 020/195] Add extended info to status output using Cyrmu --- .travis.yml | 2 + ChangeLog | 6 +- fail2ban/protocol.py | 1 + fail2ban/server/actions.py | 13 +++- fail2ban/server/banmanager.py | 112 +++++++++++++++++++++++++++ fail2ban/server/jail.py | 9 +++ fail2ban/server/jailthread.py | 6 ++ fail2ban/server/server.py | 5 +- fail2ban/server/transmitter.py | 7 +- fail2ban/tests/banmanagertestcase.py | 50 ++++++++++-- fail2ban/tests/servertestcase.py | 25 ++++++ fail2ban/tests/utils.py | 1 + 12 files changed, 227 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index bd2d294c..ce14e0ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,8 @@ before_install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi install: - pip install pyinotify + - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then travis_retry pip install dnspython; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then travis_retry pip install dnspython3; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; pip install -q coveralls; cd -; fi script: diff --git a/ChangeLog b/ChangeLog index f01ed930..3013dac8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,7 +41,11 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - Monit config for fail2ban in /files/monit - New actions: - action.d/firewallcmd-multiport and action.d/firewallcmd-allports Thanks Donald Yandt - + - New status commands: + - fail2ban-client status extended + - prints Cymru data (ASN, Country RIR) per banned IP + - Requires dnspython or dnspython3 + - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 * files/debian-initd migrated from the debian branch and should be diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 9a6ee675..dc7a7c6b 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,7 @@ protocol = [ ["start ", "starts the jail "], ["stop ", "stops the jail . The jail is removed"], ["status ", "gets the current status of "], +["status extended", "gets the current status of with extended info"], ['', "JAIL CONFIGURATION", ""], ["set idle on|off", "sets the idle state of "], ["set addignoreip ", "adds to the ignore list of "], diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index a212aaf1..f4d5c18b 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -372,9 +372,20 @@ class Actions(JailThread, Mapping): @property def status(self): - """Status of active bans, and total ban counts. + """Status of current and total ban counts and current banned IP list. """ ret = [("Currently banned", self.__banManager.size()), ("Total banned", self.__banManager.getBanTotal()), ("Banned IP list", self.__banManager.getBanList())] return ret + + @property + def statusExtended(self): + """Jail status plus banned IPs' ASN, Country and RIR + """ + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + ret = self.status +\ + [("Banned ASN list", self.__banManager.geBanListExtendedASN(cymru_info)), + ("Banned Country list", self.__banManager.geBanListExtendedCountry(cymru_info)), + ("Banned RIR list", self.__banManager.geBanListExtendedRIR(cymru_info))] + return ret diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index c21cad45..81ac7c4f 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -26,6 +26,9 @@ __license__ = "GPL" from threading import Lock +import dns.exception +import dns.resolver + from .ticket import BanTicket from .mytime import MyTime from ..helpers import getLogger @@ -118,6 +121,115 @@ class BanManager: finally: self.__lock.release() + ## + # Returns normalized value + # + # @return value or "unknown" if value is None or empty string + + @staticmethod + def handleBlankResult(value): + if value is None or len(value) == 0: + return "unknown" + else: + return value + + ## + # Returns Cymru DNS query information + # + # @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs + + def getBanListExtendedCymruInfo(self): + self.__lock.acquire() + return_dict = {"asn": [], "country": [], "rir": []} + try: + for banData in self.__banList: + ip = banData.getIP() + # Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns + # TODO: IPv6 compatibility + reversed_ip = ".".join(reversed(ip.split("."))) + question = "%s.origin.asn.cymru.com" % reversed_ip + try: + answers = dns.resolver.query(question, "TXT") + for rdata in answers: + asn, net, country, rir, changed =\ + [answer.strip("'\" ") for answer in rdata.to_text().split("|")] + asn = self.handleBlankResult(asn) + country = self.handleBlankResult(country) + rir = self.handleBlankResult(rir) + return_dict["asn"].append(self.handleBlankResult(asn)) + return_dict["country"].append(self.handleBlankResult(country)) + return_dict["rir"].append(self.handleBlankResult(rir)) + except dns.resolver.NXDOMAIN: + return_dict["asn"].append("nxdomain") + return_dict["country"].append("nxdomain") + return_dict["rir"].append("nxdomain") + except dns.exception.DNSException as dnse: + logSys.error("Unhandled DNSException querying Cymru for %s TXT" % question) + logSys.exception(dnse) + except Exception as e: + logSys.error("Unhandled Exception querying Cymru for %s TXT" % question) + logSys.exception(e) + except Exception as e: + logSys.error("Failure looking up extended Cymru info") + logSys.exception(e) + finally: + self.__lock.release() + return return_dict + + ## + # Returns list of Banned ASNs from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned ASNs + + def geBanListExtendedASN(self, cymru_info): + self.__lock.acquire() + try: + return [asn for asn in cymru_info["asn"]] + except Exception as e: + logSys.error("Failed to lookup ASN") + logSys.exception(e) + return [] + finally: + self.__lock.release() + + ## + # Returns list of Banned Countries from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned Countries + + def geBanListExtendedCountry(self, cymru_info): + self.__lock.acquire() + try: + return [country for country in cymru_info["country"]] + except Exception as e: + logSys.error("Failed to lookup Country") + logSys.exception(e) + return [] + finally: + self.__lock.release() + + ## + # Returns list of Banned RIRs from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned RIRs + + def geBanListExtendedRIR(self, cymru_info): + self.__lock.acquire() + try: + return [rir for rir in cymru_info["rir"]] + except Exception as e: + logSys.error("Failed to lookup RIR") + logSys.exception(e) + return [] + finally: + self.__lock.release() + ## # Create a ban ticket. # diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index e48e5d7b..df28945e 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -183,6 +183,15 @@ class Jail: ("Actions", self.actions.status), ] + @property + def statusExtended(self): + """The extended status of the jail. + """ + return [ + ("Filter", self.filter.status), + ("Actions", self.actions.statusExtended), + ] + def putFailTicket(self, ticket): """Add a fail ticket to the jail. diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index 2627cebf..123a2891 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -72,6 +72,12 @@ class JailThread(Thread): """ pass + @abstractproperty + def statusExtended(self): # pragma: no cover - abstract + """Abstract - Should provide extended status information. + """ + pass + def start(self): """Sets active flag and starts thread. """ diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 02f3423b..c8840608 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -322,7 +322,10 @@ class Server: def statusJail(self, name): return self.__jails[name].status - + + def statusJailExtended(self, name): + return self.__jails[name].statusExtended + # Logging ## diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 39157128..05498646 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -333,5 +333,10 @@ class Transmitter: elif len(command) == 1: name = command[0] return self.__server.statusJail(name) + elif len(command) == 2: + name = command[0] + if command[1] == "extended": + return self.__server.statusJailExtended(name) + else: + raise Exception("Invalid command (invalid status extension)") raise Exception("Invalid command (no status)") - diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 7dcb73a7..d4d106a9 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -30,7 +30,6 @@ from ..server.banmanager import BanManager from ..server.ticket import BanTicket class AddFailure(unittest.TestCase): - def setUp(self): """Call before every test case.""" self.__ticket = BanTicket('193.168.0.128', 1167605999.0) @@ -39,19 +38,58 @@ class AddFailure(unittest.TestCase): def tearDown(self): """Call after every test case.""" - + def testAdd(self): self.assertEqual(self.__banManager.size(), 1) - + def testAddDuplicate(self): self.assertFalse(self.__banManager.addBanTicket(self.__ticket)) self.assertEqual(self.__banManager.size(), 1) - + def testInListOK(self): ticket = BanTicket('193.168.0.128', 1167605999.0) self.assertTrue(self.__banManager._inBanList(ticket)) - + def testInListNOK(self): ticket = BanTicket('111.111.1.111', 1167605999.0) self.assertFalse(self.__banManager._inBanList(ticket)) - + + +class StatusExtendedCymruInfo(unittest.TestCase): + def setUp(self): + """Call before every test case.""" + self.__ban_ip = "93.184.216.34" + self.__asn = "15133" + self.__country = "EU" + self.__rir = "ripencc" + self.__ticket = BanTicket(self.__ban_ip, 1167605999.0) + self.__banManager = BanManager() + self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + + def tearDown(self): + """Call after every test case.""" + + def testCymruInfo(self): + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + if "assertDictEqual" in dir(self): + self.assertDictEqual(cymru_info, {"asn": [self.__asn], "country": [self.__country], "rir": [self.__rir]}) + else: + # Python 2.6 does not support assertDictEqual() + self.assertEqual(cymru_info["asn"], [self.__asn]) + self.assertEqual(cymru_info["country"], [self.__country]) + self.assertEqual(cymru_info["rir"], [self.__rir]) + + def testCymruInfoASN(self): + self.assertEqual( + self.__banManager.geBanListExtendedASN(self.__banManager.getBanListExtendedCymruInfo()), + [self.__asn]) + + def testCymruInfoCountry(self): + self.assertEqual( + self.__banManager.geBanListExtendedCountry(self.__banManager.getBanListExtendedCymruInfo()), + [self.__country]) + + def testCymruInfoRIR(self): + self.assertEqual( + self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()), + [self.__rir]) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index e018ed2f..3bfcc303 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -474,6 +474,27 @@ class Transmitter(TransmitterBase): ) ) + def testJailStatusExtended(self): + self.assertEqual(self.transm.proceed(["status", self.jailName, "extended"]), + (0, + [ + ('Filter', [ + ('Currently failed', 0), + ('Total failed', 0), + ('File list', [])] + ), + ('Actions', [ + ('Currently banned', 0), + ('Total banned', 0), + ('Banned IP list', []), + ('Banned ASN list', []), + ('Banned Country list', []), + ('Banned RIR list', [])] + ) + ] + ) + ) + def testAction(self): action = "TestCaseAction" cmdList = [ @@ -601,6 +622,10 @@ class Transmitter(TransmitterBase): self.assertEqual( self.transm.proceed(["status", "INVALID", "COMMAND"])[0],1) + def testStatusJailExtendedNOK(self): + self.assertEqual( + self.transm.proceed(["status", self.jailName, "INVALID_COMMAND"])[0],1) + def testJournalMatch(self): if not filtersystemd: # pragma: no cover if sys.version_info >= (2, 7): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index b56c0988..42edf369 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -107,6 +107,7 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) + tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) # ClientReaders tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) From a8f057eab749d3fbf3974edf73ff9b4b78a32a05 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:27:33 -0500 Subject: [PATCH 021/195] Add `pass` to empty methods --- fail2ban/tests/banmanagertestcase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index d4d106a9..9cf820a3 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -38,6 +38,7 @@ class AddFailure(unittest.TestCase): def tearDown(self): """Call after every test case.""" + pass def testAdd(self): self.assertEqual(self.__banManager.size(), 1) @@ -68,6 +69,7 @@ class StatusExtendedCymruInfo(unittest.TestCase): def tearDown(self): """Call after every test case.""" + pass def testCymruInfo(self): cymru_info = self.__banManager.getBanListExtendedCymruInfo() From a80ff26e151c295bb5cc1661fa5402c0115a8e97 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:40:00 -0500 Subject: [PATCH 022/195] Conditionally test fail2ban-client status extended when dnspython is installed --- fail2ban/tests/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 42edf369..dbe5cee3 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -107,7 +107,11 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) - tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) + try: + import dns + tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) + except ImportError: + pass # ClientReaders tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) From 8fd9d6a770bdfc90d59da6c1493d82e0b1a2752e Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:43:58 -0500 Subject: [PATCH 023/195] add dnspython[3] optional dependencies --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8210837e..4915c95b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Optional: - Linux >= 2.6.13 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) +- [dnspython or dnspython3](http://www.dnspython.org/) To install, just do: From 256c8711210b17a63d225610afc03a313cef311d Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:00:33 -0500 Subject: [PATCH 024/195] conditionally import dnspython --- fail2ban/server/banmanager.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 81ac7c4f..2b2fd39a 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -26,9 +26,6 @@ __license__ = "GPL" from threading import Lock -import dns.exception -import dns.resolver - from .ticket import BanTicket from .mytime import MyTime from ..helpers import getLogger @@ -139,8 +136,17 @@ class BanManager: # @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs def getBanListExtendedCymruInfo(self): - self.__lock.acquire() return_dict = {"asn": [], "country": [], "rir": []} + try: + import dns.exception + import dns.resolver + except ImportError: + logSys.error("dnspython package is required but could not be imported") + return_dict["asn"].append("error") + return_dict["country"].append("error") + return_dict["rir"].append("error") + return return_dict + self.__lock.acquire() try: for banData in self.__banList: ip = banData.getIP() From ce0fe8b907000a02eca1b8edddd897325580a1f6 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:00:49 -0500 Subject: [PATCH 025/195] test dnspython nxdomain returned --- fail2ban/tests/banmanagertestcase.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 9cf820a3..3f7fe97e 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -63,9 +63,9 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.__asn = "15133" self.__country = "EU" self.__rir = "ripencc" - self.__ticket = BanTicket(self.__ban_ip, 1167605999.0) + ticket = BanTicket(self.__ban_ip, 1167605999.0) self.__banManager = BanManager() - self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + self.assertTrue(self.__banManager.addBanTicket(ticket)) def tearDown(self): """Call after every test case.""" @@ -74,7 +74,9 @@ class StatusExtendedCymruInfo(unittest.TestCase): def testCymruInfo(self): cymru_info = self.__banManager.getBanListExtendedCymruInfo() if "assertDictEqual" in dir(self): - self.assertDictEqual(cymru_info, {"asn": [self.__asn], "country": [self.__country], "rir": [self.__rir]}) + self.assertDictEqual(cymru_info, {"asn": [self.__asn], + "country": [self.__country], + "rir": [self.__rir]}) else: # Python 2.6 does not support assertDictEqual() self.assertEqual(cymru_info["asn"], [self.__asn]) @@ -95,3 +97,18 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.assertEqual( self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()), [self.__rir]) + + def testCymruInfoNxdomain(self): + ticket = BanTicket("10.0.0.0", 1167605999.0) + self.__banManager = BanManager() + self.assertTrue(self.__banManager.addBanTicket(ticket)) + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + if "assertDictEqual" in dir(self): + self.assertDictEqual(cymru_info, {"asn": [self.__asn, "nxdomain"], + "country": [self.__country, "nxdomain"], + "rir": [self.__rir, "nxdomain"]}) + else: + # Python 2.6 does not support assertDictEqual() + self.assertEqual(cymru_info["asn"], [self.__asn, "nxdomain"]) + self.assertEqual(cymru_info["country"], [self.__country, "nxdomain"]) + self.assertEqual(cymru_info["rir"], [self.__rir, "nxdomain"]) From 03db257c434e93b80742d33ee68773471d70c617 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:18:09 -0500 Subject: [PATCH 026/195] fix test of new banManager's instance --- fail2ban/tests/banmanagertestcase.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 3f7fe97e..c4dd42cf 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -104,11 +104,11 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.assertTrue(self.__banManager.addBanTicket(ticket)) cymru_info = self.__banManager.getBanListExtendedCymruInfo() if "assertDictEqual" in dir(self): - self.assertDictEqual(cymru_info, {"asn": [self.__asn, "nxdomain"], - "country": [self.__country, "nxdomain"], - "rir": [self.__rir, "nxdomain"]}) + self.assertDictEqual(cymru_info, {"asn": ["nxdomain"], + "country": ["nxdomain"], + "rir": ["nxdomain"]}) else: # Python 2.6 does not support assertDictEqual() - self.assertEqual(cymru_info["asn"], [self.__asn, "nxdomain"]) - self.assertEqual(cymru_info["country"], [self.__country, "nxdomain"]) - self.assertEqual(cymru_info["rir"], [self.__rir, "nxdomain"]) + self.assertEqual(cymru_info["asn"], ["nxdomain"]) + self.assertEqual(cymru_info["country"], ["nxdomain"]) + self.assertEqual(cymru_info["rir"], ["nxdomain"]) From 13b2aea48a370b84075ca4ce80ea0f7ce9d69491 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 24 Jan 2015 12:22:34 -0500 Subject: [PATCH 027/195] Remove version specific package --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4915c95b..17a78ee5 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Optional: - Linux >= 2.6.13 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) -- [dnspython or dnspython3](http://www.dnspython.org/) +- [dnspython](http://www.dnspython.org/) To install, just do: From 60ac0a1a1797a9a10fe7d6b91003571de5c54a5e Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:12:04 -0500 Subject: [PATCH 028/195] Add extended info to status output using Cyrmu --- .travis.yml | 2 + ChangeLog | 6 +- fail2ban/protocol.py | 1 + fail2ban/server/actions.py | 13 +++- fail2ban/server/banmanager.py | 112 +++++++++++++++++++++++++++ fail2ban/server/jail.py | 9 +++ fail2ban/server/jailthread.py | 6 ++ fail2ban/server/server.py | 5 +- fail2ban/server/transmitter.py | 7 +- fail2ban/tests/banmanagertestcase.py | 50 ++++++++++-- fail2ban/tests/servertestcase.py | 25 ++++++ fail2ban/tests/utils.py | 1 + 12 files changed, 227 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index ead9ef46..32742761 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,8 @@ before_install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get update -qq; fi install: - travis_retry pip install pyinotify + - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then travis_retry pip install dnspython; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then travis_retry pip install dnspython3; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; travis_retry pip install -q coveralls; cd -; fi script: diff --git a/ChangeLog b/ChangeLog index f01ed930..3013dac8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,7 +41,11 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - Monit config for fail2ban in /files/monit - New actions: - action.d/firewallcmd-multiport and action.d/firewallcmd-allports Thanks Donald Yandt - + - New status commands: + - fail2ban-client status extended + - prints Cymru data (ASN, Country RIR) per banned IP + - Requires dnspython or dnspython3 + - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 * files/debian-initd migrated from the debian branch and should be diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 9a6ee675..dc7a7c6b 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,7 @@ protocol = [ ["start ", "starts the jail "], ["stop ", "stops the jail . The jail is removed"], ["status ", "gets the current status of "], +["status extended", "gets the current status of with extended info"], ['', "JAIL CONFIGURATION", ""], ["set idle on|off", "sets the idle state of "], ["set addignoreip ", "adds to the ignore list of "], diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index a212aaf1..f4d5c18b 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -372,9 +372,20 @@ class Actions(JailThread, Mapping): @property def status(self): - """Status of active bans, and total ban counts. + """Status of current and total ban counts and current banned IP list. """ ret = [("Currently banned", self.__banManager.size()), ("Total banned", self.__banManager.getBanTotal()), ("Banned IP list", self.__banManager.getBanList())] return ret + + @property + def statusExtended(self): + """Jail status plus banned IPs' ASN, Country and RIR + """ + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + ret = self.status +\ + [("Banned ASN list", self.__banManager.geBanListExtendedASN(cymru_info)), + ("Banned Country list", self.__banManager.geBanListExtendedCountry(cymru_info)), + ("Banned RIR list", self.__banManager.geBanListExtendedRIR(cymru_info))] + return ret diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index c21cad45..81ac7c4f 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -26,6 +26,9 @@ __license__ = "GPL" from threading import Lock +import dns.exception +import dns.resolver + from .ticket import BanTicket from .mytime import MyTime from ..helpers import getLogger @@ -118,6 +121,115 @@ class BanManager: finally: self.__lock.release() + ## + # Returns normalized value + # + # @return value or "unknown" if value is None or empty string + + @staticmethod + def handleBlankResult(value): + if value is None or len(value) == 0: + return "unknown" + else: + return value + + ## + # Returns Cymru DNS query information + # + # @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs + + def getBanListExtendedCymruInfo(self): + self.__lock.acquire() + return_dict = {"asn": [], "country": [], "rir": []} + try: + for banData in self.__banList: + ip = banData.getIP() + # Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns + # TODO: IPv6 compatibility + reversed_ip = ".".join(reversed(ip.split("."))) + question = "%s.origin.asn.cymru.com" % reversed_ip + try: + answers = dns.resolver.query(question, "TXT") + for rdata in answers: + asn, net, country, rir, changed =\ + [answer.strip("'\" ") for answer in rdata.to_text().split("|")] + asn = self.handleBlankResult(asn) + country = self.handleBlankResult(country) + rir = self.handleBlankResult(rir) + return_dict["asn"].append(self.handleBlankResult(asn)) + return_dict["country"].append(self.handleBlankResult(country)) + return_dict["rir"].append(self.handleBlankResult(rir)) + except dns.resolver.NXDOMAIN: + return_dict["asn"].append("nxdomain") + return_dict["country"].append("nxdomain") + return_dict["rir"].append("nxdomain") + except dns.exception.DNSException as dnse: + logSys.error("Unhandled DNSException querying Cymru for %s TXT" % question) + logSys.exception(dnse) + except Exception as e: + logSys.error("Unhandled Exception querying Cymru for %s TXT" % question) + logSys.exception(e) + except Exception as e: + logSys.error("Failure looking up extended Cymru info") + logSys.exception(e) + finally: + self.__lock.release() + return return_dict + + ## + # Returns list of Banned ASNs from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned ASNs + + def geBanListExtendedASN(self, cymru_info): + self.__lock.acquire() + try: + return [asn for asn in cymru_info["asn"]] + except Exception as e: + logSys.error("Failed to lookup ASN") + logSys.exception(e) + return [] + finally: + self.__lock.release() + + ## + # Returns list of Banned Countries from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned Countries + + def geBanListExtendedCountry(self, cymru_info): + self.__lock.acquire() + try: + return [country for country in cymru_info["country"]] + except Exception as e: + logSys.error("Failed to lookup Country") + logSys.exception(e) + return [] + finally: + self.__lock.release() + + ## + # Returns list of Banned RIRs from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned RIRs + + def geBanListExtendedRIR(self, cymru_info): + self.__lock.acquire() + try: + return [rir for rir in cymru_info["rir"]] + except Exception as e: + logSys.error("Failed to lookup RIR") + logSys.exception(e) + return [] + finally: + self.__lock.release() + ## # Create a ban ticket. # diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index e48e5d7b..df28945e 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -183,6 +183,15 @@ class Jail: ("Actions", self.actions.status), ] + @property + def statusExtended(self): + """The extended status of the jail. + """ + return [ + ("Filter", self.filter.status), + ("Actions", self.actions.statusExtended), + ] + def putFailTicket(self, ticket): """Add a fail ticket to the jail. diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index 2627cebf..123a2891 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -72,6 +72,12 @@ class JailThread(Thread): """ pass + @abstractproperty + def statusExtended(self): # pragma: no cover - abstract + """Abstract - Should provide extended status information. + """ + pass + def start(self): """Sets active flag and starts thread. """ diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 02f3423b..c8840608 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -322,7 +322,10 @@ class Server: def statusJail(self, name): return self.__jails[name].status - + + def statusJailExtended(self, name): + return self.__jails[name].statusExtended + # Logging ## diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 39157128..05498646 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -333,5 +333,10 @@ class Transmitter: elif len(command) == 1: name = command[0] return self.__server.statusJail(name) + elif len(command) == 2: + name = command[0] + if command[1] == "extended": + return self.__server.statusJailExtended(name) + else: + raise Exception("Invalid command (invalid status extension)") raise Exception("Invalid command (no status)") - diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 7dcb73a7..d4d106a9 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -30,7 +30,6 @@ from ..server.banmanager import BanManager from ..server.ticket import BanTicket class AddFailure(unittest.TestCase): - def setUp(self): """Call before every test case.""" self.__ticket = BanTicket('193.168.0.128', 1167605999.0) @@ -39,19 +38,58 @@ class AddFailure(unittest.TestCase): def tearDown(self): """Call after every test case.""" - + def testAdd(self): self.assertEqual(self.__banManager.size(), 1) - + def testAddDuplicate(self): self.assertFalse(self.__banManager.addBanTicket(self.__ticket)) self.assertEqual(self.__banManager.size(), 1) - + def testInListOK(self): ticket = BanTicket('193.168.0.128', 1167605999.0) self.assertTrue(self.__banManager._inBanList(ticket)) - + def testInListNOK(self): ticket = BanTicket('111.111.1.111', 1167605999.0) self.assertFalse(self.__banManager._inBanList(ticket)) - + + +class StatusExtendedCymruInfo(unittest.TestCase): + def setUp(self): + """Call before every test case.""" + self.__ban_ip = "93.184.216.34" + self.__asn = "15133" + self.__country = "EU" + self.__rir = "ripencc" + self.__ticket = BanTicket(self.__ban_ip, 1167605999.0) + self.__banManager = BanManager() + self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + + def tearDown(self): + """Call after every test case.""" + + def testCymruInfo(self): + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + if "assertDictEqual" in dir(self): + self.assertDictEqual(cymru_info, {"asn": [self.__asn], "country": [self.__country], "rir": [self.__rir]}) + else: + # Python 2.6 does not support assertDictEqual() + self.assertEqual(cymru_info["asn"], [self.__asn]) + self.assertEqual(cymru_info["country"], [self.__country]) + self.assertEqual(cymru_info["rir"], [self.__rir]) + + def testCymruInfoASN(self): + self.assertEqual( + self.__banManager.geBanListExtendedASN(self.__banManager.getBanListExtendedCymruInfo()), + [self.__asn]) + + def testCymruInfoCountry(self): + self.assertEqual( + self.__banManager.geBanListExtendedCountry(self.__banManager.getBanListExtendedCymruInfo()), + [self.__country]) + + def testCymruInfoRIR(self): + self.assertEqual( + self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()), + [self.__rir]) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index e018ed2f..3bfcc303 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -474,6 +474,27 @@ class Transmitter(TransmitterBase): ) ) + def testJailStatusExtended(self): + self.assertEqual(self.transm.proceed(["status", self.jailName, "extended"]), + (0, + [ + ('Filter', [ + ('Currently failed', 0), + ('Total failed', 0), + ('File list', [])] + ), + ('Actions', [ + ('Currently banned', 0), + ('Total banned', 0), + ('Banned IP list', []), + ('Banned ASN list', []), + ('Banned Country list', []), + ('Banned RIR list', [])] + ) + ] + ) + ) + def testAction(self): action = "TestCaseAction" cmdList = [ @@ -601,6 +622,10 @@ class Transmitter(TransmitterBase): self.assertEqual( self.transm.proceed(["status", "INVALID", "COMMAND"])[0],1) + def testStatusJailExtendedNOK(self): + self.assertEqual( + self.transm.proceed(["status", self.jailName, "INVALID_COMMAND"])[0],1) + def testJournalMatch(self): if not filtersystemd: # pragma: no cover if sys.version_info >= (2, 7): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index b56c0988..42edf369 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -107,6 +107,7 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) + tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) # ClientReaders tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) From ba699690573da914bdb9c8b5d5e5887303a58ff3 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:27:33 -0500 Subject: [PATCH 029/195] Add `pass` to empty methods --- fail2ban/tests/banmanagertestcase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index d4d106a9..9cf820a3 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -38,6 +38,7 @@ class AddFailure(unittest.TestCase): def tearDown(self): """Call after every test case.""" + pass def testAdd(self): self.assertEqual(self.__banManager.size(), 1) @@ -68,6 +69,7 @@ class StatusExtendedCymruInfo(unittest.TestCase): def tearDown(self): """Call after every test case.""" + pass def testCymruInfo(self): cymru_info = self.__banManager.getBanListExtendedCymruInfo() From 405f363fe87cb798ad6b6cdd36cb5a26e5e487e4 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:40:00 -0500 Subject: [PATCH 030/195] Conditionally test fail2ban-client status extended when dnspython is installed --- fail2ban/tests/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 42edf369..dbe5cee3 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -107,7 +107,11 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) - tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) + try: + import dns + tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) + except ImportError: + pass # ClientReaders tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) From ef3c942808f9472e82854122828f4db39e62d85f Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:43:58 -0500 Subject: [PATCH 031/195] add dnspython[3] optional dependencies --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8210837e..4915c95b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Optional: - Linux >= 2.6.13 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) +- [dnspython or dnspython3](http://www.dnspython.org/) To install, just do: From 07a47179a774e3d56861cee9fc231799e19f7fc9 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:00:33 -0500 Subject: [PATCH 032/195] conditionally import dnspython --- fail2ban/server/banmanager.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 81ac7c4f..2b2fd39a 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -26,9 +26,6 @@ __license__ = "GPL" from threading import Lock -import dns.exception -import dns.resolver - from .ticket import BanTicket from .mytime import MyTime from ..helpers import getLogger @@ -139,8 +136,17 @@ class BanManager: # @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs def getBanListExtendedCymruInfo(self): - self.__lock.acquire() return_dict = {"asn": [], "country": [], "rir": []} + try: + import dns.exception + import dns.resolver + except ImportError: + logSys.error("dnspython package is required but could not be imported") + return_dict["asn"].append("error") + return_dict["country"].append("error") + return_dict["rir"].append("error") + return return_dict + self.__lock.acquire() try: for banData in self.__banList: ip = banData.getIP() From a0debea56a3efb4609918f39b32497536516fb92 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:00:49 -0500 Subject: [PATCH 033/195] test dnspython nxdomain returned --- fail2ban/tests/banmanagertestcase.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 9cf820a3..3f7fe97e 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -63,9 +63,9 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.__asn = "15133" self.__country = "EU" self.__rir = "ripencc" - self.__ticket = BanTicket(self.__ban_ip, 1167605999.0) + ticket = BanTicket(self.__ban_ip, 1167605999.0) self.__banManager = BanManager() - self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + self.assertTrue(self.__banManager.addBanTicket(ticket)) def tearDown(self): """Call after every test case.""" @@ -74,7 +74,9 @@ class StatusExtendedCymruInfo(unittest.TestCase): def testCymruInfo(self): cymru_info = self.__banManager.getBanListExtendedCymruInfo() if "assertDictEqual" in dir(self): - self.assertDictEqual(cymru_info, {"asn": [self.__asn], "country": [self.__country], "rir": [self.__rir]}) + self.assertDictEqual(cymru_info, {"asn": [self.__asn], + "country": [self.__country], + "rir": [self.__rir]}) else: # Python 2.6 does not support assertDictEqual() self.assertEqual(cymru_info["asn"], [self.__asn]) @@ -95,3 +97,18 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.assertEqual( self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()), [self.__rir]) + + def testCymruInfoNxdomain(self): + ticket = BanTicket("10.0.0.0", 1167605999.0) + self.__banManager = BanManager() + self.assertTrue(self.__banManager.addBanTicket(ticket)) + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + if "assertDictEqual" in dir(self): + self.assertDictEqual(cymru_info, {"asn": [self.__asn, "nxdomain"], + "country": [self.__country, "nxdomain"], + "rir": [self.__rir, "nxdomain"]}) + else: + # Python 2.6 does not support assertDictEqual() + self.assertEqual(cymru_info["asn"], [self.__asn, "nxdomain"]) + self.assertEqual(cymru_info["country"], [self.__country, "nxdomain"]) + self.assertEqual(cymru_info["rir"], [self.__rir, "nxdomain"]) From 735c51adaefd245f215aed5381c7d8f7ca8d86ed Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:18:09 -0500 Subject: [PATCH 034/195] fix test of new banManager's instance --- fail2ban/tests/banmanagertestcase.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 3f7fe97e..c4dd42cf 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -104,11 +104,11 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.assertTrue(self.__banManager.addBanTicket(ticket)) cymru_info = self.__banManager.getBanListExtendedCymruInfo() if "assertDictEqual" in dir(self): - self.assertDictEqual(cymru_info, {"asn": [self.__asn, "nxdomain"], - "country": [self.__country, "nxdomain"], - "rir": [self.__rir, "nxdomain"]}) + self.assertDictEqual(cymru_info, {"asn": ["nxdomain"], + "country": ["nxdomain"], + "rir": ["nxdomain"]}) else: # Python 2.6 does not support assertDictEqual() - self.assertEqual(cymru_info["asn"], [self.__asn, "nxdomain"]) - self.assertEqual(cymru_info["country"], [self.__country, "nxdomain"]) - self.assertEqual(cymru_info["rir"], [self.__rir, "nxdomain"]) + self.assertEqual(cymru_info["asn"], ["nxdomain"]) + self.assertEqual(cymru_info["country"], ["nxdomain"]) + self.assertEqual(cymru_info["rir"], ["nxdomain"]) From dfe4d02f6540fdb6879500589da798b621ea5ff2 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 24 Jan 2015 12:22:34 -0500 Subject: [PATCH 035/195] Remove version specific package --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4915c95b..17a78ee5 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Optional: - Linux >= 2.6.13 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) -- [dnspython or dnspython3](http://www.dnspython.org/) +- [dnspython](http://www.dnspython.org/) To install, just do: From eb76dcd5a00079b39b1b200f3c9f12fdc4c0fa6a Mon Sep 17 00:00:00 2001 From: rumple010 Date: Sun, 25 Jan 2015 23:15:07 -0500 Subject: [PATCH 036/195] add nsupdate action Adds a new action file that uses nsupdate to dynamically update a BIND zone file with a TXT resource record representing a banned IP address. Resource record is deleted from the zone when the ban expires. --- config/action.d/nsupdate.conf | 110 ++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 config/action.d/nsupdate.conf diff --git a/config/action.d/nsupdate.conf b/config/action.d/nsupdate.conf new file mode 100644 index 00000000..9ecb6f95 --- /dev/null +++ b/config/action.d/nsupdate.conf @@ -0,0 +1,110 @@ +# Fail2Ban configuration file +# +# Author: Andrew St. Jean +# +# Use nsupdate to perform dynamic DNS updates on a BIND zone file. +# One may want to do this to update a local RBL with banned IP addresses. +# +# Options +# +# domain DNS domain that will appear in nsupdate add and delete +# commands. +# +# ttl The time to live (TTL) in seconds of the TXT resource +# record. +# +# rdata Data portion of the TXT resource record. +# +# nsupdatecmd Full path to the nsupdate command. +# +# keyfile Full path to TSIG key file used for authentication between +# nsupdate and BIND. +# +# The ban and unban commands assume nsupdate will authenticate to the BIND +# server using a TSIG key. The full path to the key file must be specified +# in the parameter. Use this command to generate your TSIG key. +# +# dnssec-keygen -a HMAC-MD5 -b 256 -n HOST +# +# Replace with some meaningful name. +# +# This command will generate two files. Specify the .private file in the +# option. Note that the .key file must also be present in the same +# directory for nsupdate to use the key. +# +# Don't forget to add the key and appropriate allow-update or update-policy +# option to your named.conf file. +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = + + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = + + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = echo | awk -F. '{print "prereq nxrrset "$4"."$3"."$2"."$1". TXT"; print "update add "$4"."$3"."$2"."$1". IN TXT \"\""; print "send"}' | -k + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = echo | awk -F. '{print "update delete "$4"."$3"."$2"."$1"."; print "send"}' | -k + +[Init] + +# Option: domain +# Notes.: DNS domain that nsupdate will update. +# Values: STRING +# +domain = + +# Option: ttl +# Notes.: time to live (TTL) of TXT resource record added by nsupdate. +# Values: NUM +# +ttl = 600 + +# Option: rdata +# Notes.: data portion of the TXT resource record added by nsupdate. +# Values: STRING +# +rdata = Your IP has been banned + +# Option: nsupdatecmd +# Notes.: specifies the full path to the nsupdate program that dynamically +# updates BIND zone files. +# Values: CMD +# +nsupdatecmd = /usr/bin/nsupdate + +# Option: keyfile +# Notes.: specifies the full path to the file containing the +# TSIG key for communicating with BIND. +# Values: STRING +# +keyfile = + From 085d0f72ed1b23e58fe7185b9af099e8a9b4f515 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 26 Jan 2015 09:19:44 -0500 Subject: [PATCH 037/195] ENH: use non-UTC date invocation (without -u) and report offset for localzone (%z) --- ChangeLog | 5 ++++- config/action.d/sendmail-common.conf | 4 ++-- config/action.d/sendmail-whois-ipjailmatches.conf | 2 +- config/action.d/sendmail-whois-ipmatches.conf | 2 +- config/action.d/sendmail-whois-lines.conf | 2 +- config/action.d/sendmail-whois-matches.conf | 2 +- config/action.d/sendmail-whois.conf | 2 +- config/action.d/sendmail.conf | 2 +- config/action.d/xarf-login-attack.conf | 2 +- 9 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index ca2601bc..86dca7f0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -48,7 +48,10 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 * files/debian-initd migrated from the debian branch and should be - suitable for manual installations now (thanks Juan Karlo de Guzman) + suitable for manual installations now (thanks Juan Karlo de + Guzman) + * action.d/{sendmail-*,xarg-login-attack}.conf - report local + timezone not UTC time/zone. Closes gh-911 ver. 0.9.1 (2014/10/29) - better, faster, stronger diff --git a/config/action.d/sendmail-common.conf b/config/action.d/sendmail-common.conf index 26dcb4c8..af0212bd 100644 --- a/config/action.d/sendmail-common.conf +++ b/config/action.d/sendmail-common.conf @@ -15,7 +15,7 @@ after = sendmail-common.local # Values: CMD # actionstart = printf %%b "Subject: [Fail2Ban] : started on `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n @@ -28,7 +28,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started on `uname -n` # Values: CMD # actionstop = printf %%b "Subject: [Fail2Ban] : stopped on `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail-whois-ipjailmatches.conf b/config/action.d/sendmail-whois-ipjailmatches.conf index 45b1f312..2de70d85 100644 --- a/config/action.d/sendmail-whois-ipjailmatches.conf +++ b/config/action.d/sendmail-whois-ipjailmatches.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail-whois-ipmatches.conf b/config/action.d/sendmail-whois-ipmatches.conf index 8193fb04..1390e7a3 100644 --- a/config/action.d/sendmail-whois-ipmatches.conf +++ b/config/action.d/sendmail-whois-ipmatches.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index 4169e82a..135632ce 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail-whois-matches.conf b/config/action.d/sendmail-whois-matches.conf index ed664766..c99d5ebf 100644 --- a/config/action.d/sendmail-whois-matches.conf +++ b/config/action.d/sendmail-whois-matches.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf index fc601277..9403a388 100644 --- a/config/action.d/sendmail-whois.conf +++ b/config/action.d/sendmail-whois.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf index 46050e11..4b088dc8 100644 --- a/config/action.d/sendmail.conf +++ b/config/action.d/sendmail.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf index 6d6a74f0..19b3167f 100644 --- a/config/action.d/xarf-login-attack.conf +++ b/config/action.d/xarf-login-attack.conf @@ -46,7 +46,7 @@ actionban = oifs=${IFS}; IFS=.;SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(di REPORTID=