From dab2ddb9dad9c1c9061bc40589d56ecb5368787b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 18 Nov 2013 07:57:16 +1100 Subject: [PATCH 01/74] ENH: recidive jail to block all protocols. Closes #440 --- ChangeLog | 1 + config/jail.conf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 2369bc21..2e1a5b91 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,7 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better - Fixes: - allow for ",milliseconds" in the custom date format of proftpd.log + - recidive jail to block all protocols. Closes gh-440 - New Features: diff --git a/config/jail.conf b/config/jail.conf index 486ea078..e9ca3313 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -408,7 +408,7 @@ maxretry = 5 enabled = false filter = recidive logpath = /var/log/fail2ban.log -action = iptables-allports[name=recidive] +action = iptables-allports[name=recidive,protocol=all] sendmail-whois-lines[name=recidive, logpath=/var/log/fail2ban.log] bantime = 604800 ; 1 week findtime = 86400 ; 1 day From 8aa20a7b0efdb8020e59db0f423d8382cf4d8d05 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 18 Nov 2013 07:59:56 +1100 Subject: [PATCH 02/74] ENH: credits for #440 recidive jail protocol=all --- ChangeLog | 2 +- THANKS | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 2e1a5b91..dfecbc51 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,7 +14,7 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better - Fixes: - allow for ",milliseconds" in the custom date format of proftpd.log - - recidive jail to block all protocols. Closes gh-440 + - recidive jail to block all protocols. Closes gh-440. Thanks Ioan Indreias - New Features: diff --git a/THANKS b/THANKS index 13303c21..ac2f4439 100644 --- a/THANKS +++ b/THANKS @@ -33,6 +33,7 @@ Georgiy Mernov Guillaume Delvit Hanno 'Rince' Wagner Iain Lea +Ioan Indreias Jonathan Kamens Jonathan Lanning Jonathan Underwood From d34d8db3d2bf361853cf7f9fa8b2cd3ca0edc707 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 22 Nov 2013 15:57:03 -0500 Subject: [PATCH 03/74] BF/ENH: include [PID] into logging msgs, remove indentation from syslog messages Otherwise leads to incorrect parsing of the log messages by syslog(-ng). See http://bugs.debian.org/730202 I also removed %(levelname)-6s from syslog messages completely since they are passed to the syslog and it is up to the configuration/admin to decide include levels into the messages or not (I have checked that at least debug level indeed goes to /var/log/debug) --- server/server.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/server.py b/server/server.py index 773fdf0b..44d8ae68 100644 --- a/server/server.py +++ b/server/server.py @@ -347,13 +347,12 @@ class Server: try: self.__loggingLock.acquire() # set a format which is simpler for console use - formatter = logging.Formatter("%(asctime)s %(name)-16s: %(levelname)-6s %(message)s") + formatter = logging.Formatter("%(name)s[%(process)d]: %(levelname)-7s %(message)s") if target == "SYSLOG": # Syslog daemons already add date to the message. - formatter = logging.Formatter("%(name)-16s: %(levelname)-6s %(message)s") + formatter = logging.Formatter("%(name)s[%(process)d]: %(message)s") facility = logging.handlers.SysLogHandler.LOG_DAEMON - hdlr = logging.handlers.SysLogHandler("/dev/log", - facility = facility) + hdlr = logging.handlers.SysLogHandler("/dev/log", facility=facility) elif target == "STDOUT": hdlr = logging.StreamHandler(sys.stdout) elif target == "STDERR": From f2c529ca7ba558e44f3f0177cea96c70c0e70d4f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 23 Nov 2013 11:33:41 +1100 Subject: [PATCH 04/74] ENH: move signal.signal(signal.SIGHUP, signal.SIG_IGN) before fork in server. closes #446 --- server/server.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server/server.py b/server/server.py index 773fdf0b..f5dbc3c8 100644 --- a/server/server.py +++ b/server/server.py @@ -409,6 +409,14 @@ class Server: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/278731 """ + # When the first child terminates, all processes in the second child + # are sent a SIGHUP, so it's ignored. + + # We need to set this in the parent process, so it gets inherited by the + # child process, and this makes sure that it is effect even if the parent + # terminates quickly. + signal.signal(signal.SIGHUP, signal.SIG_IGN) + try: # Fork a child process so the parent can exit. This will return control # to the command line or shell. This is required so that the new process @@ -431,10 +439,6 @@ class Server: # leader. os.setsid() - # When the first child terminates, all processes in the second child - # are sent a SIGHUP, so it's ignored. - signal.signal(signal.SIGHUP, signal.SIG_IGN) - try: # Fork a second child to prevent zombies. Since the first child is # a session leader without a controlling terminal, it's possible for From 9a82bc3c617c488dd376571370d08b05bf1954af Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 24 Nov 2013 18:21:02 +1100 Subject: [PATCH 05/74] BF: kernel messages can have space. Thanks ag4ve(shawn). Closes #448 --- ChangeLog | 1 + THANKS | 1 + config/filter.d/common.conf | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 24f01e9a..6636ef76 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,7 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better - Fixes: - allow for ",milliseconds" in the custom date format of proftpd.log - allow for ", referer ..." in apache-* filter for apache error logs. + - allow for spaces at the beginning of kernel messages. Closes gh-448 - New Features: diff --git a/THANKS b/THANKS index 46c7bc5e..e71eefea 100644 --- a/THANKS +++ b/THANKS @@ -8,6 +8,7 @@ be added Adrien Clerc ache +ag4ve (Shawn) Amir Caspi Andrey G. Grozin Andy Fragen diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf index b992e4b8..ae8e8b7b 100644 --- a/config/filter.d/common.conf +++ b/config/filter.d/common.conf @@ -34,7 +34,7 @@ __daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_r # Some messages have a kernel prefix with a timestamp # EXAMPLES: kernel: [769570.846956] -__kernel_prefix = kernel: \[\d+\.\d+\] +__kernel_prefix = kernel: \[ *\d+\.\d+\] __hostname = \S+ From a989787e0dace5b609028b95526c1eb212e1ebef Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 24 Nov 2013 18:43:23 +1100 Subject: [PATCH 06/74] DOC: more distro bug tracker urls --- DEVELOP | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DEVELOP b/DEVELOP index 40182ff9..3939fe47 100644 --- a/DEVELOP +++ b/DEVELOP @@ -743,10 +743,14 @@ Releasing * https://github.com/fail2ban/fail2ban/issues?sort=updated&state=open * http://bugs.debian.org/cgi-bin/pkgreport.cgi?dist=unstable;package=fail2ban + * https://bugs.launchpad.net/ubuntu/+source/fail2ban * http://bugs.sabayon.org/buglist.cgi?quicksearch=net-analyzer%2Ffail2ban + * https://bugs.archlinux.org/?project=5&cat%5B%5D=33&string=fail2ban * https://bugs.gentoo.org/buglist.cgi?query_format=advanced&short_desc=fail2ban&bug_status=UNCONFIRMED&bug_status=CONFIRMED&bug_status=IN_PROGRESS&short_desc_type=allwords * https://bugzilla.redhat.com/buglist.cgi?query_format=advanced&bug_status=NEW&bug_status=ASSIGNED&component=fail2ban&classification=Red%20Hat&classification=Fedora * http://www.freebsd.org/cgi/query-pr-summary.cgi?text=fail2ban + * https://bugs.mageia.org/buglist.cgi?quicksearch=fail2ban + * https://build.opensuse.org/package/requests/openSUSE:Factory/fail2ban # Make sure the tests pass From a6f085786ca46485557c0adbe6b6510f1a06253b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 24 Nov 2013 09:50:39 -0500 Subject: [PATCH 07/74] ENH: reintroducing levelnameinto syslog msgs, time stamp and indentation in non-syslog msgs any indentation from syslog msgs wsa removed -- no need --- server/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/server.py b/server/server.py index 44d8ae68..6ac93a54 100644 --- a/server/server.py +++ b/server/server.py @@ -347,10 +347,10 @@ class Server: try: self.__loggingLock.acquire() # set a format which is simpler for console use - formatter = logging.Formatter("%(name)s[%(process)d]: %(levelname)-7s %(message)s") + formatter = logging.Formatter("%(asctime)s %(name)-16s[%(process)d]: %(levelname)-7s %(message)s") if target == "SYSLOG": # Syslog daemons already add date to the message. - formatter = logging.Formatter("%(name)s[%(process)d]: %(message)s") + formatter = logging.Formatter("%(name)s[%(process)d]: %(levelname)s %(message)s") facility = logging.handlers.SysLogHandler.LOG_DAEMON hdlr = logging.handlers.SysLogHandler("/dev/log", facility=facility) elif target == "STDOUT": From a26d4f42b7b4b3a7359e64eba4d5997eef25db46 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 24 Nov 2013 09:59:45 -0500 Subject: [PATCH 08/74] ENH: added optional [PID] matching in recidive.conf --- config/filter.d/recidive.conf | 2 +- testcases/files/logs/recidive | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/recidive.conf b/config/filter.d/recidive.conf index b29acaf3..13d2f53a 100644 --- a/config/filter.d/recidive.conf +++ b/config/filter.d/recidive.conf @@ -27,6 +27,6 @@ _daemon = fail2ban\.actions # jail using this filter 'recidive', or change this line! _jailname = recidive -failregex = ^(%(__prefix_line)s|,\d{3} fail2ban.actions:\s+)WARNING\s+\[(?!%(_jailname)s\])(?:.*)\]\s+Ban\s+\s*$ +failregex = ^(%(__prefix_line)s|,\d{3} fail2ban.actions%(__pid_re)s?:\s+)WARNING\s+\[(?!%(_jailname)s\])(?:.*)\]\s+Ban\s+\s*$ # Author: Tom Hendrikx, modifications by Amir Caspi diff --git a/testcases/files/logs/recidive b/testcases/files/logs/recidive index 6af85137..83acc3e1 100644 --- a/testcases/files/logs/recidive +++ b/testcases/files/logs/recidive @@ -1,5 +1,7 @@ # failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4" } 2006-02-13 15:52:30,388 fail2ban.actions: WARNING [sendmail] Ban 1.2.3.4 +# failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4", "desc": "Extended with [PID]" } +2006-02-13 15:52:30,388 fail2ban.actions[123]: WARNING [sendmail] Ban 1.2.3.4 # failJSON: { "match": false } 2006-02-13 16:07:31,183 fail2ban.actions: WARNING [sendmail] Unban 1.2.3.4 # failJSON: { "match": false } From 085ebbe1de7477733b28e06204ccbb19ea2c3d09 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 24 Nov 2013 11:55:58 -0500 Subject: [PATCH 09/74] Changelog entries for the last changes --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 24f01e9a..a26257f1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,8 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better - Fixes: - allow for ",milliseconds" in the custom date format of proftpd.log - allow for ", referer ..." in apache-* filter for apache error logs. + - remove indentation of name and loglevel while logging to SYSLOG to + resolve syslog(-ng) parsing problems. Closes Debian bug #730202. - New Features: @@ -22,6 +24,7 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better * filter.d/solid-pop3d -- added thanks to Jacques Lav!gnotte on mailinglist. - Enhancements: + - loglines now also report "[PID]" after the name portion ver. 0.8.11 (2013/11/13) - loves-unittests-and-tight-DoS-free-filter-regexes From 093aee967622216c5d2d4023fb9842388f87c3b1 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 25 Nov 2013 07:54:49 +1100 Subject: [PATCH 10/74] TST: no python-2.5 any more - https://github.com/travis-ci/travis-ci/issues/1668 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2d091754..d693f975 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ # travis-ci.org definition for Fail2Ban build language: python python: - - "2.5" - "2.6" - "2.7" before_install: From dc154c792e6d09abe3592e1321581f9103dd47fe Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 25 Nov 2013 08:08:20 +1100 Subject: [PATCH 11/74] BF: add init section with name for action.d/apf. Closes #398 --- config/action.d/apf.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/action.d/apf.conf b/config/action.d/apf.conf index 9af3066d..f1d54dd2 100644 --- a/config/action.d/apf.conf +++ b/config/action.d/apf.conf @@ -41,3 +41,10 @@ actionban = apf --deny "banned by Fail2Ban " # Values: CMD # actionunban = apf --remove + +[Init] + +# Name used in APF configuration +# +name = default + From 99838440c8a7cb8c55d2479f6bf5fa32c8fc3640 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 28 Nov 2013 23:18:34 +1100 Subject: [PATCH 12/74] DOC: document rational behind 20 character jail name limit --- ChangeLog | 2 ++ server/jail.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 58f8f84a..e94a1338 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,8 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better - allow for ", referer ..." in apache-* filter for apache error logs. - allow for spaces at the beginning of kernel messages. Closes gh-448 - recidive jail to block all protocols. Closes gh-440. Thanks Ioan Indreias + - long names on jails documented based on iptables limit of 30 less + len("fail2ban-"). - New Features: diff --git a/server/jail.py b/server/jail.py index 5e60ec7f..7ce12e46 100644 --- a/server/jail.py +++ b/server/jail.py @@ -102,9 +102,11 @@ class Jail: self.__filter = FilterPyinotify(self) def setName(self, name): + # 20 based on iptable chain name limit of 30 less len('fail2ban-') if len(name) >= 20: - logSys.warning("Jail name %r might be too long and some commands " - "might not function correctly. Please shorten" + logSys.warning("Jail name %r might be too long and some commands" + " (e.g. iptables) might not function correctly." + " Please shorten" % name) self.__name = name From fb666b69ffff45d8cbd6e381e5ffccfc6419d939 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 28 Nov 2013 23:35:05 +1100 Subject: [PATCH 13/74] BF: firewall-cmd-direct-new was too long. Thanks Joel. --- ChangeLog | 2 + config/action.d/firewall-cmd-direct-new.conf | 52 -------------------- 2 files changed, 2 insertions(+), 52 deletions(-) delete mode 100644 config/action.d/firewall-cmd-direct-new.conf diff --git a/ChangeLog b/ChangeLog index 58f8f84a..7a210b2c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,8 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better - IMPORTANT incompatible changes: - Fixes: + - Rename firewall-cmd-direct-new to firewall-cmd-new to fit within jail name + name length. As per gh-395 - allow for ",milliseconds" in the custom date format of proftpd.log - allow for ", referer ..." in apache-* filter for apache error logs. - allow for spaces at the beginning of kernel messages. Closes gh-448 diff --git a/config/action.d/firewall-cmd-direct-new.conf b/config/action.d/firewall-cmd-direct-new.conf deleted file mode 100644 index 55b6762d..00000000 --- a/config/action.d/firewall-cmd-direct-new.conf +++ /dev/null @@ -1,52 +0,0 @@ -# Fail2Ban configuration file -# -# Author: Edgar Hoch -# Copied from iptables-new.conf and modified for use with firewalld by Edgar Hoch. -# It uses "firewall-cmd" instead of "iptables". -# -# Because of the --remove-rules in stop this action requires firewalld-0.3.8+ - -[INCLUDES] - -before = iptables-blocktype.conf - -[Definition] - -actionstart = firewall-cmd --direct --add-chain ipv4 filter fail2ban- - firewall-cmd --direct --add-rule ipv4 filter fail2ban- 1000 -j RETURN - firewall-cmd --direct --add-rule ipv4 filter 0 -m state --state NEW -p --dport -j fail2ban- - -actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state --state NEW -p --dport -j fail2ban- - firewall-cmd --direct --remove-rules ipv4 filter fail2ban- - firewall-cmd --direct --remove-chain ipv4 filter fail2ban- - -actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q 'fail2ban-[ \t]' - -actionban = firewall-cmd --direct --add-rule ipv4 filter fail2ban- 0 -s -j - -actionunban = firewall-cmd --direct --remove-rule ipv4 filter fail2ban- 0 -s -j - -[Init] - -# Default name of the chain -# -name = default - -# Option: port -# Notes.: specifies port to monitor -# Values: [ NUM | STRING ] -# -port = ssh - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp | icmp | all ] -# -protocol = tcp - -# Option: chain -# Notes specifies the iptables chain to which the fail2ban rules should be -# added -# Values: [ STRING ] -# -chain = INPUT_direct From 9e538927087d2d7382631063c6423af8a4a6b95e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 29 Nov 2013 19:26:24 +1100 Subject: [PATCH 14/74] BF: did remove instead of move --- config/action.d/firewalld-cmd-new.conf | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 config/action.d/firewalld-cmd-new.conf diff --git a/config/action.d/firewalld-cmd-new.conf b/config/action.d/firewalld-cmd-new.conf new file mode 100644 index 00000000..837352e9 --- /dev/null +++ b/config/action.d/firewalld-cmd-new.conf @@ -0,0 +1,52 @@ +# Fail2Ban configuration file +# +# Author: Edgar Hoch +# Copied from iptables-new.conf and modified for use with firewalld by Edgar Hoch. +# It uses "firewall-cmd" instead of "iptables". +# +# Because of the --remove-rules in stop this action requires firewalld-0.3.8+ + +[INCLUDES] + +before = iptables-blocktype.conf + +[Definition] + +actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b- + firewall-cmd --direct --add-rule ipv4 filter f2b- 1000 -j RETURN + firewall-cmd --direct --add-rule ipv4 filter 0 -m state --state NEW -p --dport -j f2b- + +actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state --state NEW -p --dport -j f2b- + firewall-cmd --direct --remove-rules ipv4 filter f2b- + firewall-cmd --direct --remove-chain ipv4 filter f2b- + +actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q 'f2b-[ \t]' + +actionban = firewall-cmd --direct --add-rule ipv4 filter f2b- 0 -s -j + +actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b- 0 -s -j + +[Init] + +# Default name of the chain +# +name = default + +# Option: port +# Notes.: specifies port to monitor +# Values: [ NUM | STRING ] +# +port = ssh + +# Option: protocol +# Notes.: internally used by config reader for interpolations. +# Values: [ tcp | udp | icmp | all ] +# +protocol = tcp + +# Option: chain +# Notes specifies the iptables chain to which the fail2ban rules should be +# added +# Values: [ STRING ] +# +chain = INPUT_direct From cade7463079a9601ffe50036ecd7a6077b2e8c39 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 29 Nov 2013 21:45:11 +1100 Subject: [PATCH 15/74] BF: jail name mysqld-syslog-iptables too long. removed -iptables. Thanks Stefan (#447) --- ChangeLog | 1 + THANKS | 1 + config/jail.conf | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 58f8f84a..02c55bb7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,7 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better - allow for ", referer ..." in apache-* filter for apache error logs. - allow for spaces at the beginning of kernel messages. Closes gh-448 - recidive jail to block all protocols. Closes gh-440. Thanks Ioan Indreias + - mysqld-syslog-iptables rule was too long. Part of gh-447. - New Features: diff --git a/THANKS b/THANKS index afc72aa3..04c8728d 100644 --- a/THANKS +++ b/THANKS @@ -66,6 +66,7 @@ Russell Odom Sebastian Arcus Sireyessire silviogarbes +Stefan Tatschner Stephen Gildea Steven Hiscocks Tom Pike diff --git a/config/jail.conf b/config/jail.conf index 433170a5..05f4b768 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -389,7 +389,7 @@ logpath = /var/log/mysqld.log maxretry = 5 -[mysqld-syslog-iptables] +[mysqld-syslog] enabled = false filter = mysqld-auth From b9b2ddf99625837ce8b76ab54ae9e2943b3159b2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 29 Nov 2013 21:47:53 +1100 Subject: [PATCH 16/74] BF: smtps not IANA standard. Closes #447 --- ChangeLog | 2 ++ config/jail.conf | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 58f8f84a..299a89c8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,8 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better - allow for ", referer ..." in apache-* filter for apache error logs. - allow for spaces at the beginning of kernel messages. Closes gh-448 - recidive jail to block all protocols. Closes gh-440. Thanks Ioan Indreias + - smtps not a IANA standard and has been removed from Arch. Replaced with + 465. Thanks Stefan. Closes gh-447 - New Features: diff --git a/config/jail.conf b/config/jail.conf index 433170a5..481f0880 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -506,7 +506,7 @@ logpath = /var/log/auth.log enabled = false filter = dovecot -action = iptables-multiport[name=dovecot, port="pop3,pop3s,imap,imaps,submission,smtps,sieve", protocol=tcp] +action = iptables-multiport[name=dovecot, port="pop3,pop3s,imap,imaps,submission,465,sieve", protocol=tcp] logpath = /var/log/mail.log @@ -514,7 +514,7 @@ logpath = /var/log/mail.log enabled = false filter = dovecot -action = iptables-multiport[name=dovecot-auth, port="pop3,pop3s,imap,imaps,submission,smtps,sieve", protocol=tcp] +action = iptables-multiport[name=dovecot-auth, port="pop3,pop3s,imap,imaps,submission,465,sieve", protocol=tcp] logpath = /var/log/secure From 86a0a5962a9ed2c8c3c5b4fd7b75adb2c740b803 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 30 Nov 2013 08:05:20 +1100 Subject: [PATCH 17/74] BF: revert to fail2ban- prefix as f2b- was intended for 0.9 --- config/action.d/firewalld-cmd-new.conf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/config/action.d/firewalld-cmd-new.conf b/config/action.d/firewalld-cmd-new.conf index 837352e9..55b6762d 100644 --- a/config/action.d/firewalld-cmd-new.conf +++ b/config/action.d/firewalld-cmd-new.conf @@ -12,19 +12,19 @@ before = iptables-blocktype.conf [Definition] -actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b- - firewall-cmd --direct --add-rule ipv4 filter f2b- 1000 -j RETURN - firewall-cmd --direct --add-rule ipv4 filter 0 -m state --state NEW -p --dport -j f2b- +actionstart = firewall-cmd --direct --add-chain ipv4 filter fail2ban- + firewall-cmd --direct --add-rule ipv4 filter fail2ban- 1000 -j RETURN + firewall-cmd --direct --add-rule ipv4 filter 0 -m state --state NEW -p --dport -j fail2ban- -actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state --state NEW -p --dport -j f2b- - firewall-cmd --direct --remove-rules ipv4 filter f2b- - firewall-cmd --direct --remove-chain ipv4 filter f2b- +actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state --state NEW -p --dport -j fail2ban- + firewall-cmd --direct --remove-rules ipv4 filter fail2ban- + firewall-cmd --direct --remove-chain ipv4 filter fail2ban- -actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q 'f2b-[ \t]' +actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q 'fail2ban-[ \t]' -actionban = firewall-cmd --direct --add-rule ipv4 filter f2b- 0 -s -j +actionban = firewall-cmd --direct --add-rule ipv4 filter fail2ban- 0 -s -j -actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b- 0 -s -j +actionunban = firewall-cmd --direct --remove-rule ipv4 filter fail2ban- 0 -s -j [Init] From 56b6bf7d25766a47febba1a079629567a63892c7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 30 Nov 2013 10:30:29 +1100 Subject: [PATCH 18/74] ENH: reduce firewalld-cmd-new -> firewallcmd-new --- config/action.d/{firewalld-cmd-new.conf => firewallcmd-new.conf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename config/action.d/{firewalld-cmd-new.conf => firewallcmd-new.conf} (100%) diff --git a/config/action.d/firewalld-cmd-new.conf b/config/action.d/firewallcmd-new.conf similarity index 100% rename from config/action.d/firewalld-cmd-new.conf rename to config/action.d/firewallcmd-new.conf From 95845b7b6571cd80385777de6fdd932612165831 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 30 Nov 2013 17:47:10 +1100 Subject: [PATCH 19/74] BF: complain action could match too many IP addresses --- ChangeLog | 1 + config/action.d/complain.conf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 3cc0f453..12fc70f1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,7 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better - smtps not a IANA standard and has been removed from Arch. Replaced with 465. Thanks Stefan. Closes gh-447 - mysqld-syslog-iptables rule was too long. Part of gh-447. + - complain action - ensure where not matching other IPs in log sample. - Enhancements: - long names on jails documented based on iptables limit of 30 less diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index ad14a87e..d1ca25c7 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -58,7 +58,7 @@ actioncheck = actionban = ADDRESSES=`whois | perl -e 'while () { next if /^changed|@(ripe|apnic)\.net/io; $m += (/abuse|trouble:|report|spam|security/io?3:0); if (/([a-z0-9_\-\.+]+@[a-z0-9\-]+(\.[[a-z0-9\-]+)+)/io) { while (s/([a-z0-9_\-\.+]+@[a-z0-9\-]+(\.[[a-z0-9\-]+)+)//io) { if ($m) { $a{lc($1)}=$m } else { $b{lc($1)}=$m } } $m=0 } else { $m && --$m } } if (%%a) {print join(",",keys(%%a))} else {print join(",",keys(%%b))}'` IP= if [ ! -z "$ADDRESSES" ]; then - (printf %%b "\n"; date '+Note: Local timezone is %%z (%%Z)'; grep '' ) | "Abuse from " $ADDRESSES + (printf %%b "\n"; date '+Note: Local timezone is %%z (%%Z)'; grep '[^0-9][^0-9]' ) | "Abuse from " $ADDRESSES fi # Option: actionunban From 0495aa098eefb1ad7111c7f98ee761e8e642c65e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 30 Nov 2013 18:01:45 +1100 Subject: [PATCH 20/74] BF: grep matches on shouldn't include other IPs --- config/action.d/ipfw.conf | 2 +- config/action.d/mail-whois-lines.conf | 2 +- config/action.d/sendmail-whois-lines.conf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/action.d/ipfw.conf b/config/action.d/ipfw.conf index 09045815..37625209 100644 --- a/config/action.d/ipfw.conf +++ b/config/action.d/ipfw.conf @@ -43,7 +43,7 @@ actionban = ipfw add tcp from to # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = ipfw delete `ipfw list | grep -i | awk '{print $1;}'` +actionunban = ipfw delete `ipfw list | grep -i "[^0-9][^0-9]" | awk '{print $1;}'` [Init] diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 758c4eff..6b7b3841 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -42,7 +42,7 @@ actionban = printf %%b "Hi,\n Here are more information about :\n `whois `\n\n Lines containing IP: in \n - `grep '\<\>' `\n\n + `grep '[^0-9][^0-9]' `\n\n Regards,\n Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index 5a331e24..a0f0a9c3 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -58,7 +58,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Here are more information about :\n `/usr/bin/whois `\n\n Lines containing IP: in \n - `grep '\<\>' `\n\n + `grep '[^0-9][^0-9]' `\n\n Regards,\n Fail2Ban" | /usr/sbin/sendmail -f From b5d6310d281c1c45112ffb285c77a858da99d6f6 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 4 Dec 2013 20:51:30 +1100 Subject: [PATCH 21/74] BF: create flushlogs command to prevent logrotation clobbering logtarget. Closes gh-458 --- ChangeLog | 2 ++ client/beautifier.py | 2 ++ common/protocol.py | 1 + files/fail2ban-logrotate | 2 +- server/server.py | 13 ++++++++++++- server/transmitter.py | 2 ++ 6 files changed, 20 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3cc0f453..fcfe82cc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,8 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better - smtps not a IANA standard and has been removed from Arch. Replaced with 465. Thanks Stefan. Closes gh-447 - mysqld-syslog-iptables rule was too long. Part of gh-447. + - add 'flushlogs' command to allow logrotation without clobbering logtarget + settings. Closes gh-458, Debian bug #697333, Redhat bug #891798. - Enhancements: - long names on jails documented based on iptables limit of 30 less diff --git a/client/beautifier.py b/client/beautifier.py index 8e690656..bc9e89b1 100644 --- a/client/beautifier.py +++ b/client/beautifier.py @@ -63,6 +63,8 @@ class Beautifier: msg = "Jail stopped" elif inC[0] == "add": msg = "Added jail " + response + elif inC[0] == "flushlogs": + msg = "logs: " + response elif inC[0:1] == ['status']: if len(inC) > 1: # Create IP list diff --git a/common/protocol.py b/common/protocol.py index 9309ce7f..278ccd53 100644 --- a/common/protocol.py +++ b/common/protocol.py @@ -43,6 +43,7 @@ protocol = [ ["get loglevel", "gets the logging level"], ["set logtarget ", "sets logging target to . Can be STDOUT, STDERR, SYSLOG or a file"], ["get logtarget", "gets logging target"], +["flushlogs", "flushes the logtarget if a file and reopens it. For log rotation."], ['', "JAIL CONTROL", ""], ["add ", "creates using "], ["start ", "starts the jail "], diff --git a/files/fail2ban-logrotate b/files/fail2ban-logrotate index 67c6364a..a09870af 100644 --- a/files/fail2ban-logrotate +++ b/files/fail2ban-logrotate @@ -13,6 +13,6 @@ missingok compress postrotate - /usr/bin/fail2ban-client set logtarget /var/log/fail2ban.log 1>/dev/null || true + /usr/bin/fail2ban-client flushlogs 1>/dev/null || true endscript } diff --git a/server/server.py b/server/server.py index 6ac93a54..1358391b 100644 --- a/server/server.py +++ b/server/server.py @@ -361,7 +361,7 @@ class Server: # Target should be a file try: open(target, "a").close() - hdlr = logging.FileHandler(target) + hdlr = logging.handlers.RotatingFileHandler(target) except IOError: logSys.error("Unable to log to " + target) logSys.info("Logging to previous target " + self.__logTarget) @@ -401,6 +401,17 @@ class Server: finally: self.__loggingLock.release() + def flushLogs(self): + if self.__logTarget not in ['STDERR', 'STDOUT', 'SYSLOG']: + for handler in logging.getLogger("fail2ban").handlers: + handler.doRollover() + return "rolled over" + else: + for handler in logging.getLogger("fail2ban").handlers: + handler.flush() + return "flushed" + + def __createDaemon(self): # pragma: no cover """ Detach a process from the controlling terminal and run it in the background as a daemon. diff --git a/server/transmitter.py b/server/transmitter.py index deaf9adf..cf65dada 100644 --- a/server/transmitter.py +++ b/server/transmitter.py @@ -92,6 +92,8 @@ class Transmitter: value = command[1] time.sleep(int(value)) return None + elif command[0] == "flushlogs": + return self.__server.flushLogs() elif command[0] == "set": return self.__commandSet(command[1:]) elif command[0] == "get": From e108de3f6d19eecb0a3860405ceb6bf37c6201ff Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 4 Dec 2013 22:27:23 +1100 Subject: [PATCH 22/74] ENH: banning an IP in the ignoreIPList now issues warning to log, but still continues --- server/filter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/filter.py b/server/filter.py index 80433c01..66d3fbf0 100644 --- a/server/filter.py +++ b/server/filter.py @@ -219,6 +219,9 @@ class Filter(JailThread): # to enable banip fail2ban-client BAN command def addBannedIP(self, ip): + if self.inIgnoreIPList(ip): + logSys.warning('Requested to manually ban an ignored IP ' + ip + '. User knows best. Proceeding to ban it.') + unixTime = MyTime.time() for i in xrange(self.failManager.getMaxRetry()): self.failManager.addFailure(FailTicket(ip, unixTime)) From 97d7f46bb757514f5ccb9140ffcdbb7fd0f70d59 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 4 Dec 2013 22:40:48 +1100 Subject: [PATCH 23/74] DOC: correct grammar - s/Here are more information/Here is more information/ --- config/action.d/mail-whois-lines.conf | 2 +- config/action.d/mail-whois.conf | 2 +- config/action.d/sendmail-whois-lines.conf | 2 +- config/action.d/sendmail-whois.conf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 758c4eff..bbf66a59 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -39,7 +39,7 @@ actioncheck = actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here are more information about :\n + Here is more information about :\n `whois `\n\n Lines containing IP: in \n `grep '\<\>' `\n\n diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf index fa133ab3..c132b086 100644 --- a/config/action.d/mail-whois.conf +++ b/config/action.d/mail-whois.conf @@ -39,7 +39,7 @@ actioncheck = actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here are more information about :\n + Here is more information about :\n `whois `\n Regards,\n Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index 5a331e24..b40b1084 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -55,7 +55,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here are more information about :\n + Here is more information about :\n `/usr/bin/whois `\n\n Lines containing IP: in \n `grep '\<\>' `\n\n diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf index a65f9875..f2b07d9e 100644 --- a/config/action.d/sendmail-whois.conf +++ b/config/action.d/sendmail-whois.conf @@ -55,7 +55,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here are more information about :\n + Here is more information about :\n `/usr/bin/whois `\n Regards,\n Fail2Ban" | /usr/sbin/sendmail -f From 4dc51e5defdc8ce56db4b71b9b6b53d654908d56 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 4 Dec 2013 22:43:06 +1100 Subject: [PATCH 24/74] BF: put notice in email if whois program could not provide more information. Closes gh-471 --- config/action.d/mail-whois-lines.conf | 2 +- config/action.d/mail-whois.conf | 2 +- config/action.d/sendmail-whois-lines.conf | 2 +- config/action.d/sendmail-whois.conf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index bbf66a59..067c5819 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -40,7 +40,7 @@ actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `whois `\n\n + `whois || echo missing whois program`\n\n Lines containing IP: in \n `grep '\<\>' `\n\n Regards,\n diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf index c132b086..e4c8450e 100644 --- a/config/action.d/mail-whois.conf +++ b/config/action.d/mail-whois.conf @@ -40,7 +40,7 @@ actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `whois `\n + `whois || echo missing whois program`\n Regards,\n Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index b40b1084..8cdafc0b 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -56,7 +56,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` The IP has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `/usr/bin/whois `\n\n + `/usr/bin/whois || echo missing whois program`\n\n Lines containing IP: in \n `grep '\<\>' `\n\n Regards,\n diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf index f2b07d9e..e428c44d 100644 --- a/config/action.d/sendmail-whois.conf +++ b/config/action.d/sendmail-whois.conf @@ -56,7 +56,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` The IP has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `/usr/bin/whois `\n + `/usr/bin/whois || echo missing whois program`\n Regards,\n Fail2Ban" | /usr/sbin/sendmail -f From e810ec009d56a51478fd740c60f2770448a7a752 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 5 Dec 2013 08:22:20 +0000 Subject: [PATCH 25/74] ENH: Added blocklist.de reporting API action --- config/action.d/blocklist_de.conf | 55 +++++++++++++++++++++++++++++++ config/jail.conf | 12 +++++++ 2 files changed, 67 insertions(+) create mode 100644 config/action.d/blocklist_de.conf diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf new file mode 100644 index 00000000..d11b175b --- /dev/null +++ b/config/action.d/blocklist_de.conf @@ -0,0 +1,55 @@ +# Fail2Ban configuration file +# +# Author: Steven Hiscocks +# +# + +# Action to report IP address to blocklist.de +# Blocklist.de must be signed up to at www.blocklist.de +# Once registered, one or more servers can be added. +# This action requires the server 'email address' and the assoicate apikey. +# +# From blocklist.de: +# www.blocklist.de is a free and voluntary service provided by a +# Fraud/Abuse-specialist, whose servers are often attacked on SSH-, +# Mail-Login-, FTP-, Webserver- and other services. +# The mission is to report all attacks to the abuse deparments of the +# infected PCs/servers to ensure that the responsible provider can inform +# the customer about the infection and disable them +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = ! curl --data-urlencode 'server=' --data 'apikey=' --data 'service=' --data 'ip=' --data-urlencode 'logs=' --data 'format=text' "https://www.blocklist.de/en/httpreports.html" | grep "status: error" + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = diff --git a/config/jail.conf b/config/jail.conf index 33d1d439..cbcfb758 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -532,3 +532,15 @@ filter = selinux-ssh action = iptables[name=SELINUX-SSH, port=ssh, protocol=tcp] logpath = /var/log/audit/audit.log maxretry = 5 + +# Report block via blocklist.de fail2ban reporting service API +# See action.d/blocklist_de.conf for more information +[ssh-blocklist] + +enabled = false +filter = sshd +action = iptables[name=SSH, port=ssh, protocol=tcp] + sendmail-whois[name=SSH, dest=you@example.com, sender=fail2ban@example.com, sendername="Fail2Ban"] + blocklist_de[email="fail2ban@example.com", apikey="xxxxxx", service=%(filter)s] +logpath = /var/log/sshd.log +maxretry = 5 From f742ed0e4bc3182f36f348345bce2858b6db4369 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 5 Dec 2013 18:06:53 +0000 Subject: [PATCH 26/74] DOC: when to use blocklist.de reporting Taken from commit 1846056606d24abe4e7d3f2e1cf56407c65b9008 --- config/action.d/blocklist_de.conf | 10 ++++++++++ config/jail.conf | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf index d11b175b..468f3fc9 100644 --- a/config/action.d/blocklist_de.conf +++ b/config/action.d/blocklist_de.conf @@ -17,6 +17,16 @@ # infected PCs/servers to ensure that the responsible provider can inform # the customer about the infection and disable them # +# IMPORTANT: +# +# Reporting an IP of abuse is a serious complaint. Make sure that it is +# serious. Fail2ban developers and network owners recommend you only use this +# action for: +# * The recidive where the IP has been banned multiple times +# * Where maxretry has been set quite high, beyond the normal user typing +# password incorrectly. +# * For filters that have a low likelyhood of receiving human errors +# [Definition] diff --git a/config/jail.conf b/config/jail.conf index cbcfb758..3b8220e5 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -533,6 +533,9 @@ action = iptables[name=SELINUX-SSH, port=ssh, protocol=tcp] logpath = /var/log/audit/audit.log maxretry = 5 +# See the IMPORTANT note in action.d/blocklist_de.conf for when to +# use this action +# # Report block via blocklist.de fail2ban reporting service API # See action.d/blocklist_de.conf for more information [ssh-blocklist] @@ -543,4 +546,4 @@ action = iptables[name=SSH, port=ssh, protocol=tcp] sendmail-whois[name=SSH, dest=you@example.com, sender=fail2ban@example.com, sendername="Fail2Ban"] blocklist_de[email="fail2ban@example.com", apikey="xxxxxx", service=%(filter)s] logpath = /var/log/sshd.log -maxretry = 5 +maxretry = 20 From a19b33cc7263c0758d6848ecbaa0e5ad4e24b962 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 5 Dec 2013 18:12:15 +0000 Subject: [PATCH 27/74] ENH: blocklist.de action added fail2ban version as user agent --- config/action.d/blocklist_de.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf index 468f3fc9..6f47d87b 100644 --- a/config/action.d/blocklist_de.conf +++ b/config/action.d/blocklist_de.conf @@ -54,7 +54,7 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ! curl --data-urlencode 'server=' --data 'apikey=' --data 'service=' --data 'ip=' --data-urlencode 'logs=' --data 'format=text' "https://www.blocklist.de/en/httpreports.html" | grep "status: error" +actionban = ! curl --data-urlencode 'server=' --data 'apikey=' --data 'service=' --data 'ip=' --data-urlencode 'logs=' --data 'format=text' --user-agent "`fail2ban-client --version | head -1`" "https://www.blocklist.de/en/httpreports.html" | grep "status: error" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the From 008952035db55c803b1b33ed78bf93c6eb74cf62 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 6 Dec 2013 08:08:11 +1100 Subject: [PATCH 28/74] BF: files/redhat-initd - as per http://pkgs.fedoraproject.org/cgit/fail2ban.git/tree/fail2ban-init.patch --- files/redhat-initd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/redhat-initd b/files/redhat-initd index 43147c95..08eb0f3c 100755 --- a/files/redhat-initd +++ b/files/redhat-initd @@ -1,6 +1,6 @@ #!/bin/bash # -# chkconfig: 345 92 08 +# chkconfig: - 92 08 # processname: fail2ban-server # config: /etc/fail2ban/fail2ban.conf # pidfile: /var/run/fail2ban/fail2ban.pid From b3c173795e1fe023adcc4a781674075d071bc756 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 6 Dec 2013 08:22:21 +0000 Subject: [PATCH 29/74] ENH: blocklist.de action error on HTTP response code 4xx --- config/action.d/blocklist_de.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf index 6f47d87b..f45882d3 100644 --- a/config/action.d/blocklist_de.conf +++ b/config/action.d/blocklist_de.conf @@ -54,7 +54,7 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ! curl --data-urlencode 'server=' --data 'apikey=' --data 'service=' --data 'ip=' --data-urlencode 'logs=' --data 'format=text' --user-agent "`fail2ban-client --version | head -1`" "https://www.blocklist.de/en/httpreports.html" | grep "status: error" +actionban = curl --fail --data-urlencode 'server=' --data 'apikey=' --data 'service=' --data 'ip=' --data-urlencode 'logs=' --data 'format=text' --user-agent "fail2ban v0.8.12" "https://www.blocklist.de/en/httpreports.html" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the From 476bbdd284836b47b77f0ccafbfbb3776a5df807 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 7 Dec 2013 10:57:05 +1100 Subject: [PATCH 30/74] TST: test case for flushlogs --- testcases/servertestcase.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/testcases/servertestcase.py b/testcases/servertestcase.py index c93326a5..af6f538d 100644 --- a/testcases/servertestcase.py +++ b/testcases/servertestcase.py @@ -25,7 +25,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import unittest, socket, time, tempfile, os, sys -from server.server import Server +from server.server import Server, logSys from server.jail import Jail from common.exceptions import UnknownJailException @@ -521,6 +521,33 @@ class TransmitterLogging(TransmitterBase): self.setGetTest("loglevel", "0", 0) self.setGetTestNOK("loglevel", "Bird") + def testFlushLogs(self): + self.assertEqual(self.transm.proceed(["flushlogs"]), (0, "flushed")) + try: + f, fn = tempfile.mkstemp("fail2ban.log") + os.close(f) + self.server.setLogLevel(2) + self.assertEqual(self.transm.proceed(["set", "logtarget", fn]), (0, fn)) + logSys.warn("Before file moved") + try: + f2, fn2 = tempfile.mkstemp("fail2ban.log") + os.close(f2) + os.rename(fn, fn2) + logSys.warn("After file moved") + self.assertEqual(self.transm.proceed(["flushlogs"]), (0, "flushed")) + logSys.warn("After flushlogs") + with open(fn2,'r') as f: + self.assertTrue(f.next().endswith("Before file moved\n")) + self.assertTrue(f.next().endswith("After file moved\n")) + self.assertRaises(StopIteration, f.next) + with open(fn,'r') as f: + self.assertTrue(f.next().endswith("After flushlogs\n")) + self.assertRaises(StopIteration, f.next) + finally: + os.remove(fn2) + finally: + os.remove(fn) + class JailTests(unittest.TestCase): From 8451f720f03c8b7535deba7b3b3993458edbe720 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 7 Dec 2013 11:04:06 +1100 Subject: [PATCH 31/74] TST: fix flushlogs and include test for STDERR flushing --- testcases/servertestcase.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testcases/servertestcase.py b/testcases/servertestcase.py index af6f538d..e6137d2f 100644 --- a/testcases/servertestcase.py +++ b/testcases/servertestcase.py @@ -522,7 +522,7 @@ class TransmitterLogging(TransmitterBase): self.setGetTestNOK("loglevel", "Bird") def testFlushLogs(self): - self.assertEqual(self.transm.proceed(["flushlogs"]), (0, "flushed")) + self.assertEqual(self.transm.proceed(["flushlogs"]), (0, "rolled over")) try: f, fn = tempfile.mkstemp("fail2ban.log") os.close(f) @@ -534,7 +534,7 @@ class TransmitterLogging(TransmitterBase): os.close(f2) os.rename(fn, fn2) logSys.warn("After file moved") - self.assertEqual(self.transm.proceed(["flushlogs"]), (0, "flushed")) + self.assertEqual(self.transm.proceed(["flushlogs"]), (0, "rolled over")) logSys.warn("After flushlogs") with open(fn2,'r') as f: self.assertTrue(f.next().endswith("Before file moved\n")) @@ -547,6 +547,8 @@ class TransmitterLogging(TransmitterBase): os.remove(fn2) finally: os.remove(fn) + self.assertEqual(self.transm.proceed(["set", "logtarget", "STDERR"]), (0, "STDERR")) + self.assertEqual(self.transm.proceed(["flushlogs"]), (0, "flushed")) class JailTests(unittest.TestCase): From 630dd91dcdd6a3f5501b2e15a06577cee32dab0c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 6 Dec 2013 18:01:36 +0000 Subject: [PATCH 32/74] BF: Add [Init] section to blocklist.de action --- config/action.d/blocklist_de.conf | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf index f45882d3..d4170cab 100644 --- a/config/action.d/blocklist_de.conf +++ b/config/action.d/blocklist_de.conf @@ -63,3 +63,24 @@ actionban = curl --fail --data-urlencode 'server=' --data 'apikey= Date: Mon, 9 Dec 2013 09:21:55 +1100 Subject: [PATCH 33/74] BF: action.d/complain - match IP at beginning and end of lines --- config/action.d/complain.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index d1ca25c7..62331f19 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -58,7 +58,7 @@ actioncheck = actionban = ADDRESSES=`whois | perl -e 'while () { next if /^changed|@(ripe|apnic)\.net/io; $m += (/abuse|trouble:|report|spam|security/io?3:0); if (/([a-z0-9_\-\.+]+@[a-z0-9\-]+(\.[[a-z0-9\-]+)+)/io) { while (s/([a-z0-9_\-\.+]+@[a-z0-9\-]+(\.[[a-z0-9\-]+)+)//io) { if ($m) { $a{lc($1)}=$m } else { $b{lc($1)}=$m } } $m=0 } else { $m && --$m } } if (%%a) {print join(",",keys(%%a))} else {print join(",",keys(%%b))}'` IP= if [ ! -z "$ADDRESSES" ]; then - (printf %%b "\n"; date '+Note: Local timezone is %%z (%%Z)'; grep '[^0-9][^0-9]' ) | "Abuse from " $ADDRESSES + (printf %%b "\n"; date '+Note: Local timezone is %%z (%%Z)'; grep -E '(^|[^0-9])([^0-9]|$)' ) | "Abuse from " $ADDRESSES fi # Option: actionunban From e8eab11615e1d753f20345339e8a7bd0a8a12db6 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 9 Dec 2013 14:45:09 +1100 Subject: [PATCH 34/74] DOC: proftp - turn off ReverseDNS --- config/filter.d/proftpd.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index bf8f9b5f..b7e13414 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -1,5 +1,7 @@ # Fail2Ban fitler for the Proftpd FTP daemon # +# Set "UseReverseDNS off" in proftpd to avoid the need for DNS. +# See: http://www.proftpd.org/docs/howto/DNS.html [INCLUDES] From db4c21acde6830eb8fbcdd34d1a5e71a11ed9e34 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 9 Dec 2013 14:46:01 +1100 Subject: [PATCH 35/74] BF/DOC: fix filename in documentation for filter.d/proftpd --- config/filter.d/proftpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index b7e13414..ac714cc1 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -1,6 +1,6 @@ # Fail2Ban fitler for the Proftpd FTP daemon # -# Set "UseReverseDNS off" in proftpd to avoid the need for DNS. +# Set "UseReverseDNS off" in proftpd.conf to avoid the need for DNS. # See: http://www.proftpd.org/docs/howto/DNS.html [INCLUDES] From 916649119ef124e952f50e6c27cf3003897f15f9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 9 Dec 2013 23:07:42 +1100 Subject: [PATCH 36/74] ENH: use format string rather than concatination on log message --- server/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/filter.py b/server/filter.py index 66d3fbf0..0ef756d2 100644 --- a/server/filter.py +++ b/server/filter.py @@ -220,7 +220,7 @@ class Filter(JailThread): def addBannedIP(self, ip): if self.inIgnoreIPList(ip): - logSys.warning('Requested to manually ban an ignored IP ' + ip + '. User knows best. Proceeding to ban it.') + logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.' % ip) unixTime = MyTime.time() for i in xrange(self.failManager.getMaxRetry()): From 66374913ec773175ef921798d3efdd4c47f27e01 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 10 Dec 2013 21:24:37 +1100 Subject: [PATCH 37/74] ENH: add squid filter --- ChangeLog | 1 + THANKS | 1 + config/filter.d/squid.conf | 13 +++++++++++++ testcases/files/logs/squid | 13 +++++++++++++ 4 files changed, 28 insertions(+) create mode 100644 config/filter.d/squid.conf create mode 100644 testcases/files/logs/squid diff --git a/ChangeLog b/ChangeLog index 90b8ba27..a0624630 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,7 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better len("fail2ban-"). - remove indentation of name and loglevel while logging to SYSLOG to resolve syslog(-ng) parsing problems. Closes Debian bug #730202. + - added squid filter. Thanks Roman Gelfand. - New Features: diff --git a/THANKS b/THANKS index 04c8728d..6d4845bb 100644 --- a/THANKS +++ b/THANKS @@ -62,6 +62,7 @@ RealRancor René Berber Robert Edeker Rolf Fokkens +Roman Gelfand Russell Odom Sebastian Arcus Sireyessire diff --git a/config/filter.d/squid.conf b/config/filter.d/squid.conf new file mode 100644 index 00000000..8cbf744f --- /dev/null +++ b/config/filter.d/squid.conf @@ -0,0 +1,13 @@ +# Fail2Ban filter for Squid attempted proxy bypasses +# +# + +[Definition] + +failregex = ^\s+\d\s\s+[A-Z]+_DENIED/403 .*$ + ^\s+\d\s\s+NONE/405 .*$ + + + +# Author: Daniel Black + diff --git a/testcases/files/logs/squid b/testcases/files/logs/squid new file mode 100644 index 00000000..fa2c593c --- /dev/null +++ b/testcases/files/logs/squid @@ -0,0 +1,13 @@ +# Logs thanks to Roman Gelfand +# +# failJSON: { "time": "2013-12-08T23:55:23", "match": true , "host": "91.188.124.227" } +1386543323.511 4 91.188.124.227 TCP_DENIED/403 4099 GET http://www.proxy-listen.de/azenv.php - HIER_NONE/- text/html + +# failJSON: { "time": "2013-12-08T23:58:20", "match": true , "host": "175.44.0.184" } +1386543500.220 5 175.44.0.184 NONE/405 3364 CONNECT error:method-not-allowed - HIER_NONE/- text/html + +# failJSON: { "time": "2013-12-09T00:08:04", "match": true , "host": "198.74.125.200" } +1386544084.763 3 198.74.125.200 TCP_DENIED/403 3722 GET http://www2t.biglobe.ne.jp/~take52/test/env.cgi - HIER_NONE/- text/html + +# failJSON: { "time": "2013-12-09T00:09:06", "match": true , "host": "175.42.91.151" } +1386544146.088 1 175.42.91.151 TCP_DENIED/403 3745 GET http://pkfsp.ru/wp-content/uploads/proxyc/engine.php - HIER_NONE/- text/html From 9d532828fcb51ca245b330c026a16da8bdecc941 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 07:44:41 +1100 Subject: [PATCH 38/74] BF: multiple _ separated values according to http://wiki.squid-cache.org/SquidFaq/SquidLogs#Squid_result_codes. Thanks Steven --- config/filter.d/squid.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/squid.conf b/config/filter.d/squid.conf index 8cbf744f..da282692 100644 --- a/config/filter.d/squid.conf +++ b/config/filter.d/squid.conf @@ -4,7 +4,7 @@ [Definition] -failregex = ^\s+\d\s\s+[A-Z]+_DENIED/403 .*$ +failregex = ^\s+\d\s\s+[A-Z_]+_DENIED/403 .*$ ^\s+\d\s\s+NONE/405 .*$ From 5688c064ad757006bb268bb1e4d1ce6c120df0cd Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 09:50:17 +1100 Subject: [PATCH 39/74] ENH: separate out log capture framework for other test cases - now utils.LogCaptureTestCase --- testcases/actiontestcase.py | 27 +++++---------------------- testcases/utils.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/testcases/actiontestcase.py b/testcases/actiontestcase.py index 14356ee5..e0587b3d 100644 --- a/testcases/actiontestcase.py +++ b/testcases/actiontestcase.py @@ -24,40 +24,23 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import unittest, time +import time import logging, sys from server.action import Action -from StringIO import StringIO +from utils import LogCaptureTestCase -class ExecuteAction(unittest.TestCase): +class ExecuteAction(LogCaptureTestCase): def setUp(self): """Call before every test case.""" self.__action = Action("Test") - - # For extended testing of what gets output into logging - # system, we will redirect it to a string - logSys = logging.getLogger("fail2ban") - - # Keep old settings - self._old_level = logSys.level - self._old_handlers = logSys.handlers - # Let's log everything into a string - self._log = StringIO() - logSys.handlers = [logging.StreamHandler(self._log)] - logSys.setLevel(getattr(logging, 'DEBUG')) + LogCaptureTestCase.setUp(self) def tearDown(self): """Call after every test case.""" - # print "O: >>%s<<" % self._log.getvalue() - logSys = logging.getLogger("fail2ban") - logSys.handlers = self._old_handlers - logSys.level = self._old_level + LogCaptureTestCase.tearDown(self) self.__action.execActionStop() - def _is_logged(self, s): - return s in self._log.getvalue() - def testNameChange(self): self.assertEqual(self.__action.getName(), "Test") self.__action.setName("Tricky Test") diff --git a/testcases/utils.py b/testcases/utils.py index 643c9ad1..b048c8c6 100644 --- a/testcases/utils.py +++ b/testcases/utils.py @@ -22,8 +22,9 @@ __author__ = "Yaroslav Halchenko" __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" -import logging, os, re, tempfile, sys, time, traceback +import unittest, logging, os, re, tempfile, sys, time, traceback from os.path import basename, dirname +from StringIO import StringIO # # Following "traceback" functions are adopted from PyMVPA distributed @@ -105,3 +106,29 @@ def mtimesleep(): # no sleep now should be necessary since polling tracks now not only # mtime but also ino and size pass + +class LogCaptureTestCase(unittest.TestCase): + + def setUp(self): + + # For extended testing of what gets output into logging + # system, we will redirect it to a string + logSys = logging.getLogger("fail2ban") + + # Keep old settings + self._old_level = logSys.level + self._old_handlers = logSys.handlers + # Let's log everything into a string + self._log = StringIO() + logSys.handlers = [logging.StreamHandler(self._log)] + logSys.setLevel(getattr(logging, 'DEBUG')) + + def tearDown(self): + """Call after every test case.""" + # print "O: >>%s<<" % self._log.getvalue() + logSys = logging.getLogger("fail2ban") + logSys.handlers = self._old_handlers + logSys.level = self._old_level + + def _is_logged(self, s): + return s in self._log.getvalue() From f4661d81779683e0ef92b3278347639e3260bb36 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 09:56:04 +1100 Subject: [PATCH 40/74] ENH: rebase LogFileMonitor on LogCaptureTestCase --- testcases/filtertestcase.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 8bc24085..6ab99d82 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -39,7 +39,7 @@ from server.failmanager import FailManagerEmpty # Useful helpers # -from utils import mtimesleep +from utils import mtimesleep, LogCaptureTestCase # yoh: per Steven Hiscocks's insight while troubleshooting # https://github.com/fail2ban/fail2ban/issues/103#issuecomment-15542836 @@ -194,11 +194,12 @@ class LogFile(unittest.TestCase): self.assertTrue(self.filter.isModified(LogFile.FILENAME)) -class LogFileMonitor(unittest.TestCase): +class LogFileMonitor(LogCaptureTestCase): """Few more tests for FilterPoll API """ def setUp(self): """Call before every test case.""" + LogCaptureTestCase.setUp(self) self.filter = self.name = 'NA' _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') @@ -208,6 +209,7 @@ class LogFileMonitor(unittest.TestCase): self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") def tearDown(self): + LogCaptureTestCase.tearDown(self) _killfile(self.file, self.name) pass From f4531e7b45c97cb03480b5e58749bad83f914231 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 10:10:31 +1100 Subject: [PATCH 41/74] TST: test cases fro filter.delFailRegex and filter.delIgnoreRegex --- testcases/filtertestcase.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 6ab99d82..b7b161d3 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -227,6 +227,16 @@ class LogFileMonitor(LogCaptureTestCase): # shorter wait time for not modified status return not self.isModified(0.4) + def testRemovingFailRegex(self): + self.filter.delFailRegex(0) + self.assertFalse(self._is_logged('Cannot remove regular expression. Index 0 is not valid')) + self.filter.delFailRegex(0) + self.assertTrue(self._is_logged('Cannot remove regular expression. Index 0 is not valid')) + + def testRemovingIgnoreRegex(self): + self.filter.delIgnoreRegex(0) + self.assertTrue(self._is_logged('Cannot remove regular expression. Index 0 is not valid')) + def testNewChangeViaIsModified(self): # it is a brand new one -- so first we think it is modified self.assertTrue(self.isModified()) From 44bbaebfe511e679e8a0dfa251e1383138f90002 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 10:15:24 +1100 Subject: [PATCH 42/74] TST: CIDR for ignoreip --- testcases/filtertestcase.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index b7b161d3..892a4a6e 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -173,6 +173,15 @@ class IgnoreIP(unittest.TestCase): self.filter.addIgnoreIP("www.epfl.ch") self.assertFalse(self.filter.inIgnoreIPList("127.177.50.10")) + def testIgnoreIPCIDR(self): + self.filter.addIgnoreIP('192.168.1.0/25') + self.assertTrue(self.filter.inIgnoreIPList('192.168.1.0')) + self.assertTrue(self.filter.inIgnoreIPList('192.168.1.1')) + self.assertTrue(self.filter.inIgnoreIPList('192.168.1.127')) + self.assertFalse(self.filter.inIgnoreIPList('192.168.1.128')) + self.assertFalse(self.filter.inIgnoreIPList('192.168.1.255')) + self.assertFalse(self.filter.inIgnoreIPList('192.168.0.255')) + class LogFile(unittest.TestCase): From 988e14d8c652a077fcb47a0ec1a88eb3ca3fdb5f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 10:17:55 +1100 Subject: [PATCH 43/74] TST: negative match for DNS lookup test added --- testcases/filtertestcase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 892a4a6e..0d2b58fb 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -163,6 +163,8 @@ class IgnoreIP(unittest.TestCase): self.filter.addIgnoreIP("www.epfl.ch") self.assertTrue(self.filter.inIgnoreIPList("128.178.50.12")) + self.assertFalse(self.filter.inIgnoreIPList("128.178.50.11")) + self.assertFalse(self.filter.inIgnoreIPList("128.178.50.13")) def testIgnoreIPNOK(self): ipList = "", "999.999.999.999", "abcdef", "192.168.0." From 60c4957a5278b033c41e71c18d2824441507ea14 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 10:21:19 +1100 Subject: [PATCH 44/74] DOC/TST: remove TODO as all regexs have samples --- testcases/samplestestcase.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index d88be5e9..6b4d4530 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -123,7 +123,6 @@ def testSampleRegexsFactory(name): regexsUsed.add(failregex) - # TODO: Remove exception handling once all regexs have samples for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()): self.assertTrue( failRegexIndex in regexsUsed, From a8b5c5b5f3f5de2c5a103c22f40672481ab33906 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 10:31:58 +1100 Subject: [PATCH 45/74] TST: check IgnoreIP happens in filter.processLine --- testcases/filtertestcase.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 0d2b58fb..63f23400 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -144,15 +144,13 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line # Actual tests # -class IgnoreIP(unittest.TestCase): +class IgnoreIP(LogCaptureTestCase): def setUp(self): """Call before every test case.""" + LogCaptureTestCase.setUp(self) self.filter = FileFilter(None) - def tearDown(self): - """Call after every test case.""" - def testIgnoreIPOK(self): ipList = "127.0.0.1", "192.168.0.1", "255.255.255.255", "99.99.99.99" for ip in ipList: @@ -184,6 +182,12 @@ class IgnoreIP(unittest.TestCase): self.assertFalse(self.filter.inIgnoreIPList('192.168.1.255')) self.assertFalse(self.filter.inIgnoreIPList('192.168.0.255')) + def testIgnoreInProcessLine(self): + self.filter.addIgnoreIP('192.168.1.0/25') + self.filter.addFailRegex('') + self.filter.processLineAndAdd('Thu Jul 11 01:21:43 2013 192.168.1.32') + self.assertTrue(self._is_logged('Ignore 192.168.1.32')) + class LogFile(unittest.TestCase): From ebf4a02004ed2d312a3329f0dd4ab36cb0028f8f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 10:43:47 +1100 Subject: [PATCH 46/74] TST: get/set use DNS on Filters --- fail2ban-testcases | 1 + testcases/filtertestcase.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/fail2ban-testcases b/fail2ban-testcases index f44e84d4..16c92c5f 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -174,6 +174,7 @@ tests.addTest(unittest.makeSuite(misctestcase.CustomDateFormatsTest)) # Filter if not opts.no_network: tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) +tests.addTest(unittest.makeSuite(filtertestcase.BasicFilter)) tests.addTest(unittest.makeSuite(filtertestcase.LogFile)) tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor)) if not opts.no_network: diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 63f23400..7b09ffed 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -31,7 +31,7 @@ import tempfile from server.jail import Jail from server.filterpoll import FilterPoll -from server.filter import FileFilter, DNSUtils +from server.filter import Filter, FileFilter, DNSUtils from server.failmanager import FailManager from server.failmanager import FailManagerEmpty @@ -144,6 +144,20 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line # Actual tests # +class BasicFilter(unittest.TestCase): + + def setUp(self): + self.filter = Filter('name') + + def testGetSetUseDNS(self): + # default is warn + self.assertEqual(self.filter.getUseDns(), 'warn') + self.filter.setUseDns(True) + self.assertEqual(self.filter.getUseDns(), 'yes') + self.filter.setUseDns(False) + self.assertEqual(self.filter.getUseDns(), 'no') + + class IgnoreIP(LogCaptureTestCase): def setUp(self): From f3c42851180f08316a5b898ad5566000ab7b4af9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 10:46:52 +1100 Subject: [PATCH 47/74] TST: no test coverage on subclass overwritten function _delLogPath --- server/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/filter.py b/server/filter.py index 0ef756d2..52bb8c4c 100644 --- a/server/filter.py +++ b/server/filter.py @@ -446,7 +446,7 @@ class FileFilter(Filter): self._delLogPath(path) return - def _delLogPath(self, path): + def _delLogPath(self, path): # pragma: no cover - overwritten function # nothing to do by default # to be overridden by backends pass From 2b89457dc94249d44295cc5e79d17e0ec0fb3522 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 10:55:06 +1100 Subject: [PATCH 48/74] TST: addBanned IP when ignore exists --- testcases/filtertestcase.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 7b09ffed..29491ae5 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -34,6 +34,7 @@ from server.filterpoll import FilterPoll from server.filter import Filter, FileFilter, DNSUtils from server.failmanager import FailManager from server.failmanager import FailManagerEmpty +from dummyjail import DummyJail # # Useful helpers @@ -163,7 +164,8 @@ class IgnoreIP(LogCaptureTestCase): def setUp(self): """Call before every test case.""" LogCaptureTestCase.setUp(self) - self.filter = FileFilter(None) + self.jail = DummyJail() + self.filter = FileFilter(self.jail) def testIgnoreIPOK(self): ipList = "127.0.0.1", "192.168.0.1", "255.255.255.255", "99.99.99.99" @@ -202,6 +204,11 @@ class IgnoreIP(LogCaptureTestCase): self.filter.processLineAndAdd('Thu Jul 11 01:21:43 2013 192.168.1.32') self.assertTrue(self._is_logged('Ignore 192.168.1.32')) + def testIgnoreAddBannedIP(self): + self.filter.addIgnoreIP('192.168.1.0/25') + self.filter.addBannedIP('192.168.1.32') + self.assertFalse(self._is_logged('Ignore 192.168.1.32')) + self.assertTrue(self._is_logged('Requested to manually ban an ignored IP 192.168.1.32. User knows best. Proceeding to ban it.')) class LogFile(unittest.TestCase): @@ -347,7 +354,6 @@ class LogFileMonitor(LogCaptureTestCase): from threading import Lock -from dummyjail import DummyJail def get_monitor_failures_testcase(Filter_): """Generator of TestCase's for different filters/backends From c13b91fa709a39e8c402d55453178af35dd20c29 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 12:08:23 +1100 Subject: [PATCH 49/74] TST: separate out DNS based IgnoreIP tests --- fail2ban-testcases | 3 ++- testcases/filtertestcase.py | 25 ++++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/fail2ban-testcases b/fail2ban-testcases index 16c92c5f..21b8fda4 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -173,7 +173,8 @@ tests.addTest(unittest.makeSuite(misctestcase.CustomDateFormatsTest)) # Filter if not opts.no_network: - tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) + tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIPDNS)) +tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) tests.addTest(unittest.makeSuite(filtertestcase.BasicFilter)) tests.addTest(unittest.makeSuite(filtertestcase.LogFile)) tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor)) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 29491ae5..46870d41 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -173,21 +173,12 @@ class IgnoreIP(LogCaptureTestCase): self.filter.addIgnoreIP(ip) self.assertTrue(self.filter.inIgnoreIPList(ip)) - # Test DNS - self.filter.addIgnoreIP("www.epfl.ch") - - self.assertTrue(self.filter.inIgnoreIPList("128.178.50.12")) - self.assertFalse(self.filter.inIgnoreIPList("128.178.50.11")) - self.assertFalse(self.filter.inIgnoreIPList("128.178.50.13")) def testIgnoreIPNOK(self): ipList = "", "999.999.999.999", "abcdef", "192.168.0." for ip in ipList: self.filter.addIgnoreIP(ip) self.assertFalse(self.filter.inIgnoreIPList(ip)) - # Test DNS - self.filter.addIgnoreIP("www.epfl.ch") - self.assertFalse(self.filter.inIgnoreIPList("127.177.50.10")) def testIgnoreIPCIDR(self): self.filter.addIgnoreIP('192.168.1.0/25') @@ -210,6 +201,22 @@ class IgnoreIP(LogCaptureTestCase): self.assertFalse(self._is_logged('Ignore 192.168.1.32')) self.assertTrue(self._is_logged('Requested to manually ban an ignored IP 192.168.1.32. User knows best. Proceeding to ban it.')) + +class IgnoreIPDNS(IgnoreIP): + + def testIgnoreIPDNSOK(self): + self.filter.addIgnoreIP("www.epfl.ch") + self.assertTrue(self.filter.inIgnoreIPList("128.178.50.12")) + + def testIgnoreIPDNSNOK(self): + # Test DNS + self.filter.addIgnoreIP("www.epfl.ch") + print self._log.getvalue() + self.assertFalse(self.filter.inIgnoreIPList("127.177.50.10")) + self.assertFalse(self.filter.inIgnoreIPList("128.178.50.11")) + self.assertFalse(self.filter.inIgnoreIPList("128.178.50.13")) + + class LogFile(unittest.TestCase): FILENAME = "testcases/files/testcase01.log" From 33d96ae4626d0b9d40afef39dfc5a02bafb7aeb1 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 12:10:44 +1100 Subject: [PATCH 50/74] TST: separate out DNS based IgnoreIP tests --- testcases/filtertestcase.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 46870d41..a4d0230f 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -211,7 +211,6 @@ class IgnoreIPDNS(IgnoreIP): def testIgnoreIPDNSNOK(self): # Test DNS self.filter.addIgnoreIP("www.epfl.ch") - print self._log.getvalue() self.assertFalse(self.filter.inIgnoreIPList("127.177.50.10")) self.assertFalse(self.filter.inIgnoreIPList("128.178.50.11")) self.assertFalse(self.filter.inIgnoreIPList("128.178.50.13")) From 5005719180bb7a344b00740851fc0b133233f0bb Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 12:34:26 +1100 Subject: [PATCH 51/74] TST: permission denied on log file --- testcases/filtertestcase.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index a4d0230f..000ff353 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -269,6 +269,11 @@ class LogFileMonitor(LogCaptureTestCase): # shorter wait time for not modified status return not self.isModified(0.4) + def testNoLogFile(self): + os.chmod(self.name, 0) + self.filter.getFailures(self.name) + self.assertTrue(self._is_logged('Unable to open %s' % self.name)) + def testRemovingFailRegex(self): self.filter.delFailRegex(0) self.assertFalse(self._is_logged('Cannot remove regular expression. Index 0 is not valid')) From a03815facf198472273918b0763d97dd61286ac7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 Dec 2013 13:07:08 +1100 Subject: [PATCH 52/74] TST: FileFilter tail tests --- server/filter.py | 3 +++ testcases/filtertestcase.py | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/server/filter.py b/server/filter.py index 52bb8c4c..b92289ef 100644 --- a/server/filter.py +++ b/server/filter.py @@ -568,6 +568,9 @@ class FileContainer: def getFileName(self): return self.__filename + def getPos(self): + return self.__pos + def open(self): self.__handler = open(self.__filename) # Set the file descriptor to be FD_CLOEXEC diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 000ff353..7b18c4bf 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -603,7 +603,13 @@ class GetFailures(unittest.TestCase): def tearDown(self): """Call after every test case.""" - + def testTail(self): + self.filter.addLogPath(LogFile.FILENAME, tail=True) + self.assertEqual(self.filter.getLogPath()[-1].getPos(), 1653) + self.filter.getLogPath()[-1].close() + self.assertEqual(self.filter.getLogPath()[-1].readline(), "") + self.filter.delLogPath(LogFile.FILENAME) + self.assertEqual(self.filter.getLogPath(),[]) def testGetFailures01(self, filename=None, failures=None): filename = filename or GetFailures.FILENAME_01 From f2c58e74c144b91c59b6f3fee9d48098a30f437e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2013 08:24:29 +0000 Subject: [PATCH 53/74] TST: check client.JailReader.setName --- testcases/clientreadertestcase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 773d5072..0e3e6132 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -114,6 +114,8 @@ class JailReaderTest(unittest.TestCase): self.assertTrue(jail.getOptions()) self.assertFalse(jail.isEnabled()) self.assertEqual(jail.getName(), 'ssh-iptables') + jail.setName('ssh-funky-blocker') + self.assertEqual(jail.getName(), 'ssh-funky-blocker') def testSplitAction(self): action = "mail-whois[name=SSH]" From 970fd5d2891f59c7ac3943d8c50715badbd1047c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2013 08:52:01 +0000 Subject: [PATCH 54/74] BF: ensure dangling symlink error message is reachable $ ls -la /tmp/f2b-tempq0ipGY/f2 lrwxrwxrwx. 1 dan dan 11 Dec 12 08:42 /tmp/f2b-tempq0ipGY/f2 -> nonexisting In [3]: os.path.exists('/tmp/f2b-tempq0ipGY/f2') Out[3]: False In [4]: os.path.lexists('/tmp/f2b-tempq0ipGY/f2') Out[4]: True --- client/jailreader.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/client/jailreader.py b/client/jailreader.py index 7fbac423..8980431e 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -65,9 +65,10 @@ class JailReader(ConfigReader): pathList = [] for p in glob.glob(path): if not os.path.exists(p): - logSys.warning("File %s doesn't even exist, thus cannot be monitored" % p) - elif not os.path.lexists(p): - logSys.warning("File %s is a dangling link, thus cannot be monitored" % p) + if os.path.lexists(p): + logSys.warning("File %s is a dangling link, thus cannot be monitored" % p) + else: + logSys.warning("File %s doesn't even exist, thus cannot be monitored" % p) else: pathList.append(p) return pathList From f84a03d6b57b9a82fc42cd8e98dd71c11ddceeb8 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2013 09:08:42 +0000 Subject: [PATCH 55/74] BF: remove nonreachable parts of code Glob ensures the file exists so only a check that a missing dangling symlink needs to be done. $ ls -la /tmp/f2b-tempq0ipGY/f2 lrwxrwxrwx. 1 dan dan 11 Dec 12 08:42 /tmp/f2b-tempq0ipGY/f2 -> xisting In [3]: os.path.exists('/tmp/f2b-tempq0ipGY/f2') Out[3]: False In [4]: os.path.lexists('/tmp/f2b-tempq0ipGY/f2') Out[4]: True --- client/jailreader.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/client/jailreader.py b/client/jailreader.py index 8980431e..1c651fe9 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -64,13 +64,10 @@ class JailReader(ConfigReader): """ pathList = [] for p in glob.glob(path): - if not os.path.exists(p): - if os.path.lexists(p): - logSys.warning("File %s is a dangling link, thus cannot be monitored" % p) - else: - logSys.warning("File %s doesn't even exist, thus cannot be monitored" % p) - else: + if os.path.exists(p): pathList.append(p) + else: + logSys.warning("File %s is a dangling link, thus cannot be monitored" % p) return pathList def getOptions(self): From cb4f1e51422bd5976f1598995996ccaa4cf29cc9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2013 09:10:12 +0000 Subject: [PATCH 56/74] TST: remove temp files in glob test --- testcases/clientreadertestcase.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 0e3e6132..7f85dbd9 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -135,6 +135,10 @@ class JailReaderTest(unittest.TestCase): self.assertEqual(JailReader._glob(os.path.join(d, '*')), [os.path.join(d, 'f1')]) # since f2 is dangling -- empty list self.assertEqual(JailReader._glob(os.path.join(d, 'f2')), []) + self.assertEqual(JailReader._glob(os.path.join(d, 'nonexisting')), []) + os.remove(os.path.join(d, 'f1')) + os.remove(os.path.join(d, 'f2')) + os.rmdir(d) class JailsReaderTest(unittest.TestCase): From 3036afca9167524e7cd54fb54fb63507f040fb12 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2013 10:13:57 +0000 Subject: [PATCH 57/74] TST: check dangling link log message --- testcases/clientreadertestcase.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 7f85dbd9..0937a5a5 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -27,6 +27,7 @@ from client.configreader import ConfigReader from client.jailreader import JailReader from client.jailsreader import JailsReader from client.configurator import Configurator +from utils import LogCaptureTestCase class ConfigReaderTest(unittest.TestCase): @@ -106,7 +107,7 @@ option = %s self.assertEqual(self._getoption(), 1) -class JailReaderTest(unittest.TestCase): +class JailReaderTest(LogCaptureTestCase): def testStockSSHJail(self): jail = JailReader('ssh-iptables', basedir='config') # we are running tests from root project dir atm @@ -127,17 +128,21 @@ class JailReaderTest(unittest.TestCase): d = tempfile.mkdtemp(prefix="f2b-temp") # Generate few files # regular file - open(os.path.join(d, 'f1'), 'w').close() + f1 = os.path.join(d, 'f1') + open(f1, 'w').close() # dangling link - os.symlink('nonexisting', os.path.join(d, 'f2')) + + f2 = os.path.join(d, 'f2') + os.symlink('nonexisting',f2) # must be only f1 - self.assertEqual(JailReader._glob(os.path.join(d, '*')), [os.path.join(d, 'f1')]) + self.assertEqual(JailReader._glob(os.path.join(d, '*')), [f1]) # since f2 is dangling -- empty list - self.assertEqual(JailReader._glob(os.path.join(d, 'f2')), []) + self.assertEqual(JailReader._glob(f2), []) + self.assertTrue(self._is_logged('File %s is a dangling link, thus cannot be monitored' % f2)) self.assertEqual(JailReader._glob(os.path.join(d, 'nonexisting')), []) - os.remove(os.path.join(d, 'f1')) - os.remove(os.path.join(d, 'f2')) + os.remove(f1) + os.remove(f2) os.rmdir(d) class JailsReaderTest(unittest.TestCase): From b18ce122dd236776d500799475b675d52d7840b8 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2013 20:07:09 +0000 Subject: [PATCH 58/74] BF/ENH: fix error when action doesn't match regex. Document unreachable code. Simplify regex --- client/jailreader.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/client/jailreader.py b/client/jailreader.py index 1c651fe9..2138acee 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -35,7 +35,7 @@ logSys = logging.getLogger("fail2ban.client.config") class JailReader(ConfigReader): - actionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") + actionCRE = re.compile("^([\w_.-]+)(?:\[(.*)\])?$") def __init__(self, name, force_enable=False, **kwargs): ConfigReader.__init__(self, **kwargs) @@ -173,12 +173,16 @@ class JailReader(ConfigReader): def splitAction(action): m = JailReader.actionCRE.match(action) d = dict() - mgroups = m.groups() + try: + mgroups = m.groups() + except AttributeError: + raise ValueError("While reading action %s we should have got 1 or " + "2 groups. Got: 0" % action) if len(mgroups) == 2: action_name, action_opts = mgroups elif len(mgroups) == 1: action_name, action_opts = mgroups[0], None - else: + else: # pragma: nocover - unreachable - regex only can capture 2 groups raise ValueError("While reading action %s we should have got up to " "2 groups. Got: %r" % (action, mgroups)) if not action_opts is None: From c6d14dcf0eedef9513b3b5fb5b028ff301908734 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2013 20:35:30 +0000 Subject: [PATCH 59/74] TST: complete coverage of splitAction --- client/jailreader.py | 2 +- testcases/clientreadertestcase.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/client/jailreader.py b/client/jailreader.py index 2138acee..131a4dd5 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -180,7 +180,7 @@ class JailReader(ConfigReader): "2 groups. Got: 0" % action) if len(mgroups) == 2: action_name, action_opts = mgroups - elif len(mgroups) == 1: + elif len(mgroups) == 1: # pragma: nocover - unreachable - .* on second group always matches action_name, action_opts = mgroups[0], None else: # pragma: nocover - unreachable - regex only can capture 2 groups raise ValueError("While reading action %s we should have got up to " diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 0937a5a5..c22f028a 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -123,7 +123,19 @@ class JailReaderTest(LogCaptureTestCase): expected = ['mail-whois', {'name': 'SSH'}] result = JailReader.splitAction(action) self.assertEqual(expected, result) + + self.assertEqual(['mail.who_is', {}], JailReader.splitAction("mail.who_is")) + self.assertEqual(['mail.who_is', {'a':'cat', 'b':'dog'}], JailReader.splitAction("mail.who_is[a=cat,b=dog]")) + self.assertEqual(['mail--ho_is', {}], JailReader.splitAction("mail--ho_is")) + + self.assertEqual(['mail--ho_is', {}], JailReader.splitAction("mail--ho_is['s']")) + self.assertTrue(self._is_logged("Invalid argument ['s'] in ''s''")) + + self.assertEqual(['mail', {'a': ','}], JailReader.splitAction("mail[a=',']")) + self.assertRaises(ValueError, JailReader.splitAction ,'mail-how[') + + def testGlob(self): d = tempfile.mkdtemp(prefix="f2b-temp") # Generate few files From 3ddf8da76e6c69e0f91c9f2fdebba96f9932eff3 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 13 Dec 2013 08:45:10 +0000 Subject: [PATCH 60/74] ENH: ensure filter is defined in jail before its read --- client/jailreader.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/client/jailreader.py b/client/jailreader.py index 131a4dd5..1b5bf1ae 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -87,15 +87,18 @@ class JailReader(ConfigReader): if self.isEnabled(): # Read filter - self.__filter = FilterReader(self.__opts["filter"], self.__name, - basedir=self.getBaseDir()) - ret = self.__filter.read() - if ret: - self.__filter.getOptions(self.__opts) + if self.__opts["filter"]: + self.__filter = FilterReader(self.__opts["filter"], self.__name, + basedir=self.getBaseDir()) + ret = self.__filter.read() + if ret: + self.__filter.getOptions(self.__opts) + else: + logSys.error("Unable to read the filter") + return False else: - logSys.error("Unable to read the filter") - return False - + logSys.warn("No filter set for jail %s" % self.__name) + # Read action for act in self.__opts["action"].split('\n'): try: From d74dd31d2301d7e0c5479192405bdc55a70b3c62 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 13 Dec 2013 10:00:34 +0000 Subject: [PATCH 61/74] BF: corrected tests for missing jail Previously tests relied on the missing filter to trigger the conditions required for a missing jail. We now handle this explicitly. --- client/configreader.py | 2 +- client/jailreader.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/client/configreader.py b/client/configreader.py index 96aab5f3..470f029f 100644 --- a/client/configreader.py +++ b/client/configreader.py @@ -112,7 +112,7 @@ class ConfigReader(SafeConfigParserWithIncludes): except NoSectionError, e: # No "Definition" section or wrong basedir logSys.error(e) - values[option[1]] = option[2] + return False except NoOptionError: if not option[2] is None: logSys.warn("'%s' not defined in '%s'. Using default one: %r" diff --git a/client/jailreader.py b/client/jailreader.py index 1b5bf1ae..d7c7b84c 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -54,7 +54,7 @@ class JailReader(ConfigReader): return ConfigReader.read(self, "jail") def isEnabled(self): - return self.__force_enable or self.__opts["enabled"] + return self.__force_enable or ( self.__opts and self.__opts["enabled"] ) @staticmethod def _glob(path): @@ -84,6 +84,8 @@ class JailReader(ConfigReader): ["string", "filter", ""], ["string", "action", ""]] self.__opts = ConfigReader.getOptions(self, self.__name, opts) + if not self.__opts: + return False if self.isEnabled(): # Read filter From 1407b955e6673c025c8bd6252f7daca1251ea477 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 13 Dec 2013 10:03:51 +0000 Subject: [PATCH 62/74] TST: more client/jailreader tests --- MANIFEST | 4 ++++ testcases/clientreadertestcase.py | 19 ++++++++++++++++++- testcases/config/action.d/brokenaction.conf | 4 ++++ testcases/config/fail2ban.conf | 5 +++++ testcases/config/filter.d/simple.conf | 4 ++++ testcases/config/jail.conf | 20 ++++++++++++++++++++ 6 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 testcases/config/action.d/brokenaction.conf create mode 100644 testcases/config/fail2ban.conf create mode 100644 testcases/config/filter.d/simple.conf create mode 100644 testcases/config/jail.conf diff --git a/MANIFEST b/MANIFEST index 0e0eb327..a7fefd5c 100644 --- a/MANIFEST +++ b/MANIFEST @@ -242,3 +242,7 @@ files/fail2ban-tmpfiles.conf files/fail2ban.service files/ipmasq-ZZZzzz_fail2ban.rul files/gen_badbots +testcases/config/jail.conf +testcases/config/fail2ban.conf +testcases/config/filter.d/simple.conf +testcases/config/action.d/brokenaction.conf diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index c22f028a..2dec428e 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -109,6 +109,22 @@ option = %s class JailReaderTest(LogCaptureTestCase): + def testJailActionEmpty(self): + jail = JailReader('emptyaction', basedir=os.path.join('testcases','config')) + self.assertTrue(jail.read()) + self.assertTrue(jail.getOptions()) + self.assertTrue(jail.isEnabled()) + self.assertTrue(self._is_logged('No filter set for jail emptyaction')) + self.assertTrue(self._is_logged('No actions were defined for emptyaction')) + + def testJailActionBrokenDef(self): + jail = JailReader('brokenactiondef', basedir=os.path.join('testcases','config')) + self.assertTrue(jail.read()) + self.assertFalse(jail.getOptions()) + self.assertTrue(jail.isEnabled()) + self.assertTrue(self._is_logged('Error in action definition joho[foo')) + self.assertTrue(self._is_logged('Caught exception: While reading action joho[foo we should have got 1 or 2 groups. Got: 0')) + def testStockSSHJail(self): jail = JailReader('ssh-iptables', basedir='config') # we are running tests from root project dir atm self.assertTrue(jail.read()) @@ -157,7 +173,7 @@ class JailReaderTest(LogCaptureTestCase): os.remove(f2) os.rmdir(d) -class JailsReaderTest(unittest.TestCase): +class JailsReaderTest(LogCaptureTestCase): def testProvidingBadBasedir(self): if not os.path.exists('/XXX'): @@ -176,6 +192,7 @@ class JailsReaderTest(unittest.TestCase): # We should not "read" some bogus jail old_comm_commands = comm_commands[:] # make a copy self.assertFalse(jails.getOptions("BOGUS")) + self.assertTrue(self._is_logged("No section: 'BOGUS'")) # and there should be no side-effects self.assertEqual(jails.convert(), old_comm_commands) diff --git a/testcases/config/action.d/brokenaction.conf b/testcases/config/action.d/brokenaction.conf new file mode 100644 index 00000000..d2c8d059 --- /dev/null +++ b/testcases/config/action.d/brokenaction.conf @@ -0,0 +1,4 @@ + +[Definition] + +actioban = hit with big stick diff --git a/testcases/config/fail2ban.conf b/testcases/config/fail2ban.conf new file mode 100644 index 00000000..36984c78 --- /dev/null +++ b/testcases/config/fail2ban.conf @@ -0,0 +1,5 @@ +[Definition] + +# 3 = INFO +loglevel = 3 + diff --git a/testcases/config/filter.d/simple.conf b/testcases/config/filter.d/simple.conf new file mode 100644 index 00000000..4a2c0bb7 --- /dev/null +++ b/testcases/config/filter.d/simple.conf @@ -0,0 +1,4 @@ + +[Definition] + +failregex = diff --git a/testcases/config/jail.conf b/testcases/config/jail.conf new file mode 100644 index 00000000..419e2e2c --- /dev/null +++ b/testcases/config/jail.conf @@ -0,0 +1,20 @@ + +[DEFAULT] +filter = simple + +[emptyaction] +enabled = true +filter = +action = + +[brokenactiondef] +enabled = true +action = joho[foo + +[brokenaction] +enabled = true +action = brokenaction + +[missingaction] +enabled = true +action = thefunkychickendance From e916fcdce4cd5a0b927644c9f79e75375e8141d0 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 13 Dec 2013 10:51:38 +0000 Subject: [PATCH 63/74] TST: test case for actions and filters missing in a jail --- testcases/clientreadertestcase.py | 9 +++++++++ testcases/config/jail.conf | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 2dec428e..e85ddff8 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -117,6 +117,15 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(self._is_logged('No filter set for jail emptyaction')) self.assertTrue(self._is_logged('No actions were defined for emptyaction')) + def testJailActionFilterMissing(self): + jail = JailReader('missingbitsjail', basedir=os.path.join('testcases','config')) + self.assertTrue(jail.read()) + self.assertFalse(jail.getOptions()) + self.assertTrue(jail.isEnabled()) + #print self._log.getvalue() + self.assertTrue(self._is_logged("Found no accessible config files for 'filter.d/catchallthebadies' under testcases/config")) + self.assertTrue(self._is_logged('Unable to read the filter')) + def testJailActionBrokenDef(self): jail = JailReader('brokenactiondef', basedir=os.path.join('testcases','config')) self.assertTrue(jail.read()) diff --git a/testcases/config/jail.conf b/testcases/config/jail.conf index 419e2e2c..4bdd74cf 100644 --- a/testcases/config/jail.conf +++ b/testcases/config/jail.conf @@ -15,6 +15,6 @@ action = joho[foo enabled = true action = brokenaction -[missingaction] -enabled = true +[missingbitsjail] +filter = catchallthebadies action = thefunkychickendance From f6fb737e6cece7d535e64a848fcc0b137519e51c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 13 Dec 2013 10:55:15 +0000 Subject: [PATCH 64/74] TST: remove commented test print --- testcases/clientreadertestcase.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index e85ddff8..96f9134f 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -122,7 +122,6 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) - #print self._log.getvalue() self.assertTrue(self._is_logged("Found no accessible config files for 'filter.d/catchallthebadies' under testcases/config")) self.assertTrue(self._is_logged('Unable to read the filter')) From 2f3648c458e675de99acac6fef0a506c49a0daac Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 13 Dec 2013 11:11:58 +0000 Subject: [PATCH 65/74] DOC: add missing jail directives --- man/jail.conf.5 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 8ea44316..6c114c56 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -63,6 +63,12 @@ Comments: use '#' for comment lines and ';' (following a space) for inline comme .SH DEFAULT The following options are applicable to all jails. Their meaning is described in the default \fIjail.conf\fR file. .TP +\fBfilter\fR +.TP +\fBlogpath\fR +.TP +\fBaction\fR +.TP \fBignoreip\fR .TP \fBbantime\fR @@ -74,6 +80,10 @@ The following options are applicable to all jails. Their meaning is described in \fBbackend\fR .TP \fBusedns\fR +.TP +\fBfailregex\fR +.TP +\fBignoreregex\fR .SH "ACTION FILES" From b147270be769ff5b456e29d56d12d27dacd95480 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 13 Dec 2013 11:36:00 +0000 Subject: [PATCH 66/74] BF: allow processing with empty filter --- client/jailreader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/jailreader.py b/client/jailreader.py index d7c7b84c..6f78275b 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -99,6 +99,7 @@ class JailReader(ConfigReader): logSys.error("Unable to read the filter") return False else: + self.__filter = None logSys.warn("No filter set for jail %s" % self.__name) # Read action @@ -168,7 +169,8 @@ class JailReader(ConfigReader): # Do not send a command if the rule is empty. if regex != '': stream.append(["set", self.__name, "addignoreregex", regex]) - stream.extend(self.__filter.convert()) + if self.__filter: + stream.extend(self.__filter.convert()) for action in self.__actions: stream.extend(action.convert()) stream.insert(0, ["add", self.__name, backend]) From 18f0e58caad68212e84f711526e3e1857ae7ae7b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 13 Dec 2013 11:41:40 +0000 Subject: [PATCH 67/74] TST: increase coverage in jailreader --- testcases/clientreadertestcase.py | 32 +++++++++++++++++++++++++++++++ testcases/config/jail.conf | 12 ++++++++++++ 2 files changed, 44 insertions(+) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 96f9134f..f55be051 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -188,6 +188,38 @@ class JailsReaderTest(LogCaptureTestCase): reader = JailsReader(basedir='/XXX') self.assertRaises(ValueError, reader.read) + def testReadTestJailConf(self): + jails = JailsReader(basedir=os.path.join('testcases','config')) + self.assertTrue(jails.read()) # opens fine + self.assertFalse(jails.getOptions()) # reads not sof ine + self.assertRaises(ValueError, jails.convert) + comm_commands = jails.convert(allow_no_files=True) + self.maxDiff = None + self.assertEqual(comm_commands, + [['add', 'emptyaction', 'auto'], + ['set', 'emptyaction', 'usedns', 'warn'], + ['set', 'emptyaction', 'addlogpath', '/var/log/messages'], + ['set', 'emptyaction', 'maxretry', 3], + ['set', 'emptyaction', 'findtime', 600], + ['set', 'emptyaction', 'bantime', 600], + ['add', 'special', 'auto'], + ['set', 'special', 'usedns', 'warn'], + ['set', 'special', 'addlogpath', '/var/log/messages'], + ['set', 'special', 'maxretry', 3], + ['set', 'special', 'addfailregex', ''], + ['set', 'special', 'findtime', 600], + ['set', 'special', 'bantime', 600], + ['add', 'missinglogfiles', 'auto'], + ['set', 'missinglogfiles', 'usedns', 'warn'], + ['set', 'missinglogfiles', 'maxretry', 3], + ['set', 'missinglogfiles', 'findtime', 600], + ['set', 'missinglogfiles', 'bantime', 600], + ['set', 'missinglogfiles', 'addfailregex', ''], + ['start', 'emptyaction'], + ['start', 'special'], + ['start', 'missinglogfiles']]) + + def testReadStockJailConf(self): jails = JailsReader(basedir='config') # we are running tests from root project dir atm self.assertTrue(jails.read()) # opens fine diff --git a/testcases/config/jail.conf b/testcases/config/jail.conf index 4bdd74cf..ab791451 100644 --- a/testcases/config/jail.conf +++ b/testcases/config/jail.conf @@ -7,6 +7,14 @@ enabled = true filter = action = +[special] +failregex = +ignoreregex = +ignoreip = + +[missinglogfiles] +logpath = /weapons/of/mass/destruction + [brokenactiondef] enabled = true action = joho[foo @@ -18,3 +26,7 @@ action = brokenaction [missingbitsjail] filter = catchallthebadies action = thefunkychickendance + +[parse_to_end_of_jail.conf] +enabled = true +action = From 13ccebe78f2c9eafc148382559a79ced32354b50 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 13 Dec 2013 23:40:51 +0000 Subject: [PATCH 68/74] BF: fix actioncheck in firewallcmd --- ChangeLog | 1 + THANKS | 1 + config/action.d/firewallcmd-new.conf | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index a0624630..0eca4e37 100644 --- a/ChangeLog +++ b/ChangeLog @@ -26,6 +26,7 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better settings. Closes gh-458, Debian bug #697333, Redhat bug #891798. - complain action - ensure where not matching other IPs in log sample. Closes gh-467 + - Fix firewall-cmd actioncheck - patch from Adam Tkac. Redhat Bug #979622 - Enhancements: - long names on jails documented based on iptables limit of 30 less diff --git a/THANKS b/THANKS index 6d4845bb..e448e09e 100644 --- a/THANKS +++ b/THANKS @@ -6,6 +6,7 @@ the project. If you have been left off, please let us know (preferably send a pull request on github with the "fix") and you will be added +Adam Tkac Adrien Clerc ache ag4ve (Shawn) diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf index 55b6762d..d3443e4e 100644 --- a/config/action.d/firewallcmd-new.conf +++ b/config/action.d/firewallcmd-new.conf @@ -20,7 +20,7 @@ actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state firewall-cmd --direct --remove-rules ipv4 filter fail2ban- firewall-cmd --direct --remove-chain ipv4 filter fail2ban- -actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q 'fail2ban-[ \t]' +actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -Eq 'fail2ban-$|fail2ban- ' actionban = firewall-cmd --direct --add-rule ipv4 filter fail2ban- 0 -s -j From b39729a2ab1abea9835fb4f5377c60d56308df9b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 14 Dec 2013 06:51:36 +0000 Subject: [PATCH 69/74] BF: fix unintential typo --- testcases/config/action.d/brokenaction.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/config/action.d/brokenaction.conf b/testcases/config/action.d/brokenaction.conf index d2c8d059..59e97b7f 100644 --- a/testcases/config/action.d/brokenaction.conf +++ b/testcases/config/action.d/brokenaction.conf @@ -1,4 +1,4 @@ [Definition] -actioban = hit with big stick +actionban = hit with big stick From 603095bc16ff08798cf16b789bc0360aa62ed112 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 14 Dec 2013 07:00:41 +0000 Subject: [PATCH 70/74] BF: errors in a jail prevents further sections from being parsed. Closes #485 --- client/jailsreader.py | 5 +++-- testcases/clientreadertestcase.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/client/jailsreader.py b/client/jailsreader.py index 00c63e3c..d32e6561 100644 --- a/client/jailsreader.py +++ b/client/jailsreader.py @@ -60,6 +60,7 @@ class JailsReader(ConfigReader): sections = [ section ] # Get the options of all jails. + parse_status = True for sec in sections: jail = JailReader(sec, basedir=self.getBaseDir(), force_enable=self.__force_enable) @@ -71,8 +72,8 @@ class JailsReader(ConfigReader): self.__jails.append(jail) else: logSys.error("Errors in jail %r. Skipping..." % sec) - return False - return True + parse_status = False + return parse_status def convert(self, allow_no_files=False): """Convert read before __opts and jails to the commands stream diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index f55be051..91689836 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -215,9 +215,37 @@ class JailsReaderTest(LogCaptureTestCase): ['set', 'missinglogfiles', 'findtime', 600], ['set', 'missinglogfiles', 'bantime', 600], ['set', 'missinglogfiles', 'addfailregex', ''], + ['add', 'brokenaction', 'auto'], + ['set', 'brokenaction', 'usedns', 'warn'], + ['set', 'brokenaction', 'addlogpath', '/var/log/messages'], + ['set', 'brokenaction', 'maxretry', 3], + ['set', 'brokenaction', 'findtime', 600], + ['set', 'brokenaction', 'bantime', 600], + ['set', 'brokenaction', 'addfailregex', ''], + ['set', 'brokenaction', 'addaction', 'brokenaction'], + ['set', + 'brokenaction', + 'actionban', + 'brokenaction', + 'hit with big stick '], + ['set', 'brokenaction', 'actionstop', 'brokenaction', ''], + ['set', 'brokenaction', 'actionstart', 'brokenaction', ''], + ['set', 'brokenaction', 'actionunban', 'brokenaction', ''], + ['set', 'brokenaction', 'actioncheck', 'brokenaction', ''], + ['add', 'parse_to_end_of_jail.conf', 'auto'], + ['set', 'parse_to_end_of_jail.conf', 'usedns', 'warn'], + ['set', 'parse_to_end_of_jail.conf', 'addlogpath', '/var/log/messages'], + ['set', 'parse_to_end_of_jail.conf', 'maxretry', 3], + ['set', 'parse_to_end_of_jail.conf', 'findtime', 600], + ['set', 'parse_to_end_of_jail.conf', 'bantime', 600], + ['set', 'parse_to_end_of_jail.conf', 'addfailregex', ''], ['start', 'emptyaction'], ['start', 'special'], - ['start', 'missinglogfiles']]) + ['start', 'missinglogfiles'], + ['start', 'brokenaction'], + ['start', 'parse_to_end_of_jail.conf'],]) + self.assertTrue(self._is_logged("Errors in jail 'missingbitsjail'. Skipping...")) + self.assertTrue(self._is_logged("No file(s) found for glob /weapons/of/mass/destruction")) def testReadStockJailConf(self): From 4ffc57e14f1254942c88c9d9f94e9ef791af5e5f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 14 Dec 2013 07:11:29 +0000 Subject: [PATCH 71/74] ENH: simplify firewallcmd-new actioncheck and provide output samples --- config/action.d/firewallcmd-new.conf | 30 +++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf index d3443e4e..7a5c03de 100644 --- a/config/action.d/firewallcmd-new.conf +++ b/config/action.d/firewallcmd-new.conf @@ -1,9 +1,5 @@ # Fail2Ban configuration file # -# Author: Edgar Hoch -# Copied from iptables-new.conf and modified for use with firewalld by Edgar Hoch. -# It uses "firewall-cmd" instead of "iptables". -# # Because of the --remove-rules in stop this action requires firewalld-0.3.8+ [INCLUDES] @@ -20,7 +16,7 @@ actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state firewall-cmd --direct --remove-rules ipv4 filter fail2ban- firewall-cmd --direct --remove-chain ipv4 filter fail2ban- -actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -Eq 'fail2ban-$|fail2ban- ' +actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -Eq 'fail2ban-( |$)' actionban = firewall-cmd --direct --add-rule ipv4 filter fail2ban- 0 -s -j @@ -50,3 +46,27 @@ protocol = tcp # Values: [ STRING ] # chain = INPUT_direct + +# DEV NOTES: +# +# Author: Edgar Hoch +# Copied from iptables-new.conf and modified for use with firewalld by Edgar Hoch. +# It uses "firewall-cmd" instead of "iptables". +# +# Output: +# +# $ firewall-cmd --direct --add-chain ipv4 filter fail2ban-name +# success +# $ firewall-cmd --direct --add-rule ipv4 filter fail2ban-name 1000 -j RETURN +# success +# $ sudo firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m state --state NEW -p tcp --dport 22 -j fail2ban-name +# success +# $ firewall-cmd --direct --get-chains ipv4 filter +# fail2ban-name +# $ firewall-cmd --direct --get-chains ipv4 filter | od -h +# 0000000 6166 6c69 6232 6e61 6e2d 6d61 0a65 +# $ firewall-cmd --direct --get-chains ipv4 filter | grep -Eq 'fail2ban-name( |$)' ; echo $? +# 0 +# $ firewall-cmd -V +# 0.3.8 + From a398c51d6c49ac5de70f9eee68f8c72c1fed10ba Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 15 Dec 2013 22:36:47 +0000 Subject: [PATCH 72/74] ENH: simplify actioncheck on firewallcmd-new a little more --- config/action.d/firewallcmd-new.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf index 7a5c03de..bae72ca2 100644 --- a/config/action.d/firewallcmd-new.conf +++ b/config/action.d/firewallcmd-new.conf @@ -16,7 +16,7 @@ actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state firewall-cmd --direct --remove-rules ipv4 filter fail2ban- firewall-cmd --direct --remove-chain ipv4 filter fail2ban- -actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -Eq 'fail2ban-( |$)' +actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q '^fail2ban-$' actionban = firewall-cmd --direct --add-rule ipv4 filter fail2ban- 0 -s -j From 5c26bcbd2b24328a4fc85be0360090ee787bf6d9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 16 Dec 2013 10:07:41 +0000 Subject: [PATCH 73/74] TST: hopefully normalise config so that consistent test results occur on travis and locally --- testcases/clientreadertestcase.py | 4 ---- testcases/config/jail.conf | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 91689836..42c3fa28 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -198,13 +198,11 @@ class JailsReaderTest(LogCaptureTestCase): self.assertEqual(comm_commands, [['add', 'emptyaction', 'auto'], ['set', 'emptyaction', 'usedns', 'warn'], - ['set', 'emptyaction', 'addlogpath', '/var/log/messages'], ['set', 'emptyaction', 'maxretry', 3], ['set', 'emptyaction', 'findtime', 600], ['set', 'emptyaction', 'bantime', 600], ['add', 'special', 'auto'], ['set', 'special', 'usedns', 'warn'], - ['set', 'special', 'addlogpath', '/var/log/messages'], ['set', 'special', 'maxretry', 3], ['set', 'special', 'addfailregex', ''], ['set', 'special', 'findtime', 600], @@ -217,7 +215,6 @@ class JailsReaderTest(LogCaptureTestCase): ['set', 'missinglogfiles', 'addfailregex', ''], ['add', 'brokenaction', 'auto'], ['set', 'brokenaction', 'usedns', 'warn'], - ['set', 'brokenaction', 'addlogpath', '/var/log/messages'], ['set', 'brokenaction', 'maxretry', 3], ['set', 'brokenaction', 'findtime', 600], ['set', 'brokenaction', 'bantime', 600], @@ -234,7 +231,6 @@ class JailsReaderTest(LogCaptureTestCase): ['set', 'brokenaction', 'actioncheck', 'brokenaction', ''], ['add', 'parse_to_end_of_jail.conf', 'auto'], ['set', 'parse_to_end_of_jail.conf', 'usedns', 'warn'], - ['set', 'parse_to_end_of_jail.conf', 'addlogpath', '/var/log/messages'], ['set', 'parse_to_end_of_jail.conf', 'maxretry', 3], ['set', 'parse_to_end_of_jail.conf', 'findtime', 600], ['set', 'parse_to_end_of_jail.conf', 'bantime', 600], diff --git a/testcases/config/jail.conf b/testcases/config/jail.conf index ab791451..525308e3 100644 --- a/testcases/config/jail.conf +++ b/testcases/config/jail.conf @@ -1,6 +1,7 @@ [DEFAULT] filter = simple +logpath = /non/exist [emptyaction] enabled = true From 729929ada98f63558ae8cd61586868173bc234ef Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 16 Dec 2013 10:21:46 +0000 Subject: [PATCH 74/74] TST: jails can occur in any order once parsed. Sort results to facilitate comparison --- testcases/clientreadertestcase.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 42c3fa28..28387703 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -190,13 +190,13 @@ class JailsReaderTest(LogCaptureTestCase): def testReadTestJailConf(self): jails = JailsReader(basedir=os.path.join('testcases','config')) - self.assertTrue(jails.read()) # opens fine - self.assertFalse(jails.getOptions()) # reads not sof ine + self.assertTrue(jails.read()) + self.assertFalse(jails.getOptions()) self.assertRaises(ValueError, jails.convert) comm_commands = jails.convert(allow_no_files=True) self.maxDiff = None - self.assertEqual(comm_commands, - [['add', 'emptyaction', 'auto'], + self.assertEqual(sorted(comm_commands), + sorted([['add', 'emptyaction', 'auto'], ['set', 'emptyaction', 'usedns', 'warn'], ['set', 'emptyaction', 'maxretry', 3], ['set', 'emptyaction', 'findtime', 600], @@ -239,7 +239,7 @@ class JailsReaderTest(LogCaptureTestCase): ['start', 'special'], ['start', 'missinglogfiles'], ['start', 'brokenaction'], - ['start', 'parse_to_end_of_jail.conf'],]) + ['start', 'parse_to_end_of_jail.conf'],])) self.assertTrue(self._is_logged("Errors in jail 'missingbitsjail'. Skipping...")) self.assertTrue(self._is_logged("No file(s) found for glob /weapons/of/mass/destruction"))